VERSION=0.23

BIN_OPT:= \
  create-graph.native \
  print-stats.native \
  bin2src.native \
  src2bin.native \
  build-fixpoint.native \
  clean-repository.native \
  buildgraph2srcgraph.native \
  annotate-strong.native \
  partial-order.native \
  find-fvs.native \
  collapse-srcgraph.native \
  optuniv.native \
  calculate-fas.native \
  buildcheck-more-problems.native \
  distcheck-more-problems.native

BIN_BYTE:=$(patsubst %.native, %.byte, $(BIN_OPT))
BIN_DEBUG:=$(patsubst %.native, %.d.byte, $(BIN_OPT))
BIN_PROF:=$(patsubst %.native, %.p.native, $(BIN_OPT))

BIN_PYTOOLS:=$(patsubst tools/%.py, %, $(filter-out tools/util.py tools/debarch.py,$(wildcard tools/*.py)))
BIN_OCTOOLS:=$(patsubst %.native, %, $(BIN_OPT))
BIN_SHTOOLS:=$(patsubst tools/%.sh, %, $(wildcard tools/*.sh))

BIN_TOOLS:=$(BIN_PYTOOLS) $(BIN_OCTOOLS) $(BIN_SHTOOLS)
MANPAGES:=$(patsubst %, doc/man/botch-%.1, $(BIN_TOOLS))

OCAMLBUILD_BEST ?= $(if $(wildcard /usr/bin/ocamlopt),native,byte)

PWD := $(shell pwd)
BUILD = $(PWD)/dose/_build
DOSELIBS = $(PWD)/dose/_build/dose3
CUDFLIBS = $(PWD)/dose/_build/cudf
DOSEBYTELIBS = \
  _build/doselibs/common.cma \
  _build/doselibs/debian.cma \
  _build/doselibs/versioning.cma \
  _build/doselibs/csw.cma \
  _build/doselibs/pef.cma \
  _build/doselibs/algo.cma \
  _build/doselibs/doseparse.cma \
  _build/doselibs/doseparseNoRpm.cma
# we turn on all warnings but turn off:
#   4  Fragile pattern matching: matching that will remain complete even if additional constructors are added to one of the variant types matched.
#   9  Missing fields in a record pattern.
#   24 Bad module name: the source file name is not a valid OCaml module name.
CFLAGS=-cflags "-w +a-4-9-24"
bindir=/usr/bin
datadir=/usr/share/botch
mandir=/usr/share/man

# we have to export PYTHONHASHSEED because otherwise networkx will create
# content with random output order
#   https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=749710
#   https://github.com/networkx/networkx/issues/1181
# on the other hand, the generated hashes are not the same on 32 vs 64 arches
#   http://bugs.python.org/issue22621
#   http://bugs.debian.org/778373
export PYTHONHASHSEED=0
# to make the sort order in the tests locale independent
export LC_COLLATE=C.UTF-8
export LC_ALL=C.UTF-8
# because dose2html outputs utf8 on standard output
export PYTHONIOENCODING=utf-8

.PHONY: all
all: $(OCAMLBUILD_BEST) wiki man

.PHONY: native
native: atdgen
	OCAMLPATH=$(BUILD) ocamlbuild -lflags -runtime-variant,_pic -classic-display -use-ocamlfind $(CFLAGS) $(BIN_OPT)

.PHONY: byte
byte: atdgen
	OCAMLPATH=$(BUILD) ocamlbuild -classic-display -use-ocamlfind $(CFLAGS) $(BIN_BYTE)

.PHONY: debug
debug: atdgen
	OCAMLPATH=$(BUILD) ocamlbuild -classic-display -use-ocamlfind $(CFLAGS) $(BIN_DEBUG)

# profiling for use of native executables with gprof
.PHONY: profile
profile: atdgen
	OCAMLPATH=$(BUILD) ocamlbuild -classic-display -use-ocamlfind $(CFLAGS) $(BIN_PROF)

doc/man/botch.1: $(MANPAGES)
	./doc/man/generate-botch-manpage.py | pod2man --section 1 --center="botch tools" --name botch > $@

.PHONY: man
man: $(MANPAGES) doc/man/botch.1

.PHONY: wiki
wiki:
	$(MAKE) -C doc/wiki

doc/man/%.1: doc/man/%.pod
	pod2man --section 1 --center="botch tools" $< > $@

.PHONY: doselib
doselib:
	(cd dose && ./configure --with-zip --with-bz2 && make clean libs $(DOSEBYTELIBS))
	rm -Rf $(DOSELIBS)
	mkdir -p $(DOSELIBS)
	cp dose/META $(DOSELIBS)
	for i in \
		dose/_build/*/common.* \
		dose/_build/*/versioning.* \
		dose/_build/*/debian.* \
		dose/_build/*/algo.* \
		dose/_build/*/csw.* \
		dose/_build/*/pef.* \
		dose/_build/*/doseparseNoRpm.* \
		dose/_build/*/doseparse.*; do \
	  if [ -e $$i ]; then \
	  cp $$i $(DOSELIBS) ; \
	  rm -f $(DOSELIBS)/*.mlpack $(DOSELIBS)/*.cmx ; \
	  fi ; \
	done
	rm -Rf $(CUDFLIBS)
	mkdir -p $(CUDFLIBS)
	cp dose/cudf/META $(CUDFLIBS)
	cp dose/_build/doselibs/cudf*.cmi $(CUDFLIBS)
	for i in dose/_build/doselibs/cudf.*; do \
	  if [ -e $$i ]; then \
	    cp $$i $(CUDFLIBS) ; \
	    rm -f $(CUDFLIBS)/*.mlpack $(CUDFLIBS)/*.cmx ; \
	  fi ; \
	done

%_j.ml: %.atd
	atdgen -j -j-std $<

%_t.ml: %.atd
	atdgen -t $<

.PHONY: atdgen
atdgen: datatypes_j.ml datatypes_t.ml

# the "while : ; do" loop around graph-difference is a temporary fix to work
# around random segmentation faults. See http://bugs.python.org/issue24605
define diff_tmp_out
	for t in tmp out; do \
		for f in tests/$(1)/$$t/* $$t/*; do basename "$$f"; done | sort | uniq | while read f; do \
			echo checking $$f; \
			case "$$f" in \
				*.xml|*.dot) \
					echo "+ ./tools/graph-difference.py \"tests/$(1)/$$t/$$f\" \"$$t/$$f\""; \
					while : ; do ./tools/graph-difference.py "tests/$(1)/$$t/$$f" "$$t/$$f"; exit=$$?; if [ $$exit -eq 139 ]; then echo segfault; continue; fi; if [ $$exit -ne 0 ]; then exit 1; else break; fi; done;; \
				*) \
					echo "+ diff -u \"tests/$(1)/$$t/$$f\" \"$$t/$$f\""; \
					cmp --silent "tests/$(1)/$$t/$$f" "$$t/$$f" || { diff -u "tests/$(1)/$$t/$$f" "$$t/$$f" | head --lines 100 && exit 1; };; \
			esac; \
		done; \
	done
endef

.PHONY: test-misc
test-misc: $(OCAMLBUILD_BEST)
	rm -rf tmp out
	mkdir -p tmp out
	$(eval packages := tests/sid-amd64-packages-20160830T000000Z)
	$(eval sources := tests/sid-sources-20160830T000000Z)
	grep-dctrl --exact-match --field Package build-essential $(packages) \
		| ./tools/latest-version.py - - > tmp/build-essential
	./bin2src.$(OCAMLBUILD_BEST) --deb-native-arch=amd64 tmp/build-essential $(sources) \
		> tmp/build-essential-src
	./create-graph.$(OCAMLBUILD_BEST) --deb-native-arch=amd64 --bg $(sources) $(packages) tmp/build-essential-src \
		> tmp/selfcontained_repo.xml
	./tools/buildgraph2packages.py tmp/selfcontained_repo.xml $(packages) \
		> tmp/packages
	./bin2src.$(OCAMLBUILD_BEST) --deb-native-arch=amd64 tmp/packages $(sources) \
		> tmp/sources
	# test botch-calcportsmetric
	./create-graph.$(OCAMLBUILD_BEST) --deb-drop-b-d-indep --deb-native-arch=amd64 tmp/packages tmp/sources --strongtype > tmp/strongbuildgraph.xml
	./buildgraph2srcgraph.$(OCAMLBUILD_BEST) tmp/strongbuildgraph.xml --deb-drop-b-d-indep --deb-native-arch=amd64 tmp/packages tmp/sources > tmp/strongsrcgraph.xml
	./create-graph.$(OCAMLBUILD_BEST) --deb-drop-b-d-indep --deb-native-arch=amd64 tmp/packages tmp/sources --closuretype > tmp/closurebuildgraph.xml
	./buildgraph2srcgraph.$(OCAMLBUILD_BEST) tmp/closurebuildgraph.xml --deb-drop-b-d-indep --deb-native-arch=amd64 tmp/packages tmp/sources > tmp/closuresrcgraph.xml
	./tools/calcportsmetric.py tmp/strongsrcgraph.xml tmp/closuresrcgraph.xml > out/importance_metric.txt
	# test botch-multiarch-interpreter-problem
	dose-ceve --deb-drop-b-d-indep --deb-native-arch=amd64 -G pkg -T grml deb://tmp/packages debsrc://tmp/sources > tmp/ma_interpreter.xml
	./tools/multiarch-interpreter-problem.py --packages tmp/packages tmp/ma_interpreter.xml > out/ma_interpreter.txt
	# test y-u-b-d-transitive-essential
	./tools/y-u-b-d-transitive-essential.sh --debug --verbose --develop --tmp tmp --output out amd64 tmp/packages tmp/sources acl
	# remove svg because graphviz changes its output format slightly every other version
	rm out/acl_2.2.52-3_n20.svg
	# test botch-buildcheck-more-problems
	./buildcheck-more-problems.$(OCAMLBUILD_BEST) --verbose --progress --checkonly=plasma-desktop,haskell-hledger-ui --deb-native-arch=amd64 --explain --failures $(packages) $(sources) > out/buildcheck.yaml || [ $$? -eq 1 ]
	# test botch-buildcheck-more-problems cross because we want to be able
	# to check yaml output that contains a depchain with an implicit
	# dependency on a package that is Essential:yes (and thus there is no
	# "depends" field)
	./tools/convert-arch.py amd64 armhf tmp/packages - | grep-dctrl -X \( --not -FArchitecture all --and --not -FMulti-Arch foreign \) > tmp/packages_armhf_noall_nomaforeign
	./buildcheck-more-problems.$(OCAMLBUILD_BEST) --verbose --progress --checkonly=apache2,ant --deb-native-arch=amd64 --deb-host-arch=armhf --explain --failures tmp/packages_armhf_noall_nomaforeign tests/crossbuild-essential-armhf tmp/packages tmp/sources > out/buildcheck_cross.yaml || [ $$? -eq 1 ]
	# test botch-distcheck-more-problems
	./distcheck-more-problems.$(OCAMLBUILD_BEST) --verbose --progress --checkonly=task-kde-desktop:amd64,libghc-hledger-dev:amd64 --deb-native-arch=amd64 --explain --failures deb://$(packages) > out/distcheck.yaml || [ $$? -eq 1 ]
	# test dose2html
	./tools/dose2html.py --srcsdir=out --wwwroot=out/ --packages=$(packages) out/distcheck.yaml out/distcheck.html
	./tools/dose2html.py --srcsdir=out --wwwroot=out/ --packages=$(packages) out/buildcheck.yaml out/buildcheck.html
	./tools/dose2html.py --srcsdir=out --wwwroot=out/ --packages=tmp/packages_armhf_noall_nomaforeign --packages=tests/crossbuild-essential-armhf --packages=tmp/packages out/buildcheck_cross.yaml out/buildcheck_cross.html
	# test networkx dot read/write because there constantly seem to be
	# problems with that:
	#   - endless loop for graphs with attributes in pygraphviz
	#        https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=794444
	#        https://github.com/pygraphviz/pygraphviz/issues/65
	#   - pygraphviz cannot handle graph attributes
	#        https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=819739
	#   - networkx cannot store graph in dot format where a node name contains a colon
	#        https://github.com/networkx/networkx/issues/2050
	#   - python3 support
	#        https://github.com/erocarrera/pydot/issues/76
	#   - dead pydot upstream and too many forks
	#        https://github.com/carlos-jenkins/pydotplus/issues/1
	#   - exclusive use of non-packaged pydotplus fork by networkx
	#        https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=819480
	#   - undeterministic output order
	#        https://github.com/networkx/networkx/issues/1181
	#        https://github.com/networkx/networkx/issues/1267
	#   - unicode encode/decode problems
	#        https://github.com/erocarrera/pydot/issues/24
	#   - networkx API breakages: removal of nx.write_dot and nx.read_dot
	#        http://networkx.readthedocs.io/en/stable/reference/api_1.11.html
	#   - pygraphviz API breakages: AGraph dropped the file parameter
	#        https://github.com/pygraphviz/pygraphviz/commit/21d9153c25ff5d8047c46f48f043f0788ab10da5
	#   - pygraphviz segfauls when reading from a io.StringIO
	#        https://github.com/pygraphviz/pygraphviz/issues/101
	#   - pygraphviz adds the input filename as a graph attribute
	#        https://github.com/pygraphviz/pygraphviz/issues/102
	#   - networkx doesn't parse dot data from memory
	#        https://github.com/networkx/networkx/issues/2249
	dose-ceve --deb-drop-b-d-indep -c "src:acl" --deb-native-arch=amd64 -G pkg -T dot deb://tmp/packages debsrc://tmp/sources > tmp/aclbd.dot
	./tools/graph-shortest-path.py tmp/aclbd.dot --all --source __ID__:"src:acl (= 2.2.52-3)" --target __ID__:"multiarch-support:amd64 (= 2.23-5)" > out/acl-ma-path.dot
	./tools/graph-ancestors.py --target __ID__:"perl:amd64 (= 5.22.2-3)" tmp/aclbd.dot > out/acl_pred.dot
	./tools/graph-descendants.py --source __ID__:"perl:amd64 (= 5.22.2-3)" tmp/aclbd.dot > out/acl_succ.dot
	./tools/graph-neighborhood.py --depth=2 --center __ID__:"perl:amd64 (= 5.22.2-3)" tmp/aclbd.dot > out/acl_neighbors.dot
	# test y-u-no-bootstrap
	# Remove yui-compressor to make src:doxygen bd-uninstallable
	grep-dctrl --not --field Package yui-compressor tmp/packages > tmp/packages_noyui
	./tools/y-u-no-bootstrap.sh --debug --verbose --develop --tmp tmp --output out amd64 tmp/packages_noyui tmp/sources
	# remove svg because graphviz changes its output format slightly every other version
	rm out/doxygen_1.8.11-3.svg
	# verify results
	$(call diff_tmp_out,misc)

.PHONY: test-default
test-default: $(OCAMLBUILD_BEST)
	rm -rf tmp out
	./tools/native.sh --debug --verbose --develop --output out --tmp tmp --jobs=2 --deb-drop-b-d-indep amd64 tests/sid-amd64-packages-20160830T000000Z tests/sid-sources-20160830T000000Z
	$(call diff_tmp_out,default)

.PHONY: test-selfcontained
test-selfcontained: $(OCAMLBUILD_BEST)
	rm -rf tmp out
	./tools/native.sh --debug --verbose --develop --output out --tmp tmp --deb-drop-b-d-indep --optgraph --latest --clean --self-contained --optuniv --sapsb --strong --no-drop amd64 tests/sid-amd64-packages-20160830T000000Z tests/sid-sources-20160830T000000Z
	$(call diff_tmp_out,selfcontained)

.PHONY: test-cross
test-cross: $(OCAMLBUILD_BEST)
	rm -rf tmp out
	mkdir out
	cp tests/cross-ma.diff out/ma.diff
	./tools/cross.sh --debug --verbose --develop --output=out --tmp=tmp --deb-drop-b-d-indep --optgraph --jobs=2 amd64 armhf tests/sid-amd64-packages-20160830T000000Z tests/sid-sources-20160830T000000Z
	$(call diff_tmp_out,cross)

.PHONY: test-man
test-man: $(OCAMLBUILD_BEST)
	# test whether there is no manpage without a program by
	# normalizing program and manpage names and only printing
	# those names which are not duplicate but unique
	ls tools/*.py *.$(OCAMLBUILD_BEST) tools/*.sh doc/man/botch-*.pod \
		| grep -v tools/util.py \
		| grep -v tools/debarch.py \
		| while read pkg; do \
			pkg=`basename $$pkg .$(OCAMLBUILD_BEST)`; \
			pkg=`basename $$pkg .py`; \
			pkg=`basename $$pkg .sh`; \
			pkg=`basename $$pkg .pod`; \
			echo $${pkg#botch-}; \
		done | sort | uniq -u | while read pkg; do \
			echo doc/man/botch-$${pkg}.pod is superfluous >&2; \
			exit 1; \
		done
	# check that every pod file includes a synopsis and a description
	# section and that the synopsis includes the right program name
	for f in doc/man/*.pod; do \
		grep -q '^=head1 SYNOPSIS$$' "$$f"; \
		grep -q '^=head1 DESCRIPTION$$' "$$f"; \
		name=$$(basename "$$f" .pod); \
		ret=0; \
		awk '/^=head1 SYNOPSIS$$/{flag=1;next}/^=head1 DESCRIPTION$$/{flag=0}flag' \
			"$$f" | grep -q '^=item B<'"$$name"'> ' || ret=1; \
		if [ $$ret -ne 0 ]; then \
			echo "wrong synopsis in $$f" >&2; \
			exit 1; \
		fi; \
	done
	# now that we know that there is one man page for every program, check
	# whether the man page documents all the program options
	ls tools/*.py *.$(OCAMLBUILD_BEST) tools/*.sh \
		| grep -v tools/util.py \
		| grep -v tools/debarch.py \
		| grep -v buildcheck-more-problems | grep -v distcheck-more-problems \
		| while read pkg; do \
			shortopts=`./$$pkg --help 2>&1 | sed -ne 's/^ \+-\([^ =-]\).*$$/\1/p' | sort`; \
			longopts=`./$$pkg --help 2>&1 | sed -ne 's/^ \+\(-[^ =-]\+,\)\? --\([^ =]\+\)\([ =].*\)\?$$/\2/p' | sort`; \
			echo "compare \`./$$pkg --help\` with man page..."; \
			man=`basename $$pkg .$(OCAMLBUILD_BEST)`; \
			man=`basename $$man .py`; \
			man=`basename $$man .sh`; \
			man=`basename $$man .pod`; \
			longman=`sed -ne 's/=item B<[^>]*--\([^>=]\+\)=\?>.*$$/\1/p' doc/man/botch-$${man}.pod | sort`; \
			shortman=`sed -ne 's/=item B<-\([^>,-]\)[^>]*>.*$$/\1/p' doc/man/botch-$${man}.pod | sort`; \
			if [ "$$shortopts" != "$$shortman" ]; then \
				echo "\`./$$pkg --help\` and man page ./$$man disagree:"; \
				echo "$$shortopts != $$shortman"; \
				exit 1; \
			fi; \
			if [ "$$longopts" != "$$longman" ]; then \
				echo "\`./$$pkg --help\` and man page ./$$man disagree:"; \
				echo "$$longopts != $$longman"; \
				exit 1; \
			fi; \
		done

.PHONY: test-python
test-python:
	# FIXME: add more tests
	#./tests.py
	#OCAMLPATH=$(BUILD) ocamlbuild -classic-display -use-ocamlfind $(CFLAGS) tests.$(OCAMLBUILD_BEST)
	#./tests.$(OCAMLBUILD_BEST)
	pyflakes3 tools/*.py
	# E402 is triggered by the sys.path.append() statement in front of import of utils
	# see https://github.com/PyCQA/pycodestyle/issues/264
	# W503 is not PEP8 compliant
	# E203 is not PEP8 compliant
	pycodestyle --max-line-length=88 --ignore=E402,E203,W503 tools/*.py
	black --check tools/*.py

.PHONY: test
test: test-python test-man test-default test-selfcontained test-cross test-misc

.PHONY: install-man
install-man: man
	mkdir -p $(DESTDIR)$(mandir)/man1/
	cp $(MANPAGES) $(DESTDIR)$(mandir)/man1/
	cp doc/man/botch.1 $(DESTDIR)$(mandir)/man1/

.PHONY: install-wiki
install-wiki: wiki
	$(MAKE) -C doc/wiki install

.PHONY: install-bin
install-bin: $(OCAMLBUILD_BEST)
	mkdir -p $(DESTDIR)$(bindir)
	for octool in $(BIN_OCTOOLS); do \
		cp _build/$$octool.$(OCAMLBUILD_BEST) $(DESTDIR)$(bindir)/botch-$$octool; \
	done
	for shtool in $(BIN_SHTOOLS); do \
		cp tools/$$shtool.sh $(DESTDIR)$(bindir)/botch-$$shtool; \
	done
	for pytool in $(BIN_PYTOOLS); do \
		cp tools/$$pytool.py $(DESTDIR)$(bindir)/botch-$$pytool; \
	done
	mkdir -p $(DESTDIR)$(datadir)
	cp -r droppable $(DESTDIR)$(datadir)
	cp tools/util.py $(DESTDIR)$(datadir)
	cp tools/debarch.py $(DESTDIR)$(datadir)

.PHONY: install
install: install-bin install-man install-wiki

.PHONY: clean
clean:
	ocamlbuild -clean
	rm -f datatypes_j.ml datatypes_j.mli datatypes.ml datatypes.mli datatypes_t.ml datatypes_t.mli
	rm -f tools/*.pyc
	rm -f *.native *.byte *.p.native *.d.byte
	rm -f doc/man/*.1
	$(MAKE) -C doc/wiki clean

.PHONY: distclean
distclean: clean
	rm -rf tools/__pycache__ tmp out

.PHONY: doseclean
doseclean:
	(cd dose && ocamlbuild -clean)

.PHONY: tarball
tarball:
	$(eval tmpdir := $(shell mktemp --directory))
	git archive --worktree-attributes --prefix=botch-$(VERSION)/ -o $(tmpdir)/botch-$(VERSION).tar HEAD
	git -C doc/wiki archive --worktree-attributes --prefix=botch-$(VERSION)/doc/wiki/ -o $(tmpdir)/botch-doc.tar HEAD
	git -C tests archive --worktree-attributes --prefix=botch-$(VERSION)/tests/ -o $(tmpdir)/botch-tests.tar HEAD
	# tar --concatenate seems to only take two files as input so we can
	# neither replace the temporary files with a pipe nor combine both of
	# the below commands into one... :(
	tar --concatenate -f $(tmpdir)/botch-$(VERSION).tar $(tmpdir)/botch-doc.tar
	tar --concatenate -f $(tmpdir)/botch-$(VERSION).tar $(tmpdir)/botch-tests.tar
	# testing shows that in this case, -9e compresses better than -9
	xz --verbose -c9e $(tmpdir)/botch-$(VERSION).tar > ../botch-$(VERSION).tar.xz
	rm -rf $(tmpdir)

.PHONY: upload
upload: tarball
	gpg -u 8FBD83E1 --armor --detach-sig ../botch-$(VERSION).tar.xz
	scp ../botch-$(VERSION).tar.xz.asc ../botch-$(VERSION).tar.xz mmfn:/var/www/botch/
	ssh mmfn tree -s --noreport -P "botch-*.tar.xz*" -o /var/www/botch/index.html -v -L 1 -H "." -T "botch-releases" /var/www/botch/

# Behave like POSIX. In particular, if this target is mentioned then recipes
# will be invoked as if the shell had been passed the -e flag: the first
# failing command in a recipe will cause the recipe to fail immediately.
.POSIX:
