Makefile: First try at rewriting the ccan Makefile.

This change contains a simpler Makefile replacement with only 62 lines
of directives, 10 rules, and a 13 line support script for dependencies. The
build dependencies have been minimised and in some cases, corrected.

FEATURES:
* All targets can be built from a clean tree in one invocation.
* Parallel builds (tested with -j32 on 8 cores).
* Auto discovery of modules via _info files.
* Hopefully complete dependencies via a simplified generator.
* CFLAGS are respected and appended to compile flags.
* LINTFLAGS can be set to add check options (e.g. LINTFLAGS=-v).
* 'make clean' doesn't build anything before cleaning now.
* 'make quiet=1' builds quietly. 'make check quiet=1 -j N' produces
  summary output like the former summary target.
* Non-phony test targets; tests are rebuilt only when dirty. Targets are:
  check, fastcheck and fullcheck, the latter runs in non-summary mode.
* 'make <module>.[check|fastcheck|fullcheck]' runs tests for single modules.

TODO:
* Support Makefile-web and any other scattered targets

NOTES:
* The changes to dependency generation expose a circular
  dependency between asort and order which is not fixed here.
* Tests always run their dependent tests. With -j support and
  minimised rebuilds via tighter dependencies, its not worth avoiding.
* Some targets have been dropped as uneeded (e.g. distclean, tools).
Signed-off-by: Jon Griffiths <jon_p_griffiths@yahoo.com>

.gitignore

 TAGS
-.depends
 .valgrind_suppressions
 *.d
 *.o
-libccan.a
+*.ok
 config.h
+config.h.tmp
 *~
 tools/ccan_depends
 tools/doc_extract
 tools/namespacize
 tools/run_tests
 tools/ccanlint/ccanlint
-tools/ccanlint/generated-testlist
 tools/modfiles
-inter-depends
-test-depends
-lib-depends
 tools/_infotojson/infotojson
 tools/ccanlint/test/run-file_analysis
 tools/configurator/configurator

Makefile

-# Hacky makefile to compile everything and run the tests in some kind
-# of sane order.
-
-# Main targets:
-# 
-# check: run tests on all ccan modules (use 'make check V=--verbose' for more)
-#        Includes building libccan.a.
-# libccan.a: A library with all the ccan modules in it.
-# tools: build useful tools in tools/ dir.
-#        Especially tools/ccanlint/ccanlint and tools/namespacize.
-# distclean: destroy everything back to pristine state
-
-# Where make scores puts the results
-SCOREDIR=scores/$(shell whoami)/$(shell uname -s)-$(shell uname -m)-$(CC)-$(shell git describe --always --dirty)
-CCANLINT=tools/ccanlint/ccanlint --deps-fail-ignore
-CCANLINT_FAST=$(CCANLINT) -x tests_pass_valgrind -x tests_compile_coverage
-
-default: all_info libccan.a
-
-ALL_DEPENDS=$(patsubst %, ccan/%/.depends, $(MODS))
-
-# By default, we skip modules with external deps (or plaform specific)
-MODS_EXCLUDE:=altstack generator jmap jset nfs ogg_to_pcm tal/talloc wwviaudio
-# This randomly fails, and reliably fails under Jenkins :(
-MODS_FLAKY:=altstack
-MODS_RELIABLE=$(filter-out $(MODS_FLAKY),$(MODS))
-
-include Makefile-ccan
-
-fastcheck: $(MODS_RELIABLE:%=summary-fastcheck/%)
-
-check: $(MODS_RELIABLE:%=summary-check/%)
-
-distclean: clean
-	rm -f $(ALL_DEPENDS)
-
-scores: $(SCOREDIR)/SUMMARY
-
-$(SCOREDIR)/SUMMARY: $(MODS:%=$(SCOREDIR)/%.score)
-	git describe --always > $@
-	uname -a >> $@
-	$(CC) -v >> $@
-	cat $^ | grep 'Total score:' >> $@
-
-$(SCOREDIR)/%.score: ccan/%/_info tools/ccanlint/ccanlint $(OBJFILES)
-	mkdir -p `dirname $@`
-	$(CCANLINT) -v -s ccan/$* > $@ || true
-
-$(ALL_DEPENDS): %/.depends: %/_info tools/ccan_depends
-	tools/ccan_depends $* > $@ || ( rm -f $@; exit 1 )
-
-# Actual dependencies are created in inter-depends
-check/%: tools/ccanlint/ccanlint
-	$(CCANLINT) ccan/$*
-
-fastcheck/%: tools/ccanlint/ccanlint
-	$(CCANLINT_FAST) ccan/$*
-
-# Doesn't test dependencies, doesn't print verbose fail results.
-summary-check/%: tools/ccanlint/ccanlint $(OBJFILES)
-	$(CCANLINT) -s ccan/$*
-
-summary-fastcheck/%: tools/ccanlint/ccanlint $(OBJFILES)
-	$(CCANLINT_FAST) -s ccan/$*
-
-ccan/%/info: ccan/%/_info config.h
-	$(CC) $(CCAN_CFLAGS) -I. -o $@ -x c $<
-
-all_info: $(MODS:%=ccan/%/info)
-
-clean: tools-clean
-	rm -f `find * -name '*.o'` `find * -name '.depends'` `find * -name '*.a'`  `find * -name info` `find * -name '*.d'` `find ccan -name '*-Makefile'`
-	rm -f config.h
-	rm -f inter-depends lib-depends test-depends
-
-# Creates a dependency from the tests to the object files which it needs.
-inter-depends: $(ALL_DEPENDS) Makefile
-	for f in $(ALL_DEPENDS); do echo check-$$(basename $$(dirname $$f) ): $$(for dir in $$(cat $$f) $$(dirname $$f); do [ "$$(echo $$dir/*.c)" = "$$dir/*.c" ] || echo ccan/"$$(basename $$dir)".o; done); done > $@
-
-# Creates dependencies between tests, so if foo depends on bar, bar is tested
-# first 
-test-depends: $(ALL_DEPENDS) Makefile
-	for f in $(ALL_DEPENDS); do echo check/`basename \`dirname $$f\``: `sed -n 's,ccan/\(.*\),check/\1,p' < $$f`; done > $@
-
-TAGS: FORCE
-	find * -name '*.[ch]' | xargs etags
-
-FORCE:
-
-# Ensure we don't end up with empty file if configurator fails!
-config.h: tools/configurator/configurator Makefile Makefile-ccan
-	tools/configurator/configurator $(CC) $(CCAN_CFLAGS) > $@.tmp && mv $@.tmp $@
-
-include tools/Makefile
--include inter-depends
--include test-depends
--include Makefile-web
+# Makefile for CCAN
+
+# 'make quiet=1' builds silently
+QUIETEN.1 := @
+PRE := $(QUIETEN.$(quiet))
+
+all::
+
+# Our flags for building
+WARN_CFLAGS := -Wall -Wstrict-prototypes -Wold-style-definition -Wundef \
+ -Wmissing-prototypes -Wmissing-declarations -Wpointer-arith -Wwrite-strings
+DEP_CFLAGS = -MMD -MP -MF$(@:%=%.d) -MT$@
+CCAN_CFLAGS := -g3 -ggdb $(WARN_CFLAGS) -DCCAN_STR_DEBUG=1 -I. $(CFLAGS)
+
+# Anything with an _info file is a module ...
+INFO_SRCS := $(wildcard ccan/*/_info ccan/*/*/_info)
+ALL_INFOS := $(INFO_SRCS:%_info=%info)
+ALL_MODULES := $(ALL_INFOS:%/info=%)
+
+# ... Except stuff that needs external dependencies, which we exclude
+EXCLUDE := altstack generator jmap jset nfs ogg_to_pcm tal/talloc wwviaudio
+MODULES:= $(filter-out $(EXCLUDE:%=ccan/%), $(ALL_MODULES))
+
+# Sources are C files in each module, objects the resulting .o files
+SRCS := $(wildcard $(MODULES:%=%/*.c))
+OBJS := $(SRCS:%.c=%.o)
+DEPS := $(OBJS:%=%.d)
+
+# We build all object files using our CCAN_CFLAGS, after config.h
+%.o : %.c config.h
+	$(PRE)$(CC) $(CCAN_CFLAGS) $(DEP_CFLAGS) -c $< -o $@
+
+# _info files are compiled into executables and don't need dependencies
+%info : %_info config.h
+	$(PRE)$(CC) $(CCAN_CFLAGS) -I. -o $@ -x c $<
+
+# config.h is built by configurator which has no ccan dependencies
+CONFIGURATOR := tools/configurator/configurator
+$(CONFIGURATOR): $(CONFIGURATOR).c
+	$(PRE)$(CC) $(CCAN_CFLAGS) $(DEP_CFLAGS) $< -o $@
+config.h: $(CONFIGURATOR) Makefile
+	$(PRE)$(CONFIGURATOR) $(CC) $(CCAN_CFLAGS) >$@.tmp && mv $@.tmp $@
+
+# Tools
+TOOLS := tools/ccan_depends tools/doc_extract tools/namespacize tools/modfiles
+TOOLS_SRCS := $(filter-out $(TOOLS:%=%.c), $(wildcard tools/*.c))
+TOOLS_DEPS := $(TOOLS_SRCS:%.c=%.d) $(TOOLS:%=%.d)
+TOOLS_CCAN_MODULES := err foreach hash htable list noerr opt rbuf \
+    read_write_all str take tal tal/grab_file tal/link tal/path tal/str time
+TOOLS_CCAN_SRCS := $(wildcard $(TOOLS_CCAN_MODULES:%=ccan/%/*.c))
+TOOLS_OBJS := $(TOOLS_SRCS:%.c=%.o) $(TOOLS_CCAN_SRCS:%.c=%.o)
+tools/% : tools/%.c $(TOOLS_OBJS)
+	$(PRE)$(CC) $(CCAN_CFLAGS) $(DEP_CFLAGS) $< $(TOOLS_OBJS) -lm -o $@
+
+# ccanlint
+LINT := tools/ccanlint/ccanlint
+LINT_OPTS.ok := -s
+LINT_OPTS.fast.ok := -s -x tests_pass_valgrind -x tests_compile_coverage
+LINT_SRCS := $(filter-out $(LINT).c, $(wildcard tools/ccanlint/*.c tools/ccanlint/tests/*.c))
+LINT_DEPS := $(LINT_SRCS:%.c=%.d) $(LINT).d
+LINT_CCAN_MODULES := asort autodata dgraph ilog lbalance ptr_valid strmap
+LINT_CCAN_SRCS := $(wildcard $(LINT_CCAN_MODULES:%=ccan/%/*.c))
+LINT_OBJS := $(LINT_SRCS:%.c=%.o) $(LINT_CCAN_SRCS:%.c=%.o) $(TOOLS_OBJS)
+$(LINT): $(LINT).c $(LINT_OBJS)
+	$(PRE)$(CC) $(CCAN_CFLAGS) $(DEP_CFLAGS) $(LINT).c $(LINT_OBJS) -lm -o $@
+
+# We generate dependencies for tests into a .d file
+%/.d: %/info tools/gen_deps.sh tools/ccan_depends
+	$(PRE)tools/gen_deps.sh $* > $@ || rm -f $@
+TEST_DEPS := $(MODULES:%=%/.d)
+
+# We produce .ok files when the tests succeed
+%.ok: $(LINT)
+	$(PRE)$(LINT) $(LINT_OPTS$(notdir $@)) --deps-fail-ignore $(LINTFLAGS) $(dir $*) && touch $@
+
+check: $(MODULES:%=%/.ok)
+fastcheck: $(MODULES:%=%/.fast.ok)
+fullcheck: $(MODULES:%=%/.full.ok)
+
+ifeq ($(strip $(filter clean config.h, $(MAKECMDGOALS))),)
+-include $(DEPS) $(LINT_DEPS) $(TOOLS_DEPS) $(TEST_DEPS)
+endif
+
+# Default target: object files, info files and tools
+all:: $(OBJS) $(ALL_INFOS) $(CONFIGURATOR) $(LINT) $(TOOLS)
+
+.PHONY: clean TAGS
+clean:
+	$(PRE)find . -name "*.d" -o -name "*.o" -o -name "*.ok" | xargs -n 256 rm -f
+	$(PRE)rm -f $(CONFIGURATOR) $(LINT) $(TOOLS) TAGS config.h config.h.d $(ALL_INFOS)
+
+# 'make TAGS' builds etags
+TAGS:
+	$(PRE)find * -name '*.[ch]' | xargs etags

Makefile-ccan

-# Example makefile which makes a "libccan.a" of everything under ccan/.
-# For simple projects you could just do:
-#	SRCFILES += $(wildcard ccan/*/*.c)
-
-#CCAN_CFLAGS=-g -O3 -Wall -Wstrict-prototypes -Wold-style-definition -Wmissing-prototypes -Wmissing-declarations -Wpointer-arith -Wwrite-strings -Wundef -DCCAN_STR_DEBUG=1
-CCAN_CFLAGS=-g3 -ggdb -Wall -Wstrict-prototypes -Wold-style-definition -Wmissing-prototypes -Wmissing-declarations -Wpointer-arith -Wwrite-strings -Wundef -DCCAN_STR_DEBUG=1
-CFLAGS = $(CCAN_CFLAGS) -I. $(DEPGEN)
-
-MODS := a_star \
-	aga \
-	agar \
-	alignof \
-	altstack \
-	antithread \
-	antithread/alloc \
-	argcheck \
-	array_size \
-	asearch \
-	asort \
-	asprintf \
-	autodata \
-	avl \
-	base64 \
-	bdelta \
-	bitmap \
-	block_pool \
-	breakpoint \
-	btree \
-	build_assert \
-	bytestring \
-	cast \
-	ccan_tokenizer \
-	cdump \
-	charset \
-	check_type \
-	ciniparser \
-	compiler \
-	container_of \
-	cppmagic \
-	cpuid \
-	crc \
-	crcsync \
-	crypto/ripemd160 \
-	crypto/sha256 \
-	crypto/sha512 \
-	crypto/shachain \
-	crypto/siphash24 \
-	daemonize \
-	daemon_with_notify \
-	darray \
-	deque \
-	dgraph \
-	endian \
-	eratosthenes \
-	err \
-	failtest \
-	foreach \
-	generator \
-	grab_file \
-	hash \
-	heap \
-	htable \
-	idtree \
-	ilog \
-	invbloom \
-	io \
-	isaac \
-	iscsi \
-	jacobson_karels \
-	jmap \
-	jset \
-	json \
-	lbalance \
-	likely \
-	list \
-	lpq \
-	lqueue \
-	lstack \
-	md4 \
-	mem \
-	minmax \
-	net \
-	nfs \
-	noerr \
-	ntdb \
-	objset \
-	ogg_to_pcm \
-	opt \
-	order \
-	permutation \
-	pipecmd \
-	pr_log \
-	ptrint \
-	ptr_valid \
-	pushpull \
-	rbtree \
-	rbuf \
-	read_write_all \
-	rfc822 \
-	rszshm \
-	short_types \
-	siphash \
-	sparse_bsearch \
-	str \
-	str/hex \
-	strgrp \
-	stringbuilder \
-	stringmap \
-	strmap \
-	strset \
-	structeq \
-	take \
-	tal \
-	tal/grab_file \
-	tal/link \
-	tal/path \
-	tal/stack \
-	tal/str \
-	tal/talloc \
-	talloc \
-	tally \
-	tap \
-	tcon \
-	time \
-	timer \
-	tlist \
-	tlist2 \
-	ttxml \
-	typesafe_cb \
-	version \
-	wwviaudio \
-	xstring
-
-# Anything with C files needs building; dir leaves / on, sort uniquifies
-MODS_WITH_SRC = $(patsubst ccan/%/, %, $(sort $(foreach m, $(MODS), $(dir $(wildcard ccan/$m/*.c)))))
-
-default: libccan.a
-
-# Automatic dependency generation: makes ccan/*/*.d files.
-DEPGEN=-MMD
--include $(foreach m, $(MODS), ccan/$(m)/*.d)
-
-DIRS=$(patsubst %, ccan/%, $(filter-out $(MODS_EXCLUDE), $(MODS_WITH_SRC)))
-
-# Generate everyone's separate Makefiles.
--include $(foreach dir, $(DIRS), $(dir)-Makefile)
-
-ccan/%-Makefile:
-	@echo $@: $(wildcard ccan/$*/*.[ch]) ccan/$*/_info > $@
-	@echo ccan/$*.o: $(patsubst %.c, %.o, $(wildcard ccan/$*/*.c)) >> $@
-
-# We compile all the ccan/foo/*.o files together into ccan/foo.o
-OBJFILES=$(DIRS:=.o)
-
-# We create all the .o files and link them together.
-$(OBJFILES): %.o:
-	$(LD) -r -o $@ $^
-
-libccan.a: $(OBJFILES)
-	$(AR) r $@ $(OBJFILES)

tools/Makefile

-ALL_TOOLS = tools/configurator/configurator tools/ccan_depends tools/doc_extract tools/namespacize tools/ccanlint/ccanlint tools/modfiles
-LDLIBS = -lrt
-DEP_OBJS = ccan/err/err.o \
-	ccan/foreach/foreach.o \
-	ccan/hash/hash.o \
-	ccan/htable/htable.o \
-	ccan/list/list.o \
-	ccan/noerr/noerr.o \
-	ccan/opt/opt.o \
-	ccan/opt/helpers.o \
-	ccan/opt/parse.o \
-	ccan/opt/usage.o \
-	ccan/rbuf/rbuf.o \
-	ccan/read_write_all/read_write_all.o \
-	ccan/str/debug.o \
-	ccan/str/str.o \
-	ccan/take/take.o \
-	ccan/tal/tal.o \
-	ccan/tal/grab_file/grab_file.o \
-	ccan/tal/link/link.o \
-	ccan/tal/path/path.o \
-	ccan/tal/str/str.o \
-	ccan/time/time.o \
-	tools/read_config_header.o \
-	tools/ccan_dir.o \
-	tools/compile.o \
-	tools/depends.o \
-	tools/tools.o
-
-.PHONY: tools
-tools: $(ALL_TOOLS)
-
-tools/ccan_depends.o: config.h
-
-tools/ccan_depends: tools/ccan_depends.o $(DEP_OBJS)
-
-tools/doc_extract: tools/doc_extract.o tools/doc_extract-core.o $(DEP_OBJS)
-
-tools/namespacize: tools/namespacize.o $(DEP_OBJS)
-
-tools/namespacize.o tools/depends.o: tools/tools.h
-
-tools/configurator/configurator: tools/configurator/configurator.c
-
-tools/modfiles: tools/modfiles.o tools/manifest.o $(DEP_OBJS)
-
-tools-clean: ccanlint-clean
-	rm -f $(ALL_TOOLS)
-
-include tools/ccanlint/Makefile

tools/ccanlint/Makefile

-TEST_CFILES := $(wildcard tools/ccanlint/tests/*.c)
-TEST_OBJS := $(TEST_CFILES:.c=.o)
-
-CORE_OBJS := \
-	ccan/asort/asort.o \
-	ccan/autodata/autodata.o \
-	ccan/dgraph/dgraph.o \
-	ccan/foreach/foreach.o \
-	ccan/hash/hash.o \
-	ccan/htable/htable.o \
-	ccan/ilog/ilog.o \
-	ccan/lbalance/lbalance.o \
-	ccan/list/list.o \
-	ccan/noerr/noerr.o \
-	ccan/opt/helpers.o \
-	ccan/opt/opt.o \
-	ccan/opt/parse.o \
-	ccan/opt/usage.o \
-	ccan/ptr_valid/ptr_valid.o \
-	ccan/rbuf/rbuf.o \
-	ccan/read_write_all/read_write_all.o \
-	ccan/str/str.o ccan/str/debug.o \
-	ccan/strmap/strmap.o \
-	ccan/take/take.o \
-	ccan/tal/tal.o \
-	ccan/tal/grab_file/grab_file.o \
-	ccan/tal/link/link.o \
-	ccan/tal/path/path.o \
-	ccan/tal/str/str.o \
-	ccan/time/time.o \
-	tools/ccanlint/async.o \
-	tools/ccanlint/ccanlint.o \
-	tools/ccanlint/file_analysis.o \
-	tools/ccanlint/licenses.o \
-	tools/ccan_dir.o \
-	tools/compile.o \
-	tools/depends.o \
-	tools/doc_extract-core.o \
-	tools/manifest.o \
-	tools/read_config_header.o \
-	tools/tools.o
-
-OBJS := $(CORE_OBJS) $(TEST_OBJS)
-
-$(CORE_OBJS): config.h
-
-tools/ccanlint/ccanlint: $(OBJS)
-
-ccanlint-clean:
-	rm -f tools/ccanlint/ccanlint

tools/gen_deps.sh

+#! /bin/sh
+
+# Compute the test dependencies for a ccan module. Usage:
+# tools/gen_deps.sh ccan/path/to/module
+
+path=$1
+module=`echo $path | sed 's/^ccan\///g'`
+
+# The test depends on the test sources ...
+test_srcs=`ls $path/test/*.[ch] 2>/dev/null | tr '\n' ' '`
+
+# ... and the object files of our module (rather than the sources, so
+#     that we pick up the resursive dependencies for the objects)
+module_objs=`ls $path/*.c 2>/dev/null | sed 's/.c$/.o/g' | tr '\n' ' '`
+
+# ... and on the modules this test uses having passed their tests
+deps=$(echo `$path/info testdepends` `$path/info depends` | tr ' ' '\n' | \
+       sort | uniq | sed -e 's/$/\/.ok/g' -e '/^\/.ok$/d' | tr '\n' ' ')
+
+# Print the test targets and target aliases
+echo "${module}_ok_deps := $test_srcs $module_objs $deps"
+echo "$path/.ok: \$(${module}_ok_deps)"
+echo "$path/.fast.ok: \$(${module}_ok_deps:%.ok=%.fast.ok)"
+echo "$path/.full.ok: \$(${module}_ok_deps:%.ok=%.full.ok)"
+echo "${module}.check: $path/.ok"
+echo "${module}.fastcheck: $path/.fast.ok"
+echo "${module}.fullcheck: $path/.full.ok"