ccanlint: Add -k option to keep results.

Particularly useful for building tests standalone.

tools/ccanlint/ccanlint.c

@@ -41,12 +41,13 @@ static unsigned int timeout;
 
 static void usage(const char *name)
 {
-	fprintf(stderr, "Usage: %s [-s] [-n] [-v] [-t <ms>] [-d <dirname>] [-x <tests>]\n"
+	fprintf(stderr, "Usage: %s [-s] [-n] [-v] [-t <ms>] [-d <dirname>] [-x <tests>] [-k <test>]*\n"
 		"   -v: verbose mode\n"
 		"   -s: simply give one line summary\n"
 		"   -d: use this directory instead of the current one\n"
 		"   -n: do not compile anything\n"
 		"   -l: list tests ccanlint performs\n"
+		"   -k: keep results of this test (can be used multiple times)\n"
 		"   -x: exclude tests (e.g. -x trailing_whitespace,valgrind)\n"
 		"   -t: ignore (terminate) tests that are slower than this\n",
 		name);
@@ -131,7 +132,7 @@ static bool run_test(struct ccanlint *i,
 	}
 
 	timeleft = timeout ? timeout : default_timeout_ms;
-	result = i->check(m, &timeleft);
+	result = i->check(m, i->keep_results, &timeleft);
 	if (timeout && timeleft == 0) {
 		skip = "timeout";
 		goto skip;
@@ -284,6 +285,29 @@ static void init_tests(void)
 	}
 }
 
+static struct ccanlint *find_test(const char *key)
+{
+	struct ccanlint *i;
+
+	list_for_each(&compulsory_tests, i, list)
+		if (streq(i->key, key))
+			return i;
+
+	list_for_each(&normal_tests, i, list)
+		if (streq(i->key, key))
+			return i;
+
+	return NULL;
+}
+
+static void keep_test(const char *testname)
+{
+	struct ccanlint *i = find_test(testname);
+	if (!i)
+		errx(1, "No test %s to --keep", testname);
+	i->keep_results = true;
+}
+
 static void print_tests(struct list_head *tests, const char *type)
 {
 	struct ccanlint *i;
@@ -301,7 +325,6 @@ static void print_tests(struct list_head *tests, const char *type)
 
 static void list_tests(void)
 {
-	init_tests();
 	print_tests(&compulsory_tests, "Compulsory");
 	print_tests(&normal_tests, "Normal");
 	exit(0);
@@ -316,11 +339,13 @@ int main(int argc, char *argv[])
 	struct ccanlint *i;
 	const char *prefix = "", *dir = ".";
 	
+	init_tests();
+
 	exclude = btree_new(btree_strcmp);
 
 	/* I'd love to use long options, but that's not standard. */
 	/* FIXME: getopt_long ccan package? */
-	while ((c = getopt(argc, argv, "sd:vnlx:t:")) != -1) {
+	while ((c = getopt(argc, argv, "sd:vnlx:t:k:")) != -1) {
 		switch (c) {
 		case 'd':
 			dir = optarg;
@@ -339,6 +364,9 @@ int main(int argc, char *argv[])
 		case 'n':
 			safe_mode = true;
 			break;
+		case 'k':
+			keep_test(optarg);
+			break;
 		case 'x': {
 			char **exclude_strs = strsplit(NULL, optarg, ",", NULL);
 			size_t i;
@@ -361,8 +389,6 @@ int main(int argc, char *argv[])
 
 	m = get_manifest(talloc_autofree_context(), dir);
 
-	init_tests();
-
 	/* If you don't pass the compulsory tests, you don't even get a score */
 	if (verbose)
 		printf("Compulsory tests:\n");

tools/ccanlint/ccanlint.h

@@ -51,8 +51,9 @@ struct ccanlint {
 	const char *(*can_run)(struct manifest *m);
 
 	/* If this returns non-NULL, it means the check failed.
+	 * keep is set if you should keep the results.
 	 * If timeleft is set to 0, means it timed out. */
-	void *(*check)(struct manifest *m, unsigned int *timeleft);
+	void *(*check)(struct manifest *m, bool keep, unsigned int *timeleft);
 
 	/* The non-NULL return from check is passed to one of these: */
 
@@ -74,6 +75,8 @@ struct ccanlint {
 	bool skip;
 	/* Did we fail a dependency?  If so, skip and mark as fail. */
 	bool skip_fail;
+	/* Did the user want to keep these results? */
+	bool keep_results;
 };
 
 /* Ask the user a yes/no question: the answer is NO if there's an error. */

tools/ccanlint/compulsory_tests/build.c

@@ -33,7 +33,9 @@ static char *obj_list(const struct manifest *m)
 	return list;
 }
 
-static void *do_build(struct manifest *m, unsigned int *timeleft)
+static void *do_build(struct manifest *m,
+		      bool keep,
+		      unsigned int *timeleft)
 {
 	char *filename, *err;
 
@@ -43,7 +45,7 @@ static void *do_build(struct manifest *m, unsigned int *timeleft)
 		return NULL;
 	}
 	filename = link_objects(m, obj_list(m), &err);
-	if (filename) {
+	if (filename && keep) {
 		char *realname = talloc_asprintf(m, "%s.o", m->dir);
 		/* We leave this object file around, all built. */
 		if (!move_file(filename, realname))

tools/ccanlint/compulsory_tests/build_objs.c

@@ -21,7 +21,8 @@ static const char *can_build(struct manifest *m)
 	return NULL;
 }
 
-static void *check_objs_build(struct manifest *m, unsigned int *timeleft)
+static void *check_objs_build(struct manifest *m,
+			      bool keep, unsigned int *timeleft)
 {
 	char *report = NULL;
 	struct ccan_file *i;
@@ -33,8 +34,10 @@ static void *check_objs_build(struct manifest *m, unsigned int *timeleft)
 		/* One point for each obj file. */
 		build_objs.total_score++;
 
-		i->compiled = compile_object(m, fullfile, ccan_dir, &err);
-		if (!i->compiled) {
+		i->compiled = maybe_temp_file(m, "", keep, fullfile);
+		err = compile_object(m, fullfile, ccan_dir, i->compiled);
+		if (err) {
+			talloc_free(i->compiled);
 			if (report)
 				report = talloc_append_string(report, err);
 			else
@@ -50,7 +53,7 @@ static const char *describe_objs_build(struct manifest *m, void *check_result)
 }
 
 struct ccanlint build_objs = {
-	.key = "build-objs",
+	.key = "build-objects",
 	.name = "Module object files can be built",
 	.check = check_objs_build,
 	.describe = describe_objs_build,

tools/ccanlint/compulsory_tests/check_build.c

@@ -45,10 +45,12 @@ static char *lib_list(const struct manifest *m)
 	return ret;
 }
 
-static void *check_use_build(struct manifest *m, unsigned int *timeleft)
+static void *check_use_build(struct manifest *m,
+			     bool keep,
+			     unsigned int *timeleft)
 {
 	char *contents;
-	char *tmpfile, *err;
+	char *tmpfile;
 	int fd;
 
 	tmpfile = temp_file(m, ".c");
@@ -71,10 +73,8 @@ static void *check_use_build(struct manifest *m, unsigned int *timeleft)
 	}
 	close(fd);
 
-	if (!compile_and_link(m, tmpfile, ccan_dir, obj_list(m), "",
-			      lib_list(m), &err))
-		return err;
-	return NULL;
+	return compile_and_link(m, tmpfile, ccan_dir, obj_list(m), "",
+				lib_list(m), temp_file(m, ""));
 }
 
 static const char *describe_use_build(struct manifest *m, void *check_result)

tools/ccanlint/compulsory_tests/check_depends_built.c

@@ -35,7 +35,9 @@ static bool expect_obj_file(const char *dir)
 	return has_c_files;
 }
 
-static void *check_depends_built(struct manifest *m, unsigned int *timeleft)
+static void *check_depends_built(struct manifest *m,
+				 bool keep,
+				 unsigned int *timeleft)
 {
 	struct ccan_file *i;
 	struct stat st;

tools/ccanlint/compulsory_tests/check_depends_exist.c

@@ -31,7 +31,9 @@ static char *add_dep(char *sofar, struct manifest *m, const char *dep)
 	return sofar;
 }
 
-static void *check_depends_exist(struct manifest *m, unsigned int *timeleft)
+static void *check_depends_exist(struct manifest *m,
+				 bool keep,
+				 unsigned int *timeleft)
 {
 	unsigned int i;
 	char *report = NULL;

tools/ccanlint/compulsory_tests/check_includes_build.c

@@ -22,20 +22,37 @@ static const char *can_build(struct manifest *m)
 	return NULL;
 }
 
-static void *check_includes_build(struct manifest *m, unsigned int *timeleft)
+static struct ccan_file *main_header(struct manifest *m)
+{
+	struct ccan_file *f;
+
+	list_for_each(&m->h_files, f, list) {
+		if (strstarts(f->name, m->basename)
+		    && strlen(f->name) == strlen(m->basename) + 2)
+			return f;
+	}
+	/* Should not happen: we depend on has_main_header */
+	return NULL;
+}
+
+static void *check_includes_build(struct manifest *m,
+				  bool keep,
+				  unsigned int *timeleft)
 {
 	char *contents;
-	char *tmpfile, *err;
+	char *tmpsrc, *tmpobj;
 	int fd;
+	struct ccan_file *mainh = main_header(m);
 
-	tmpfile = temp_file(m, ".c");
+	tmpsrc = maybe_temp_file(m, "-included.c", keep, mainh->fullname);
+	tmpobj = maybe_temp_file(m, ".o", keep, tmpsrc);
 
-	fd = open(tmpfile, O_WRONLY | O_CREAT | O_EXCL, 0600);
+	fd = open(tmpsrc, O_WRONLY | O_CREAT | O_EXCL, 0600);
 	if (fd < 0)
-		return talloc_asprintf(m, "Creating temporary file: %s",
-				       strerror(errno));
+		return talloc_asprintf(m, "Creating temporary file %s: %s",
+				       tmpsrc, strerror(errno));
 
-	contents = talloc_asprintf(tmpfile, "#include <ccan/%s/%s.h>\n",
+	contents = talloc_asprintf(tmpsrc, "#include <ccan/%s/%s.h>\n",
 				   m->basename, m->basename);
 	if (write(fd, contents, strlen(contents)) != strlen(contents)) {
 		close(fd);
@@ -43,9 +60,7 @@ static void *check_includes_build(struct manifest *m, unsigned int *timeleft)
 	}
 	close(fd);
 
-	if (compile_object(m, tmpfile, ccan_dir, &err))
-		return NULL;
-	return err;
+	return compile_object(m, tmpsrc, ccan_dir, tmpobj);
 }
 
 static const char *describe_includes_build(struct manifest *m,
@@ -65,4 +80,4 @@ struct ccanlint includes_build = {
 	.can_run = can_build,
 };
 
-REGISTER_TEST(includes_build, &depends_exist, NULL);
+REGISTER_TEST(includes_build, &depends_exist, &has_main_header, NULL);

tools/ccanlint/compulsory_tests/compile_test_helpers.c

@@ -21,25 +21,24 @@ static const char *can_build(struct manifest *m)
 	return NULL;
 }
 
-static char *compile(struct manifest *m, struct ccan_file *cfile)
+static char *compile(struct manifest *m, 
+		     bool keep,
+		     struct ccan_file *cfile)
 {
-	char *err;
-	char *fullfile = talloc_asprintf(m, "%s/%s", m->dir, cfile->name);
-
-	cfile->compiled = compile_object(m, fullfile, ccan_dir, &err);
-	if (cfile->compiled)
-		return NULL;
-	return err;
+	cfile->compiled = maybe_temp_file(m, "", keep, cfile->fullname);
+	return compile_object(m, cfile->fullname, ccan_dir, cfile->compiled);
 }
 
-static void *do_compile_test_helpers(struct manifest *m, unsigned int *timeleft)
+static void *do_compile_test_helpers(struct manifest *m,
+				     bool keep,
+				     unsigned int *timeleft)
 {
 	char *cmdout = NULL;
 	struct ccan_file *i;
 
 	list_for_each(&m->other_test_c_files, i, list) {
 		compile_tests.total_score++;
-		cmdout = compile(m, i);
+		cmdout = compile(m, keep, i);
 		if (cmdout)
 			return talloc_asprintf(m,
 					       "Failed to compile helper C"

tools/ccanlint/compulsory_tests/compile_tests.c

@@ -60,18 +60,23 @@ static char *lib_list(const struct manifest *m)
 }
 
 static char *compile(const void *ctx,
-		     struct manifest *m, struct ccan_file *file, bool fail,
-		     bool link_with_module)
+		     struct manifest *m,
+		     struct ccan_file *file,
+		     bool fail,
+		     bool link_with_module,
+		     bool keep)
 {
 	char *errmsg;
 
-	file->compiled = compile_and_link(ctx, file->fullname, ccan_dir,
+	file->compiled = maybe_temp_file(ctx, "", keep, file->fullname);
+	errmsg = compile_and_link(ctx, file->fullname, ccan_dir,
 				  obj_list(m, link_with_module),
 				  fail ? "-DFAIL" : "",
-					  lib_list(m), &errmsg);
-	if (!file->compiled)
+				  lib_list(m), file->compiled);
+	if (errmsg) {
+		talloc_free(file->compiled);
 		return errmsg;
-	talloc_steal(ctx, file->compiled);
+	}
 	return NULL;
 }
 
@@ -82,7 +87,9 @@ struct compile_tests_result {
 	const char *output;
 };
 
-static void *do_compile_tests(struct manifest *m, unsigned int *timeleft)
+static void *do_compile_tests(struct manifest *m,
+			      bool keep,
+			      unsigned int *timeleft)
 {
 	struct list_head *list = talloc(m, struct list_head);
 	char *cmdout;
@@ -93,7 +100,7 @@ static void *do_compile_tests(struct manifest *m, unsigned int *timeleft)
 
 	list_for_each(&m->compile_ok_tests, i, list) {
 		compile_tests.total_score++;
-		cmdout = compile(list, m, i, false, false);
+		cmdout = compile(list, m, i, false, false, keep);
 		if (cmdout) {
 			res = talloc(list, struct compile_tests_result);
 			res->filename = i->name;
@@ -105,7 +112,7 @@ static void *do_compile_tests(struct manifest *m, unsigned int *timeleft)
 
 	list_for_each(&m->run_tests, i, list) {
 		compile_tests.total_score++;
-		cmdout = compile(m, m, i, false, false);
+		cmdout = compile(m, m, i, false, false, keep);
 		if (cmdout) {
 			res = talloc(list, struct compile_tests_result);
 			res->filename = i->name;
@@ -117,7 +124,7 @@ static void *do_compile_tests(struct manifest *m, unsigned int *timeleft)
 
 	list_for_each(&m->api_tests, i, list) {
 		compile_tests.total_score++;
-		cmdout = compile(m, m, i, false, true);
+		cmdout = compile(m, m, i, false, true, keep);
 		if (cmdout) {
 			res = talloc(list, struct compile_tests_result);
 			res->filename = i->name;
@@ -129,7 +136,7 @@ static void *do_compile_tests(struct manifest *m, unsigned int *timeleft)
 
 	list_for_each(&m->compile_fail_tests, i, list) {
 		compile_tests.total_score++;
-		cmdout = compile(list, m, i, false, false);
+		cmdout = compile(list, m, i, false, false, false);
 		if (cmdout) {
 			res = talloc(list, struct compile_tests_result);
 			res->filename = i->name;
@@ -137,7 +144,7 @@ static void *do_compile_tests(struct manifest *m, unsigned int *timeleft)
 			res->output = talloc_steal(res, cmdout);
 			list_add_tail(list, &res->list);
 		} else {
-			cmdout = compile(list, m, i, true, false);
+			cmdout = compile(list, m, i, true, false, false);
 			if (!cmdout) {
 				res = talloc(list, struct compile_tests_result);
 				res->filename = i->name;
@@ -184,7 +191,7 @@ static const char *describe_compile_tests(struct manifest *m,
 }
 
 struct ccanlint compile_tests = {
-	.key = "compile",
+	.key = "compile-tests",
 	.name = "Module tests compile",
 	.score = score_compile_tests,
 	.check = do_compile_tests,

tools/ccanlint/compulsory_tests/has_info.c

@@ -12,7 +12,9 @@
 #include <ccan/noerr/noerr.h>
 #include <ccan/talloc/talloc.h>
 
-static void *check_has_info(struct manifest *m, unsigned int *timeleft)
+static void *check_has_info(struct manifest *m,
+			    bool keep,
+			    unsigned int *timeleft)
 {
 	if (m->info_file)
 		return NULL;

tools/ccanlint/compulsory_tests/has_main_header.c

@@ -12,7 +12,9 @@
 #include <ccan/talloc/talloc.h>
 #include <ccan/noerr/noerr.h>
 
-static void *check_has_main_header(struct manifest *m, unsigned int *timeleft)
+static void *check_has_main_header(struct manifest *m,
+				   bool keep,
+				   unsigned int *timeleft)
 {
 	struct ccan_file *f;
 

tools/ccanlint/compulsory_tests/has_tests.c

@@ -12,7 +12,9 @@
 
 static char test_is_not_dir[] = "test is not a directory";
 
-static void *check_has_tests(struct manifest *m, unsigned int *timeleft)
+static void *check_has_tests(struct manifest *m,
+			     bool keep,
+			     unsigned int *timeleft)
 {
 	struct stat st;
 	char *test_dir = talloc_asprintf(m, "%s/test", m->dir);

tools/ccanlint/compulsory_tests/run_tests.c

@@ -27,7 +27,9 @@ struct run_tests_result {
 	const char *output;
 };
 
-static void *do_run_tests(struct manifest *m, unsigned int *timeleft)
+static void *do_run_tests(struct manifest *m,
+			  bool keep,
+			  unsigned int *timeleft)
 {
 	struct list_head *list = talloc(m, struct list_head);
 	struct run_tests_result *res;

tools/ccanlint/tests/has_info_documentation.c

@@ -23,6 +23,7 @@ struct info_docs
 };
 
 static void *check_has_info_documentation(struct manifest *m,
+					  bool keep,
 					  unsigned int *timeleft)
 {
 	struct list_head *infodocs = get_ccan_file_docs(m->info_file);

tools/ccanlint/tests/idempotent.c

@@ -112,7 +112,9 @@ static char *report_idem(struct ccan_file *f, char *sofar)
 	return sofar;
 }
 
-static void *check_idempotent(struct manifest *m, unsigned int *timeleft)
+static void *check_idempotent(struct manifest *m,
+			      bool keep,
+			      unsigned int *timeleft)
 {
 	struct ccan_file *f;
 	char *report = NULL;

tools/ccanlint/tests/run_tests_valgrind.c

@@ -31,7 +31,9 @@ struct run_tests_result {
 	const char *output;
 };
 
-static void *do_run_tests_vg(struct manifest *m, unsigned int *timeleft)
+static void *do_run_tests_vg(struct manifest *m,
+			     bool keep,
+			     unsigned int *timeleft)
 {
 	struct list_head *list = talloc(m, struct list_head);
 	struct run_tests_result *res;
@@ -128,7 +130,7 @@ static void run_under_debugger_vg(struct manifest *m, void *check_result)
 }
 
 struct ccanlint run_tests_vg = {
-	.key = "valgrind",
+	.key = "valgrind-tests",
 	.name = "Module's run and api tests succeed under valgrind",
 	.score = score_run_tests_vg,
 	.check = do_run_tests_vg,

tools/ccanlint/tests/trailing_whitespace.c

@@ -20,6 +20,7 @@ static char *report_on_trailing_whitespace(const char *line)
 }
 
 static void *check_trailing_whitespace(struct manifest *m,
+				       bool keep,
 				       unsigned int *timeleft)
 {
 	char *report;

tools/compile.c

@@ -17,32 +17,18 @@ char *link_objects(const void *ctx, const char *objs, char **errmsg)
 
 /* Compile a single C file to an object file.  Returns errmsg if fails. */
 char *compile_object(const void *ctx, const char *cfile, const char *ccandir,
-		     char **errmsg)
+		     const char *outfile)
 {
-	char *file = temp_file(ctx, ".o");
-
-	*errmsg = run_command(ctx, NULL, "cc " CFLAGS " -I%s -c -o %s %s",
-			      ccandir, file, cfile);
-	if (*errmsg) {
-		talloc_free(file);
-		return NULL;
-	}
-	return file;
+	return run_command(ctx, NULL, "cc " CFLAGS " -I%s -c -o %s %s",
+			   ccandir, outfile, cfile);
 }
 
 /* Compile and link single C file, with object files.
- * Returns name of result, or NULL (and fills in errmsg). */
+ * Returns error message or NULL on success. */
 char *compile_and_link(const void *ctx, const char *cfile, const char *ccandir,
 		       const char *objs, const char *extra_cflags,
-		       const char *libs, char **errmsg)
+		       const char *libs, const char *outfile)
 {
-	char *file = temp_file(ctx, "");
-
-	*errmsg = run_command(ctx, NULL, "cc " CFLAGS " -I%s %s -o %s %s %s %s",
-			      ccandir, extra_cflags, file, cfile, objs, libs);
-	if (*errmsg) {
-		talloc_free(file);
-		return NULL;
-	}
-	return file;
+	return run_command(ctx, NULL, "cc " CFLAGS " -I%s %s -o %s %s %s %s",
+			   ccandir, extra_cflags, outfile, cfile, objs, libs);
 }

tools/depends.c

@@ -39,7 +39,7 @@ lines_from_cmd(const void *ctx, unsigned int *num, char *format, ...)
  * temp_file helps here. */
 static char *compile_info(const void *ctx, const char *dir)
 {
-	char *info_c_file, *info, *errmsg, *ccandir;
+	char *info_c_file, *info, *ccandir, *compiled;
 	size_t len;
 	int fd;
 
@@ -60,8 +60,12 @@ static char *compile_info(const void *ctx, const char *dir)
 
 	ccandir = talloc_dirname(ctx, dir);
 	*strrchr(ccandir, '/') = '\0';
-	return compile_and_link(ctx, info_c_file, ccandir, "", "", "",
-				&errmsg);
+
+	compiled = temp_file(ctx, "");
+	if (compile_and_link(ctx, info_c_file, ccandir, "", "", "",
+			     compiled))
+		return NULL;
+	return compiled;
 }
 
 static char **get_one_deps(const void *ctx, const char *dir,

tools/tools.c

@@ -199,6 +199,18 @@ char *temp_file(const void *ctx, const char *extension)
 	return talloc_asprintf(ctx, "%s/%u%s", tmpdir, count++, extension);
 }
 
+char *maybe_temp_file(const void *ctx, const char *extension, bool keep,
+		      const char *srcname)
+{
+	size_t baselen;
+
+	if (!keep)
+		return temp_file(ctx, extension);
+
+	baselen = strrchr(srcname, '.') - srcname;
+	return talloc_asprintf(ctx, "%.*s%s", baselen, srcname, extension);
+}
+
 bool move_file(const char *oldname, const char *newname)
 {
 	char *contents;

tools/tools.h

@@ -40,11 +40,16 @@ bool move_file(const char *oldname, const char *newname);
 char *link_objects(const void *ctx, const char *objs, char **errmsg);
 /* Compile a single C file to an object file.  Returns errmsg if fails. */
 char *compile_object(const void *ctx, const char *cfile, const char *ccandir,
-		     char **errmsg);
-/* Compile and link single C file, with object files, libs, etc. */
+		     const char *outfile);
+/* Compile and link single C file, with object files, libs, etc.  NULL on
+ * success, error output on fail. */
 char *compile_and_link(const void *ctx, const char *cfile, const char *ccandir,
 		       const char *objs, const char *extra_cflags,
-		       const char *libs, char **errmsg);
+		       const char *libs, const char *outfile);
+
+/* If keep is false, return a temporary file.  Otherwise, base it on srcname */
+char *maybe_temp_file(const void *ctx, const char *extension, bool keep,
+		      const char *srcname);
 
 /* Default wait for run_command.  Should never time out. */
 extern const unsigned int default_timeout_ms;