[PATCH] kbuild: external module support

From: Sam Ravnborg <sam@ravnborg.org>

Based on initial patch from Andreas Gruenbacher there is now better support
for building external modules with kbuild.

The preferred syntax is now:
make -C $KERNELSRC M=$PWD

but the old syntax:
make -C $KERNELSRC SUBDIRS=$PWD modules
will remain supported.

The major differences compared to before are that:
1) No attempt is made to neither check nor update any files in $KERNELSRC
2) Module versions are now supported

During stage 2 of kernel compilation where the modules are built, a new file
Module.symvers is created.  This file contains the version for all symbols
exported by the kernel and any module compiled within the kernel tree.

When the external module is build the Module.symvers file is being read and
symbol versions are used from that file.

The purpose of avoiding any updates in the kernel src is that usually in a
distribution the kernel src will be read-only, and there is no need to try to
update it.  And when building an external module the focus is on the module,
not the kernel.

I expect the distributions will start using something like this:

kernel src - with no generated files. Not even .config:
/usr/src/linux-<version>

Output from build:
/lib/modules/linux-<version>/build

where build is a real directory with relevant output files and the
appropriate .config.

I have some Documentation in the pipe-line, but wants to see how this
approach is received before completing it.

This patch is made on top of the previously posted patch to divide
make clean in three steps.

And you may need to edit the following line in the patch to make it apply:
 %docs: scripts_basic FORCE
to
 %docs: scripts FORCE

scripts/Makefile.modpost

@@ -13,12 +13,6 @@ include scripts/Makefile.lib
 __modules := $(shell head -q -n1 /dev/null $(wildcard $(MODVERDIR)/*.mod))
 modules := $(patsubst %.o,%.ko,$(wildcard $(__modules:.ko=.o)))
 
-ifneq ($(filter-out $(modules),$(__modules)),)
-  $(warning Trouble: $(filter-out $(modules),$(__modules)))
-  $(warning *** Uh-oh, you have stale module entries. You messed with SUBDIRS,)
-  $(warning     do not complain if something goes wrong.)
-endif
-
 __modversions: $(modules)
 	@:
 
@@ -55,9 +49,11 @@ $(modules:.ko=.mod.c): __modpost ;
 # Extract all checksums for all exported symbols
 
 quiet_cmd_modpost = MODPOST
-      cmd_modpost = scripts/modpost $(filter-out FORCE,$^)
+      cmd_modpost = scripts/modpost \
+	$(if $(filter vmlinux,$^),-o,-i) $(objtree)/Module.symvers \
+	$(filter-out FORCE,$^)
 
-__modpost: $(wildcard vmlinux) $(modules:.ko=.o) FORCE
+__modpost: $(if $(KBUILD_EXTMOD),,$(wildcard vmlinux)) $(modules:.ko=.o) FORCE
 	$(call if_changed,modpost)
 
 targets += __modpost

scripts/modpost.c

@@ -16,7 +16,7 @@
 
 /* Are we using CONFIG_MODVERSIONS? */
 int modversions = 0;
-/* Do we have vmlinux? */
+/* Warn about undefined symbols? (do so if we have vmlinux) */
 int have_vmlinux = 0;
 
 void
@@ -58,6 +58,17 @@ void *do_nofail(void *ptr, const char *file, int line, const char *expr)
 
 static struct module *modules;
 
+struct module *
+find_module(char *modname)
+{
+	struct module *mod;
+
+	for (mod = modules; mod; mod = mod->next)
+		if (strcmp(mod->name, modname) == 0)
+			break;
+	return mod;
+}
+
 struct module *
 new_module(char *modname)
 {
@@ -181,7 +192,7 @@ grab_file(const char *filename, unsigned long *size)
 	int fd;
 
 	fd = open(filename, O_RDONLY);
-	if (fstat(fd, &st) != 0)
+	if (fd < 0 || fstat(fd, &st) != 0)
 		return NULL;
 
 	*size = st.st_size;
@@ -402,6 +413,8 @@ read_symbols(char *modname)
 		/* May not have this if !CONFIG_MODULE_UNLOAD: fake it.
 		   If it appears, we'll get the real CRC. */
 		add_exported_symbol("cleanup_module", mod, &fake_crc);
+		add_exported_symbol("struct_module", mod, &fake_crc);
+		mod->skip = 1;
 	}
 
 	for (sym = info.symtab_start; sym < info.symtab_stop; sym++) {
@@ -604,19 +617,106 @@ write_if_changed(struct buffer *b, const char *fname)
 	fclose(file);
 }
 
+void
+read_dump(const char *fname)
+{
+	unsigned long size, pos = 0;
+	void *file = grab_file(fname, &size);
+	char *line;
+
+        if (!file) {
+                perror(fname);
+                abort();
+        }
+
+	while ((line = get_next_line(&pos, file, size))) {
+		char *symname, *modname, *d;
+		unsigned int crc;
+		struct module *mod;
+
+		if (!(symname = strchr(line, '\t')))
+			goto fail;
+		*symname++ = '\0';
+		if (!(modname = strchr(symname, '\t')))
+			goto fail;
+		*modname++ = '\0';
+		if (strchr(modname, '\t'))
+			goto fail;
+		crc = strtoul(line, &d, 16);
+		if (*symname == '\0' || *modname == '\0' || *d != '\0')
+			goto fail;
+
+		if (!(mod = find_module(modname))) {
+			if (is_vmlinux(modname)) {
+				modversions = 1;
+				have_vmlinux = 1;
+			}
+			mod = new_module(NOFAIL(strdup(modname)));
+			mod->skip = 1;
+		}
+		add_exported_symbol(symname, mod, &crc);
+	}
+	return;
+fail:
+	fatal("parse error in symbol dump file\n");
+}
+
+void
+write_dump(const char *fname)
+{
+	struct buffer buf = { };
+	struct symbol *symbol;
+	int n;
+
+	for (n = 0; n < SYMBOL_HASH_SIZE ; n++) {
+		symbol = symbolhash[n];
+		while (symbol) {
+			symbol = symbol->next;
+		}
+	}
+
+	for (n = 0; n < SYMBOL_HASH_SIZE ; n++) {
+		symbol = symbolhash[n];
+		while (symbol) {
+			buf_printf(&buf, "0x%08x\t%s\t%s\n", symbol->crc,
+				symbol->name, symbol->module->name);
+			symbol = symbol->next;
+		}
+	}
+	write_if_changed(&buf, fname);
+}
+
 int
 main(int argc, char **argv)
 {
 	struct module *mod;
 	struct buffer buf = { };
 	char fname[SZ];
+	char *dump_read = NULL, *dump_write = NULL;
+	int opt;
 
-	for (; argv[1]; argv++) {
-		read_symbols(argv[1]);
+	while ((opt = getopt(argc, argv, "i:o:")) != -1) {
+		switch(opt) {
+			case 'i':
+				dump_read = optarg;
+				break;
+			case 'o':
+				dump_write = optarg;
+				break;
+			default:
+				exit(1);
+		}
+	}
+
+	if (dump_read)
+		read_dump(dump_read);
+
+	while (optind < argc) {
+		read_symbols(argv[optind++]);
 	}
 
 	for (mod = modules; mod; mod = mod->next) {
-		if (is_vmlinux(mod->name))
+		if (mod->skip)
 			continue;
 
 		buf.pos = 0;
@@ -629,6 +729,10 @@ main(int argc, char **argv)
 		sprintf(fname, "%s.mod.c", mod->name);
 		write_if_changed(&buf, fname);
 	}
+
+	if (dump_write)
+		write_dump(dump_write);
+
 	return 0;
 }
 

scripts/modpost.h

@@ -73,6 +73,7 @@ struct module {
 	const char *name;
 	struct symbol *unres;
 	int seen;
+	int skip;
 	struct buffer dev_table_buf;
 };