[PATCH] Use CPU_UP_PREPARE properly

From: Rusty Russell <rusty@rustcorp.com.au>

The cpu hotplug code actually provides two notifiers: CPU_UP_PREPARE
which preceeds the online and can fail, and CPU_ONLINE which can't.

Current usage is only done at boot, so this distinction doesn't
matter, but it's a bad example to set.  This also means that the
migration threads do not have to be higher priority than the
others, since they are ready to go before any CPU_ONLINE callbacks
are done.

This patch is experimental but fairly straight foward: I haven't been
able to test it since extracting it from the hotplug cpu code, so it's
possible I screwed something up.

kernel/sched.c

@@ -2806,33 +2806,39 @@ static int migration_call(struct notifier_block *nfb, unsigned long action,
 	struct task_struct *p;
 
 	switch (action) {
-	case CPU_ONLINE:
+	case CPU_UP_PREPARE:
 		p = kthread_create(migration_thread, hcpu, "migration/%d",cpu);
 		if (IS_ERR(p))
 			return NOTIFY_BAD;
 		kthread_bind(p, cpu);
 		cpu_rq(cpu)->migration_thread = p;
-		wake_up_process(p);
+		break;
+	case CPU_ONLINE:
+		/* Strictly unneccessary, as first user will wake it. */
+		wake_up_process(cpu_rq(cpu)->migration_thread);
 		break;
 	}
 	return NOTIFY_OK;
 }
 
-/* Want this before the other threads, so they can use set_cpus_allowed. */
-static struct notifier_block migration_notifier = {
-	.notifier_call = &migration_call,
-	.priority = 10,
+/*
+ * We want this after the other threads, so they can use set_cpus_allowed
+ * from their CPU_OFFLINE callback
+ */
+static struct notifier_block __devinitdata migration_notifier = {
+	.notifier_call = migration_call,
+	.priority = -10,
 };
 
-__init int migration_init(void)
+int __init migration_init(void)
 {
+	void *cpu = (void *)(long)smp_processor_id();
 	/* Start one for boot CPU. */
-	migration_call(&migration_notifier, CPU_ONLINE,
-		       (void *)(long)smp_processor_id());
+	migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
+	migration_call(&migration_notifier, CPU_ONLINE, cpu);
 	register_cpu_notifier(&migration_notifier);
 	return 0;
 }
-
 #endif
 
 /*

kernel/softirq.c

@@ -340,7 +340,10 @@ static int __devinit cpu_callback(struct notifier_block *nfb,
 	int hotcpu = (unsigned long)hcpu;
 	struct task_struct *p;
 
-	if (action == CPU_ONLINE) {
+	switch (action) {
+	case CPU_UP_PREPARE:
+		BUG_ON(per_cpu(tasklet_vec, hotcpu).list);
+		BUG_ON(per_cpu(tasklet_hi_vec, hotcpu).list);
 		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 		if (IS_ERR(p)) {
 			printk("ksoftirqd for %i failed\n", hotcpu);
@@ -348,7 +351,11 @@ static int __devinit cpu_callback(struct notifier_block *nfb,
 		}
 		per_cpu(ksoftirqd, hotcpu) = p;
 		kthread_bind(p, hotcpu);
-		wake_up_process(p);
+  		per_cpu(ksoftirqd, hotcpu) = p;
+ 		break;
+	case CPU_ONLINE:
+		wake_up_process(per_cpu(ksoftirqd, hotcpu));
+		break;
  	}
 	return NOTIFY_OK;
 }
@@ -359,7 +366,9 @@ static struct notifier_block __devinitdata cpu_nfb = {
 
 __init int spawn_ksoftirqd(void)
 {
-	cpu_callback(&cpu_nfb, CPU_ONLINE, (void *)(long)smp_processor_id());
+	void *cpu = (void *)(long)smp_processor_id();
+	cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
+	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 	register_cpu_notifier(&cpu_nfb);
 	return 0;
 }