Commit 83b519e8 authored by Pekka Enberg's avatar Pekka Enberg

slab: setup allocators earlier in the boot sequence

This patch makes kmalloc() available earlier in the boot sequence so we can get
rid of some bootmem allocations. The bulk of the changes are due to
kmem_cache_init() being called with interrupts disabled which requires some
changes to allocator boostrap code.

Note: 32-bit x86 does WP protect test in mem_init() so we must setup traps
before we call mem_init() during boot as reported by Ingo Molnar:

  We have a hard crash in the WP-protect code:

  [    0.000000] Checking if this processor honours the WP bit even in supervisor mode...BUG: Int 14: CR2 ffcff000
  [    0.000000]      EDI 00000188  ESI 00000ac7  EBP c17eaf9c  ESP c17eaf8c
  [    0.000000]      EBX 000014e0  EDX 0000000e  ECX 01856067  EAX 00000001
  [    0.000000]      err 00000003  EIP c10135b1   CS 00000060  flg 00010002
  [    0.000000] Stack: c17eafa8 c17fd410 c16747bc c17eafc4 c17fd7e5 000011fd f8616000 c18237cc
  [    0.000000]        00099800 c17bb000 c17eafec c17f1668 000001c5 c17f1322 c166e039 c1822bf0
  [    0.000000]        c166e033 c153a014 c18237cc 00020800 c17eaff8 c17f106a 00020800 01ba5003
  [    0.000000] Pid: 0, comm: swapper Not tainted 2.6.30-tip-02161-g7a74539-dirty #52203
  [    0.000000] Call Trace:
  [    0.000000]  [<c15357c2>] ? printk+0x14/0x16
  [    0.000000]  [<c10135b1>] ? do_test_wp_bit+0x19/0x23
  [    0.000000]  [<c17fd410>] ? test_wp_bit+0x26/0x64
  [    0.000000]  [<c17fd7e5>] ? mem_init+0x1ba/0x1d8
  [    0.000000]  [<c17f1668>] ? start_kernel+0x164/0x2f7
  [    0.000000]  [<c17f1322>] ? unknown_bootoption+0x0/0x19c
  [    0.000000]  [<c17f106a>] ? __init_begin+0x6a/0x6f
Acked-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Acked-by Linus Torvalds <torvalds@linux-foundation.org>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Matt Mackall <mpm@selenic.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Yinghai Lu <yinghai@kernel.org>
Signed-off-by: default avatarPekka Enberg <penberg@cs.helsinki.fi>
parent c91c4773
...@@ -574,6 +574,28 @@ asmlinkage void __init start_kernel(void) ...@@ -574,6 +574,28 @@ asmlinkage void __init start_kernel(void)
setup_nr_cpu_ids(); setup_nr_cpu_ids();
smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */
build_all_zonelists();
page_alloc_init();
printk(KERN_NOTICE "Kernel command line: %s\n", boot_command_line);
parse_early_param();
parse_args("Booting kernel", static_command_line, __start___param,
__stop___param - __start___param,
&unknown_bootoption);
/*
* These use large bootmem allocations and must precede
* kmem_cache_init()
*/
pidhash_init();
vmalloc_init();
vfs_caches_init_early();
sort_main_extable();
trap_init();
/*
* Set up kernel memory allocators
*/
mem_init();
kmem_cache_init();
/* /*
* Set up the scheduler prior starting any interrupts (such as the * Set up the scheduler prior starting any interrupts (such as the
* timer interrupt). Full topology setup happens at smp_init() * timer interrupt). Full topology setup happens at smp_init()
...@@ -585,25 +607,15 @@ asmlinkage void __init start_kernel(void) ...@@ -585,25 +607,15 @@ asmlinkage void __init start_kernel(void)
* fragile until we cpu_idle() for the first time. * fragile until we cpu_idle() for the first time.
*/ */
preempt_disable(); preempt_disable();
build_all_zonelists();
page_alloc_init();
printk(KERN_NOTICE "Kernel command line: %s\n", boot_command_line);
parse_early_param();
parse_args("Booting kernel", static_command_line, __start___param,
__stop___param - __start___param,
&unknown_bootoption);
if (!irqs_disabled()) { if (!irqs_disabled()) {
printk(KERN_WARNING "start_kernel(): bug: interrupts were " printk(KERN_WARNING "start_kernel(): bug: interrupts were "
"enabled *very* early, fixing it\n"); "enabled *very* early, fixing it\n");
local_irq_disable(); local_irq_disable();
} }
sort_main_extable();
trap_init();
rcu_init(); rcu_init();
/* init some links before init_ISA_irqs() */ /* init some links before init_ISA_irqs() */
early_irq_init(); early_irq_init();
init_IRQ(); init_IRQ();
pidhash_init();
init_timers(); init_timers();
hrtimers_init(); hrtimers_init();
softirq_init(); softirq_init();
...@@ -645,14 +657,10 @@ asmlinkage void __init start_kernel(void) ...@@ -645,14 +657,10 @@ asmlinkage void __init start_kernel(void)
initrd_start = 0; initrd_start = 0;
} }
#endif #endif
vmalloc_init();
vfs_caches_init_early();
cpuset_init_early(); cpuset_init_early();
page_cgroup_init(); page_cgroup_init();
mem_init();
enable_debug_pagealloc(); enable_debug_pagealloc();
cpu_hotplug_init(); cpu_hotplug_init();
kmem_cache_init();
kmemtrace_init(); kmemtrace_init();
debug_objects_mem_init(); debug_objects_mem_init();
idr_init_cache(); idr_init_cache();
......
This diff is collapsed.
...@@ -2557,13 +2557,16 @@ static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s, ...@@ -2557,13 +2557,16 @@ static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s,
if (gfp_flags & SLUB_DMA) if (gfp_flags & SLUB_DMA)
flags = SLAB_CACHE_DMA; flags = SLAB_CACHE_DMA;
down_write(&slub_lock); /*
* This function is called with IRQs disabled during early-boot on
* single CPU so there's no need to take slub_lock here.
*/
if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN, if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN,
flags, NULL)) flags, NULL))
goto panic; goto panic;
list_add(&s->list, &slab_caches); list_add(&s->list, &slab_caches);
up_write(&slub_lock);
if (sysfs_slab_add(s)) if (sysfs_slab_add(s))
goto panic; goto panic;
return s; return s;
...@@ -3021,7 +3024,7 @@ void __init kmem_cache_init(void) ...@@ -3021,7 +3024,7 @@ void __init kmem_cache_init(void)
* kmem_cache_open for slab_state == DOWN. * kmem_cache_open for slab_state == DOWN.
*/ */
create_kmalloc_cache(&kmalloc_caches[0], "kmem_cache_node", create_kmalloc_cache(&kmalloc_caches[0], "kmem_cache_node",
sizeof(struct kmem_cache_node), GFP_KERNEL); sizeof(struct kmem_cache_node), GFP_NOWAIT);
kmalloc_caches[0].refcount = -1; kmalloc_caches[0].refcount = -1;
caches++; caches++;
...@@ -3034,16 +3037,16 @@ void __init kmem_cache_init(void) ...@@ -3034,16 +3037,16 @@ void __init kmem_cache_init(void)
/* Caches that are not of the two-to-the-power-of size */ /* Caches that are not of the two-to-the-power-of size */
if (KMALLOC_MIN_SIZE <= 64) { if (KMALLOC_MIN_SIZE <= 64) {
create_kmalloc_cache(&kmalloc_caches[1], create_kmalloc_cache(&kmalloc_caches[1],
"kmalloc-96", 96, GFP_KERNEL); "kmalloc-96", 96, GFP_NOWAIT);
caches++; caches++;
create_kmalloc_cache(&kmalloc_caches[2], create_kmalloc_cache(&kmalloc_caches[2],
"kmalloc-192", 192, GFP_KERNEL); "kmalloc-192", 192, GFP_NOWAIT);
caches++; caches++;
} }
for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) { for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
create_kmalloc_cache(&kmalloc_caches[i], create_kmalloc_cache(&kmalloc_caches[i],
"kmalloc", 1 << i, GFP_KERNEL); "kmalloc", 1 << i, GFP_NOWAIT);
caches++; caches++;
} }
...@@ -3080,7 +3083,7 @@ void __init kmem_cache_init(void) ...@@ -3080,7 +3083,7 @@ void __init kmem_cache_init(void)
/* Provide the correct kmalloc names now that the caches are up */ /* Provide the correct kmalloc names now that the caches are up */
for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++)
kmalloc_caches[i]. name = kmalloc_caches[i]. name =
kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i); kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
register_cpu_notifier(&slab_notifier); register_cpu_notifier(&slab_notifier);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment