[PATCH] Change mmu_gathers into per-cpu data

From: Martin Hicks <mort@wildopensource.com>

Here is a patch that changes mmu_gathers into a per-cpu resource.  It
includes the changes for all arches except ia64.  I've sent a separate patch
to David Mosberger for ia64.

arch/alpha/mm/init.c

@@ -34,7 +34,7 @@
 #include <asm/console.h>
 #include <asm/tlb.h>
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 extern void die_if_kernel(char *,struct pt_regs *,long);
 

arch/i386/mm/init.c

@@ -41,7 +41,7 @@
 #include <asm/tlbflush.h>
 #include <asm/sections.h>
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 unsigned long highstart_pfn, highend_pfn;
 
 static int do_test_wp_bit(void);

arch/m68k/mm/init.c

@@ -33,7 +33,7 @@
 #endif
 #include <asm/tlb.h>
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 /*
  * ZERO_PAGE is a special page that is used for zero-initialized

arch/mips/mm/init.c

@@ -42,7 +42,7 @@
 #include <asm/mmu_context.h>
 #include <asm/tlb.h>
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 extern void prom_free_prom_memory(void);
 

arch/mips64/mm/init.c

@@ -35,7 +35,7 @@
 #include <asm/mmu_context.h>
 #include <asm/tlb.h>
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 void pgd_init(unsigned long page)
 {

arch/parisc/mm/init.c

@@ -23,7 +23,7 @@
 #include <asm/tlb.h>
 #include <asm/pdc_chassis.h>
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 extern char _text;	/* start of kernel code, defined by linker */
 extern int  data_start;

arch/ppc/mm/init.c

@@ -53,7 +53,7 @@
 #endif
 #define MAX_LOW_MEM	CONFIG_LOWMEM_SIZE
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 unsigned long total_memory;
 unsigned long total_lowmem;

arch/ppc64/mm/init.c

@@ -93,7 +93,7 @@ unsigned long __max_memory;
 /* This is declared as we are using the more or less generic 
  * include/asm-ppc64/tlb.h file -- tgall
  */
-struct mmu_gather     mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 void show_mem(void)
 {

arch/s390/mm/init.c

@@ -38,7 +38,7 @@
 #include <asm/tlb.h>
 #include <asm/tlbflush.h>
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 pgd_t swapper_pg_dir[PTRS_PER_PGD] __attribute__((__aligned__(PAGE_SIZE)));
 char  empty_zero_page[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE)));

arch/sh/mm/init.c

@@ -36,7 +36,7 @@
 #include <asm/io.h>
 #include <asm/tlb.h>
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 /*
  * Cache of MMU context last used.

arch/sparc/mm/init.c

@@ -32,7 +32,7 @@
 #include <asm/pgalloc.h>	/* bug in asm-generic/tlb.h: check_pgt_cache */
 #include <asm/tlb.h>
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 unsigned long *sparc_valid_addr_bitmap;
 

arch/sparc64/mm/init.c

@@ -36,7 +36,7 @@
 #include <asm/tlb.h>
 #include <asm/spitfire.h>
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 extern void device_scan(void);
 

arch/um/kernel/mem.c

@@ -45,7 +45,7 @@ extern char __init_begin, __init_end;
 extern long physmem_size;
 
 /* Not changed by UML */
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 /* Changed during early boot */
 int kmalloc_ok = 0;

arch/x86_64/mm/init.c

@@ -41,7 +41,7 @@
 
 #define Dprintk(x...)
 
-struct mmu_gather mmu_gathers[NR_CPUS];
+DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 /*
  * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the

include/asm-arm/tlb.h

@@ -33,13 +33,13 @@ struct mmu_gather {
 	unsigned int		avoided_flushes;
 };
 
-extern struct mmu_gather mmu_gathers[NR_CPUS];
+DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 static inline struct mmu_gather *
 tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush)
 {
 	int cpu = smp_processor_id();
-	struct mmu_gather *tlb = &mmu_gathers[cpu];
+	struct mmu_gather *tlb = &per_cpu(mmu_gathers, cpu);
 
 	tlb->mm = mm;
 	tlb->freed = 0;

include/asm-generic/tlb.h

@@ -44,7 +44,7 @@ struct mmu_gather {
 };
 
 /* Users of the generic TLB shootdown code must declare this storage space. */
-extern struct mmu_gather	mmu_gathers[NR_CPUS];
+DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
 
 /* tlb_gather_mmu
  *	Return a pointer to an initialized struct mmu_gather.
@@ -52,7 +52,7 @@ extern struct mmu_gather	mmu_gathers[NR_CPUS];
 static inline struct mmu_gather *
 tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush)
 {
-	struct mmu_gather *tlb = &mmu_gathers[smp_processor_id()];
+	struct mmu_gather *tlb = &per_cpu(mmu_gathers, smp_processor_id());
 
 	tlb->mm = mm;