[SPARC64]: Turn cpu_data into per-cpu data.

arch/sparc64/kernel/ebus.c

@@ -13,6 +13,7 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/interrupt.h>
+#include <linux/delay.h>
 
 #include <asm/system.h>
 #include <asm/page.h>

arch/sparc64/kernel/head.S

@@ -628,70 +628,21 @@ spitfire_vpte_base:
 #undef KERN_HIGHBITS
 #undef KERN_LOWBITS
 
-	/* Setup Interrupt globals */
-	wrpr	%o1, (PSTATE_IG|PSTATE_IE), %pstate
-#ifndef CONFIG_SMP
-	sethi	%hi(__up_workvec), %g5
-	or	%g5, %lo(__up_workvec), %g6
-#else
-	/* By definition of where we are, this is boot_cpu. */
-	brz,pt	%i0, not_starfire
-	 sethi	%hi(0x1fff4000), %g1
-	or	%g1, %lo(0x1fff4000), %g1
-	sllx	%g1, 12, %g1
-	or	%g1, 0xd0, %g1
-	lduwa	[%g1] ASI_PHYS_BYPASS_EC_E, %g1
-	b,pt	%xcc, set_worklist
-	 nop
-
-not_starfire:
-	BRANCH_IF_JALAPENO(g1,g5,is_jalapeno)
-	BRANCH_IF_ANY_CHEETAH(g1,g5,is_cheetah)
-
-	ba,pt	%xcc, not_cheetah
-	 nop
-
-is_jalapeno:
-	ldxa		[%g0] ASI_JBUS_CONFIG, %g1
-	srlx		%g1, 17, %g1
-	ba,pt		%xcc, set_worklist
-	 and		%g1, 0x1f, %g1		! 5bit JBUS ID
-
-is_cheetah:
-	ldxa		[%g0] ASI_SAFARI_CONFIG, %g1
-	srlx		%g1, 17, %g1
-	ba,pt		%xcc, set_worklist
-	 and		%g1, 0x3ff, %g1		! 10bit Safari Agent ID
-
-not_cheetah:
-	ldxa	[%g0] ASI_UPA_CONFIG, %g1
-	srlx	%g1, 17, %g1
-	and	%g1, 0x1f, %g1
-
-	/* In theory this is: &(cpu_data[boot_cpu_id].irq_worklists[0]) */
-set_worklist:
-	sethi	%hi(cpu_data), %g5
-	or	%g5, %lo(cpu_data), %g5
-	sllx	%g1, 7, %g1
-	add	%g5, %g1, %g5
-	add	%g5, 64, %g6
-#endif
-
 	/* Kill PROM timer */
-	sethi	%hi(0x80000000), %g1
-	sllx	%g1, 32, %g1
-	wr	%g1, 0, %tick_cmpr
+	sethi	%hi(0x80000000), %o2
+	sllx	%o2, 32, %o2
+	wr	%o2, 0, %tick_cmpr
 
-	BRANCH_IF_ANY_CHEETAH(g1,g5,1f)
+	BRANCH_IF_ANY_CHEETAH(o2,o3,1f)
 
 	ba,pt	%xcc, 2f
 	 nop
 
 	/* Disable STICK_INT interrupts. */
 1:
-	sethi	%hi(0x80000000), %g1
-	sllx	%g1, 32, %g1
-	wr	%g1, %asr25
+	sethi	%hi(0x80000000), %o2
+	sllx	%o2, 32, %o2
+	wr	%o2, %asr25
 
 	/* Ok, we're done setting up all the state our trap mechanims needs,
 	 * now get back into normal globals and let the PROM know what is up.
@@ -700,6 +651,9 @@ set_worklist:
 	wrpr	%g0, %g0, %wstate
 	wrpr	%o1, PSTATE_IE, %pstate
 
+	call	init_irqwork_curcpu
+	 nop
+
 	sethi	%hi(sparc64_ttable_tl0), %g5
 	call	prom_set_trap_table
 	 mov	%g5, %o0

arch/sparc64/kernel/irq.c

@@ -36,6 +36,7 @@
 #include <asm/starfire.h>
 #include <asm/uaccess.h>
 #include <asm/cache.h>
+#include <asm/cpudata.h>
 
 #ifdef CONFIG_SMP
 static void distribute_irqs(void);
@@ -56,12 +57,18 @@ static void distribute_irqs(void);
 
 struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES)));
 
-#ifndef CONFIG_SMP
-unsigned int __up_workvec[16] __attribute__ ((aligned (SMP_CACHE_BYTES)));
-#define irq_work(__cpu, __pil)	&(__up_workvec[(void)(__cpu), (__pil)])
-#else
-#define irq_work(__cpu, __pil)	&(cpu_data[(__cpu)].irq_worklists[(__pil)])
-#endif
+/* This has to be in the main kernel image, it cannot be
+ * turned into per-cpu data.  The reason is that the main
+ * kernel image is locked into the TLB and this structure
+ * is accessed from the vectored interrupt trap handler.  If
+ * access to this structure takes a TLB miss it could cause
+ * the 5-level sparc v9 trap stack to overflow.
+ */
+struct irq_work_struct {
+	unsigned int	irq_worklists[16];
+};
+struct irq_work_struct __irq_work[NR_CPUS];
+#define irq_work(__cpu, __pil)	&(__irq_work[(__cpu)].irq_worklists[(__pil)])
 
 #ifdef CONFIG_PCI
 /* This is a table of physical addresses used to deal with IBF_DMA_SYNC.
@@ -706,7 +713,7 @@ static inline void redirect_intr(int cpu, struct ino_bucket *bp)
 		goto out;
 
 	/* Voo-doo programming. */
-	if (cpu_data[buddy].idle_volume < FORWARD_VOLUME)
+	if (cpu_data(buddy).idle_volume < FORWARD_VOLUME)
 		goto out;
 
 	/* This just so happens to be correct on Cheetah
@@ -1083,15 +1090,30 @@ void enable_prom_timer(void)
 	prom_timers->count0 = 0;
 }
 
+void init_irqwork_curcpu(void)
+{
+	register struct irq_work_struct *workp asm("o2");
+	unsigned long tmp;
+
+	memset(__irq_work + smp_processor_id(), 0, sizeof(*workp));
+
+	/* Set interrupt globals.  */
+	workp = &__irq_work[smp_processor_id()];
+	__asm__ __volatile__(
+	"rdpr	%%pstate, %0\n\t"
+	"wrpr	%0, %1, %%pstate\n\t"
+	"mov	%2, %%g6\n\t"
+	"wrpr	%0, 0x0, %%pstate\n\t"
+	: "=&r" (tmp)
+	: "i" (PSTATE_IG | PSTATE_IE), "r" (workp));
+}
+
 /* Only invoked on boot processor. */
 void __init init_IRQ(void)
 {
 	map_prom_timers();
 	kill_prom_timer();
 	memset(&ivector_table[0], 0, sizeof(ivector_table));
-#ifndef CONFIG_SMP
-	memset(&__up_workvec[0], 0, sizeof(__up_workvec));
-#endif
 
 	/* We need to clear any IRQ's pending in the soft interrupt
 	 * registers, a spurious one could be left around from the

arch/sparc64/kernel/process.c

@@ -41,6 +41,7 @@
 #include <asm/elf.h>
 #include <asm/fpumacro.h>
 #include <asm/head.h>
+#include <asm/cpudata.h>
 
 /* #define VERBOSE_SHOWREGS */
 
@@ -84,8 +85,8 @@ int cpu_idle(void)
 /*
  * the idle loop on a UltraMultiPenguin...
  */
-#define idle_me_harder()	(cpu_data[smp_processor_id()].idle_volume += 1)
-#define unidle_me()		(cpu_data[smp_processor_id()].idle_volume = 0)
+#define idle_me_harder()	(cpu_data(smp_processor_id()).idle_volume += 1)
+#define unidle_me()		(cpu_data(smp_processor_id()).idle_volume = 0)
 int cpu_idle(void)
 {
 	set_thread_flag(TIF_POLLING_NRFLAG);

arch/sparc64/kernel/smp.c

@@ -25,6 +25,7 @@
 #include <asm/atomic.h>
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
+#include <asm/cpudata.h>
 
 #include <asm/irq.h>
 #include <asm/page.h>
@@ -41,7 +42,7 @@
 extern int linux_num_cpus;
 extern void calibrate_delay(void);
 
-cpuinfo_sparc cpu_data[NR_CPUS];
+DEFINE_PER_CPU(cpuinfo_sparc, __cpu_data) = { 0 };
 
 /* Please don't make this stuff initdata!!!  --DaveM */
 static unsigned char boot_cpu_id;
@@ -73,32 +74,29 @@ void smp_bogo(struct seq_file *m)
 			seq_printf(m,
 				   "Cpu%dBogo\t: %lu.%02lu\n"
 				   "Cpu%dClkTck\t: %016lx\n",
-				   i, cpu_data[i].udelay_val / (500000/HZ),
-				   (cpu_data[i].udelay_val / (5000/HZ)) % 100,
-				   i, cpu_data[i].clock_tick);
+				   i, cpu_data(i).udelay_val / (500000/HZ),
+				   (cpu_data(i).udelay_val / (5000/HZ)) % 100,
+				   i, cpu_data(i).clock_tick);
 }
 
 void __init smp_store_cpu_info(int id)
 {
-	int i, cpu_node;
+	int cpu_node;
 
 	/* multiplier and counter set by
 	   smp_setup_percpu_timer()  */
-	cpu_data[id].udelay_val			= loops_per_jiffy;
+	cpu_data(id).udelay_val			= loops_per_jiffy;
 
 	cpu_find_by_mid(id, &cpu_node);
-	cpu_data[id].clock_tick = prom_getintdefault(cpu_node,
+	cpu_data(id).clock_tick = prom_getintdefault(cpu_node,
 						     "clock-frequency", 0);
 
-	cpu_data[id].pgcache_size		= 0;
-	cpu_data[id].pte_cache[0]		= NULL;
-	cpu_data[id].pte_cache[1]		= NULL;
-	cpu_data[id].pgdcache_size		= 0;
-	cpu_data[id].pgd_cache			= NULL;
-	cpu_data[id].idle_volume		= 1;
-
-	for (i = 0; i < 16; i++)
-		cpu_data[id].irq_worklists[i] = 0;
+	cpu_data(id).pgcache_size		= 0;
+	cpu_data(id).pte_cache[0]		= NULL;
+	cpu_data(id).pte_cache[1]		= NULL;
+	cpu_data(id).pgdcache_size		= 0;
+	cpu_data(id).pgd_cache			= NULL;
+	cpu_data(id).idle_volume		= 1;
 }
 
 static void smp_setup_percpu_timer(void);
@@ -1039,8 +1037,8 @@ void smp_promstop_others(void)
 
 extern void sparc64_do_profile(struct pt_regs *regs);
 
-#define prof_multiplier(__cpu)		cpu_data[(__cpu)].multiplier
-#define prof_counter(__cpu)		cpu_data[(__cpu)].counter
+#define prof_multiplier(__cpu)		cpu_data(__cpu).multiplier
+#define prof_counter(__cpu)		cpu_data(__cpu).counter
 
 void smp_percpu_timer_interrupt(struct pt_regs *regs)
 {
@@ -1303,7 +1301,7 @@ void __init smp_cpus_done(unsigned int max_cpus)
 
 	for (i = 0; i < NR_CPUS; i++) {
 		if (cpu_online(i))
-			bogosum += cpu_data[i].udelay_val;
+			bogosum += cpu_data(i).udelay_val;
 	}
 	printk("Total of %ld processors activated "
 	       "(%lu.%02lu BogoMIPS).\n",
@@ -1316,3 +1314,27 @@ void __init smp_cpus_done(unsigned int max_cpus)
 	 */
 	smp_tune_scheduling();
 }
+
+/* This needn't do anything as we do not sleep the cpu
+ * inside of the idler task, so an interrupt is not needed
+ * to get a clean fast response.
+ *
+ * XXX Reverify this assumption... -DaveM
+ *
+ * Addendum: We do want it to do something for the signal
+ *           delivery case, we detect that by just seeing
+ *           if we are trying to send this to an idler or not.
+ */
+void smp_send_reschedule(int cpu)
+{
+	if (cpu_data(cpu).idle_volume == 0)
+		smp_receive_signal(cpu);
+}
+
+/* This is a nop because we capture all other cpus
+ * anyways when making the PROM active.
+ */
+void smp_send_stop(void)
+{
+}
+

arch/sparc64/kernel/sparc64_ksyms.c

@@ -21,6 +21,7 @@
 #include <linux/fs_struct.h>
 #include <linux/mm.h>
 #include <linux/socket.h>
+#include <linux/percpu.h>
 #include <net/compat.h>
 
 #include <asm/oplib.h>
@@ -55,6 +56,7 @@
 #include <asm/a.out.h>
 #include <asm/ns87303.h>
 #include <asm/timer.h>
+#include <asm/cpudata.h>
 
 struct poll {
 	int fd;
@@ -143,7 +145,7 @@ EXPORT_SYMBOL_NOVERS(mcount);
 #endif
 
 /* Per-CPU information table */
-EXPORT_SYMBOL(cpu_data);
+EXPORT_PER_CPU_SYMBOL(__cpu_data);
 
 /* CPU online map and active count.  */
 EXPORT_SYMBOL(cpu_online_map);

arch/sparc64/kernel/time.c

@@ -42,6 +42,7 @@
 #include <asm/starfire.h>
 #include <asm/smp.h>
 #include <asm/sections.h>
+#include <asm/cpudata.h>
 
 spinlock_t mostek_lock = SPIN_LOCK_UNLOCKED;
 spinlock_t rtc_lock = SPIN_LOCK_UNLOCKED;
@@ -1013,7 +1014,7 @@ unsigned long sparc64_get_clock_tick(unsigned int cpu)
 	if (ft->clock_tick_ref)
 		return ft->clock_tick_ref;
 #ifdef CONFIG_SMP
-	return cpu_data[cpu].clock_tick;
+	return cpu_data(cpu).clock_tick;
 #else
 	return up_clock_tick;
 #endif
@@ -1031,16 +1032,16 @@ static int sparc64_cpufreq_notifier(struct notifier_block *nb, unsigned long val
 #ifdef CONFIG_SMP
 	if (!ft->ref_freq) {
 		ft->ref_freq = freq->old;
-		ft->udelay_val_ref = cpu_data[cpu].udelay_val;
-		ft->clock_tick_ref = cpu_data[cpu].clock_tick;
+		ft->udelay_val_ref = cpu_data(cpu).udelay_val;
+		ft->clock_tick_ref = cpu_data(cpu).clock_tick;
 	}
 	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
 	    (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) {
-		cpu_data[cpu].udelay_val =
+		cpu_data(cpu).udelay_val =
 			cpufreq_scale(ft->udelay_val_ref,
 				      ft->ref_freq,
 				      freq->new);
-		cpu_data[cpu].clock_tick =
+		cpu_data(cpu).clock_tick =
 			cpufreq_scale(ft->clock_tick_ref,
 				      ft->ref_freq,
 				      freq->new);

arch/sparc64/kernel/trampoline.S

@@ -246,20 +246,12 @@ startup_continue:
 	wrpr		%o1, 0x0, %pstate
 	ldx		[%g6 + TI_TASK], %g4
 
-	/* Setup interrupt globals, we are always SMP. */
-	wrpr		%o1, PSTATE_IG, %pstate
-
-	/* Get our UPA MID. */
-	ldub		[%o2 + TI_CPU], %g1
-	sethi		%hi(cpu_data), %g5
-	or		%g5, %lo(cpu_data), %g5
+	wrpr		%g0, 0, %wstate
 
-	/* In theory this is: &(cpu_data[this_upamid].irq_worklists[0]) */
-	sllx		%g1, 7, %g1
-	add		%g5, %g1, %g1
-	add		%g1, 64, %g6
+	call		init_irqwork_curcpu
+	 nop
 
-	wrpr		%g0, 0, %wstate
+	rdpr		%pstate, %o1
 	or		%o1, PSTATE_IE, %o1
 	wrpr		%o1, 0, %pstate
 

drivers/serial/sunsab.c

@@ -32,6 +32,7 @@
 #include <linux/console.h>
 #include <linux/spinlock.h>
 #include <linux/slab.h>
+#include <linux/delay.h>
 #include <linux/init.h>
 
 #include <asm/io.h>

include/asm-sparc64/cpudata.h

+/* cpudata.h: Per-cpu parameters.
+ *
+ * Copyright (C) 2003 David S. Miller (davem@redhat.com)
+ */
+
+#ifndef _SPARC64_CPUDATA_H
+#define _SPARC64_CPUDATA_H
+
+#include <linux/percpu.h>
+
+typedef struct {
+	/* Dcache line 1 */
+	unsigned int	__pad0;		/* bh_count moved to irq_stat for consistency. KAO */
+	unsigned int	multiplier;
+	unsigned int	counter;
+	unsigned int	idle_volume;
+	unsigned long	clock_tick;	/* %tick's per second */
+	unsigned long	udelay_val;
+
+	/* Dcache line 2 */
+	unsigned int	pgcache_size;
+	unsigned int	pgdcache_size;
+	unsigned long	*pte_cache[2];
+	unsigned long	*pgd_cache;
+} cpuinfo_sparc;
+
+DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data);
+#define cpu_data(__cpu)	per_cpu(__cpu_data, (__cpu))
+
+#endif /* _SPARC64_CPUDATA_H */

include/asm-sparc64/delay.h

@@ -9,6 +9,7 @@
 
 #include <linux/config.h>
 #include <linux/param.h>
+#include <asm/cpudata.h>
 
 #ifndef __ASSEMBLY__
 
@@ -59,7 +60,7 @@ extern __inline__ void __ndelay(unsigned long usecs, unsigned long lps)
 }
 
 #ifdef CONFIG_SMP
-#define __udelay_val cpu_data[smp_processor_id()].udelay_val
+#define __udelay_val cpu_data(smp_processor_id()).udelay_val
 #else
 #define __udelay_val loops_per_jiffy
 #endif

include/asm-sparc64/pgalloc.h

@@ -10,11 +10,12 @@
 #include <asm/page.h>
 #include <asm/spitfire.h>
 #include <asm/pgtable.h>
+#include <asm/cpudata.h>
 
 /* Page table allocation/freeing. */
 #ifdef CONFIG_SMP
 /* Sliiiicck */
-#define pgt_quicklists	cpu_data[smp_processor_id()]
+#define pgt_quicklists	cpu_data(smp_processor_id())
 #else
 extern struct pgtable_cache_struct {
 	unsigned long *pgd_cache;

include/asm-sparc64/processor.h

@@ -20,7 +20,6 @@
 #include <asm/ptrace.h>
 #include <asm/segment.h>
 #include <asm/page.h>
-#include <asm/delay.h>
 
 /* Bus types */
 #define EISA_bus 0

include/asm-sparc64/smp.h

@@ -23,30 +23,6 @@
 
 #ifndef __ASSEMBLY__
 
-/* Per processor Sparc parameters we need. */
-
-/* Keep this a multiple of 64-bytes for cache reasons. */
-typedef struct {
-	/* Dcache line 1 */
-	unsigned int	__pad0;		/* bh_count moved to irq_stat for consistency. KAO */
-	unsigned int	multiplier;
-	unsigned int	counter;
-	unsigned int	idle_volume;
-	unsigned long	clock_tick;	/* %tick's per second */
-	unsigned long	udelay_val;
-
-	/* Dcache line 2 */
-	unsigned int	pgcache_size;
-	unsigned int	pgdcache_size;
-	unsigned long	*pte_cache[2];
-	unsigned long	*pgd_cache;
-
-	/* Dcache lines 3 and 4 */
-	unsigned int	irq_worklists[16];
-} ____cacheline_aligned cpuinfo_sparc;
-
-extern cpuinfo_sparc cpu_data[NR_CPUS];
-
 /*
  *	Private routines/data
  */
@@ -97,28 +73,6 @@ static __inline__ int hard_smp_processor_id(void)
 
 #define smp_processor_id() (current_thread_info()->cpu)
 
-/* This needn't do anything as we do not sleep the cpu
- * inside of the idler task, so an interrupt is not needed
- * to get a clean fast response.
- *
- * XXX Reverify this assumption... -DaveM
- *
- * Addendum: We do want it to do something for the signal
- *           delivery case, we detect that by just seeing
- *           if we are trying to send this to an idler or not.
- */
-static __inline__ void smp_send_reschedule(int cpu)
-{
-	extern void smp_receive_signal(int);
-	if (cpu_data[cpu].idle_volume == 0)
-		smp_receive_signal(cpu);
-}
-
-/* This is a nop as well because we capture all other cpus
- * anyways when making the PROM active.
- */
-static __inline__ void smp_send_stop(void) { }
-
 #endif /* !(__ASSEMBLY__) */
 
 #endif /* !(CONFIG_SMP) */