SPARC64: Initial Cheetah+ cpu support.

arch/sparc64/kernel/chmc.c

@@ -424,7 +424,7 @@ static int __init chmc_init(void)
 	int index;
 
 	/* This driver is only for cheetah platforms. */
-	if (tlb_type != cheetah)
+	if (tlb_type != cheetah && tlb_type != cheetah_plus)
 		return -ENODEV;
 
 	index = probe_for_string("memory-controller", 0);

arch/sparc64/kernel/cpu.c

@@ -33,6 +33,7 @@ struct cpu_fp_info linux_sparc_fpu[] = {
   { 0x17, 0x12, 0, "UltraSparc IIi integrated FPU"},
   { 0x17, 0x13, 0, "UltraSparc IIe integrated FPU"},
   { 0x3e, 0x14, 0, "UltraSparc III integrated FPU"},
+  { 0x3e, 0x15, 0, "UltraSparc III+ integrated FPU"},
 };
 
 #define NSPARCFPU  (sizeof(linux_sparc_fpu)/sizeof(struct cpu_fp_info))
@@ -44,6 +45,7 @@ struct cpu_iu_info linux_sparc_chips[] = {
   { 0x17, 0x12, "TI UltraSparc IIi"},
   { 0x17, 0x13, "TI UltraSparc IIe"},
   { 0x3e, 0x14, "TI UltraSparc III (Cheetah)"},
+  { 0x3e, 0x15, "TI UltraSparc III+ (Cheetah+)"},
 };
 
 #define NSPARCCHIPS  (sizeof(linux_sparc_chips)/sizeof(struct cpu_iu_info))

arch/sparc64/kernel/devices.c

@@ -63,7 +63,8 @@ void __init device_scan(void)
 				if (tlb_type == spitfire) {
 					prom_getproperty(scan, "upa-portid",
 							 (char *) &thismid, sizeof(thismid));
-				} else if (tlb_type == cheetah) {
+				} else if (tlb_type == cheetah ||
+					   tlb_type == cheetah_plus) {
 					prom_getproperty(scan, "portid",
 							 (char *) &thismid, sizeof(thismid));
 				}

arch/sparc64/kernel/dtlb_base.S

@@ -73,7 +73,7 @@ from_tl1_trap:
 	CREATE_VPTE_OFFSET1(%g4, %g6)			! Create VPTE offset
 	be,pn		%xcc, 3f			! Yep, special processing
 	 CREATE_VPTE_OFFSET2(%g4, %g6)			! Create VPTE offset
-	cmp		%g5, 3				! Last trap level?
+	cmp		%g5, 4				! Last trap level?
 	be,pn		%xcc, longpath			! Yep, cannot risk VPTE miss
 	 nop						! delay slot
 

arch/sparc64/kernel/entry.S

@@ -1074,6 +1074,161 @@ cheetah_deferred_trap_vector_tl1:
 	jmpl		%g2 + %lo(cheetah_deferred_trap), %g0
 	 mov		1, %g1
 
+	/* Cheetah+ specific traps. These are for the new I/D cache parity
+	 * error traps.  The first argument to cheetah_plus_parity_handler
+	 * is encoded as follows:
+	 *
+	 * Bit0:	0=dcache,1=icache
+	 * Bit1:	0=recoverable,1=unrecoverable
+	 */
+	.globl		cheetah_plus_dcpe_trap_vector, cheetah_plus_dcpe_trap_vector_tl1
+cheetah_plus_dcpe_trap_vector:
+	membar		#Sync
+	ba,pt		%xcc, etrap
+	 rd		%pc, %g7
+	mov		0x0, %o0
+	call		cheetah_plus_parity_error
+	 add		%sp, STACK_BIAS + REGWIN_SZ, %o1
+	ba,pt		%xcc, rtrap
+	 clr		%l6
+cheetah_plus_dcpe_trap_vector_tl1:
+	membar		#Sync
+	wrpr		PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate
+	sethi		%hi(do_dcpe_tl1), %g3
+	jmpl		%g3 + %lo(do_dcpe_tl1), %g0
+	 nop
+	nop
+	nop
+	nop
+
+	.globl		cheetah_plus_icpe_trap_vector, cheetah_plus_icpe_trap_vector_tl1
+cheetah_plus_icpe_trap_vector:
+	membar		#Sync
+	ba,pt		%xcc, etrap
+	 rd		%pc, %g7
+	mov		0x1, %o0
+	call		cheetah_plus_parity_error
+	 add		%sp, STACK_BIAS + REGWIN_SZ, %o1
+	ba,pt		%xcc, rtrap
+	 clr		%l6
+cheetah_plus_icpe_trap_vector_tl1:
+	membar		#Sync
+	wrpr		PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate
+	sethi		%hi(do_icpe_tl1), %g3
+	jmpl		%g3 + %lo(do_icpe_tl1), %g0
+	 nop
+	nop
+	nop
+	nop
+
+	/* If we take one of these traps when tl >= 1, then we
+	 * jump to interrupt globals.  If some trap level above us
+	 * was also using interrupt globals, we cannot recover.
+	 * We may use all interrupt global registers except %g6.
+	 */
+	.globl		do_dcpe_tl1, do_icpe_tl1
+do_dcpe_tl1:
+	rdpr		%tl, %g1		! Save original trap level
+	mov		1, %g2			! Setup TSTATE checking loop
+	sethi		%hi(TSTATE_IG), %g3	! TSTATE mask bit
+1:	wrpr		%g2, %tl		! Set trap level to check
+	rdpr		%tstate, %g4		! Read TSTATE for this level
+	andcc		%g4, %g3, %g0		! Interrupt globals in use?
+	bne,a,pn	%xcc, do_dcpe_tl1_fatal	! Yep, irrecoverable
+	 wrpr		%g1, %tl		! Restore original trap level
+	add		%g2, 1, %g2		! Next trap level
+	cmp		%g2, %g1		! Hit them all yet?
+	ble,pt		%icc, 1b		! Not yet
+	 nop
+	wrpr		%g1, %tl		! Restore original trap level
+do_dcpe_tl1_nonfatal:	/* Ok we may use interrupt globals safely. */
+	/* Reset D-cache parity */
+	sethi		%hi(1 << 16), %g1	! D-cache size
+	mov		(1 << 5), %g2		! D-cache line size
+	sub		%g1, %g2, %g1		! Move down 1 cacheline
+1:	srl		%g1, 14, %g3		! Compute UTAG
+	membar		#Sync
+	stxa		%g3, [%g1] ASI_DCACHE_UTAG
+	membar		#Sync
+	sub		%g2, 8, %g3		! 64-bit data word within line
+2:	membar		#Sync
+	stxa		%g0, [%g1 + %g3] ASI_DCACHE_DATA
+	membar		#Sync
+	subcc		%g3, 8, %g3		! Next 64-bit data word
+	bge,pt		%icc, 2b
+	 nop
+	subcc		%g1, %g2, %g1		! Next cacheline
+	bge,pt		%icc, 1b
+	 nop
+	ba,pt		%xcc, dcpe_icpe_tl1_common
+	 nop
+
+do_dcpe_tl1_fatal:
+	sethi		%hi(1f), %g7
+	ba,pt		%xcc, etraptl1
+1:	or		%g7, %lo(1b), %g7
+	mov		0x2, %o0
+	call		cheetah_plus_parity_error
+	 add		%sp, STACK_BIAS + REGWIN_SZ, %o1
+	ba,pt		%xcc, rtrap
+	 clr		%l6
+
+do_icpe_tl1:
+	rdpr		%tl, %g1		! Save original trap level
+	mov		1, %g2			! Setup TSTATE checking loop
+	sethi		%hi(TSTATE_IG), %g3	! TSTATE mask bit
+1:	wrpr		%g2, %tl		! Set trap level to check
+	rdpr		%tstate, %g4		! Read TSTATE for this level
+	andcc		%g4, %g3, %g0		! Interrupt globals in use?
+	bne,a,pn	%xcc, do_icpe_tl1_fatal	! Yep, irrecoverable
+	 wrpr		%g1, %tl		! Restore original trap level
+	add		%g2, 1, %g2		! Next trap level
+	cmp		%g2, %g1		! Hit them all yet?
+	ble,pt		%icc, 1b		! Not yet
+	 nop
+	wrpr		%g1, %tl		! Restore original trap level
+do_icpe_tl1_nonfatal:	/* Ok we may use interrupt globals safely. */
+	/* Flush I-cache */
+	sethi		%hi(1 << 15), %g1	! I-cache size
+	mov		(1 << 5), %g2		! I-cache line size
+	sub		%g1, %g2, %g1
+1:	or		%g1, (2 << 3), %g3
+	stxa		%g0, [%g3] ASI_IC_TAG
+	membar		#Sync
+	subcc		%g1, %g2, %g1
+	bge,pt		%icc, 1b
+	 nop
+	ba,pt		%xcc, dcpe_icpe_tl1_common
+	 nop
+
+do_icpe_tl1_fatal:
+	sethi		%hi(1f), %g7
+	ba,pt		%xcc, etraptl1
+1:	or		%g7, %lo(1b), %g7
+	mov		0x3, %o0
+	call		cheetah_plus_parity_error
+	 add		%sp, STACK_BIAS + REGWIN_SZ, %o1
+	ba,pt		%xcc, rtrap
+	 clr		%l6
+	
+dcpe_icpe_tl1_common:
+	/* Flush D-cache, re-enable D/I caches in DCU and finally
+	 * retry the trapping instruction.
+	 */
+	sethi		%hi(1 << 16), %g1	! D-cache size
+	mov		(1 << 5), %g2		! D-cache line size
+	sub		%g1, %g2, %g1
+1:	stxa		%g0, [%g1] ASI_DCACHE_TAG
+	membar		#Sync
+	subcc		%g1, %g2, %g1
+	bge,pt		%icc, 1b
+	 nop
+	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g1
+	or		%g1, (DCU_DC | DCU_IC), %g1
+	stxa		%g1, [%g0] ASI_DCU_CONTROL_REG
+	membar		#Sync
+	retry
+
 	/* Cheetah FECC trap handling, we get here from tl{0,1}_fecc
 	 * in the trap table.  That code has done a memory barrier
 	 * and has disabled both the I-cache and D-cache in the DCU
@@ -1639,12 +1794,17 @@ do_gettimeofday:	/* %o0 = timevalp */
 	or	%g2, %lo(xtime), %g2
 	or	%g1, %lo(timer_tick_compare), %g1
 1:	rdpr	%ver, %o2
-	sethi	%hi(0x003e0014), %o1
+	sethi	%hi(CHEETAH_ID), %o1
 	srlx	%o2, 32, %o2
-	or	%o1, %lo(0x003e0014), %o1
+	or	%o1, %lo(CHEETAH_ID), %o1
 	membar	#Sync
 	ldda	[%g2] ASI_NUCLEUS_QUAD_LDD, %o4
 	cmp	%o2, %o1
+	be,a,pn	%xcc, 3f
+	 rd	%asr24, %o1
+	sethi	%hi(CHEETAH_PLUS_ID), %o1
+	or	%o1, %lo(CHEETAH_PLUS_ID), %o1
+	cmp	%o2, %o1
 	bne,pt	%xcc, 2f
 	 nop
 	ba,pt	%xcc, 3f

arch/sparc64/kernel/head.S

@@ -79,24 +79,34 @@ sparc_ramdisk_size:
 	 */
 sparc64_boot:
 	rdpr	%ver, %g1
-	sethi	%hi(0x003e0014), %g5
+	sethi	%hi(CHEETAH_ID), %g5
 	srlx	%g1, 32, %g1
-	or	%g5, %lo(0x003e0014), %g5
+	or	%g5, %lo(CHEETAH_ID), %g5
+	cmp	%g1, %g5
+	be,pn	%icc, cheetah_boot
+	 sethi	%hi(CHEETAH_PLUS_ID), %g5
+	or	%g5, %lo(CHEETAH_PLUS_ID), %g5
 	cmp	%g1, %g5
 	bne,pt	%icc, spitfire_boot
 	 nop
 
+cheetah_plus_boot:
+	/* Preserve OBP choosen DCU and DCR register settings.  */
+	ba,pt	%xcc, cheetah_generic_boot
+	 nop
+
 cheetah_boot:
 	mov	DCR_BPE | DCR_RPE | DCR_SI | DCR_IFPOE | DCR_MS, %g1
 	wr	%g1, %asr18
 
-	sethi	%uhi(DCU_ME | DCU_RE | /*DCU_PE |*/ DCU_HPE | DCU_SPE | DCU_SL | DCU_WE), %g5
-	or	%g5, %ulo(DCU_ME | DCU_RE | /*DCU_PE |*/ DCU_HPE | DCU_SPE | DCU_SL | DCU_WE), %g5
+	sethi	%uhi(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g5
+	or	%g5, %ulo(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g5
 	sllx	%g5, 32, %g5
 	or	%g5, DCU_DM | DCU_IM | DCU_DC | DCU_IC, %g5
 	stxa	%g5, [%g0] ASI_DCU_CONTROL_REG
 	membar	#Sync
 
+cheetah_generic_boot:
 	mov	TSB_EXTENSION_P, %g3
 	stxa	%g0, [%g3] ASI_DMMU
 	stxa	%g0, [%g3] ASI_IMMU
@@ -149,6 +159,7 @@ cheetah_boot:
 	 add	%l0, (1 << 3), %l0
 
 cheetah_got_tlbentry:
+	ldxa	[%l0] ASI_ITLB_DATA_ACCESS, %g0
 	ldxa	[%l0] ASI_ITLB_DATA_ACCESS, %g1
 	membar	#Sync
 	and	%g1, %g3, %g1
@@ -200,6 +211,36 @@ cheetah_got_tlbentry:
 	blu,pt	%xcc, 1b
 	 add	%l0, (1 << 3), %l0
 
+	/* On Cheetah+, have to check second DTLB.  */
+	rdpr	%ver, %g1
+	srlx	%g1, 32, %g1
+	sethi	%hi(CHEETAH_PLUS_ID), %l0
+	or	%l0, %lo(CHEETAH_PLUS_ID), %l0
+	cmp	%g1, %l0
+	bne,pt	%icc, 9f
+	 nop
+
+	set	3 << 16, %l0
+1:	ldxa	[%l0] ASI_DTLB_TAG_READ, %g1
+	membar	#Sync
+	andn	%g1, %l2, %g1
+	cmp	%g1, %g3
+	blu,pn	%xcc, 2f
+	 cmp	%g1, %g7
+	bgeu,pn	%xcc, 2f
+	 nop
+	stxa	%g0, [%l7] ASI_DMMU
+	membar	#Sync
+	stxa	%g0, [%l0] ASI_DTLB_DATA_ACCESS
+	membar	#Sync
+	
+2:	and	%l0, (511 << 3), %g1
+	cmp	%g1, (511 << 3)
+	blu,pt	%xcc, 1b
+	 add	%l0, (1 << 3), %l0
+
+9:
+
 	/* Now lock the TTE we created into ITLB-0 and DTLB-0,
 	 * entry 15 (and maybe 14 too).
 	 */
@@ -429,20 +470,26 @@ sun4u_init:
 	membar	#Sync
 
 	rdpr	%ver, %g1
-	sethi	%hi(0x003e0014), %g5
+	sethi	%hi(CHEETAH_ID), %g5
 	srlx	%g1, 32, %g1
-	or	%g5, %lo(0x003e0014), %g5
+	or	%g5, %lo(CHEETAH_ID), %g5
+	cmp	%g1, %g5
+	be,pn	%icc, cheetah_tlb_fixup
+	 sethi	%hi(CHEETAH_PLUS_ID), %g5
+	or	%g5, %lo(CHEETAH_PLUS_ID), %g5
 	cmp	%g1, %g5
 	bne,pt	%icc, spitfire_tlb_fixup
 	 nop
 
 cheetah_tlb_fixup:
 	set	(0 << 16) | (15 << 3), %g7
+	ldxa	[%g7] ASI_ITLB_DATA_ACCESS, %g0
 	ldxa	[%g7] ASI_ITLB_DATA_ACCESS, %g1
 	andn	%g1, (_PAGE_G), %g1
 	stxa	%g1, [%g7] ASI_ITLB_DATA_ACCESS
 	membar	#Sync
 
+	ldxa	[%g7] ASI_DTLB_DATA_ACCESS, %g0
 	ldxa	[%g7] ASI_DTLB_DATA_ACCESS, %g1
 	andn	%g1, (_PAGE_G), %g1
 	stxa	%g1, [%g7] ASI_DTLB_DATA_ACCESS
@@ -452,9 +499,17 @@ cheetah_tlb_fixup:
 	flush	%g3
 	membar	#Sync
 
-	/* Set TLB type to cheetah. */
-	mov	1, %g2
-	sethi	%hi(tlb_type), %g5
+	sethi	%hi(CHEETAH_PLUS_ID), %g5
+	or	%g5, %lo(CHEETAH_PLUS_ID), %g5
+	rdpr	%ver, %g2
+	srlx	%g2, 32, %g2
+	cmp	%g2, %g5
+	bne,a,pt %icc, 1f
+	 mov	1, %g2		/* Set TLB type to cheetah. */
+
+	mov	2, %g2		/* Set TLB type to cheetah+. */
+
+1:	sethi	%hi(tlb_type), %g5
 	stw	%g2, [%g5 + %lo(tlb_type)]
 
 	/* Patch copy/page operations to cheetah optimized versions. */
@@ -462,6 +517,8 @@ cheetah_tlb_fixup:
 	 nop
 	call	cheetah_patch_pgcopyops
 	 nop
+	call	cheetah_patch_cachetlbops
+	 nop
 
 	ba,pt	%xcc, tlb_fixup_done
 	 nop
@@ -575,18 +632,23 @@ setup_tba:	/* i0 = is_starfire */
 	or	%g2, KERN_LOWBITS, %g2
 
 	rdpr		%ver, %g3
-	sethi		%hi(0x003e0014), %g7
+	sethi		%hi(CHEETAH_ID), %g7
 	srlx		%g3, 32, %g3
-	or		%g7, %lo(0x003e0014), %g7
+	or		%g7, %lo(CHEETAH_ID), %g7
+	cmp		%g3, %g7
+	be,pn		%icc, cheetah_vpte_base
+	sethi		%hi(CHEETAH_PLUS_ID), %g7
+	or		%g7, %lo(CHEETAH_PLUS_ID), %g7
 	cmp		%g3, %g7
-	bne,pt		%icc, 1f
+	bne,pt		%icc, spitfire_vpte_base
 	 nop
 
+cheetah_vpte_base:
 	sethi		%uhi(VPTE_BASE_CHEETAH), %g3
 	or		%g3, %ulo(VPTE_BASE_CHEETAH), %g3
 	ba,pt		%xcc, 2f
 	 sllx		%g3, 32, %g3
-1:
+spitfire_vpte_base:
 	sethi		%uhi(VPTE_BASE_SPITFIRE), %g3
 	or		%g3, %ulo(VPTE_BASE_SPITFIRE), %g3
 	sllx		%g3, 32, %g3
@@ -614,13 +676,18 @@ setup_tba:	/* i0 = is_starfire */
 
 not_starfire:
 	rdpr		%ver, %g1
-	sethi		%hi(0x003e0014), %g5
+	sethi		%hi(CHEETAH_ID), %g5
 	srlx		%g1, 32, %g1
-	or		%g7, %lo(0x003e0014), %g5
+	or		%g5, %lo(CHEETAH_ID), %g5
+	cmp		%g1, %g5
+	be,pn		%icc, is_cheetah
+	 sethi		%hi(CHEETAH_PLUS_ID), %g5
+	or		%g5, %lo(CHEETAH_PLUS_ID), %g5
 	cmp		%g1, %g5
 	bne,pt		%icc, not_cheetah
 	 nop
 
+is_cheetah:
 	ldxa		[%g0] ASI_SAFARI_CONFIG, %g1
 	srlx		%g1, 17, %g1
 	ba,pt		%xcc, set_worklist
@@ -644,14 +711,19 @@ set_worklist:
 	wr	%g0, 0, %tick_cmpr
 
 	rdpr		%ver, %g1
-	sethi		%hi(0x003e0014), %g5
+	sethi		%hi(CHEETAH_ID), %g5
 	srlx		%g1, 32, %g1
-	or		%g7, %lo(0x003e0014), %g5
+	or		%g5, %lo(CHEETAH_ID), %g5
 	cmp		%g1, %g5
-	bne,pt		%icc, 1f
+	be,pn		%icc, 1f
+	 sethi		%hi(CHEETAH_PLUS_ID), %g5
+	or		%g5, %lo(CHEETAH_PLUS_ID), %g5
+	cmp		%g1, %g5
+	bne,pt		%icc, 2f
 	 nop
 
 	/* Disable STICK_INT interrupts. */
+1:
 	sethi		%hi(0x80000000), %g1
 	sllx		%g1, 32, %g1
 	wr		%g1, %asr25
@@ -659,7 +731,7 @@ set_worklist:
 	/* Ok, we're done setting up all the state our trap mechanims needs,
 	 * now get back into normal globals and let the PROM know what is up.
 	 */
-1:
+2:
 	wrpr	%g0, %g0, %wstate
 	wrpr	%o1, PSTATE_IE, %pstate
 

arch/sparc64/kernel/irq.c

@@ -162,7 +162,7 @@ void enable_irq(unsigned int irq)
 	if (imap == 0UL)
 		return;
 
-	if (tlb_type == cheetah) {
+	if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		/* We set it to our Safari AID. */
 		__asm__ __volatile__("ldxa [%%g0] %1, %0"
 				     : "=r" (tid)
@@ -1068,7 +1068,7 @@ static int retarget_one_irq(struct irqaction *p, int goal_cpu)
 			goal_cpu = 0;
 	}
 
-	if (tlb_type == cheetah) {
+	if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		tid = goal_cpu << 26;
 		tid &= IMAP_AID_SAFARI;
 	} else if (this_is_starfire == 0) {

arch/sparc64/kernel/ptrace.c

@@ -570,7 +570,7 @@ asmlinkage void do_ptrace(struct pt_regs *regs)
 	{
 		unsigned long va;
 
-		if (tlb_type == cheetah) {
+		if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 			for (va = 0; va < (1 << 16); va += (1 << 5))
 				spitfire_put_dcache_tag(va, 0x0);
 			/* No need to mess with I-cache on Cheetah. */

arch/sparc64/kernel/setup.c

@@ -193,7 +193,7 @@ int prom_callback(long *args)
 
 			if (tlb_type == spitfire)
 				tte = spitfire_get_dtlb_data(SPITFIRE_HIGHEST_LOCKED_TLBENT);
-			else if (tlb_type == cheetah)
+			else if (tlb_type == cheetah || tlb_type == cheetah_plus)
 				tte = cheetah_get_ldtlb_data(CHEETAH_HIGHEST_LOCKED_TLBENT);
 
 			res = PROM_TRUE;

arch/sparc64/kernel/smp.c

@@ -558,9 +558,9 @@ extern unsigned long xcall_flush_tlb_page;
 extern unsigned long xcall_flush_tlb_mm;
 extern unsigned long xcall_flush_tlb_range;
 extern unsigned long xcall_flush_tlb_kernel_range;
-extern unsigned long xcall_flush_tlb_all;
-extern unsigned long xcall_tlbcachesync;
-extern unsigned long xcall_flush_cache_all;
+extern unsigned long xcall_flush_tlb_all_spitfire;
+extern unsigned long xcall_flush_tlb_all_cheetah;
+extern unsigned long xcall_flush_cache_all_spitfire;
 extern unsigned long xcall_report_regs;
 extern unsigned long xcall_receive_signal;
 extern unsigned long xcall_flush_dcache_page_cheetah;
@@ -676,13 +676,19 @@ void smp_report_regs(void)
 
 void smp_flush_cache_all(void)
 {
-	smp_cross_call(&xcall_flush_cache_all, 0, 0, 0);
-	__flush_cache_all();
+	/* Cheetah need do nothing. */
+	if (tlb_type == spitfire) {
+		smp_cross_call(&xcall_flush_cache_all_spitfire, 0, 0, 0);
+		__flush_cache_all();
+	}
 }
 
 void smp_flush_tlb_all(void)
 {
-	smp_cross_call(&xcall_flush_tlb_all, 0, 0, 0);
+	if (tlb_type == spitfire)
+		smp_cross_call(&xcall_flush_tlb_all_spitfire, 0, 0, 0);
+	else
+		smp_cross_call(&xcall_flush_tlb_all_cheetah, 0, 0, 0);
 	__flush_tlb_all();
 }
 

arch/sparc64/kernel/trampoline.S

@@ -34,13 +34,22 @@ sparc64_cpu_startup:
 	flushw
 
 	rdpr		%ver, %g1
-	sethi		%hi(0x003e0014), %g5
+	sethi		%hi(CHEETAH_ID), %g5
 	srlx		%g1, 32, %g1
-	or		%g5, %lo(0x003e0014), %g5
+	or		%g5, %lo(CHEETAH_ID), %g5
+	cmp		%g1, %g5
+	be,pn		%icc, cheetah_startup
+	 sethi		%hi(CHEETAH_PLUS_ID), %g5
+	or		%g5, %lo(CHEETAH_PLUS_ID), %g5
 	cmp		%g1, %g5
 	bne,pt		%icc, spitfire_startup
 	 nop
 
+cheetah_plus_startup:
+	/* Preserve OBP choosen DCU and DCR register settings.  */
+	ba,pt	%xcc, cheetah_generic_startup
+	 nop
+
 cheetah_startup:
 	mov		DCR_BPE | DCR_RPE | DCR_SI | DCR_IFPOE | DCR_MS, %g1
 	wr		%g1, %asr18
@@ -52,6 +61,7 @@ cheetah_startup:
 	stxa		%g5, [%g0] ASI_DCU_CONTROL_REG
 	membar		#Sync
 
+cheetah_generic_startup:
 	mov	TSB_EXTENSION_P, %g3
 	stxa	%g0, [%g3] ASI_DMMU
 	stxa	%g0, [%g3] ASI_IMMU
@@ -118,9 +128,14 @@ startup_continue:
 	stx		%g2, [%sp + 2047 + 128 + 0x30]
 
 	rdpr		%ver, %g1
-	sethi		%hi(0x003e0014), %g5
+	sethi		%hi(CHEETAH_ID), %g5
 	srlx		%g1, 32, %g1
-	or		%g5, %lo(0x003e0014), %g5
+	or		%g5, %lo(CHEETAH_ID), %g5
+	cmp		%g1, %g5
+	be,a,pn		%icc, 1f
+	 mov		15, %g2
+	sethi		%hi(CHEETAH_PLUS_ID), %g5
+	or		%g5, %lo(CHEETAH_PLUS_ID), %g5
 	cmp		%g1, %g5
 	bne,a,pt	%icc, 1f
 	 mov		63, %g2
@@ -153,9 +168,14 @@ startup_continue:
 	stx		%g2, [%sp + 2047 + 128 + 0x30]
 
 	rdpr		%ver, %g1
-	sethi		%hi(0x003e0014), %g5
+	sethi		%hi(CHEETAH_ID), %g5
 	srlx		%g1, 32, %g1
-	or		%g5, %lo(0x003e0014), %g5
+	or		%g5, %lo(CHEETAH_ID), %g5
+	cmp		%g1, %g5
+	be,a,pn		%icc, 1f
+	 mov		15, %g2
+	sethi		%hi(CHEETAH_PLUS_ID), %g5
+	or		%g5, %lo(CHEETAH_PLUS_ID), %g5
 	cmp		%g1, %g5
 	bne,a,pt	%icc, 1f
 	 mov		63, %g2
@@ -225,13 +245,18 @@ startup_continue:
 	or		%g2, KERN_LOWBITS, %g2
 
 	rdpr		%ver, %g3
-	sethi		%hi(0x003e0014), %g7
+	sethi		%hi(CHEETAH_ID), %g7
 	srlx		%g3, 32, %g3
-	or		%g7, %lo(0x003e0014), %g7
+	or		%g7, %lo(CHEETAH_ID), %g7
+	cmp		%g3, %g7
+	be,pn		%icc, 9f
+	 sethi		%hi(CHEETAH_PLUS_ID), %g7
+	or		%g7, %lo(CHEETAH_PLUS_ID), %g7
 	cmp		%g3, %g7
 	bne,pt		%icc, 1f
 	 nop
 
+9:
 	sethi		%uhi(VPTE_BASE_CHEETAH), %g3
 	or		%g3, %ulo(VPTE_BASE_CHEETAH), %g3
 	ba,pt		%xcc, 2f

arch/sparc64/kernel/traps.c

@@ -167,7 +167,7 @@ static void clean_and_reenable_l1_caches(void)
 					    LSU_CONTROL_IM | LSU_CONTROL_DM),
 				     "i" (ASI_LSU_CONTROL)
 				     : "memory");
-	} else if (tlb_type == cheetah) {
+	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		/* Flush D-cache */
 		for (va = 0; va < (1 << 16); va += (1 << 5)) {
 			__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
@@ -201,7 +201,7 @@ void do_dae(struct pt_regs *regs)
 		pci_poke_faulted = 1;
 
 		/* Why the fuck did they have to change this? */
-		if (tlb_type == cheetah)
+		if (tlb_type == cheetah || tlb_type == cheetah_plus)
 			regs->tpc += 4;
 
 		regs->tnpc = regs->tpc + 4;
@@ -390,10 +390,14 @@ static __inline__ struct cheetah_err_info *cheetah_get_error_log(unsigned long a
 	return p;
 }
 
+extern unsigned int tl0_icpe[], tl1_icpe[];
+extern unsigned int tl0_dcpe[], tl1_dcpe[];
 extern unsigned int tl0_fecc[], tl1_fecc[];
 extern unsigned int tl0_cee[], tl1_cee[];
 extern unsigned int tl0_iae[], tl1_iae[];
 extern unsigned int tl0_dae[], tl1_dae[];
+extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
+extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
 extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
 extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
 extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
@@ -487,6 +491,12 @@ void __init cheetah_ecache_flush_init(void)
 	memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
 	memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
 	memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
+	if (tlb_type == cheetah_plus) {
+		memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
+		memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
+		memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
+		memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
+	}
 	flushi(PAGE_OFFSET);
 }
 
@@ -567,9 +577,22 @@ unsigned long __init cheetah_tune_scheduling(void)
  *
  * So we must only flush the I-cache when it is disabled.
  */
+static void __cheetah_flush_icache(void)
+{
+	unsigned long i;
+
+	/* Clear the valid bits in all the tags. */
+	for (i = 0; i < (1 << 15); i += (1 << 5)) {
+		__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
+				     "membar #Sync"
+				     : /* no outputs */
+				     : "r" (i | (2 << 3)), "i" (ASI_IC_TAG));
+	}
+}
+
 static void cheetah_flush_icache(void)
 {
-	unsigned long dcu_save, i;
+	unsigned long dcu_save;
 
 	/* Save current DCU, disable I-cache. */
 	__asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
@@ -580,13 +603,7 @@ static void cheetah_flush_icache(void)
 			     : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
 			     : "g1");
 
-	/* Clear the valid bits in all the tags. */
-	for (i = 0; i < (1 << 16); i += (1 << 5)) {
-		__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
-				     "membar #Sync"
-				     : /* no outputs */
-				     : "r" (i | (2 << 3)), "i" (ASI_IC_TAG));
-	}
+	__cheetah_flush_icache();
 
 	/* Restore DCU register */
 	__asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
@@ -607,6 +624,34 @@ static void cheetah_flush_dcache(void)
 	}
 }
 
+/* In order to make the even parity correct we must do two things.
+ * First, we clear DC_data_parity and set DC_utag to an appropriate value.
+ * Next, we clear out all 32-bytes of data for that line.  Data of
+ * all-zero + tag parity value of zero == correct parity.
+ */
+static void cheetah_plus_zap_dcache_parity(void)
+{
+	unsigned long i;
+
+	for (i = 0; i < (1 << 16); i += (1 << 5)) {
+		unsigned long tag = (i >> 14);
+		unsigned long j;
+
+		__asm__ __volatile__("membar	#Sync\n\t"
+				     "stxa	%0, [%1] %2\n\t"
+				     "membar	#Sync"
+				     : /* no outputs */
+				     : "r" (tag), "r" (i),
+				       "i" (ASI_DCACHE_UTAG));
+		for (j = i; j < i + (1 << 5); j += (1 << 3))
+			__asm__ __volatile__("membar	#Sync\n\t"
+					     "stxa	%%g0, [%0] %1\n\t"
+					     "membar	#Sync"
+					     : /* no outputs */
+					     : "r" (j), "i" (ASI_DCACHE_DATA));
+	}
+}
+
 /* Conversion tables used to frob Cheetah AFSR syndrome values into
  * something palatable to the memory controller driver get_unumber
  * routine.
@@ -1327,6 +1372,46 @@ void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned
 		panic("Irrecoverable deferred error trap.\n");
 }
 
+/* Handle a D/I cache parity error trap.  TYPE is encoded as:
+ *
+ * Bit0:	0=dcache,1=icache
+ * Bit1:	0=recoverable,1=unrecoverable
+ *
+ * The hardware has disabled both the I-cache and D-cache in
+ * the %dcr register.  
+ */
+void cheetah_plus_parity_error(int type, struct pt_regs *regs)
+{
+	if (type & 0x1)
+		__cheetah_flush_icache();
+	else
+		cheetah_plus_zap_dcache_parity();
+	cheetah_flush_dcache();
+
+	/* Re-enable I-cache/D-cache */
+	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
+			     "or %%g1, %1, %%g1\n\t"
+			     "stxa %%g1, [%%g0] %0\n\t"
+			     "membar #Sync"
+			     : /* no outputs */
+			     : "i" (ASI_DCU_CONTROL_REG),
+			       "i" (DCU_DC | DCU_IC)
+			     : "g1");
+
+	if (type & 0x2) {
+		printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
+		       smp_processor_id(),
+		       (type & 0x1) ? 'I' : 'D',
+		       regs->tpc);
+		panic("Irrecoverable Cheetah+ parity error.");
+	}
+
+	printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
+	       smp_processor_id(),
+	       (type & 0x1) ? 'I' : 'D',
+	       regs->tpc);
+}
+
 void do_fpe_common(struct pt_regs *regs)
 {
 	if (regs->tstate & TSTATE_PRIV) {

arch/sparc64/kernel/ttable.S

@@ -7,6 +7,8 @@
 #include <linux/config.h>
 
 	.globl	sparc64_ttable_tl0, sparc64_ttable_tl1
+	.globl	tl0_icpe, tl1_icpe
+	.globl	tl0_dcpe, tl1_dcpe
 	.globl	tl0_fecc, tl1_fecc
 	.globl	tl0_cee, tl1_cee
 	.globl	tl0_iae, tl1_iae
@@ -79,7 +81,9 @@ tl0_damiss:
 tl0_daprot:
 #include	"dtlb_prot.S"
 tl0_fecc:	BTRAP(0x70)	/* Fast-ECC on Cheetah */
-tl0_resv071:	BTRAP(0x71) BTRAP(0x72) BTRAP(0x73) BTRAP(0x74) BTRAP(0x75)
+tl0_dcpe:	BTRAP(0x71)	/* D-cache Parity Error on Cheetah+ */
+tl0_icpe:	BTRAP(0x72)	/* I-cache Parity Error on Cheetah+ */
+tl0_resv073:	BTRAP(0x73) BTRAP(0x74) BTRAP(0x75)
 tl0_resv076:	BTRAP(0x76) BTRAP(0x77) BTRAP(0x78) BTRAP(0x79) BTRAP(0x7a) BTRAP(0x7b)
 tl0_resv07c:	BTRAP(0x7c) BTRAP(0x7d) BTRAP(0x7e) BTRAP(0x7f)
 tl0_s0n:	SPILL_0_NORMAL
@@ -235,8 +239,10 @@ tl1_damiss:
 #include	"dtlb_backend.S"
 tl1_daprot:
 #include	"dtlb_prot.S"
-tl1_fecc:	BTRAP(0x70)	/* Fast-ECC on Cheetah */
-tl1_resc071:	BTRAPTL1(0x71) BTRAPTL1(0x72) BTRAPTL1(0x73)
+tl1_fecc:	BTRAPTL1(0x70)	/* Fast-ECC on Cheetah */
+tl1_dcpe:	BTRAPTL1(0x71)	/* D-cache Parity Error on Cheetah+ */
+tl1_icpe:	BTRAPTL1(0x72)	/* I-cache Parity Error on Cheetah+ */
+tl1_resv073:	BTRAPTL1(0x73)
 tl1_resv074:	BTRAPTL1(0x74) BTRAPTL1(0x75) BTRAPTL1(0x76) BTRAPTL1(0x77)
 tl1_resv078:	BTRAPTL1(0x78) BTRAPTL1(0x79) BTRAPTL1(0x7a) BTRAPTL1(0x7b)
 tl1_resv07c:	BTRAPTL1(0x7c) BTRAPTL1(0x7d) BTRAPTL1(0x7e) BTRAPTL1(0x7f)

arch/sparc64/lib/blockops.S

@@ -66,14 +66,14 @@ cheetah_patch_1:
 	ldx		[%g6 + TI_FLAGS], %g3
 
 	/* Spitfire Errata #32 workaround */
-	mov		0x8, %o4
+	mov		PRIMARY_CONTEXT, %o4
 	stxa		%g0, [%o4] ASI_DMMU
 	membar		#Sync
 
 	ldxa		[%o3] ASI_DTLB_TAG_READ, %o4
 
 	/* Spitfire Errata #32 workaround */
-	mov		0x8, %o5
+	mov		PRIMARY_CONTEXT, %o5
 	stxa		%g0, [%o5] ASI_DMMU
 	membar		#Sync
 
@@ -85,14 +85,14 @@ cheetah_patch_1:
 	add		%o3, (TLBTEMP_ENTSZ), %o3
 
 	/* Spitfire Errata #32 workaround */
-	mov		0x8, %g5
+	mov		PRIMARY_CONTEXT, %g5
 	stxa		%g0, [%g5] ASI_DMMU
 	membar		#Sync
 
 	ldxa		[%o3] ASI_DTLB_TAG_READ, %g5
 
 	/* Spitfire Errata #32 workaround */
-	mov		0x8, %g7
+	mov		PRIMARY_CONTEXT, %g7
 	stxa		%g0, [%g7] ASI_DMMU
 	membar		#Sync
 
@@ -109,7 +109,9 @@ cheetah_patch_1:
 	rdpr		%ver, %g3
 	sllx		%g3, 16, %g3
 	srlx		%g3, 32 + 16, %g3
-	cmp		%g3, 0x14
+	cmp		%g3, 0x14			! CHEETAH_ID
+	be,pn		%icc, cheetah_copy_user_page
+	 cmp		%g3, 0x15			! CHEETAH_PLUS_ID
 	bne,pt		%icc, spitfire_copy_user_page
 	 nop
 
@@ -357,14 +359,14 @@ cheetah_patch_2:
 	wrpr		%g3, PSTATE_IE, %pstate
 
 	/* Spitfire Errata #32 workaround */
-	mov		0x8, %g5
+	mov		PRIMARY_CONTEXT, %g5
 	stxa		%g0, [%g5] ASI_DMMU
 	membar		#Sync
 
 	ldxa		[%o3] ASI_DTLB_TAG_READ, %g5
 
 	/* Spitfire Errata #32 workaround */
-	mov		0x8, %g7
+	mov		PRIMARY_CONTEXT, %g7
 	stxa		%g0, [%g7] ASI_DMMU
 	membar		#Sync
 
@@ -409,7 +411,8 @@ clear_page_common:
 	retl
 	 nop
 
-1:	stxa		%g5, [%o2] ASI_DMMU
+1:
+	stxa		%g5, [%o2] ASI_DMMU
 	stxa		%g7, [%o3] ASI_DTLB_DATA_ACCESS
 	membar		#Sync
 	jmpl		%o7 + 0x8, %g0

arch/sparc64/mm/init.c

@@ -345,6 +345,8 @@ void mmu_info(struct seq_file *m)
 {
 	if (tlb_type == cheetah)
 		seq_printf(m, "MMU Type\t: Cheetah\n");
+	else if (tlb_type == cheetah_plus)
+		seq_printf(m, "MMU Type\t: Cheetah+\n");
 	else if (tlb_type == spitfire)
 		seq_printf(m, "MMU Type\t: Spitfire\n");
 	else
@@ -514,6 +516,7 @@ static void inherit_prom_mappings(void)
 		break;
 
 	case cheetah:
+	case cheetah_plus:
 		phys_page = cheetah_get_litlb_data(sparc64_highest_locked_tlbent());
 		break;
 	};
@@ -539,7 +542,7 @@ static void inherit_prom_mappings(void)
 			"i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
 			"i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
 			: "memory");
-	} else if (tlb_type == cheetah) {
+	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		/* Lock this into i/d tlb-0 entry 11 */
 		__asm__ __volatile__(
 			"stxa	%%g0, [%2] %3\n\t"
@@ -684,9 +687,9 @@ static void __flush_nucleus_vptes(void)
 				spitfire_put_dtlb_data(i, 0x0UL);
 			}
 		}
-	} else if (tlb_type == cheetah) {
+	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		for (i = 0; i < 512; i++) {
-			unsigned long tag = cheetah_get_dtlb_tag(i);
+			unsigned long tag = cheetah_get_dtlb_tag(i, 2);
 
 			if ((tag & ~PAGE_MASK) == 0 &&
 			    (tag & PAGE_MASK) >= prom_reserved_base) {
@@ -694,7 +697,21 @@ static void __flush_nucleus_vptes(void)
 						     "membar #Sync"
 						     : /* no outputs */
 						     : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
-				cheetah_put_dtlb_data(i, 0x0UL);
+				cheetah_put_dtlb_data(i, 0x0UL, 2);
+			}
+
+			if (tlb_type != cheetah_plus)
+				continue;
+
+			tag = cheetah_get_dtlb_tag(i, 3);
+
+			if ((tag & ~PAGE_MASK) == 0 &&
+			    (tag & PAGE_MASK) >= prom_reserved_base) {
+				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
+						     "membar #Sync"
+						     : /* no outputs */
+						     : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
+				cheetah_put_dtlb_data(i, 0x0UL, 3);
 			}
 		}
 	} else {
@@ -743,7 +760,7 @@ void prom_world(int enter)
 				if (tlb_type == spitfire)
 					spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
 							       prom_dtlb[i].tlb_data);
-				else if (tlb_type == cheetah)
+				else if (tlb_type == cheetah || tlb_type == cheetah_plus)
 					cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
 							       prom_dtlb[i].tlb_data);
 			}
@@ -756,7 +773,7 @@ void prom_world(int enter)
 				if (tlb_type == spitfire)
 					spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
 							       prom_itlb[i].tlb_data);
-				else if (tlb_type == cheetah)
+				else if (tlb_type == cheetah || tlb_type == cheetah_plus)
 					cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
 							       prom_itlb[i].tlb_data);
 			}
@@ -891,7 +908,7 @@ void inherit_locked_prom_mappings(int save_p)
 					break;
 			}
 		}
-	} else if (tlb_type == cheetah) {
+	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		int high = CHEETAH_HIGHEST_LOCKED_TLBENT - bigkernel;
 
 		for (i = 0; i < high; i++) {
@@ -963,7 +980,7 @@ void prom_reload_locked(void)
 			if (tlb_type == spitfire)
 				spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
 						       prom_dtlb[i].tlb_data);
-			else if (tlb_type == cheetah)
+			else if (tlb_type == cheetah || tlb_type == cheetah_plus)
 				cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
 						      prom_dtlb[i].tlb_data);
 		}
@@ -1064,7 +1081,7 @@ void __flush_tlb_all(void)
 				spitfire_put_itlb_data(i, 0x0UL);
 			}
 		}
-	} else if (tlb_type == cheetah) {
+	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		cheetah_flush_dtlb_all();
 		cheetah_flush_itlb_all();
 	}
@@ -1208,7 +1225,7 @@ void sparc_ultra_dump_itlb(void)
 				slot+2,
 				spitfire_get_itlb_tag(slot+2), spitfire_get_itlb_data(slot+2));
 		}
-	} else if (tlb_type == cheetah) {
+	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		printk ("Contents of itlb0:\n");
 		for (slot = 0; slot < 16; slot+=2) {
 			printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
@@ -1246,7 +1263,7 @@ void sparc_ultra_dump_dtlb(void)
 				slot+2,
 				spitfire_get_dtlb_tag(slot+2), spitfire_get_dtlb_data(slot+2));
 		}
-	} else if (tlb_type == cheetah) {
+	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		printk ("Contents of dtlb0:\n");
 		for (slot = 0; slot < 16; slot+=2) {
 			printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
@@ -1259,9 +1276,19 @@ void sparc_ultra_dump_dtlb(void)
 		for (slot = 0; slot < 512; slot+=2) {
 			printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
 				slot,
-				cheetah_get_dtlb_tag(slot), cheetah_get_dtlb_data(slot),
+				cheetah_get_dtlb_tag(slot, 2), cheetah_get_dtlb_data(slot, 2),
 				slot+1,
-				cheetah_get_dtlb_tag(slot+1), cheetah_get_dtlb_data(slot+1));
+				cheetah_get_dtlb_tag(slot+1, 2), cheetah_get_dtlb_data(slot+1, 2));
+		}
+		if (tlb_type == cheetah_plus) {
+			printk ("Contents of dtlb3:\n");
+			for (slot = 0; slot < 512; slot+=2) {
+				printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
+					slot,
+					cheetah_get_dtlb_tag(slot, 3), cheetah_get_dtlb_data(slot, 3),
+					slot+1,
+					cheetah_get_dtlb_tag(slot+1, 3), cheetah_get_dtlb_data(slot+1, 3));
+			}
 		}
 	}
 }
@@ -1439,7 +1466,7 @@ void __init paging_init(void)
 			  "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (60 << 3)
 			: "memory");
 		}
-	} else if (tlb_type == cheetah) {
+	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		__asm__ __volatile__(
 	"	stxa	%1, [%0] %3\n"
 	"	stxa	%2, [%5] %4\n"
@@ -1745,7 +1772,7 @@ void __init mem_init(void)
 	       initpages << (PAGE_SHIFT-10), 
 	       PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
 
-	if (tlb_type == cheetah)
+	if (tlb_type == cheetah || tlb_type == cheetah_plus)
 		cheetah_ecache_flush_init();
 }
 

arch/sparc64/mm/ultra.S

@@ -14,21 +14,12 @@
 #include <asm/head.h>
 #include <asm/thread_info.h>
 
-	/* Basically, all this madness has to do with the
-	 * fact that Cheetah does not support IMMU flushes
-	 * out of the secondary context.  Someone needs to
-	 * throw a south lake birthday party for the folks
+	/* Basically, most of the Spitfire vs. Cheetah madness
+	 * has to do with the fact that Cheetah does not support
+	 * IMMU flushes out of the secondary context.  Someone needs
+	 * to throw a south lake birthday party for the folks
 	 * in Microelectronics who refused to fix this shit.
 	 */
-#define BRANCH_IF_CHEETAH(tmp1, tmp2, label)		\
-	rdpr	%ver, %tmp1;				\
-	sethi	%hi(0x003e0014), %tmp2;			\
-	srlx	%tmp1, 32, %tmp1;			\
-	or	%tmp2, %lo(0x003e0014), %tmp2;		\
-	cmp	%tmp1, %tmp2;				\
-	be,pn	%icc, label;				\
-	 nop;						\
-	nop;
 
 	/* This file is meant to be read efficiently by the CPU, not humans.
 	 * Staraj sie tego nikomu nie pierdolnac...
@@ -37,9 +28,7 @@
 	.align		32
 	.globl		__flush_tlb_page, __flush_tlb_mm, __flush_tlb_range
 __flush_tlb_page: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=page&PAGE_MASK, %o2=SECONDARY_CONTEXT */
-/*IC1*/	BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_page)
-__spitfire_flush_tlb_page:
-/*IC2*/	ldxa		[%o2] ASI_DMMU, %g2
+	ldxa		[%o2] ASI_DMMU, %g2
 	cmp		%g2, %o0
 	bne,pn		%icc, __spitfire_flush_tlb_page_slow
 	 or		%o1, 0x10, %g3
@@ -47,27 +36,17 @@ __spitfire_flush_tlb_page:
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 	retl
 	 flush		%g6
-__cheetah_flush_tlb_page:
-/*IC3*/	rdpr		%pstate, %g5
-	andn		%g5, PSTATE_IE, %g2
-	wrpr		%g2, 0x0, %pstate
-	wrpr		%g0, 1, %tl
-	mov		PRIMARY_CONTEXT, %o2
-	ldxa		[%o2] ASI_DMMU, %g2
-	stxa		%o0, [%o2] ASI_DMMU
-	stxa		%g0, [%o1] ASI_DMMU_DEMAP
-/*IC4*/	stxa		%g0, [%o1] ASI_IMMU_DEMAP
-	stxa		%g2, [%o2] ASI_DMMU
-	flush		%g6
-	wrpr		%g0, 0, %tl
-	retl
-	 wrpr		%g5, 0x0, %pstate
 	nop
 	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
 __flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
-/*IC5*/	BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_mm)
-__spitfire_flush_tlb_mm:
-/*IC6*/	ldxa		[%o1] ASI_DMMU, %g2
+	ldxa		[%o1] ASI_DMMU, %g2
 	cmp		%g2, %o0
 	bne,pn		%icc, __spitfire_flush_tlb_mm_slow
 	 mov		0x50, %g3
@@ -75,30 +54,20 @@ __spitfire_flush_tlb_mm:
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 	retl
 	 flush		%g6
-__cheetah_flush_tlb_mm:
-/*IC7*/	rdpr		%pstate, %g5
-	andn		%g5, PSTATE_IE, %g2
-	wrpr		%g2, 0x0, %pstate
-	wrpr		%g0, 1, %tl
-	mov		PRIMARY_CONTEXT, %o2
-	mov		0x40, %g3
-	ldxa		[%o2] ASI_DMMU, %g2
-	stxa		%o0, [%o2] ASI_DMMU
-/*IC8*/	stxa		%g0, [%g3] ASI_DMMU_DEMAP
-	stxa		%g0, [%g3] ASI_IMMU_DEMAP
-	stxa		%g2, [%o2] ASI_DMMU
-	flush		%g6
-	wrpr		%g0, 0, %tl
-	retl
-	 wrpr		%g5, 0x0, %pstate
 	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
 __flush_tlb_range: /* %o0=(ctx&TAG_CONTEXT_BITS), %o1=start&PAGE_MASK, %o2=SECONDARY_CONTEXT,
 		    * %o3=end&PAGE_MASK, %o4=PAGE_SIZE, %o5=(end - start)
 		    */
-/*IC9*/	BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_range)
-__spitfire_flush_tlb_range:
 #define TLB_MAGIC	207 /* Students, do you know how I calculated this?  -DaveM */
-/*IC10*/cmp		%o5, %o4
+	cmp		%o5, %o4
 	bleu,pt		%xcc, __flush_tlb_page
 	 srlx		%o5, PAGE_SHIFT, %g5
 	cmp		%g5, TLB_MAGIC
@@ -107,7 +76,7 @@ __spitfire_flush_tlb_range:
 	ldxa		[%o2] ASI_DMMU, %g2
 	cmp		%g2, %o0
 __spitfire_flush_tlb_range_page_by_page:
-/*IC11*/bne,pn		%icc, __spitfire_flush_tlb_range_pbp_slow
+	bne,pn		%icc, __spitfire_flush_tlb_range_pbp_slow
 	 sub		%o5, %o4, %o5
 1:	stxa		%g0, [%g5 + %o5] ASI_DMMU_DEMAP
 	stxa		%g0, [%g5 + %o5] ASI_IMMU_DEMAP
@@ -116,10 +85,9 @@ __spitfire_flush_tlb_range_page_by_page:
 	retl
 	 flush		%g6
 __spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */
-/*IC12*/rdpr		%pstate, %g1
+	rdpr		%pstate, %g1
 	wrpr		%g1, PSTATE_IE, %pstate
 	mov		TLB_TAG_ACCESS, %g3
-	/* XXX Spitfire dependency... */
 	mov		((SPITFIRE_HIGHEST_LOCKED_TLBENT-1) << 3), %g2
 
 	/* Spitfire Errata #32 workaround. */
@@ -131,7 +99,7 @@ __spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */
 	and		%o4, TAG_CONTEXT_BITS, %o5
 	cmp		%o5, %o0
 	bne,pt		%icc, 2f
-/*IC13*/ andn		%o4, TAG_CONTEXT_BITS, %o4
+	 andn		%o4, TAG_CONTEXT_BITS, %o4
 	cmp		%o4, %o1
 	blu,pt		%xcc, 2f
 	 cmp		%o4, %o3
@@ -139,7 +107,7 @@ __spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */
 2:	 ldxa		[%g2] ASI_DTLB_TAG_READ, %o4
 	and		%o4, TAG_CONTEXT_BITS, %o5
 	cmp		%o5, %o0
-/*IC14*/andn		%o4, TAG_CONTEXT_BITS, %o4
+	andn		%o4, TAG_CONTEXT_BITS, %o4
 	bne,pt		%icc, 3f
 	 cmp		%o4, %o1
 	blu,pt		%xcc, 3f
@@ -147,7 +115,7 @@ __spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */
 	blu,pn		%xcc, 5f
 	 nop
 3:	brnz,pt		%g2, 1b
-/*IC15*/ sub		%g2, (1 << 3), %g2
+	 sub		%g2, (1 << 3), %g2
 	retl
 	 wrpr		%g1, 0x0, %pstate
 4:	stxa		%g0, [%g3] ASI_IMMU
@@ -163,7 +131,7 @@ __spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */
 	 nop
 
 5:	stxa		%g0, [%g3] ASI_DMMU
-/*IC16*/stxa		%g0, [%g2] ASI_DTLB_DATA_ACCESS
+	stxa		%g0, [%g2] ASI_DTLB_DATA_ACCESS
 	flush		%g6
 
 	/* Spitfire Errata #32 workaround. */
@@ -175,33 +143,6 @@ __spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */
 	 nop
 
 	.align		32
-__cheetah_flush_tlb_range:
-	cmp		%o5, %o4
-	bleu,pt		%xcc, __cheetah_flush_tlb_page
-	 nop
-/*IC17*/rdpr		%pstate, %g5
-	andn		%g5, PSTATE_IE, %g2
-	wrpr		%g2, 0x0, %pstate
-	wrpr		%g0, 1, %tl
-	mov		PRIMARY_CONTEXT, %o2
-	sub		%o5, %o4, %o5
-	ldxa		[%o2] ASI_DMMU, %g2
-	stxa		%o0, [%o2] ASI_DMMU
-
-/*IC18*/
-1:	stxa		%g0, [%o1 + %o5] ASI_DMMU_DEMAP
-	stxa		%g0, [%o1 + %o5] ASI_IMMU_DEMAP
-	membar		#Sync
-	brnz,pt		%o5, 1b
-	 sub		%o5, %o4, %o5
-
-	stxa		%g2, [%o2] ASI_DMMU
-	flush		%g6
-	wrpr		%g0, 0, %tl
-	retl
-/*IC19*/ wrpr		%g5, 0x0, %pstate
-
-	.align		32
 	.globl		__flush_tlb_kernel_range
 __flush_tlb_kernel_range:	/* %o0=start, %o1=end */
 	cmp		%o0, %o1
@@ -226,7 +167,7 @@ __spitfire_flush_tlb_mm_slow:
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 	flush		%g6
 	stxa		%g2, [%o1] ASI_DMMU
-/*IC18*/flush		%g6
+	flush		%g6
 	retl
 	 wrpr		%g1, 0, %pstate
 
@@ -236,7 +177,7 @@ __spitfire_flush_tlb_page_slow:
 	stxa		%o0, [%o2] ASI_DMMU
 	stxa		%g0, [%g3] ASI_DMMU_DEMAP
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
-/*IC20*/flush		%g6
+	flush		%g6
 	stxa		%g2, [%o2] ASI_DMMU
 	flush		%g6
 	retl
@@ -246,7 +187,7 @@ __spitfire_flush_tlb_range_pbp_slow:
 	rdpr		%pstate, %g1
 	wrpr		%g1, PSTATE_IE, %pstate
 	stxa		%o0, [%o2] ASI_DMMU
-/*IC21*/
+
 2:	stxa		%g0, [%g5 + %o5] ASI_DMMU_DEMAP
 	stxa		%g0, [%g5 + %o5] ASI_IMMU_DEMAP
 	brnz,pt		%o5, 2b
@@ -255,7 +196,7 @@ __spitfire_flush_tlb_range_pbp_slow:
 	stxa		%g2, [%o2] ASI_DMMU
 	flush		%g6
 	retl
-/*IC22*/ wrpr		%g1, 0x0, %pstate
+	 wrpr		%g1, 0x0, %pstate
 
 /*
  * The following code flushes one page_size worth.
@@ -318,32 +259,6 @@ iflush2:sub		%o1, 0x20, %g3
 	flush		%g6
 	ba,a,pt		%xcc, 3b
 
-	.align		64
-	.globl		__flush_dcache_page
-__flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
-	sethi		%uhi(PAGE_OFFSET), %g1
-	sllx		%g1, 32, %g1
-	sub		%o0, %g1, %o0
-	
-	rdpr		%ver, %g1
-	sethi		%hi(0x003e0014), %g2
-	srlx		%g1, 32, %g1
-	or		%g2, %lo(0x003e0014), %g2
-	cmp		%g1, %g2
-	bne,pt		%icc, flush_dcpage_spitfire
-	 nop
-
-flush_dcpage_cheetah:
-	sethi		%hi(PAGE_SIZE), %o4
-1:	subcc		%o4, (1 << 5), %o4
-	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
-	bne,pt		%icc, 1b
-	 nop
-	membar		#Sync
-	/* I-cache flush never needed on Cheetah, see callers. */
-	retl
-	 nop
-
 #if (PAGE_SHIFT == 13)
 #define DTAG_MASK 0x3
 #elif (PAGE_SHIFT == 16)
@@ -354,7 +269,12 @@ flush_dcpage_cheetah:
 #define DTAG_MASK 0x3ff
 #endif
 
-flush_dcpage_spitfire:
+	.align		64
+	.globl		__flush_dcache_page
+__flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
+	sethi		%uhi(PAGE_OFFSET), %g1
+	sllx		%g1, 32, %g1
+	sub		%o0, %g1, %o0
 	clr		%o4
 	srlx		%o0, 11, %o0
 	sethi		%hi(1 << 14), %o2
@@ -435,6 +355,121 @@ __update_mmu_cache:	/* %o0=hw_context, %o1=address, %o2=pte, %o3=fault_code */
 	 or		%o5, %o0, %o5
 	ba,a,pt		%xcc, __prefill_itlb
 
+	/* Cheetah specific versions, patched at boot time.  */
+__cheetah_flush_tlb_page: /* 14 insns */
+	rdpr		%pstate, %g5
+	andn		%g5, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o2
+	ldxa		[%o2] ASI_DMMU, %g2
+	stxa		%o0, [%o2] ASI_DMMU
+	stxa		%g0, [%o1] ASI_DMMU_DEMAP
+	stxa		%g0, [%o1] ASI_IMMU_DEMAP
+	stxa		%g2, [%o2] ASI_DMMU
+	flush		%g6
+	wrpr		%g0, 0, %tl
+	retl
+	 wrpr		%g5, 0x0, %pstate
+
+__cheetah_flush_tlb_mm: /* 15 insns */
+	rdpr		%pstate, %g5
+	andn		%g5, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o2
+	mov		0x40, %g3
+	ldxa		[%o2] ASI_DMMU, %g2
+	stxa		%o0, [%o2] ASI_DMMU
+	stxa		%g0, [%g3] ASI_DMMU_DEMAP
+	stxa		%g0, [%g3] ASI_IMMU_DEMAP
+	stxa		%g2, [%o2] ASI_DMMU
+	flush		%g6
+	wrpr		%g0, 0, %tl
+	retl
+	 wrpr		%g5, 0x0, %pstate
+
+__cheetah_flush_tlb_range: /* 20 insns */
+	cmp		%o5, %o4
+	blu,pt		%xcc, 9f
+	 rdpr		%pstate, %g5
+	andn		%g5, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o2
+	sub		%o5, %o4, %o5
+	ldxa		[%o2] ASI_DMMU, %g2
+	stxa		%o0, [%o2] ASI_DMMU
+1:	stxa		%g0, [%o1 + %o5] ASI_DMMU_DEMAP
+	stxa		%g0, [%o1 + %o5] ASI_IMMU_DEMAP
+	membar		#Sync
+	brnz,pt		%o5, 1b
+	 sub		%o5, %o4, %o5
+	stxa		%g2, [%o2] ASI_DMMU
+	flush		%g6
+	wrpr		%g0, 0, %tl
+9:	retl
+	 wrpr		%g5, 0x0, %pstate
+
+flush_dcpage_cheetah: /* 9 insns */
+	sethi		%uhi(PAGE_OFFSET), %g1
+	sllx		%g1, 32, %g1
+	sub		%o0, %g1, %o0
+	sethi		%hi(PAGE_SIZE), %o4
+1:	subcc		%o4, (1 << 5), %o4
+	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
+	bne,pt		%icc, 1b
+	 nop
+	membar		#Sync
+	retl		/* I-cache flush never needed on Cheetah, see callers. */
+	 nop
+
+cheetah_patch_one:
+1:	lduw		[%o1], %g1
+	stw		%g1, [%o0]
+	flush		%o0
+	subcc		%o2, 1, %o2
+	add		%o1, 4, %o1
+	bne,pt		%icc, 1b
+	 add		%o0, 4, %o0
+	retl
+	 nop
+
+	.globl		cheetah_patch_cachetlbops
+cheetah_patch_cachetlbops:
+	save		%sp, -128, %sp
+
+	sethi		%hi(__flush_tlb_page), %o0
+	or		%o0, %lo(__flush_tlb_page), %o0
+	sethi		%hi(__cheetah_flush_tlb_page), %o1
+	or		%o1, %lo(__cheetah_flush_tlb_page), %o1
+	call		cheetah_patch_one
+	 mov		14, %o2
+
+	sethi		%hi(__flush_tlb_mm), %o0
+	or		%o0, %lo(__flush_tlb_mm), %o0
+	sethi		%hi(__cheetah_flush_tlb_mm), %o1
+	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
+	call		cheetah_patch_one
+	 mov		15, %o2
+
+	sethi		%hi(__flush_tlb_range), %o0
+	or		%o0, %lo(__flush_tlb_range), %o0
+	sethi		%hi(__cheetah_flush_tlb_range), %o1
+	or		%o1, %lo(__cheetah_flush_tlb_range), %o1
+	call		cheetah_patch_one
+	 mov		20, %o2
+
+	sethi		%hi(__flush_dcache_page), %o0
+	or		%o0, %lo(__flush_dcache_page), %o0
+	sethi		%hi(flush_dcpage_cheetah), %o1
+	or		%o1, %lo(flush_dcpage_cheetah), %o1
+	call		cheetah_patch_one
+	 mov		9, %o2
+
+	ret
+	 restore
+
 #ifdef CONFIG_SMP
 	/* These are all called by the slaves of a cross call, at
 	 * trap level 1, with interrupts fully disabled.
@@ -618,10 +653,8 @@ errata32_hwbug:
 	.text
 
 	/* These two are not performance critical... */
-	.globl		xcall_flush_tlb_all
-xcall_flush_tlb_all:
-	BRANCH_IF_CHEETAH(g2, g3, __cheetah_xcall_flush_tlb_all)
-__spitfire_xcall_flush_tlb_all:
+	.globl		xcall_flush_tlb_all_spitfire
+xcall_flush_tlb_all_spitfire:
 	/* Spitfire Errata #32 workaround. */
 	sethi		%hi(errata32_hwbug), %g4
 	stx		%g0, [%g4 + %lo(errata32_hwbug)]
@@ -663,16 +696,15 @@ __spitfire_xcall_flush_tlb_all:
 	flush		%g6
 	retry
 
-__cheetah_xcall_flush_tlb_all:
+	.globl		xcall_flush_tlb_all_cheetah
+xcall_flush_tlb_all_cheetah:
 	mov		0x80, %g2
 	stxa		%g0, [%g2] ASI_DMMU_DEMAP
 	stxa		%g0, [%g2] ASI_IMMU_DEMAP
 	retry
 
-	.globl		xcall_flush_cache_all
-xcall_flush_cache_all:
-	BRANCH_IF_CHEETAH(g2, g3, __cheetah_xcall_flush_cache_all)
-__spitfire_xcall_flush_cache_all:
+	.globl		xcall_flush_cache_all_spitfire
+xcall_flush_cache_all_spitfire:
 	sethi		%hi(16383), %g2
 	or		%g2, %lo(16383), %g2
 	clr		%g3
@@ -685,13 +717,6 @@ __spitfire_xcall_flush_cache_all:
 	flush		%g6
 	retry
 
-	/* Cheetah's caches are fully coherent in the sense that
-	 * caches are flushed here.  We need to verify this and
-	 * really just not even send out the xcall at the top level.
-	 */
-__cheetah_xcall_flush_cache_all:
-	retry
-
 	/* These just get rescheduled to PIL vectors. */
 	.globl		xcall_call_function
 xcall_call_function:

include/asm-sparc64/asi.h

@@ -40,6 +40,7 @@
 #define ASI_WCACHE_DATA		0x39 /* (III) WCache data RAM diag		*/
 #define ASI_WCACHE_TAG		0x3a /* (III) WCache tag RAM diag		*/
 #define ASI_WCACHE_SNOOP_TAG	0x3b /* (III) WCache snoop tag RAM diag		*/
+#define ASI_SRAM_FAST_INIT	0x40 /* (III+) Fast SRAM init			*/
 #define ASI_DCACHE_INVALIDATE	0x42 /* (III) DCache Invalidate diag		*/
 #define ASI_DCACHE_UTAG		0x43 /* (III) DCache uTag diag			*/
 #define ASI_DCACHE_SNOOP_TAG	0x44 /* (III) DCache snoop tag RAM diag		*/

include/asm-sparc64/dcr.h

@@ -2,11 +2,13 @@
 #ifndef _SPARC64_DCR_H
 #define _SPARC64_DCR_H
 
-/* UltraSparc-III Dispatch Control Register, ASR 0x12 */
+/* UltraSparc-III/III+ Dispatch Control Register, ASR 0x12 */
+#define DCR_DPE		0x0000000000001000 /* III+: D$ Parity Error Enable	*/
 #define DCR_OBS		0x0000000000000fc0 /* Observability Bus Controls	*/
 #define DCR_BPE		0x0000000000000020 /* Branch Predict Enable		*/
 #define DCR_RPE		0x0000000000000010 /* Return Address Prediction Enable	*/
 #define DCR_SI		0x0000000000000008 /* Single Instruction Disable	*/
+#define DCR_IPE		0x0000000000000004 /* III+: I$ Parity Error Enable	*/
 #define DCR_IFPOE	0x0000000000000002 /* IRQ FP Operation Enable		*/
 #define DCR_MS		0x0000000000000001 /* Multi-Scalar dispatch		*/
 

include/asm-sparc64/elf.h

@@ -89,7 +89,8 @@ typedef struct {
 #define ELF_HWCAP	((HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | \
 			  HWCAP_SPARC_SWAP | HWCAP_SPARC_MULDIV | \
 			  HWCAP_SPARC_V9) | \
-			 ((tlb_type == cheetah) ? HWCAP_SPARC_ULTRA3 : 0))
+			 ((tlb_type == cheetah || tlb_type == cheetah_plus) ? \
+			  HWCAP_SPARC_ULTRA3 : 0))
 
 /* This yields a string that ld.so will use to load implementation
    specific libraries for optimization.  This is more specific in

include/asm-sparc64/head.h

@@ -8,4 +8,7 @@
 
 #define	PTREGS_OFF	(STACK_BIAS + REGWIN_SZ)
 
+#define CHEETAH_ID	0x003e0014
+#define CHEETAH_PLUS_ID	0x003e0015
+
 #endif /* !(_SPARC64_HEAD_H) */

include/asm-sparc64/smp.h

@@ -91,7 +91,7 @@ static inline int any_online_cpu(unsigned long mask)
 
 extern __inline__ int hard_smp_processor_id(void)
 {
-	if (tlb_type == cheetah) {
+	if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		unsigned long safari_config;
 		__asm__ __volatile__("ldxa [%%g0] %1, %0"
 				     : "=r" (safari_config)

include/asm-sparc64/spitfire.h

@@ -22,6 +22,7 @@
 #define TSB_EXTENSION_P		0x0000000000000048 /* Ultra-III and later		*/
 #define TSB_EXTENSION_S		0x0000000000000050 /* Ultra-III and later, D-TLB only	*/
 #define TSB_EXTENSION_N		0x0000000000000058 /* Ultra-III and later		*/
+#define TLB_TAG_ACCESS_EXT	0x0000000000000060 /* Ultra-III+ and later		*/
 
 /* These registers only exist as one entity, and are accessed
  * via ASI_DMMU only.
@@ -38,12 +39,13 @@
 
 enum ultra_tlb_layout {
 	spitfire = 0,
-	cheetah = 1
+	cheetah = 1,
+	cheetah_plus = 2,
 };
 
 extern enum ultra_tlb_layout tlb_type;
 
-#define SPARC64_USE_STICK	(tlb_type == cheetah)
+#define SPARC64_USE_STICK	(tlb_type != spitfire)
 
 #define CHEETAH_HIGHEST_LOCKED_TLBENT	(16 - 1)
 
@@ -433,35 +435,35 @@ extern __inline__ void cheetah_put_litlb_data(int entry, unsigned long data)
 			       "i" (ASI_ITLB_DATA_ACCESS));
 }
 
-extern __inline__ unsigned long cheetah_get_dtlb_data(int entry)
+extern __inline__ unsigned long cheetah_get_dtlb_data(int entry, int tlb)
 {
 	unsigned long data;
 
 	__asm__ __volatile__("ldxa	[%1] %2, %%g0\n\t"
 			     "ldxa	[%1] %2, %0"
 			     : "=r" (data)
-			     : "r" ((2 << 16) | (entry << 3)), "i" (ASI_DTLB_DATA_ACCESS));
+			     : "r" ((tlb << 16) | (entry << 3)), "i" (ASI_DTLB_DATA_ACCESS));
 
 	return data;
 }
 
-extern __inline__ unsigned long cheetah_get_dtlb_tag(int entry)
+extern __inline__ unsigned long cheetah_get_dtlb_tag(int entry, int tlb)
 {
 	unsigned long tag;
 
 	__asm__ __volatile__("ldxa	[%1] %2, %0"
 			     : "=r" (tag)
-			     : "r" ((2 << 16) | (entry << 3)), "i" (ASI_DTLB_TAG_READ));
+			     : "r" ((tlb << 16) | (entry << 3)), "i" (ASI_DTLB_TAG_READ));
 	return tag;
 }
 
-extern __inline__ void cheetah_put_dtlb_data(int entry, unsigned long data)
+extern __inline__ void cheetah_put_dtlb_data(int entry, unsigned long data, int tlb)
 {
 	__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
 			     "membar	#Sync"
 			     : /* No outputs */
 			     : "r" (data),
-			       "r" ((2 << 16) | (entry << 3)),
+			       "r" ((tlb << 16) | (entry << 3)),
 			       "i" (ASI_DTLB_DATA_ACCESS));
 }