Blackfin arch: Faster C implementation of no-MPU CPLB handler

This is a mixture ofcMichael McTernan's patch and the existing cplb-mpu code.

We ditch the old cplb-nompu implementation, which is a good example of
why a good algorithm in a HLL is preferrable to a bad algorithm written in
assembly.  Rather than try to construct a table of all posible CPLBs and
search it, we just create a (smaller) table of memory regions and
their attributes.  Some of the data structures are now unified for both
the mpu and nompu cases.  A lot of needless complexity in cplbinit.c is
removed.

Further optimizations:
  * compile cplbmgr.c with a lot of -ffixed-reg options, and omit saving
    these registers on the stack when entering a CPLB exception.
  * lose cli/nop/nop/sti sequences for some workarounds - these don't
  * make
    sense in an exception context

Additional code unification should be possible after this.

[Mike Frysinger <vapier.adi@gmail.com>:
 - convert CPP if statements to C if statements
 - remove redundant statements
 - use a do...while loop rather than a for loop to get slightly better
   optimization and to avoid gcc "may be used uninitialized" warnings ...
   we know that the [id]cplb_nr_bounds variables will never be 0, so this
   is OK
 - the no-mpu code was the last user of MAX_MEM_SIZE and with that rewritten,
   we can punt it
 - add some BUG_ON() checks to make sure we dont overflow the small
   cplb_bounds array
 - add i/d cplb entries for the bootrom because there is functions/data in
   there we want to access
 - we do not need a NULL trailing entry as any time we access the bounds
   arrays, we use the nr_bounds variable
]
Signed-off-by: Michael McTernan <mmcternan@airvana.com>
Signed-off-by: Mike Frysinger <vapier.adi@gmail.com>
Signed-off-by: Bernd Schmidt <bernds_cb1@t-online.de>
Signed-off-by: Bryan Wu <cooloney@kernel.org>

arch/blackfin/Kconfig

@@ -524,14 +524,6 @@ config MEM_SDGCTL
 	default 0x0
 endmenu
 
-config MAX_MEM_SIZE
-	int "Max SDRAM Memory Size in MBytes"
-	depends on !MPU
-	default 512
-	help
-	  This is the max memory size that the kernel will create CPLB
-	  tables for.  Your system will not be able to handle any more.
-
 #
 # Max & Min Speeds for various Chips
 #

arch/blackfin/include/asm/context.S

@@ -357,3 +357,42 @@
 	SYSCFG = [sp++];
 	csync;
 .endm
+
+.macro save_context_cplb
+	[--sp] = (R7:0, P5:0);
+	[--sp] = fp;
+
+	[--sp] = a0.x;
+	[--sp] = a0.w;
+	[--sp] = a1.x;
+	[--sp] = a1.w;
+
+	[--sp] = LC0;
+	[--sp] = LC1;
+	[--sp] = LT0;
+	[--sp] = LT1;
+	[--sp] = LB0;
+	[--sp] = LB1;
+
+	[--sp] = RETS;
+.endm
+
+.macro restore_context_cplb
+	RETS = [sp++];
+
+	LB1 = [sp++];
+	LB0 = [sp++];
+	LT1 = [sp++];
+	LT0 = [sp++];
+	LC1 = [sp++];
+	LC0 = [sp++];
+
+	a1.w = [sp++];
+	a1.x = [sp++];
+	a0.w = [sp++];
+	a0.x = [sp++];
+
+	fp = [sp++];
+
+	(R7:0, P5:0) = [SP++];
+.endm

arch/blackfin/include/asm/cplb-mpu.h

-/*
- * File:         include/asm-blackfin/cplbinit.h
- * Based on:
- * Author:
- *
- * Created:
- * Description:
- *
- * Modified:
- *               Copyright 2004-2006 Analog Devices Inc.
- *
- * Bugs:         Enter bugs at http://blackfin.uclinux.org/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see the file COPYING, or write
- * to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
- */
-#ifndef __ASM_BFIN_CPLB_MPU_H
-#define __ASM_BFIN_CPLB_MPU_H
-#include <linux/threads.h>
-
-struct cplb_entry {
-	unsigned long data, addr;
-};
-
-struct mem_region {
-	unsigned long start, end;
-	unsigned long dcplb_data;
-	unsigned long icplb_data;
-};
-
-extern struct cplb_entry dcplb_tbl[NR_CPUS][MAX_CPLBS];
-extern struct cplb_entry icplb_tbl[NR_CPUS][MAX_CPLBS];
-extern int first_switched_icplb;
-extern int first_mask_dcplb;
-extern int first_switched_dcplb;
-
-extern int nr_dcplb_miss[], nr_icplb_miss[], nr_icplb_supv_miss[];
-extern int nr_dcplb_prot[], nr_cplb_flush[];
-
-extern int page_mask_order;
-extern int page_mask_nelts;
-
-extern unsigned long *current_rwx_mask[NR_CPUS];
-
-extern void flush_switched_cplbs(unsigned int);
-extern void set_mask_dcplbs(unsigned long *, unsigned int);
-
-extern void __noreturn panic_cplb_error(int seqstat, struct pt_regs *);
-
-#endif /* __ASM_BFIN_CPLB_MPU_H */

arch/blackfin/include/asm/cplb.h

@@ -116,4 +116,8 @@
 #define CPLB_INOCACHE   	CPLB_USER_RD | CPLB_VALID
 #define CPLB_IDOCACHE   	CPLB_INOCACHE | CPLB_L1_CHBL
 
+#define FAULT_RW        (1 << 16)
+#define FAULT_USERSUPV  (1 << 17)
+#define FAULT_CPLBBITS  0x0000ffff
+
 #endif				/* _CPLB_H */

arch/blackfin/include/asm/cplbinit.h

@@ -32,96 +32,56 @@
 
 #include <asm/blackfin.h>
 #include <asm/cplb.h>
+#include <linux/threads.h>
 
-#ifdef CONFIG_MPU
-
-#include <asm/cplb-mpu.h>
-extern void bfin_icache_init(struct cplb_entry *icplb_tbl);
-extern void bfin_dcache_init(struct cplb_entry *icplb_tbl);
-
+#ifdef CONFIG_CPLB_SWITCH_TAB_L1
+# define PDT_ATTR __attribute__((l1_data))
 #else
+# define PDT_ATTR
+#endif
 
-#define INITIAL_T 0x1
-#define SWITCH_T  0x2
-#define I_CPLB    0x4
-#define D_CPLB    0x8
+struct cplb_entry {
+	unsigned long data, addr;
+};
 
-#define ASYNC_MEMORY_CPLB_COVERAGE  ((ASYNC_BANK0_SIZE + ASYNC_BANK1_SIZE + \
-				ASYNC_BANK2_SIZE + ASYNC_BANK3_SIZE) / SIZE_4M)
+struct cplb_boundary {
+	unsigned long eaddr; /* End of this region.  */
+	unsigned long data; /* CPLB data value.  */
+};
 
-#define CPLB_MEM CONFIG_MAX_MEM_SIZE
+extern struct cplb_boundary dcplb_bounds[];
+extern struct cplb_boundary icplb_bounds[];
+extern int dcplb_nr_bounds, icplb_nr_bounds;
 
-/*
-* Number of required data CPLB switchtable entries
-* MEMSIZE / 4 (we mostly install 4M page size CPLBs
-* approx 16 for smaller 1MB page size CPLBs for allignment purposes
-* 1 for L1 Data Memory
-* possibly 1 for L2 Data Memory
-* 1 for CONFIG_DEBUG_HUNT_FOR_ZERO
-* 1 for ASYNC Memory
-*/
-#define MAX_SWITCH_D_CPLBS (((CPLB_MEM / 4) + 16 + 1 + 1 + 1 \
-				 + ASYNC_MEMORY_CPLB_COVERAGE) * 2)
+extern struct cplb_entry dcplb_tbl[NR_CPUS][MAX_CPLBS];
+extern struct cplb_entry icplb_tbl[NR_CPUS][MAX_CPLBS];
+extern int first_switched_icplb;
+extern int first_switched_dcplb;
 
-/*
-* Number of required instruction CPLB switchtable entries
-* MEMSIZE / 4 (we mostly install 4M page size CPLBs
-* approx 12 for smaller 1MB page size CPLBs for allignment purposes
-* 1 for L1 Instruction Memory
-* possibly 1 for L2 Instruction Memory
-* 1 for CONFIG_DEBUG_HUNT_FOR_ZERO
-*/
-#define MAX_SWITCH_I_CPLBS (((CPLB_MEM / 4) + 12 + 1 + 1 + 1) * 2)
-
-/* Number of CPLB table entries, used for cplb-nompu. */
-#define CPLB_TBL_ENTRIES (16 * 4)
-
-enum {
-	ZERO_P, L1I_MEM, L1D_MEM, L2_MEM, SDRAM_KERN, SDRAM_RAM_MTD, SDRAM_DMAZ,
-	RES_MEM, ASYNC_MEM, OCB_ROM
-};
+extern int nr_dcplb_miss[], nr_icplb_miss[], nr_icplb_supv_miss[];
+extern int nr_dcplb_prot[], nr_cplb_flush[];
 
-struct cplb_desc {
-	u32 start; /* start address */
-	u32 end; /* end address */
-	u32 psize; /* prefered size if any otherwise 1MB or 4MB*/
-	u16 attr;/* attributes */
-	u16 i_conf;/* I-CPLB DATA */
-	u16 d_conf;/* D-CPLB DATA */
-	u16 valid;/* valid */
-	const s8 name[30];/* name */
-};
+#ifdef CONFIG_MPU
 
-struct cplb_tab {
-  u_long *tab;
-	u16 pos;
-	u16 size;
-};
+extern int first_mask_dcplb;
 
-extern u_long icplb_tables[NR_CPUS][CPLB_TBL_ENTRIES+1];
-extern u_long dcplb_tables[NR_CPUS][CPLB_TBL_ENTRIES+1];
+extern int page_mask_order;
+extern int page_mask_nelts;
 
-/* Till here we are discussing about the static memory management model.
- * However, the operating envoronments commonly define more CPLB
- * descriptors to cover the entire addressable memory than will fit into
- * the available on-chip 16 CPLB MMRs. When this happens, the below table
- * will be used which will hold all the potentially required CPLB descriptors
- *
- * This is how Page descriptor Table is implemented in uClinux/Blackfin.
- */
+extern unsigned long *current_rwx_mask[NR_CPUS];
 
-extern u_long ipdt_tables[NR_CPUS][MAX_SWITCH_I_CPLBS+1];
-extern u_long dpdt_tables[NR_CPUS][MAX_SWITCH_D_CPLBS+1];
-#ifdef CONFIG_CPLB_INFO
-extern u_long ipdt_swapcount_tables[NR_CPUS][MAX_SWITCH_I_CPLBS];
-extern u_long dpdt_swapcount_tables[NR_CPUS][MAX_SWITCH_D_CPLBS];
-#endif
-extern void bfin_icache_init(u_long icplbs[]);
-extern void bfin_dcache_init(u_long dcplbs[]);
+extern void flush_switched_cplbs(unsigned int);
+extern void set_mask_dcplbs(unsigned long *, unsigned int);
+
+extern void __noreturn panic_cplb_error(int seqstat, struct pt_regs *);
 
 #endif /* CONFIG_MPU */
 
+extern void bfin_icache_init(struct cplb_entry *icplb_tbl);
+extern void bfin_dcache_init(struct cplb_entry *icplb_tbl);
+
 #if defined(CONFIG_BFIN_DCACHE) || defined(CONFIG_BFIN_ICACHE)
+extern void generate_cplb_tables_all(void);
 extern void generate_cplb_tables_cpu(unsigned int cpu);
 #endif
 #endif

arch/blackfin/include/asm/entry.h

@@ -53,9 +53,11 @@
 /* This one pushes RETI without using CLI.  Interrupts are enabled.  */
 #define SAVE_CONTEXT_SYSCALL	save_context_syscall
 #define SAVE_CONTEXT		save_context_with_interrupts
+#define SAVE_CONTEXT_CPLB	save_context_cplb
 
 #define RESTORE_ALL_SYS		restore_context_no_interrupts
 #define RESTORE_CONTEXT		restore_context_with_interrupts
+#define RESTORE_CONTEXT_CPLB	restore_context_cplb
 
 #endif				/* __ASSEMBLY__ */
 #endif				/* __BFIN_ENTRY_H */

arch/blackfin/kernel/cplb-mpu/Makefile

@@ -3,3 +3,8 @@
 #
 
 obj-y := cplbinit.o cacheinit.o cplbmgr.o
+
+CFLAGS_cplbmgr.o := -ffixed-I0 -ffixed-I1 -ffixed-I2 -ffixed-I3 \
+		    -ffixed-L0 -ffixed-L1 -ffixed-L2 -ffixed-L3 \
+		    -ffixed-M0 -ffixed-M1 -ffixed-M2 -ffixed-M3 \
+		    -ffixed-B0 -ffixed-B1 -ffixed-B2 -ffixed-B3

arch/blackfin/kernel/cplb-mpu/cplbinit.c

@@ -107,3 +107,7 @@ void __init generate_cplb_tables_cpu(unsigned int cpu)
 	while (i_i < MAX_CPLBS)
 		icplb_tbl[cpu][i_i++].data = 0;
 }
+
+void generate_cplb_tables_all(void)
+{
+}

arch/blackfin/kernel/cplb-mpu/cplbmgr.c

@@ -25,8 +25,13 @@
 #include <asm/cplbinit.h>
 #include <asm/mmu_context.h>
 
-#define FAULT_RW	(1 << 16)
-#define FAULT_USERSUPV	(1 << 17)
+/*
+ * WARNING
+ *
+ * This file is compiled with certain -ffixed-reg options.  We have to
+ * make sure not to call any functions here that could clobber these
+ * registers.
+ */
 
 int page_mask_nelts;
 int page_mask_order;

arch/blackfin/kernel/cplb-nompu/Makefile

@@ -2,4 +2,9 @@
 # arch/blackfin/kernel/cplb-nompu/Makefile
 #
 
-obj-y := cplbinit.o cacheinit.o cplbhdlr.o cplbmgr.o
+obj-y := cplbinit.o cacheinit.o cplbmgr.o
+
+CFLAGS_cplbmgr.o := -ffixed-I0 -ffixed-I1 -ffixed-I2 -ffixed-I3 \
+		    -ffixed-L0 -ffixed-L1 -ffixed-L2 -ffixed-L3 \
+		    -ffixed-M0 -ffixed-M1 -ffixed-M2 -ffixed-M3 \
+		    -ffixed-B0 -ffixed-B1 -ffixed-B2 -ffixed-B3

arch/blackfin/kernel/cplb-nompu/cacheinit.c

@@ -25,19 +25,15 @@
 #include <asm/cplbinit.h>
 
 #if defined(CONFIG_BFIN_ICACHE)
-void __cpuinit bfin_icache_init(u_long icplb[])
+void __cpuinit bfin_icache_init(struct cplb_entry *icplb_tbl)
 {
-	unsigned long *table = icplb;
 	unsigned long ctrl;
 	int i;
 
+	SSYNC();
 	for (i = 0; i < MAX_CPLBS; i++) {
-		unsigned long addr = *table++;
-		unsigned long data = *table++;
-		if (addr == (unsigned long)-1)
-			break;
-		bfin_write32(ICPLB_ADDR0 + i * 4, addr);
-		bfin_write32(ICPLB_DATA0 + i * 4, data);
+		bfin_write32(ICPLB_ADDR0 + i * 4, icplb_tbl[i].addr);
+		bfin_write32(ICPLB_DATA0 + i * 4, icplb_tbl[i].data);
 	}
 	ctrl = bfin_read_IMEM_CONTROL();
 	ctrl |= IMC | ENICPLB;
@@ -47,24 +43,20 @@ void __cpuinit bfin_icache_init(u_long icplb[])
 #endif
 
 #if defined(CONFIG_BFIN_DCACHE)
-void __cpuinit bfin_dcache_init(u_long dcplb[])
+void __cpuinit bfin_dcache_init(struct cplb_entry *dcplb_tbl)
 {
-	unsigned long *table = dcplb;
 	unsigned long ctrl;
 	int i;
 
+	SSYNC();
 	for (i = 0; i < MAX_CPLBS; i++) {
-		unsigned long addr = *table++;
-		unsigned long data = *table++;
-		if (addr == (unsigned long)-1)
-			break;
-		bfin_write32(DCPLB_ADDR0 + i * 4, addr);
-		bfin_write32(DCPLB_DATA0 + i * 4, data);
+		bfin_write32(DCPLB_ADDR0 + i * 4, dcplb_tbl[i].addr);
+		bfin_write32(DCPLB_DATA0 + i * 4, dcplb_tbl[i].data);
 	}
+
 	ctrl = bfin_read_DMEM_CONTROL();
 	ctrl |= DMEM_CNTR;
 	bfin_write_DMEM_CONTROL(ctrl);
-
 	SSYNC();
 }
 #endif

arch/blackfin/kernel/cplb-nompu/cplbhdlr.S

-/*
- * File:         arch/blackfin/mach-common/cplbhdlr.S
- * Based on:
- * Author:       LG Soft India
- *
- * Created:      ?
- * Description:  CPLB exception handler
- *
- * Modified:
- *               Copyright 2004-2006 Analog Devices Inc.
- *
- * Bugs:         Enter bugs at http://blackfin.uclinux.org/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see the file COPYING, or write
- * to the Free Software Foundation, Inc.,
- * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
- */
-
-#include <linux/linkage.h>
-#include <asm/cplb.h>
-#include <asm/entry.h>
-
-#ifdef CONFIG_EXCPT_IRQ_SYSC_L1
-.section .l1.text
-#else
-.text
-#endif
-
-.type _cplb_mgr, STT_FUNC;
-.type _panic_cplb_error, STT_FUNC;
-
-.align 2
-
-ENTRY(__cplb_hdr)
-	R2 = SEQSTAT;
-
-	/* Mask the contents of SEQSTAT and leave only EXCAUSE in R2 */
-	R2 <<= 26;
-	R2 >>= 26;
-
-	R1 = 0x23; /* Data access CPLB protection violation */
-	CC = R2 == R1;
-	IF !CC JUMP .Lnot_data_write;
-	R0 = 2;		/* is a write to data space*/
-	JUMP .Lis_icplb_miss;
-
-.Lnot_data_write:
-	R1 = 0x2C; /* CPLB miss on an instruction fetch */
-	CC = R2 == R1;
-	R0 = 0;		/* is_data_miss == False*/
-	IF CC JUMP .Lis_icplb_miss;
-
-	R1 = 0x26;
-	CC = R2 == R1;
-	IF !CC JUMP .Lunknown;
-
-	R0 = 1;		/* is_data_miss == True*/
-
-.Lis_icplb_miss:
-
-#if defined(CONFIG_BFIN_ICACHE) || defined(CONFIG_BFIN_DCACHE)
-# if defined(CONFIG_BFIN_ICACHE) && !defined(CONFIG_BFIN_DCACHE)
-	R1 = CPLB_ENABLE_ICACHE;
-# endif
-# if !defined(CONFIG_BFIN_ICACHE) && defined(CONFIG_BFIN_DCACHE)
-	R1 = CPLB_ENABLE_DCACHE;
-# endif
-# if defined(CONFIG_BFIN_ICACHE) && defined(CONFIG_BFIN_DCACHE)
-	R1 = CPLB_ENABLE_DCACHE | CPLB_ENABLE_ICACHE;
-# endif
-#else
-	R1 = 0;
-#endif
-
-	[--SP] = RETS;
-	CALL _cplb_mgr;
-	RETS = [SP++];
-	CC = R0 == 0;
-	IF !CC JUMP .Lnot_replaced;
-	RTS;
-
-/*
- * Diagnostic exception handlers
- */
-.Lunknown:
-	R0 = CPLB_UNKNOWN_ERR;
-	JUMP .Lcplb_error;
-
-.Lnot_replaced:
-	CC = R0 == CPLB_NO_UNLOCKED;
-	IF !CC JUMP .Lnext_check;
-	R0 = CPLB_NO_UNLOCKED;
-	JUMP .Lcplb_error;
-
-.Lnext_check:
-	CC = R0 == CPLB_NO_ADDR_MATCH;
-	IF !CC JUMP .Lnext_check2;
-	R0 = CPLB_NO_ADDR_MATCH;
-	JUMP .Lcplb_error;
-
-.Lnext_check2:
-	CC = R0 == CPLB_PROT_VIOL;
-	IF !CC JUMP .Lstrange_return_from_cplb_mgr;
-	R0 = CPLB_PROT_VIOL;
-	JUMP .Lcplb_error;
-
-.Lstrange_return_from_cplb_mgr:
-	IDLE;
-	CSYNC;
-	JUMP .Lstrange_return_from_cplb_mgr;
-
-.Lcplb_error:
-	R1 = sp;
-	SP += -12;
-	call _panic_cplb_error;
-	SP += 12;
-	JUMP.L _handle_bad_cplb;
-
-ENDPROC(__cplb_hdr)

arch/blackfin/kernel/cplb-nompu/cplbmgr.c

+/*
+ * File:         arch/blackfin/kernel/cplb-nompu-c/cplbmgr.c
+ * Based on:     arch/blackfin/kernel/cplb-mpu/cplbmgr.c
+ * Author:       Michael McTernan <mmcternan@airvana.com>
+ *
+ * Created:      01Nov2008
+ * Description:  CPLB miss handler.
+ *
+ * Modified:
+ *               Copyright 2008 Airvana Inc.
+ *               Copyright 2004-2007 Analog Devices Inc.
+ *
+ * Bugs:         Enter bugs at http://blackfin.uclinux.org/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <asm/blackfin.h>
+#include <asm/cplbinit.h>
+#include <asm/cplb.h>
+#include <asm/mmu_context.h>
+
+/*
+ * WARNING
+ *
+ * This file is compiled with certain -ffixed-reg options.  We have to
+ * make sure not to call any functions here that could clobber these
+ * registers.
+ */
+
+int nr_dcplb_miss[NR_CPUS], nr_icplb_miss[NR_CPUS];
+int nr_dcplb_supv_miss[NR_CPUS], nr_icplb_supv_miss[NR_CPUS];
+int nr_cplb_flush[NR_CPUS], nr_dcplb_prot[NR_CPUS];
+
+#ifdef CONFIG_EXCPT_IRQ_SYSC_L1
+#define MGR_ATTR __attribute__((l1_text))
+#else
+#define MGR_ATTR
+#endif
+
+/*
+ * We're in an exception handler.  The normal cli nop nop workaround
+ * isn't going to do very much, as the only thing that can interrupt
+ * us is an NMI, and the cli isn't going to stop that.
+ */
+#define NOWA_SSYNC __asm__ __volatile__ ("ssync;")
+
+/* Anomaly handlers provide SSYNCs, so avoid extra if anomaly is present */
+#if ANOMALY_05000125
+
+#define bfin_write_DMEM_CONTROL_SSYNC(v)    bfin_write_DMEM_CONTROL(v)
+#define bfin_write_IMEM_CONTROL_SSYNC(v)    bfin_write_IMEM_CONTROL(v)
+
+#else
+
+#define bfin_write_DMEM_CONTROL_SSYNC(v) \
+    do { NOWA_SSYNC; bfin_write_DMEM_CONTROL(v); NOWA_SSYNC; } while (0)
+#define bfin_write_IMEM_CONTROL_SSYNC(v) \
+    do { NOWA_SSYNC; bfin_write_IMEM_CONTROL(v); NOWA_SSYNC; } while (0)
+
+#endif
+
+static inline void write_dcplb_data(int cpu, int idx, unsigned long data,
+				    unsigned long addr)
+{
+	unsigned long ctrl = bfin_read_DMEM_CONTROL();
+	bfin_write_DMEM_CONTROL_SSYNC(ctrl & ~ENDCPLB);
+	bfin_write32(DCPLB_DATA0 + idx * 4, data);
+	bfin_write32(DCPLB_ADDR0 + idx * 4, addr);
+	bfin_write_DMEM_CONTROL_SSYNC(ctrl);
+
+#ifdef CONFIG_CPLB_INFO
+	dcplb_tbl[cpu][idx].addr = addr;
+	dcplb_tbl[cpu][idx].data = data;
+#endif
+}
+
+static inline void write_icplb_data(int cpu, int idx, unsigned long data,
+				    unsigned long addr)
+{
+	unsigned long ctrl = bfin_read_IMEM_CONTROL();
+
+	bfin_write_IMEM_CONTROL_SSYNC(ctrl & ~ENICPLB);
+	bfin_write32(ICPLB_DATA0 + idx * 4, data);
+	bfin_write32(ICPLB_ADDR0 + idx * 4, addr);
+	bfin_write_IMEM_CONTROL_SSYNC(ctrl);
+
+#ifdef CONFIG_CPLB_INFO
+	icplb_tbl[cpu][idx].addr = addr;
+	icplb_tbl[cpu][idx].data = data;
+#endif
+}
+
+/*
+ * Given the contents of the status register, return the index of the
+ * CPLB that caused the fault.
+ */
+static inline int faulting_cplb_index(int status)
+{
+	int signbits = __builtin_bfin_norm_fr1x32(status & 0xFFFF);
+	return 30 - signbits;
+}
+
+/*
+ * Given the contents of the status register and the DCPLB_DATA contents,
+ * return true if a write access should be permitted.
+ */
+static inline int write_permitted(int status, unsigned long data)
+{
+	if (status & FAULT_USERSUPV)
+		return !!(data & CPLB_SUPV_WR);
+	else
+		return !!(data & CPLB_USER_WR);
+}
+
+/* Counters to implement round-robin replacement.  */
+static int icplb_rr_index[NR_CPUS] PDT_ATTR;
+static int dcplb_rr_index[NR_CPUS] PDT_ATTR;
+
+/*
+ * Find an ICPLB entry to be evicted and return its index.
+ */
+static int evict_one_icplb(int cpu)
+{
+	int i = first_switched_icplb + icplb_rr_index[cpu];
+	if (i >= MAX_CPLBS) {
+		i -= MAX_CPLBS - first_switched_icplb;
+		icplb_rr_index[cpu] -= MAX_CPLBS - first_switched_icplb;
+	}
+	icplb_rr_index[cpu]++;
+	return i;
+}
+
+static int evict_one_dcplb(int cpu)
+{
+	int i = first_switched_dcplb + dcplb_rr_index[cpu];
+	if (i >= MAX_CPLBS) {
+		i -= MAX_CPLBS - first_switched_dcplb;
+		dcplb_rr_index[cpu] -= MAX_CPLBS - first_switched_dcplb;
+	}
+	dcplb_rr_index[cpu]++;
+	return i;
+}
+
+MGR_ATTR static int icplb_miss(int cpu)
+{
+	unsigned long addr = bfin_read_ICPLB_FAULT_ADDR();
+	int status = bfin_read_ICPLB_STATUS();
+	int idx;
+	unsigned long i_data, base, addr1, eaddr;
+
+	nr_icplb_miss[cpu]++;
+	if (unlikely(status & FAULT_USERSUPV))
+		nr_icplb_supv_miss[cpu]++;
+
+	base = 0;
+	for (idx = 0; idx < icplb_nr_bounds; idx++) {
+		eaddr = icplb_bounds[idx].eaddr;
+		if (addr < eaddr)
+			break;
+		base = eaddr;
+	}
+	if (unlikely(idx == icplb_nr_bounds))
+		return CPLB_NO_ADDR_MATCH;
+
+	i_data = icplb_bounds[idx].data;
+	if (unlikely(i_data == 0))
+		return CPLB_NO_ADDR_MATCH;
+
+	addr1 = addr & ~(SIZE_4M - 1);
+	addr &= ~(SIZE_1M - 1);
+	i_data |= PAGE_SIZE_1MB;
+	if (addr1 >= base && (addr1 + SIZE_4M) <= eaddr) {
+		/*
+		 * This works because
+		 * (PAGE_SIZE_4MB & PAGE_SIZE_1MB) == PAGE_SIZE_1MB.
+		 */
+		i_data |= PAGE_SIZE_4MB;
+		addr = addr1;
+	}
+
+	/* Pick entry to evict */
+	idx = evict_one_icplb(cpu);
+
+	write_icplb_data(cpu, idx, i_data, addr);
+
+	return CPLB_RELOADED;
+}
+
+MGR_ATTR static int dcplb_miss(int cpu)
+{
+	unsigned long addr = bfin_read_DCPLB_FAULT_ADDR();
+	int status = bfin_read_DCPLB_STATUS();
+	int idx;
+	unsigned long d_data, base, addr1, eaddr;
+
+	nr_dcplb_miss[cpu]++;
+	if (unlikely(status & FAULT_USERSUPV))
+		nr_dcplb_supv_miss[cpu]++;
+
+	base = 0;
+	for (idx = 0; idx < dcplb_nr_bounds; idx++) {
+		eaddr = dcplb_bounds[idx].eaddr;
+		if (addr < eaddr)
+			break;
+		base = eaddr;
+	}
+	if (unlikely(idx == dcplb_nr_bounds))
+		return CPLB_NO_ADDR_MATCH;
+
+	d_data = dcplb_bounds[idx].data;
+	if (unlikely(d_data == 0))
+		return CPLB_NO_ADDR_MATCH;
+
+	addr1 = addr & ~(SIZE_4M - 1);
+	addr &= ~(SIZE_1M - 1);
+	d_data |= PAGE_SIZE_1MB;
+	if (addr1 >= base && (addr1 + SIZE_4M) <= eaddr) {
+		/*
+		 * This works because
+		 * (PAGE_SIZE_4MB & PAGE_SIZE_1MB) == PAGE_SIZE_1MB.
+		 */
+		d_data |= PAGE_SIZE_4MB;
+		addr = addr1;
+	}
+
+	/* Pick entry to evict */
+	idx = evict_one_dcplb(cpu);
+
+	write_dcplb_data(cpu, idx, d_data, addr);
+
+	return CPLB_RELOADED;
+}
+
+MGR_ATTR static noinline int dcplb_protection_fault(int cpu)
+{
+	int status = bfin_read_DCPLB_STATUS();
+
+	nr_dcplb_prot[cpu]++;
+
+	if (likely(status & FAULT_RW)) {
+		int idx = faulting_cplb_index(status);
+		unsigned long regaddr = DCPLB_DATA0 + idx * 4;
+		unsigned long data = bfin_read32(regaddr);
+
+		/* Check if fault is to dirty a clean page */
+		if (!(data & CPLB_WT) && !(data & CPLB_DIRTY) &&
+		    write_permitted(status, data)) {
+
+			dcplb_tbl[cpu][idx].data = data;
+			bfin_write32(regaddr, data);
+			return CPLB_RELOADED;
+		}
+	}
+
+	return CPLB_PROT_VIOL;
+}
+
+MGR_ATTR int cplb_hdr(int seqstat, struct pt_regs *regs)
+{
+	int cause = seqstat & 0x3f;
+	unsigned int cpu = smp_processor_id();
+	switch (cause) {
+	case 0x2C:
+		return icplb_miss(cpu);
+	case 0x26:
+		return dcplb_miss(cpu);
+	default:
+		if (unlikely(cause == 0x23))
+			return dcplb_protection_fault(cpu);
+
+		return CPLB_UNKNOWN_ERR;
+	}
+}

arch/blackfin/kernel/cplbinfo.c

@@ -20,8 +20,6 @@ static char const page_strtbl[][3] = { "1K", "4K", "1M", "4M" };
 #define page(flags)    (((flags) & 0x30000) >> 16)
 #define strpage(flags) page_strtbl[page(flags)]
 
-#ifdef CONFIG_MPU
-
 struct cplbinfo_data {
 	loff_t pos;
 	char cplb_type;
@@ -75,88 +73,6 @@ static void cplbinfo_seq_init(struct cplbinfo_data *cdata, unsigned int cpu)
 	}
 }
 
-#else
-
-struct cplbinfo_data {
-	loff_t pos;
-	char cplb_type;
-	u32 mem_control;
-	unsigned long *pdt_tables, *pdt_swapcount;
-	unsigned long cplb_addr, cplb_data;
-};
-
-extern int page_size_table[];
-
-static int cplb_find_entry(unsigned long addr_tbl, unsigned long data_tbl,
-                           unsigned long addr_find, unsigned long data_find)
-{
-	int i;
-
-	for (i = 0; i < 16; ++i) {
-		unsigned long cplb_addr = bfin_read32(addr_tbl + i * 4);
-		unsigned long cplb_data = bfin_read32(data_tbl + i * 4);
-		if (addr_find >= cplb_addr &&
-		    addr_find < cplb_addr + page_size_table[page(cplb_data)] &&
-		    cplb_data == data_find)
-			return i;
-	}
-
-	return -1;
-}
-
-static void cplbinfo_print_header(struct seq_file *m)
-{
-	seq_printf(m, "Address\t\tData\tSize\tValid\tLocked\tSwapin\tiCount\toCount\n");
-}
-
-static int cplbinfo_nomore(struct cplbinfo_data *cdata)
-{
-	return cdata->pdt_tables[cdata->pos * 2] == 0xffffffff;
-}
-
-static int cplbinfo_show(struct seq_file *m, void *p)
-{
-	struct cplbinfo_data *cdata;
-	unsigned long data, addr;
-	int entry;
-	loff_t pos;
-
-	cdata = p;
-	pos = cdata->pos * 2;
-	addr = cdata->pdt_tables[pos];
-	data = cdata->pdt_tables[pos + 1];
-	entry = cplb_find_entry(cdata->cplb_addr, cdata->cplb_data, addr, data);
-
-	seq_printf(m,
-		"0x%08lx\t0x%05lx\t%s\t%c\t%c\t%2d\t%ld\t%ld\n",
-		addr, data, strpage(data),
-		(data & CPLB_VALID) ? 'Y' : 'N',
-		(data & CPLB_LOCK) ? 'Y' : 'N', entry,
-		cdata->pdt_swapcount[pos],
-		cdata->pdt_swapcount[pos + 1]);
-
-	return 0;
-}
-
-static void cplbinfo_seq_init(struct cplbinfo_data *cdata, unsigned int cpu)
-{
-	if (cdata->cplb_type == 'I') {
-		cdata->mem_control = bfin_read_IMEM_CONTROL();
-		cdata->pdt_tables = ipdt_tables[cpu];
-		cdata->pdt_swapcount = ipdt_swapcount_tables[cpu];
-		cdata->cplb_addr = ICPLB_ADDR0;
-		cdata->cplb_data = ICPLB_DATA0;
-	} else {
-		cdata->mem_control = bfin_read_DMEM_CONTROL();
-		cdata->pdt_tables = dpdt_tables[cpu];
-		cdata->pdt_swapcount = dpdt_swapcount_tables[cpu];
-		cdata->cplb_addr = DCPLB_ADDR0;
-		cdata->cplb_data = DCPLB_DATA0;
-	}
-}
-
-#endif
-
 static void *cplbinfo_start(struct seq_file *m, loff_t *pos)
 {
 	struct cplbinfo_data *cdata = m->private;

arch/blackfin/kernel/setup.c

@@ -88,6 +88,7 @@ void __init generate_cplb_tables(void)
 {
 	unsigned int cpu;
 
+	generate_cplb_tables_all();
 	/* Generate per-CPU I&D CPLB tables */
 	for (cpu = 0; cpu < num_possible_cpus(); ++cpu)
 		generate_cplb_tables_cpu(cpu);
@@ -97,19 +98,11 @@ void __init generate_cplb_tables(void)
 void __cpuinit bfin_setup_caches(unsigned int cpu)
 {
 #ifdef CONFIG_BFIN_ICACHE
-#ifdef CONFIG_MPU
 	bfin_icache_init(icplb_tbl[cpu]);
-#else
-	bfin_icache_init(icplb_tables[cpu]);
-#endif
 #endif
 
 #ifdef CONFIG_BFIN_DCACHE
-#ifdef CONFIG_MPU
 	bfin_dcache_init(dcplb_tbl[cpu]);
-#else
-	bfin_dcache_init(dcplb_tables[cpu]);
-#endif
 #endif
 
 	/*

arch/blackfin/mach-common/entry.S

@@ -112,24 +112,21 @@ ENTRY(_ex_dcplb_viol)
 ENTRY(_ex_dcplb_miss)
 ENTRY(_ex_icplb_miss)
 	(R7:6,P5:4) = [sp++];
-	ASTAT = [sp++];
-	SAVE_ALL_SYS
-#ifdef CONFIG_MPU
+	/* We leave the previously pushed ASTAT on the stack.  */
+	SAVE_CONTEXT_CPLB
+
 	/* We must load R1 here, _before_ DEBUG_HWTRACE_SAVE, since that
 	 * will change the stack pointer.  */
 	R0 = SEQSTAT;
 	R1 = SP;
-#endif
+
 	DEBUG_HWTRACE_SAVE(p5, r7)
-#ifdef CONFIG_MPU
+
 	sp += -12;
 	call _cplb_hdr;
 	sp += 12;
 	CC = R0 == 0;
 	IF !CC JUMP _handle_bad_cplb;
-#else
-	call __cplb_hdr;
-#endif
 
 #ifdef CONFIG_DEBUG_DOUBLEFAULT
 	/* While we were processing this, did we double fault? */
@@ -143,7 +140,8 @@ ENTRY(_ex_icplb_miss)
 #endif
 
 	DEBUG_HWTRACE_RESTORE(p5, r7)
-	RESTORE_ALL_SYS
+	RESTORE_CONTEXT_CPLB
+	ASTAT = [SP++];
 	SP = EX_SCRATCH_REG;
 	rtx;
 ENDPROC(_ex_icplb_miss)
@@ -298,9 +296,8 @@ ENTRY(_handle_bad_cplb)
 	 * the stack to get ready so, we can fall through - we
 	 * need to make a CPLB exception look like a normal exception
 	 */
-
-	RESTORE_ALL_SYS
-	[--sp] = ASTAT;
+	RESTORE_CONTEXT_CPLB
+	/* ASTAT is still on the stack, where it is needed.  */
 	[--sp] = (R7:6,P5:4);
 
 ENTRY(_ex_replaceable)

arch/blackfin/mm/init.c

@@ -119,16 +119,6 @@ asmlinkage void init_pda(void)
 
 	cpu_pda[cpu].ex_stack = exception_stack[cpu + 1];
 
-#ifdef CONFIG_MPU
-#else
-	cpu_pda[cpu].ipdt = ipdt_tables[cpu];
-	cpu_pda[cpu].dpdt = dpdt_tables[cpu];
-#ifdef CONFIG_CPLB_INFO
-	cpu_pda[cpu].ipdt_swapcount = ipdt_swapcount_tables[cpu];
-	cpu_pda[cpu].dpdt_swapcount = dpdt_swapcount_tables[cpu];
-#endif
-#endif
-
 #ifdef CONFIG_SMP
 	cpu_pda[cpu].imask = 0x1f;
 #endif