Commit 1b3a6e97 authored by Thiemo Seufer's avatar Thiemo Seufer Committed by Ralf Baechle

Fix 64bit SMP TLB handler and stack frame handling, optimize 32bit SMP

TLB handlers a bit, match definitions in pgtable-{32,64}.h better.
Signed-off-by: default avatarThiemo Seufer <ths@networkno.de>
Signed-off-by: default avatarRalf Baechle <ralf@linux-mips.org>
parent 7c2740f1
...@@ -157,6 +157,7 @@ NESTED(kernel_entry, 16, sp) # kernel entry point ...@@ -157,6 +157,7 @@ NESTED(kernel_entry, 16, sp) # kernel entry point
LONG_S a2, fw_arg2 LONG_S a2, fw_arg2
LONG_S a3, fw_arg3 LONG_S a3, fw_arg3
MTC0 zero, CP0_CONTEXT # clear context register
PTR_LA $28, init_thread_union PTR_LA $28, init_thread_union
PTR_ADDIU sp, $28, _THREAD_SIZE - 32 PTR_ADDIU sp, $28, _THREAD_SIZE - 32
set_saved_sp sp, t0, t1 set_saved_sp sp, t0, t1
......
...@@ -91,7 +91,7 @@ enum opcode { ...@@ -91,7 +91,7 @@ enum opcode {
insn_addu, insn_addiu, insn_and, insn_andi, insn_beq, insn_addu, insn_addiu, insn_and, insn_andi, insn_beq,
insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl, insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0, insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0,
insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32, insn_dsll, insn_dsll32, insn_dsra, insn_dsrl,
insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld, insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld,
insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0, insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0,
insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll, insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
...@@ -134,7 +134,6 @@ static __initdata struct insn insn_table[] = { ...@@ -134,7 +134,6 @@ static __initdata struct insn insn_table[] = {
{ insn_dsll32, M(spec_op,0,0,0,0,dsll32_op), RT | RD | RE }, { insn_dsll32, M(spec_op,0,0,0,0,dsll32_op), RT | RD | RE },
{ insn_dsra, M(spec_op,0,0,0,0,dsra_op), RT | RD | RE }, { insn_dsra, M(spec_op,0,0,0,0,dsra_op), RT | RD | RE },
{ insn_dsrl, M(spec_op,0,0,0,0,dsrl_op), RT | RD | RE }, { insn_dsrl, M(spec_op,0,0,0,0,dsrl_op), RT | RD | RE },
{ insn_dsrl32, M(spec_op,0,0,0,0,dsrl32_op), RT | RD | RE },
{ insn_dsubu, M(spec_op,0,0,0,0,dsubu_op), RS | RT | RD }, { insn_dsubu, M(spec_op,0,0,0,0,dsubu_op), RS | RT | RD },
{ insn_eret, M(cop0_op,cop_op,0,0,0,eret_op), 0 }, { insn_eret, M(cop0_op,cop_op,0,0,0,eret_op), 0 },
{ insn_j, M(j_op,0,0,0,0,0), JIMM }, { insn_j, M(j_op,0,0,0,0,0), JIMM },
...@@ -366,7 +365,6 @@ I_u2u1u3(_dsll); ...@@ -366,7 +365,6 @@ I_u2u1u3(_dsll);
I_u2u1u3(_dsll32); I_u2u1u3(_dsll32);
I_u2u1u3(_dsra); I_u2u1u3(_dsra);
I_u2u1u3(_dsrl); I_u2u1u3(_dsrl);
I_u2u1u3(_dsrl32);
I_u3u1u2(_dsubu); I_u3u1u2(_dsubu);
I_0(_eret); I_0(_eret);
I_u1(_j); I_u1(_j);
...@@ -944,34 +942,29 @@ build_get_pmde64(u32 **p, struct label **l, struct reloc **r, ...@@ -944,34 +942,29 @@ build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
/* No i_nop needed here, since the next insn doesn't touch TMP. */ /* No i_nop needed here, since the next insn doesn't touch TMP. */
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
# ifdef CONFIG_BUILD_ELF64
/* /*
* 64 bit SMP has the lower part of &pgd_current[smp_processor_id()] * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
* stored in CONTEXT. * stored in CONTEXT.
*/ */
if (in_compat_space_p(pgdc)) { i_dmfc0(p, ptr, C0_CONTEXT);
i_dmfc0(p, ptr, C0_CONTEXT); i_dsrl(p, ptr, ptr, 23);
i_dsra(p, ptr, ptr, 23); i_LA_mostly(p, tmp, pgdc);
i_ld(p, ptr, 0, ptr); i_daddu(p, ptr, ptr, tmp);
} else { i_dmfc0(p, tmp, C0_BADVADDR);
#ifdef CONFIG_BUILD_ELF64 i_ld(p, ptr, rel_lo(pgdc), ptr);
i_dmfc0(p, ptr, C0_CONTEXT); # else
i_dsrl(p, ptr, ptr, 23); /*
i_dsll(p, ptr, ptr, 3); * 64 bit SMP running in compat space has the lower part of
i_LA_mostly(p, tmp, pgdc); * &pgd_current[smp_processor_id()] stored in CONTEXT.
i_daddu(p, ptr, ptr, tmp); */
i_dmfc0(p, tmp, C0_BADVADDR); if (!in_compat_space_p(pgdc))
i_ld(p, ptr, rel_lo(pgdc), ptr); panic("Invalid page directory address!");
#else
i_dmfc0(p, ptr, C0_CONTEXT); i_dmfc0(p, ptr, C0_CONTEXT);
i_lui(p, tmp, rel_highest(pgdc)); i_dsra(p, ptr, ptr, 23);
i_dsll(p, ptr, ptr, 9); i_ld(p, ptr, 0, ptr);
i_daddiu(p, tmp, tmp, rel_higher(pgdc)); # endif
i_dsrl32(p, ptr, ptr, 0);
i_and(p, ptr, ptr, tmp);
i_dmfc0(p, tmp, C0_BADVADDR);
i_ld(p, ptr, 0, ptr);
#endif
}
#else #else
i_LA_mostly(p, ptr, pgdc); i_LA_mostly(p, ptr, pgdc);
i_ld(p, ptr, rel_lo(pgdc), ptr); i_ld(p, ptr, rel_lo(pgdc), ptr);
...@@ -1028,7 +1021,6 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) ...@@ -1028,7 +1021,6 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
i_mfc0(p, ptr, C0_CONTEXT); i_mfc0(p, ptr, C0_CONTEXT);
i_LA_mostly(p, tmp, pgdc); i_LA_mostly(p, tmp, pgdc);
i_srl(p, ptr, ptr, 23); i_srl(p, ptr, ptr, 23);
i_sll(p, ptr, ptr, 2);
i_addu(p, ptr, tmp, ptr); i_addu(p, ptr, tmp, ptr);
#else #else
i_LA_mostly(p, ptr, pgdc); i_LA_mostly(p, ptr, pgdc);
......
...@@ -30,7 +30,7 @@ extern unsigned long pgd_current[]; ...@@ -30,7 +30,7 @@ extern unsigned long pgd_current[];
#ifdef CONFIG_32BIT #ifdef CONFIG_32BIT
#define TLBMISS_HANDLER_SETUP() \ #define TLBMISS_HANDLER_SETUP() \
write_c0_context((unsigned long) smp_processor_id() << 23); \ write_c0_context((unsigned long) smp_processor_id() << 25); \
TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir) TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir)
#endif #endif
#if defined(CONFIG_64BIT) && !defined(CONFIG_BUILD_ELF64) #if defined(CONFIG_64BIT) && !defined(CONFIG_BUILD_ELF64)
...@@ -40,7 +40,7 @@ extern unsigned long pgd_current[]; ...@@ -40,7 +40,7 @@ extern unsigned long pgd_current[];
#endif #endif
#if defined(CONFIG_64BIT) && defined(CONFIG_BUILD_ELF64) #if defined(CONFIG_64BIT) && defined(CONFIG_BUILD_ELF64)
#define TLBMISS_HANDLER_SETUP() \ #define TLBMISS_HANDLER_SETUP() \
write_c0_context((unsigned long) smp_processor_id() << 23); \ write_c0_context((unsigned long) smp_processor_id() << 26); \
TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir) TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir)
#endif #endif
......
...@@ -114,7 +114,7 @@ extern pmd_t invalid_pmd_table[PTRS_PER_PMD]; ...@@ -114,7 +114,7 @@ extern pmd_t invalid_pmd_table[PTRS_PER_PMD];
extern pmd_t empty_bad_pmd_table[PTRS_PER_PMD]; extern pmd_t empty_bad_pmd_table[PTRS_PER_PMD];
/* /*
* Empty pmd entries point to the invalid_pte_table. * Empty pgd/pmd entries point to the invalid_pte_table.
*/ */
static inline int pmd_none(pmd_t pmd) static inline int pmd_none(pmd_t pmd)
{ {
...@@ -156,7 +156,8 @@ static inline void pud_clear(pud_t *pudp) ...@@ -156,7 +156,8 @@ static inline void pud_clear(pud_t *pudp)
pud_val(*pudp) = ((unsigned long) invalid_pmd_table); pud_val(*pudp) = ((unsigned long) invalid_pmd_table);
} }
#define pte_page(x) pfn_to_page((unsigned long)((pte_val(x) >> PAGE_SHIFT))) #define pte_page(x) pfn_to_page(pte_pfn(x))
#ifdef CONFIG_CPU_VR41XX #ifdef CONFIG_CPU_VR41XX
#define pte_pfn(x) ((unsigned long)((x).pte >> (PAGE_SHIFT + 2))) #define pte_pfn(x) ((unsigned long)((x).pte >> (PAGE_SHIFT + 2)))
#define pfn_pte(pfn, prot) __pte(((pfn) << (PAGE_SHIFT + 2)) | pgprot_val(prot)) #define pfn_pte(pfn, prot) __pte(((pfn) << (PAGE_SHIFT + 2)) | pgprot_val(prot))
...@@ -167,12 +168,14 @@ static inline void pud_clear(pud_t *pudp) ...@@ -167,12 +168,14 @@ static inline void pud_clear(pud_t *pudp)
#define __pgd_offset(address) pgd_index(address) #define __pgd_offset(address) pgd_index(address)
#define __pud_offset(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) #define __pud_offset(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
#define __pmd_offset(address) pmd_index(address)
#define page_pte(page) page_pte_prot(page, __pgprot(0)) #define page_pte(page) page_pte_prot(page, __pgprot(0))
/* to find an entry in a kernel page-table-directory */ /* to find an entry in a kernel page-table-directory */
#define pgd_offset_k(address) pgd_offset(&init_mm, 0) #define pgd_offset_k(address) pgd_offset(&init_mm, 0)
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
/* to find an entry in a page-table-directory */ /* to find an entry in a page-table-directory */
#define pgd_offset(mm,addr) ((mm)->pgd + pgd_index(addr)) #define pgd_offset(mm,addr) ((mm)->pgd + pgd_index(addr))
...@@ -185,8 +188,7 @@ static inline unsigned long pud_page(pud_t pud) ...@@ -185,8 +188,7 @@ static inline unsigned long pud_page(pud_t pud)
/* Find an entry in the second-level page table.. */ /* Find an entry in the second-level page table.. */
static inline pmd_t *pmd_offset(pud_t * pud, unsigned long address) static inline pmd_t *pmd_offset(pud_t * pud, unsigned long address)
{ {
return (pmd_t *) pud_page(*pud) + return (pmd_t *) pud_page(*pud) + pmd_index(address);
((address >> PMD_SHIFT) & (PTRS_PER_PMD - 1));
} }
/* Find an entry in the third-level page table.. */ /* Find an entry in the third-level page table.. */
......
...@@ -60,7 +60,6 @@ ...@@ -60,7 +60,6 @@
mfc0 k0, CP0_CONTEXT mfc0 k0, CP0_CONTEXT
lui k1, %hi(kernelsp) lui k1, %hi(kernelsp)
srl k0, k0, 23 srl k0, k0, 23
sll k0, k0, 2
addu k1, k0 addu k1, k0
LONG_L k1, %lo(kernelsp)(k1) LONG_L k1, %lo(kernelsp)(k1)
#endif #endif
...@@ -76,12 +75,12 @@ ...@@ -76,12 +75,12 @@
#endif #endif
#if defined(CONFIG_64BIT) && defined(CONFIG_BUILD_ELF64) #if defined(CONFIG_64BIT) && defined(CONFIG_BUILD_ELF64)
MFC0 k1, CP0_CONTEXT MFC0 k1, CP0_CONTEXT
dsrl k1, 23
dsll k1, k1, 3
lui k0, %highest(kernelsp) lui k0, %highest(kernelsp)
dsrl k1, 23
daddiu k0, %higher(kernelsp) daddiu k0, %higher(kernelsp)
dsll k0, k0, 16 dsll k0, k0, 16
daddiu k0, %hi(kernelsp) daddiu k0, %hi(kernelsp)
dsll k0, k0, 16
daddu k1, k1, k0 daddu k1, k1, k0
LONG_L k1, %lo(kernelsp)(k1) LONG_L k1, %lo(kernelsp)(k1)
#endif #endif
...@@ -91,7 +90,6 @@ ...@@ -91,7 +90,6 @@
#ifdef CONFIG_32BIT #ifdef CONFIG_32BIT
mfc0 \temp, CP0_CONTEXT mfc0 \temp, CP0_CONTEXT
srl \temp, 23 srl \temp, 23
sll \temp, 2
LONG_S \stackp, kernelsp(\temp) LONG_S \stackp, kernelsp(\temp)
#endif #endif
#if defined(CONFIG_64BIT) && !defined(CONFIG_BUILD_ELF64) #if defined(CONFIG_64BIT) && !defined(CONFIG_BUILD_ELF64)
...@@ -102,8 +100,8 @@ ...@@ -102,8 +100,8 @@
LONG_S \stackp, %lo(kernelsp)(\temp) LONG_S \stackp, %lo(kernelsp)(\temp)
#endif #endif
#if defined(CONFIG_64BIT) && defined(CONFIG_BUILD_ELF64) #if defined(CONFIG_64BIT) && defined(CONFIG_BUILD_ELF64)
lw \temp, TI_CPU(gp) MFC0 \temp, CP0_CONTEXT
dsll \temp, 3 dsrl \temp, 23
LONG_S \stackp, kernelsp(\temp) LONG_S \stackp, kernelsp(\temp)
#endif #endif
.endm .endm
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment