Commit 8b234274 authored by David S. Miller's avatar David S. Miller

[SPARC64]: More TLB/TSB handling fixes.

The SUN4V convention with non-shared TSBs is that the context
bit of the TAG is clear.  So we have to choose an "invalid"
bit and initialize new TSBs appropriately.  Otherwise a zero
TAG looks "valid".

Make sure, for the window fixup cases, that we use the right
global registers and that we don't potentially trample on
the live global registers in etrap/rtrap handling (%g2 and
%g6) and that we put the missing virtual address properly
in %g5.
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 7adb37fe
...@@ -2,10 +2,10 @@ ...@@ -2,10 +2,10 @@
ldxa [%g0] ASI_DMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer ldxa [%g0] ASI_DMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer
ldxa [%g0] ASI_DMMU, %g6 ! Get TAG TARGET ldxa [%g0] ASI_DMMU, %g6 ! Get TAG TARGET
srlx %g6, 48, %g5 ! Get context srlx %g6, 48, %g5 ! Get context
sllx %g6, 22, %g6 ! Zero out context
brz,pn %g5, kvmap_dtlb ! Context 0 processing brz,pn %g5, kvmap_dtlb ! Context 0 processing
nop ! Delay slot (fill me) srlx %g6, 22, %g6 ! Delay slot
TSB_LOAD_QUAD(%g1, %g4) ! Load TSB entry TSB_LOAD_QUAD(%g1, %g4) ! Load TSB entry
nop ! Push branch to next I$ line
cmp %g4, %g6 ! Compare TAG cmp %g4, %g6 ! Compare TAG
/* DTLB ** ICACHE line 2: TSB compare and TLB load */ /* DTLB ** ICACHE line 2: TSB compare and TLB load */
......
...@@ -2,25 +2,25 @@ ...@@ -2,25 +2,25 @@
ldxa [%g0] ASI_IMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer ldxa [%g0] ASI_IMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer
ldxa [%g0] ASI_IMMU, %g6 ! Get TAG TARGET ldxa [%g0] ASI_IMMU, %g6 ! Get TAG TARGET
srlx %g6, 48, %g5 ! Get context srlx %g6, 48, %g5 ! Get context
sllx %g6, 22, %g6 ! Zero out context
brz,pn %g5, kvmap_itlb ! Context 0 processing brz,pn %g5, kvmap_itlb ! Context 0 processing
nop ! Delay slot (fill me) srlx %g6, 22, %g6 ! Delay slot
TSB_LOAD_QUAD(%g1, %g4) ! Load TSB entry TSB_LOAD_QUAD(%g1, %g4) ! Load TSB entry
cmp %g4, %g6 ! Compare TAG cmp %g4, %g6 ! Compare TAG
sethi %hi(PAGE_EXEC), %g4 ! Setup exec check
/* ITLB ** ICACHE line 2: TSB compare and TLB load */ /* ITLB ** ICACHE line 2: TSB compare and TLB load */
sethi %hi(PAGE_EXEC), %g4 ! Setup exec check
ldx [%g4 + %lo(PAGE_EXEC)], %g4 ldx [%g4 + %lo(PAGE_EXEC)], %g4
bne,pn %xcc, tsb_miss_itlb ! Miss bne,pn %xcc, tsb_miss_itlb ! Miss
mov FAULT_CODE_ITLB, %g3 mov FAULT_CODE_ITLB, %g3
andcc %g5, %g4, %g0 ! Executable? andcc %g5, %g4, %g0 ! Executable?
be,pn %xcc, tsb_do_fault be,pn %xcc, tsb_do_fault
nop ! Delay slot, fill me nop ! Delay slot, fill me
stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load TLB nop
retry ! Trap done
/* ITLB ** ICACHE line 3: */ /* ITLB ** ICACHE line 3: */
nop stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load TLB
nop retry ! Trap done
nop nop
nop nop
nop nop
......
...@@ -52,8 +52,10 @@ kvmap_itlb_vmalloc_addr: ...@@ -52,8 +52,10 @@ kvmap_itlb_vmalloc_addr:
/* Load and check PTE. */ /* Load and check PTE. */
ldxa [%g5] ASI_PHYS_USE_EC, %g5 ldxa [%g5] ASI_PHYS_USE_EC, %g5
mov 1, %g7
sllx %g7, TSB_TAG_INVALID_BIT, %g7
brgez,a,pn %g5, kvmap_itlb_longpath brgez,a,pn %g5, kvmap_itlb_longpath
KTSB_STORE(%g1, %g0) KTSB_STORE(%g1, %g7)
KTSB_WRITE(%g1, %g5, %g6) KTSB_WRITE(%g1, %g5, %g6)
...@@ -146,8 +148,10 @@ kvmap_dtlb_vmalloc_addr: ...@@ -146,8 +148,10 @@ kvmap_dtlb_vmalloc_addr:
/* Load and check PTE. */ /* Load and check PTE. */
ldxa [%g5] ASI_PHYS_USE_EC, %g5 ldxa [%g5] ASI_PHYS_USE_EC, %g5
mov 1, %g7
sllx %g7, TSB_TAG_INVALID_BIT, %g7
brgez,a,pn %g5, kvmap_dtlb_longpath brgez,a,pn %g5, kvmap_dtlb_longpath
KTSB_STORE(%g1, %g0) KTSB_STORE(%g1, %g7)
KTSB_WRITE(%g1, %g5, %g6) KTSB_WRITE(%g1, %g5, %g6)
...@@ -215,8 +219,8 @@ kvmap_dtlb_longpath: ...@@ -215,8 +219,8 @@ kvmap_dtlb_longpath:
wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
.section .sun4v_2insn_patch, "ax" .section .sun4v_2insn_patch, "ax"
.word 661b .word 661b
nop SET_GL(1)
nop ldxa [%g0] ASI_SCRATCHPAD, %g5
.previous .previous
rdpr %tl, %g3 rdpr %tl, %g3
...@@ -226,7 +230,7 @@ kvmap_dtlb_longpath: ...@@ -226,7 +230,7 @@ kvmap_dtlb_longpath:
ldxa [%g4] ASI_DMMU, %g5 ldxa [%g4] ASI_DMMU, %g5
.section .sun4v_2insn_patch, "ax" .section .sun4v_2insn_patch, "ax"
.word 661b .word 661b
mov %g4, %g5 ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
nop nop
.previous .previous
......
...@@ -16,15 +16,14 @@ ...@@ -16,15 +16,14 @@
ldx [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \ ldx [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \
ldx [BASE + HV_FAULT_D_CTX_OFFSET], CTX; ldx [BASE + HV_FAULT_D_CTX_OFFSET], CTX;
/* DEST = (CTX << 48) | (VADDR >> 22) /* DEST = (VADDR >> 22)
* *
* Branch to ZERO_CTX_LABEL is context is zero. * Branch to ZERO_CTX_LABEL is context is zero.
*/ */
#define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, TMP, ZERO_CTX_LABEL) \ #define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \
srlx VADDR, 22, TMP; \ srlx VADDR, 22, DEST; \
sllx CTX, 48, DEST; \
brz,pn CTX, ZERO_CTX_LABEL; \ brz,pn CTX, ZERO_CTX_LABEL; \
or DEST, TMP, DEST; nop;
/* Create TSB pointer. This is something like: /* Create TSB pointer. This is something like:
* *
...@@ -53,7 +52,7 @@ sun4v_itlb_miss: ...@@ -53,7 +52,7 @@ sun4v_itlb_miss:
ldxa [%g1] ASI_SCRATCHPAD, %g1 ldxa [%g1] ASI_SCRATCHPAD, %g1
LOAD_ITLB_INFO(%g2, %g4, %g5) LOAD_ITLB_INFO(%g2, %g4, %g5)
COMPUTE_TAG_TARGET(%g6, %g4, %g5, %g3, kvmap_itlb_4v) COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7) COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7)
/* Load TSB tag/pte into %g2/%g3 and compare the tag. */ /* Load TSB tag/pte into %g2/%g3 and compare the tag. */
...@@ -72,15 +71,15 @@ sun4v_itlb_miss: ...@@ -72,15 +71,15 @@ sun4v_itlb_miss:
* *
* %g3: PTE * %g3: PTE
* %g4: vaddr * %g4: vaddr
* %g6: TAG TARGET (only "CTX << 48" part matters)
*/ */
sun4v_itlb_load: sun4v_itlb_load:
ldxa [%g0] ASI_SCRATCHPAD, %g6
mov %o0, %g1 ! save %o0 mov %o0, %g1 ! save %o0
mov %o1, %g2 ! save %o1 mov %o1, %g2 ! save %o1
mov %o2, %g5 ! save %o2 mov %o2, %g5 ! save %o2
mov %o3, %g7 ! save %o3 mov %o3, %g7 ! save %o3
mov %g4, %o0 ! vaddr mov %g4, %o0 ! vaddr
srlx %g6, 48, %o1 ! ctx ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 ! ctx
mov %g3, %o2 ! PTE mov %g3, %o2 ! PTE
mov HV_MMU_IMMU, %o3 ! flags mov HV_MMU_IMMU, %o3 ! flags
ta HV_MMU_MAP_ADDR_TRAP ta HV_MMU_MAP_ADDR_TRAP
...@@ -101,7 +100,7 @@ sun4v_dtlb_miss: ...@@ -101,7 +100,7 @@ sun4v_dtlb_miss:
ldxa [%g1] ASI_SCRATCHPAD, %g1 ldxa [%g1] ASI_SCRATCHPAD, %g1
LOAD_DTLB_INFO(%g2, %g4, %g5) LOAD_DTLB_INFO(%g2, %g4, %g5)
COMPUTE_TAG_TARGET(%g6, %g4, %g5, %g3, kvmap_dtlb_4v) COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7) COMPUTE_TSB_PTR(%g1, %g4, %g3, %g7)
/* Load TSB tag/pte into %g2/%g3 and compare the tag. */ /* Load TSB tag/pte into %g2/%g3 and compare the tag. */
...@@ -115,15 +114,15 @@ sun4v_dtlb_miss: ...@@ -115,15 +114,15 @@ sun4v_dtlb_miss:
* *
* %g3: PTE * %g3: PTE
* %g4: vaddr * %g4: vaddr
* %g6: TAG TARGET (only "CTX << 48" part matters)
*/ */
sun4v_dtlb_load: sun4v_dtlb_load:
ldxa [%g0] ASI_SCRATCHPAD, %g6
mov %o0, %g1 ! save %o0 mov %o0, %g1 ! save %o0
mov %o1, %g2 ! save %o1 mov %o1, %g2 ! save %o1
mov %o2, %g5 ! save %o2 mov %o2, %g5 ! save %o2
mov %o3, %g7 ! save %o3 mov %o3, %g7 ! save %o3
mov %g4, %o0 ! vaddr mov %g4, %o0 ! vaddr
srlx %g6, 48, %o1 ! ctx ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 ! ctx
mov %g3, %o2 ! PTE mov %g3, %o2 ! PTE
mov HV_MMU_DMMU, %o3 ! flags mov HV_MMU_DMMU, %o3 ! flags
ta HV_MMU_MAP_ADDR_TRAP ta HV_MMU_MAP_ADDR_TRAP
...@@ -136,16 +135,18 @@ sun4v_dtlb_load: ...@@ -136,16 +135,18 @@ sun4v_dtlb_load:
retry retry
sun4v_dtlb_prot: sun4v_dtlb_prot:
SET_GL(1)
/* Load MMU Miss base into %g2. */ /* Load MMU Miss base into %g2. */
ldxa [%g0] ASI_SCRATCHPAD, %g2 ldxa [%g0] ASI_SCRATCHPAD, %g5
ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5 ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
rdpr %tl, %g1 rdpr %tl, %g1
cmp %g1, 1 cmp %g1, 1
bgu,pn %xcc, winfix_trampoline bgu,pn %xcc, winfix_trampoline
nop nop
ba,pt %xcc, sparc64_realfault_common ba,pt %xcc, sparc64_realfault_common
mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
/* Called from trap table with TAG TARGET placed into /* Called from trap table with TAG TARGET placed into
* %g6, SCRATCHPAD_UTSBREG1 contents in %g1, and * %g6, SCRATCHPAD_UTSBREG1 contents in %g1, and
...@@ -189,7 +190,8 @@ sun4v_itlb_error: ...@@ -189,7 +190,8 @@ sun4v_itlb_error:
sethi %hi(sun4v_err_itlb_vaddr), %g1 sethi %hi(sun4v_err_itlb_vaddr), %g1
stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)] stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
sethi %hi(sun4v_err_itlb_ctx), %g1 sethi %hi(sun4v_err_itlb_ctx), %g1
srlx %g6, 48, %o1 ! ctx ldxa [%g0] ASI_SCRATCHPAD, %g6
ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1
stx %o1, [%g1 + %lo(sun4v_err_itlb_ctx)] stx %o1, [%g1 + %lo(sun4v_err_itlb_ctx)]
sethi %hi(sun4v_err_itlb_pte), %g1 sethi %hi(sun4v_err_itlb_pte), %g1
stx %g3, [%g1 + %lo(sun4v_err_itlb_pte)] stx %g3, [%g1 + %lo(sun4v_err_itlb_pte)]
...@@ -214,7 +216,8 @@ sun4v_dtlb_error: ...@@ -214,7 +216,8 @@ sun4v_dtlb_error:
sethi %hi(sun4v_err_dtlb_vaddr), %g1 sethi %hi(sun4v_err_dtlb_vaddr), %g1
stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)] stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
sethi %hi(sun4v_err_dtlb_ctx), %g1 sethi %hi(sun4v_err_dtlb_ctx), %g1
srlx %g6, 48, %o1 ! ctx ldxa [%g0] ASI_SCRATCHPAD, %g6
ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1
stx %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)] stx %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)]
sethi %hi(sun4v_err_dtlb_pte), %g1 sethi %hi(sun4v_err_dtlb_pte), %g1
stx %g3, [%g1 + %lo(sun4v_err_dtlb_pte)] stx %g3, [%g1 + %lo(sun4v_err_dtlb_pte)]
......
...@@ -36,7 +36,7 @@ tsb_miss_itlb: ...@@ -36,7 +36,7 @@ tsb_miss_itlb:
/* At this point we have: /* At this point we have:
* %g4 -- missing virtual address * %g4 -- missing virtual address
* %g1 -- TSB entry address * %g1 -- TSB entry address
* %g6 -- TAG TARGET ((vaddr >> 22) | (ctx << 48)) * %g6 -- TAG TARGET (vaddr >> 22)
*/ */
tsb_miss_page_table_walk: tsb_miss_page_table_walk:
TRAP_LOAD_PGD_PHYS(%g7, %g5) TRAP_LOAD_PGD_PHYS(%g7, %g5)
...@@ -50,8 +50,10 @@ tsb_reload: ...@@ -50,8 +50,10 @@ tsb_reload:
/* Load and check PTE. */ /* Load and check PTE. */
ldxa [%g5] ASI_PHYS_USE_EC, %g5 ldxa [%g5] ASI_PHYS_USE_EC, %g5
mov 1, %g7
sllx %g7, TSB_TAG_INVALID_BIT, %g7
brgez,a,pn %g5, tsb_do_fault brgez,a,pn %g5, tsb_do_fault
TSB_STORE(%g1, %g0) TSB_STORE(%g1, %g7)
/* If it is larger than the base page size, don't /* If it is larger than the base page size, don't
* bother putting it into the TSB. * bother putting it into the TSB.
...@@ -62,8 +64,10 @@ tsb_reload: ...@@ -62,8 +64,10 @@ tsb_reload:
sethi %hi(_PAGE_SZBITS), %g7 sethi %hi(_PAGE_SZBITS), %g7
ldx [%g7 + %lo(_PAGE_SZBITS)], %g7 ldx [%g7 + %lo(_PAGE_SZBITS)], %g7
cmp %g2, %g7 cmp %g2, %g7
mov 1, %g7
sllx %g7, TSB_TAG_INVALID_BIT, %g7
bne,a,pn %xcc, tsb_tlb_reload bne,a,pn %xcc, tsb_tlb_reload
TSB_STORE(%g1, %g0) TSB_STORE(%g1, %g7)
TSB_WRITE(%g1, %g5, %g6) TSB_WRITE(%g1, %g5, %g6)
...@@ -136,7 +140,7 @@ tsb_do_fault: ...@@ -136,7 +140,7 @@ tsb_do_fault:
.section .sun4v_2insn_patch, "ax" .section .sun4v_2insn_patch, "ax"
.word 661b .word 661b
SET_GL(1) SET_GL(1)
ldxa [%g0] ASI_SCRATCHPAD, %g2 ldxa [%g0] ASI_SCRATCHPAD, %g4
.previous .previous
bne,pn %xcc, tsb_do_itlb_fault bne,pn %xcc, tsb_do_itlb_fault
...@@ -150,7 +154,7 @@ tsb_do_dtlb_fault: ...@@ -150,7 +154,7 @@ tsb_do_dtlb_fault:
ldxa [%g4] ASI_DMMU, %g5 ldxa [%g4] ASI_DMMU, %g5
.section .sun4v_2insn_patch, "ax" .section .sun4v_2insn_patch, "ax"
.word 661b .word 661b
ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5 ldx [%g4 + HV_FAULT_D_ADDR_OFFSET], %g5
nop nop
.previous .previous
...@@ -217,8 +221,9 @@ tsb_flush: ...@@ -217,8 +221,9 @@ tsb_flush:
bne,pn %icc, 1b bne,pn %icc, 1b
membar #LoadLoad membar #LoadLoad
cmp %g1, %o1 cmp %g1, %o1
mov 1, %o3
bne,pt %xcc, 2f bne,pt %xcc, 2f
clr %o3 sllx %o3, TSB_TAG_INVALID_BIT, %o3
TSB_CAS_TAG(%o0, %g1, %o3) TSB_CAS_TAG(%o0, %g1, %o3)
cmp %g1, %o3 cmp %g1, %o3
bne,pn %xcc, 1b bne,pn %xcc, 1b
......
...@@ -296,7 +296,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t p ...@@ -296,7 +296,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t p
tsb = &mm->context.tsb[(address >> PAGE_SHIFT) & tsb = &mm->context.tsb[(address >> PAGE_SHIFT) &
(mm->context.tsb_nentries - 1UL)]; (mm->context.tsb_nentries - 1UL)];
tag = (address >> 22UL) | CTX_HWBITS(mm->context) << 48UL; tag = (address >> 22UL);
tsb_insert(tsb, tag, pte_val(pte)); tsb_insert(tsb, tag, pte_val(pte));
} }
} }
...@@ -1110,6 +1110,8 @@ void __init paging_init(void) ...@@ -1110,6 +1110,8 @@ void __init paging_init(void)
kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL; kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
kern_size = (unsigned long)&_end - (unsigned long)KERNBASE; kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;
memset(swapper_tsb, 0x40, sizeof(swapper_tsb));
if (tlb_type == hypervisor) if (tlb_type == hypervisor)
sun4v_pgprot_init(); sun4v_pgprot_init();
else else
......
...@@ -20,9 +20,9 @@ static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long nentries ...@@ -20,9 +20,9 @@ static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long nentries
return vaddr & (nentries - 1); return vaddr & (nentries - 1);
} }
static inline int tag_compare(unsigned long tag, unsigned long vaddr, unsigned long context) static inline int tag_compare(unsigned long tag, unsigned long vaddr)
{ {
return (tag == ((vaddr >> 22) | (context << 48))); return (tag == (vaddr >> 22));
} }
/* TSB flushes need only occur on the processor initiating the address /* TSB flushes need only occur on the processor initiating the address
...@@ -38,8 +38,8 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) ...@@ -38,8 +38,8 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
unsigned long hash = tsb_hash(v, KERNEL_TSB_NENTRIES); unsigned long hash = tsb_hash(v, KERNEL_TSB_NENTRIES);
struct tsb *ent = &swapper_tsb[hash]; struct tsb *ent = &swapper_tsb[hash];
if (tag_compare(ent->tag, v, 0)) { if (tag_compare(ent->tag, v)) {
ent->tag = 0UL; ent->tag = (1UL << TSB_TAG_INVALID_BIT);
membar_storeload_storestore(); membar_storeload_storestore();
} }
} }
...@@ -50,14 +50,9 @@ void flush_tsb_user(struct mmu_gather *mp) ...@@ -50,14 +50,9 @@ void flush_tsb_user(struct mmu_gather *mp)
struct mm_struct *mm = mp->mm; struct mm_struct *mm = mp->mm;
struct tsb *tsb = mm->context.tsb; struct tsb *tsb = mm->context.tsb;
unsigned long nentries = mm->context.tsb_nentries; unsigned long nentries = mm->context.tsb_nentries;
unsigned long ctx, base; unsigned long base;
int i; int i;
if (unlikely(!CTX_VALID(mm->context)))
return;
ctx = CTX_HWBITS(mm->context);
if (tlb_type == cheetah_plus || tlb_type == hypervisor) if (tlb_type == cheetah_plus || tlb_type == hypervisor)
base = __pa(tsb); base = __pa(tsb);
else else
...@@ -71,7 +66,7 @@ void flush_tsb_user(struct mmu_gather *mp) ...@@ -71,7 +66,7 @@ void flush_tsb_user(struct mmu_gather *mp)
hash = tsb_hash(v, nentries); hash = tsb_hash(v, nentries);
ent = base + (hash * sizeof(struct tsb)); ent = base + (hash * sizeof(struct tsb));
tag = (v >> 22UL) | (ctx << 48UL); tag = (v >> 22UL);
tsb_flush(ent, tag); tsb_flush(ent, tag);
} }
...@@ -243,7 +238,8 @@ static void copy_tsb(struct tsb *old_tsb, unsigned long old_size, ...@@ -243,7 +238,8 @@ static void copy_tsb(struct tsb *old_tsb, unsigned long old_size,
"i" (ASI_NUCLEUS_QUAD_LDD)); "i" (ASI_NUCLEUS_QUAD_LDD));
} }
if (!tag || (tag & (1UL << TSB_TAG_LOCK_BIT))) if (tag & ((1UL << TSB_TAG_LOCK_BIT) |
(1UL << TSB_TAG_INVALID_BIT)))
continue; continue;
/* We only put base page size PTEs into the TSB, /* We only put base page size PTEs into the TSB,
...@@ -315,10 +311,13 @@ void tsb_grow(struct mm_struct *mm, unsigned long rss, gfp_t gfp_flags) ...@@ -315,10 +311,13 @@ void tsb_grow(struct mm_struct *mm, unsigned long rss, gfp_t gfp_flags)
break; break;
} }
page = alloc_pages(gfp_flags | __GFP_ZERO, get_order(size)); page = alloc_pages(gfp_flags, get_order(size));
if (unlikely(!page)) if (unlikely(!page))
return; return;
/* Mark all tags as invalid. */
memset(page_address(page), 0x40, size);
if (size == max_tsb_size) if (size == max_tsb_size)
mm->context.tsb_rss_limit = ~0UL; mm->context.tsb_rss_limit = ~0UL;
else else
......
...@@ -12,6 +12,8 @@ ...@@ -12,6 +12,8 @@
* *
* ldxa [%g0] ASI_{D,I}MMU_TSB_8KB_PTR, %g1 * ldxa [%g0] ASI_{D,I}MMU_TSB_8KB_PTR, %g1
* ldxa [%g0] ASI_{D,I}MMU, %g6 * ldxa [%g0] ASI_{D,I}MMU, %g6
* sllx %g6, 22, %g6
* srlx %g6, 22, %g6
* ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 * ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4
* cmp %g4, %g6 * cmp %g4, %g6
* bne,pn %xcc, tsb_miss_{d,i}tlb * bne,pn %xcc, tsb_miss_{d,i}tlb
...@@ -29,6 +31,9 @@ ...@@ -29,6 +31,9 @@
* ------------------------------------------------- * -------------------------------------------------
* 63 61 60 48 47 42 41 0 * 63 61 60 48 47 42 41 0
* *
* But actually, since we use per-mm TSB's, we zero out the CONTEXT
* field.
*
* Like the powerpc hashtables we need to use locking in order to * Like the powerpc hashtables we need to use locking in order to
* synchronize while we update the entries. PTE updates need locking * synchronize while we update the entries. PTE updates need locking
* as well. * as well.
...@@ -42,6 +47,9 @@ ...@@ -42,6 +47,9 @@
#define TSB_TAG_LOCK_BIT 47 #define TSB_TAG_LOCK_BIT 47
#define TSB_TAG_LOCK_HIGH (1 << (TSB_TAG_LOCK_BIT - 32)) #define TSB_TAG_LOCK_HIGH (1 << (TSB_TAG_LOCK_BIT - 32))
#define TSB_TAG_INVALID_BIT 46
#define TSB_TAG_INVALID_HIGH (1 << (TSB_TAG_INVALID_BIT - 32))
#define TSB_MEMBAR membar #StoreStore #define TSB_MEMBAR membar #StoreStore
/* Some cpus support physical address quad loads. We want to use /* Some cpus support physical address quad loads. We want to use
......
...@@ -184,20 +184,20 @@ ...@@ -184,20 +184,20 @@
ldxa [%g0] ASI_SCRATCHPAD, %g2; \ ldxa [%g0] ASI_SCRATCHPAD, %g2; \
ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4; \ ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4; \
ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5; \ ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5; \
srlx %g4, 22, %g7; \ srlx %g4, 22, %g6; \
sllx %g5, 48, %g6; \
ba,pt %xcc, sun4v_itsb_miss; \ ba,pt %xcc, sun4v_itsb_miss; \
or %g6, %g7, %g6; \ nop; \
nop; \
nop; nop;
#define SUN4V_DTSB_MISS \ #define SUN4V_DTSB_MISS \
ldxa [%g0] ASI_SCRATCHPAD, %g2; \ ldxa [%g0] ASI_SCRATCHPAD, %g2; \
ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4; \ ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4; \
ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5; \ ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5; \
srlx %g4, 22, %g7; \ srlx %g4, 22, %g6; \
sllx %g5, 48, %g6; \
ba,pt %xcc, sun4v_dtsb_miss; \ ba,pt %xcc, sun4v_dtsb_miss; \
or %g6, %g7, %g6; \ nop; \
nop; \
nop; nop;
/* Before touching these macros, you owe it to yourself to go and /* Before touching these macros, you owe it to yourself to go and
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment