Commit d665767e authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman

powerpc/64s/radix: Implement _tlbie(l)_va_range flush functions

Move the barriers and range iteration down into the _tlbie* level,
which improves readability.
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent 14001c60
...@@ -85,7 +85,7 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric) ...@@ -85,7 +85,7 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
} }
static inline void __tlbiel_va(unsigned long va, unsigned long pid, static inline void __tlbiel_va(unsigned long va, unsigned long pid,
unsigned long ap, unsigned long ric) unsigned long ap, unsigned long ric)
{ {
unsigned long rb,rs,prs,r; unsigned long rb,rs,prs,r;
...@@ -101,13 +101,28 @@ static inline void __tlbiel_va(unsigned long va, unsigned long pid, ...@@ -101,13 +101,28 @@ static inline void __tlbiel_va(unsigned long va, unsigned long pid,
} }
static inline void _tlbiel_va(unsigned long va, unsigned long pid, static inline void _tlbiel_va(unsigned long va, unsigned long pid,
unsigned long ap, unsigned long ric) unsigned long psize, unsigned long ric)
{ {
unsigned long ap = mmu_get_ap(psize);
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
__tlbiel_va(va, pid, ap, ric); __tlbiel_va(va, pid, ap, ric);
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
} }
static inline void _tlbiel_va_range(unsigned long start, unsigned long end,
unsigned long pid, unsigned long page_size,
unsigned long psize)
{
unsigned long addr;
unsigned long ap = mmu_get_ap(psize);
asm volatile("ptesync": : :"memory");
for (addr = start; addr < end; addr += page_size)
__tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
asm volatile("ptesync": : :"memory");
}
static inline void __tlbie_va(unsigned long va, unsigned long pid, static inline void __tlbie_va(unsigned long va, unsigned long pid,
unsigned long ap, unsigned long ric) unsigned long ap, unsigned long ric)
{ {
...@@ -125,13 +140,27 @@ static inline void __tlbie_va(unsigned long va, unsigned long pid, ...@@ -125,13 +140,27 @@ static inline void __tlbie_va(unsigned long va, unsigned long pid,
} }
static inline void _tlbie_va(unsigned long va, unsigned long pid, static inline void _tlbie_va(unsigned long va, unsigned long pid,
unsigned long ap, unsigned long ric) unsigned long psize, unsigned long ric)
{ {
unsigned long ap = mmu_get_ap(psize);
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
__tlbie_va(va, pid, ap, ric); __tlbie_va(va, pid, ap, ric);
asm volatile("eieio; tlbsync; ptesync": : :"memory"); asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
static inline void _tlbie_va_range(unsigned long start, unsigned long end,
unsigned long pid, unsigned long page_size,
unsigned long psize)
{
unsigned long addr;
unsigned long ap = mmu_get_ap(psize);
asm volatile("ptesync": : :"memory");
for (addr = start; addr < end; addr += page_size)
__tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
/* /*
* Base TLB flushing operations: * Base TLB flushing operations:
...@@ -174,12 +203,11 @@ void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmadd ...@@ -174,12 +203,11 @@ void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmadd
int psize) int psize)
{ {
unsigned long pid; unsigned long pid;
unsigned long ap = mmu_get_ap(psize);
preempt_disable(); preempt_disable();
pid = mm->context.id; pid = mm->context.id;
if (pid != MMU_NO_CONTEXT) if (pid != MMU_NO_CONTEXT)
_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB); _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
preempt_enable(); preempt_enable();
} }
...@@ -239,7 +267,6 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, ...@@ -239,7 +267,6 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
int psize) int psize)
{ {
unsigned long pid; unsigned long pid;
unsigned long ap = mmu_get_ap(psize);
pid = mm->context.id; pid = mm->context.id;
if (unlikely(pid == MMU_NO_CONTEXT)) if (unlikely(pid == MMU_NO_CONTEXT))
...@@ -247,9 +274,9 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, ...@@ -247,9 +274,9 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
preempt_disable(); preempt_disable();
if (!mm_is_thread_local(mm)) if (!mm_is_thread_local(mm))
_tlbie_va(vmaddr, pid, ap, RIC_FLUSH_TLB); _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
else else
_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB); _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
preempt_enable(); preempt_enable();
} }
...@@ -336,9 +363,7 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, ...@@ -336,9 +363,7 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
unsigned long end, int psize) unsigned long end, int psize)
{ {
unsigned long pid; unsigned long pid;
unsigned long addr;
bool local; bool local;
unsigned long ap = mmu_get_ap(psize);
unsigned long page_size = 1UL << mmu_psize_defs[psize].shift; unsigned long page_size = 1UL << mmu_psize_defs[psize].shift;
pid = mm->context.id; pid = mm->context.id;
...@@ -354,17 +379,10 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, ...@@ -354,17 +379,10 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
else else
_tlbie_pid(pid, RIC_FLUSH_TLB); _tlbie_pid(pid, RIC_FLUSH_TLB);
} else { } else {
asm volatile("ptesync": : :"memory");
for (addr = start; addr < end; addr += page_size) {
if (local)
__tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
else
__tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
}
if (local) if (local)
asm volatile("ptesync": : :"memory"); _tlbiel_va_range(start, end, pid, page_size, psize);
else else
asm volatile("eieio; tlbsync; ptesync": : :"memory"); _tlbie_va_range(start, end, pid, page_size, psize);
} }
preempt_enable(); preempt_enable();
} }
...@@ -372,7 +390,6 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, ...@@ -372,7 +390,6 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
{ {
unsigned long ap = mmu_get_ap(mmu_virtual_psize);
unsigned long pid, end; unsigned long pid, end;
bool local; bool local;
...@@ -395,19 +412,11 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) ...@@ -395,19 +412,11 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
_tlbie_pid(pid, RIC_FLUSH_PWC); _tlbie_pid(pid, RIC_FLUSH_PWC);
/* Then iterate the pages */ /* Then iterate the pages */
asm volatile("ptesync": : :"memory");
end = addr + HPAGE_PMD_SIZE; end = addr + HPAGE_PMD_SIZE;
for (; addr < end; addr += PAGE_SIZE) {
if (local)
_tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
else
_tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
}
if (local) if (local)
asm volatile("ptesync": : :"memory"); _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize);
else else
asm volatile("eieio; tlbsync; ptesync": : :"memory"); _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize);
preempt_enable(); preempt_enable();
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment