Commit 76946074 authored by Anton Blanchard's avatar Anton Blanchard

ppc64: rework ppc64 hashtable management

parent 962a7e51
......@@ -62,7 +62,6 @@
HTAB htab_data = {NULL, 0, 0, 0, 0};
extern unsigned long _SDR1;
extern unsigned long klimit;
#define KB (1024)
#define MB (1024*KB)
......@@ -77,10 +76,9 @@ loop_forever(void)
static inline void
create_pte_mapping(unsigned long start, unsigned long end,
unsigned long mode, unsigned long mask, int large)
unsigned long mode, int large)
{
unsigned long addr;
HPTE *htab = (HPTE *)__v2a(htab_data.htab);
unsigned int step;
if (large)
......@@ -89,14 +87,33 @@ create_pte_mapping(unsigned long start, unsigned long end,
step = 4*KB;
for (addr = start; addr < end; addr += step) {
unsigned long vpn, hash, hpteg;
unsigned long vsid = get_kernel_vsid(addr);
unsigned long va = (vsid << 28) | (addr & 0xfffffff);
int ret;
if (large)
vpn = va >> LARGE_PAGE_SHIFT;
else
vpn = va >> PAGE_SHIFT;
hash = hpt_hash(vpn, large);
hpteg = ((hash & htab_data.htab_hash_mask)*HPTES_PER_GROUP);
if (naca->platform == PLATFORM_PSERIES_LPAR)
pSeries_lpar_make_pte(htab, va,
(unsigned long)__v2a(addr), mode, mask, large);
ret = pSeries_lpar_hpte_insert(hpteg, va,
(unsigned long)__v2a(addr) >> PAGE_SHIFT,
0, mode, 1, large);
else
pSeries_make_pte(htab, va,
(unsigned long)__v2a(addr), mode, mask, large);
ret = pSeries_hpte_insert(hpteg, va,
(unsigned long)__v2a(addr) >> PAGE_SHIFT,
0, mode, 1, large);
if (ret == -1) {
ppc64_terminate_msg(0x20, "create_pte_mapping");
loop_forever();
}
}
}
......@@ -105,7 +122,7 @@ htab_initialize(void)
{
unsigned long table, htab_size_bytes;
unsigned long pteg_count;
unsigned long mode_rw, mask;
unsigned long mode_rw;
/*
* Calculate the required size of the htab. We want the number of
......@@ -146,19 +163,18 @@ htab_initialize(void)
}
mode_rw = _PAGE_ACCESSED | _PAGE_COHERENT | PP_RWXX;
mask = pteg_count-1;
/* XXX we currently map kernel text rw, should fix this */
if (cpu_has_largepage() && naca->physicalMemorySize > 256*MB) {
create_pte_mapping((unsigned long)KERNELBASE,
KERNELBASE + 256*MB, mode_rw, mask, 0);
KERNELBASE + 256*MB, mode_rw, 0);
create_pte_mapping((unsigned long)KERNELBASE + 256*MB,
KERNELBASE + (naca->physicalMemorySize),
mode_rw, mask, 1);
mode_rw, 1);
} else {
create_pte_mapping((unsigned long)KERNELBASE,
KERNELBASE+(naca->physicalMemorySize),
mode_rw, mask, 0);
mode_rw, 0);
}
}
#undef KB
......@@ -204,7 +220,7 @@ static inline unsigned long computeHptePP(unsigned long pte)
* to be valid via Linux page tables, return 1. If handled return 0
*/
int __hash_page(unsigned long ea, unsigned long access, unsigned long vsid,
pte_t *ptep, unsigned long trap)
pte_t *ptep, unsigned long trap, int local)
{
unsigned long va, vpn;
unsigned long newpp, prpn;
......@@ -212,8 +228,15 @@ int __hash_page(unsigned long ea, unsigned long access, unsigned long vsid,
long slot;
pte_t old_pte, new_pte;
/* XXX fix for large ptes */
int large = 0;
/* Search the Linux page table for a match with va */
va = (vsid << 28) | (ea & 0x0fffffff);
if (large)
vpn = va >> LARGE_PAGE_SHIFT;
else
vpn = va >> PAGE_SHIFT;
/*
......@@ -276,16 +299,14 @@ int __hash_page(unsigned long ea, unsigned long access, unsigned long vsid,
/* There MIGHT be an HPTE for this pte */
unsigned long hash, slot, secondary;
/* XXX fix large pte flag */
hash = hpt_hash(vpn, 0);
hash = hpt_hash(vpn, large);
secondary = (pte_val(old_pte) & _PAGE_SECONDARY) >> 15;
if (secondary)
hash = ~hash;
slot = (hash & htab_data.htab_hash_mask) * HPTES_PER_GROUP;
slot += (pte_val(old_pte) & _PAGE_GROUP_IX) >> 12;
/* XXX fix large pte flag */
if (ppc_md.hpte_updatepp(slot, newpp, va, 0) == -1)
if (ppc_md.hpte_updatepp(slot, newpp, va, large, local) == -1)
pte_val(old_pte) &= ~_PAGE_HPTEFLAGS;
else
if (!pte_same(old_pte, new_pte))
......@@ -293,8 +314,7 @@ int __hash_page(unsigned long ea, unsigned long access, unsigned long vsid,
}
if (likely(!(pte_val(old_pte) & _PAGE_HASHPTE))) {
/* XXX fix large pte flag */
unsigned long hash = hpt_hash(vpn, 0);
unsigned long hash = hpt_hash(vpn, large);
unsigned long hpte_group;
prpn = pte_val(old_pte) >> PTE_SHIFT;
......@@ -309,18 +329,16 @@ int __hash_page(unsigned long ea, unsigned long access, unsigned long vsid,
/* copy appropriate flags from linux pte */
hpteflags = (pte_val(new_pte) & 0x1f8) | newpp;
/* XXX fix large pte flag */
slot = ppc_md.hpte_insert(hpte_group, vpn, prpn, 0,
hpteflags, 0, 0);
slot = ppc_md.hpte_insert(hpte_group, va, prpn, 0,
hpteflags, 0, large);
/* Primary is full, try the secondary */
if (slot == -1) {
pte_val(new_pte) |= 1 << 15;
hpte_group = ((~hash & htab_data.htab_hash_mask) *
HPTES_PER_GROUP) & ~0x7UL;
/* XXX fix large pte flag */
slot = ppc_md.hpte_insert(hpte_group, vpn, prpn,
1, hpteflags, 0, 0);
slot = ppc_md.hpte_insert(hpte_group, va, prpn,
1, hpteflags, 0, large);
if (slot == -1) {
if (mftb() & 0x1)
hpte_group = ((hash & htab_data.htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
......@@ -351,6 +369,8 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
struct mm_struct *mm;
pte_t *ptep;
int ret;
int user_region = 0;
int local = 0;
/* Check for invalid addresses. */
if (!IS_VALID_EA(ea))
......@@ -358,6 +378,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
switch (REGION_ID(ea)) {
case USER_REGION_ID:
user_region = 1;
mm = current->mm;
if (mm == NULL)
return 1;
......@@ -401,8 +422,12 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
* from modifying entries while we search and update
*/
spin_lock(&mm->page_table_lock);
if (user_region && (mm->cpu_vm_mask == (1 << smp_processor_id())))
local = 1;
ptep = find_linux_pte(pgdir, ea);
ret = __hash_page(ea, access, vsid, ptep, trap);
ret = __hash_page(ea, access, vsid, ptep, trap, local);
spin_unlock(&mm->page_table_lock);
return ret;
......
......@@ -22,46 +22,6 @@
#include <asm/tlbflush.h>
#include <asm/tlb.h>
/*
* Create a pte. Used during initialization only.
* We assume the PTE will fit in the primary PTEG.
*/
void pSeries_make_pte(HPTE *htab, unsigned long va, unsigned long pa,
int mode, unsigned long hash_mask, int large)
{
HPTE *hptep;
unsigned long hash, i;
unsigned long vpn;
if (large)
vpn = va >> LARGE_PAGE_SHIFT;
else
vpn = va >> PAGE_SHIFT;
hash = hpt_hash(vpn, large);
hptep = htab + ((hash & hash_mask)*HPTES_PER_GROUP);
for (i = 0; i < 8; ++i, ++hptep) {
if (hptep->dw0.dw0.v == 0) { /* !valid */
hptep->dw1.dword1 = pa | mode;
hptep->dw0.dword0 = 0;
hptep->dw0.dw0.avpn = va >> 23;
hptep->dw0.dw0.bolted = 1; /* bolted */
if (large) {
hptep->dw0.dw0.l = 1;
hptep->dw0.dw0.avpn &= ~0x1UL;
}
hptep->dw0.dw0.v = 1; /* make valid */
return;
}
}
/* We should _never_ get here and too early to call xmon. */
while(1)
;
}
#define HPTE_LOCK_BIT 3
static inline void pSeries_lock_hpte(HPTE *hptep)
......@@ -72,7 +32,7 @@ static inline void pSeries_lock_hpte(HPTE *hptep)
if (!test_and_set_bit(HPTE_LOCK_BIT, word))
break;
while(test_bit(HPTE_LOCK_BIT, word))
barrier();
cpu_relax();
}
}
......@@ -86,11 +46,10 @@ static inline void pSeries_unlock_hpte(HPTE *hptep)
static spinlock_t pSeries_tlbie_lock = SPIN_LOCK_UNLOCKED;
static long pSeries_hpte_insert(unsigned long hpte_group, unsigned long vpn,
long pSeries_hpte_insert(unsigned long hpte_group, unsigned long va,
unsigned long prpn, int secondary,
unsigned long hpteflags, int bolted, int large)
{
unsigned long avpn = vpn >> 11;
unsigned long arpn = physRpn_to_absRpn(prpn);
HPTE *hptep = htab_data.htab + hpte_group;
Hpte_dword0 dw0;
......@@ -120,13 +79,15 @@ static long pSeries_hpte_insert(unsigned long hpte_group, unsigned long vpn,
lhpte.dw1.flags.flags = hpteflags;
lhpte.dw0.dword0 = 0;
lhpte.dw0.dw0.avpn = avpn;
lhpte.dw0.dw0.avpn = va >> 23;
lhpte.dw0.dw0.h = secondary;
lhpte.dw0.dw0.bolted = bolted;
lhpte.dw0.dw0.v = 1;
if (large)
if (large) {
lhpte.dw0.dw0.l = 1;
lhpte.dw0.dw0.avpn &= ~0x1UL;
}
hptep->dw1.dword1 = lhpte.dw1.dword1;
......@@ -150,17 +111,10 @@ static long pSeries_hpte_remove(unsigned long hpte_group)
Hpte_dword0 dw0;
int i;
int slot_offset;
unsigned long vsid, group, pi, pi_high;
unsigned long slot;
unsigned long flags;
int large;
unsigned long va;
/* pick a random slot to start at */
/* pick a random entry to start at */
slot_offset = mftb() & 0x7;
udbg_printf("remove_hpte in %d\n", slot_offset);
for (i = 0; i < HPTES_PER_GROUP; i++) {
hptep = htab_data.htab + hpte_group + slot_offset;
dw0 = hptep->dw0.dw0;
......@@ -181,30 +135,9 @@ static long pSeries_hpte_remove(unsigned long hpte_group)
if (i == HPTES_PER_GROUP)
return -1;
large = dw0.l;
/* Invalidate the hpte. NOTE: this also unlocks it */
hptep->dw0.dword0 = 0;
/* Invalidate the tlb */
vsid = dw0.avpn >> 5;
slot = hptep - htab_data.htab;
group = slot >> 3;
if (dw0.h)
group = ~group;
pi = (vsid ^ group) & 0x7ff;
pi_high = (dw0.avpn & 0x1f) << 11;
pi |= pi_high;
if (large)
va = pi << LARGE_PAGE_SHIFT;
else
va = pi << PAGE_SHIFT;
spin_lock_irqsave(&pSeries_tlbie_lock, flags);
_tlbie(va, large);
spin_unlock_irqrestore(&pSeries_tlbie_lock, flags);
return i;
}
......@@ -259,41 +192,40 @@ static long pSeries_hpte_find(unsigned long vpn)
}
static long pSeries_hpte_updatepp(unsigned long slot, unsigned long newpp,
unsigned long va, int large)
unsigned long va, int large, int local)
{
HPTE *hptep = htab_data.htab + slot;
Hpte_dword0 dw0;
unsigned long vpn, avpn;
unsigned long avpn = va >> 23;
unsigned long flags;
int ret = 0;
if (large)
vpn = va >> LARGE_PAGE_SHIFT;
else
vpn = va >> PAGE_SHIFT;
avpn = vpn >> 11;
avpn &= ~0x1UL;
pSeries_lock_hpte(hptep);
dw0 = hptep->dw0.dw0;
/* Even if we miss, we need to invalidate the TLB */
if ((dw0.avpn != avpn) || !dw0.v) {
pSeries_unlock_hpte(hptep);
udbg_printf("updatepp missed\n");
return -1;
}
ret = -1;
} else {
set_pp_bit(newpp, hptep);
pSeries_unlock_hpte(hptep);
}
/* Ensure it is out of the tlb too */
/* XXX use tlbiel where possible */
if (cpu_has_tlbiel() && !large && local) {
_tlbiel(va);
} else {
spin_lock_irqsave(&pSeries_tlbie_lock, flags);
_tlbie(va, large);
spin_unlock_irqrestore(&pSeries_tlbie_lock, flags);
}
return 0;
return ret;
}
/*
......@@ -322,7 +254,6 @@ static void pSeries_hpte_updateboltedpp(unsigned long newpp, unsigned long ea)
set_pp_bit(newpp, hptep);
/* Ensure it is out of the tlb too */
/* XXX use tlbiel where possible */
spin_lock_irqsave(&pSeries_tlbie_lock, flags);
_tlbie(va, 0);
spin_unlock_irqrestore(&pSeries_tlbie_lock, flags);
......@@ -333,28 +264,23 @@ static void pSeries_hpte_invalidate(unsigned long slot, unsigned long va,
{
HPTE *hptep = htab_data.htab + slot;
Hpte_dword0 dw0;
unsigned long vpn, avpn;
unsigned long avpn = va >> 23;
unsigned long flags;
if (large)
vpn = va >> LARGE_PAGE_SHIFT;
else
vpn = va >> PAGE_SHIFT;
avpn = vpn >> 11;
avpn &= ~0x1UL;
pSeries_lock_hpte(hptep);
dw0 = hptep->dw0.dw0;
/* Even if we miss, we need to invalidate the TLB */
if ((dw0.avpn != avpn) || !dw0.v) {
pSeries_unlock_hpte(hptep);
udbg_printf("invalidate missed\n");
return;
}
} else {
/* Invalidate the hpte. NOTE: this also unlocks it */
hptep->dw0.dword0 = 0;
}
/* Invalidate the tlb */
if (cpu_has_tlbiel() && !large && local) {
......@@ -374,6 +300,7 @@ static void pSeries_flush_hash_range(unsigned long context,
HPTE *hptep;
Hpte_dword0 dw0;
struct ppc64_tlb_batch *batch = &ppc64_tlb_batch[smp_processor_id()];
/* XXX fix for large ptes */
unsigned long large = 0;
......@@ -399,22 +326,24 @@ static void pSeries_flush_hash_range(unsigned long context,
slot += (pte_val(batch->pte[i]) & _PAGE_GROUP_IX) >> 12;
hptep = htab_data.htab + slot;
avpn = vpn >> 11;
avpn = va >> 23;
if (large)
avpn &= ~0x1UL;
pSeries_lock_hpte(hptep);
dw0 = hptep->dw0.dw0;
/* Even if we miss, we need to invalidate the TLB */
if ((dw0.avpn != avpn) || !dw0.v) {
pSeries_unlock_hpte(hptep);
udbg_printf("invalidate missed\n");
continue;
} else {
/* Invalidate the hpte. NOTE: this also unlocks it */
hptep->dw0.dword0 = 0;
}
j++;
/* Invalidate the hpte. NOTE: this also unlocks it */
hptep->dw0.dword0 = 0;
}
if (cpu_has_tlbiel() && !large && local) {
......@@ -457,7 +386,6 @@ void hpte_init_pSeries(void)
ppc_md.hpte_updateboltedpp = pSeries_hpte_updateboltedpp;
ppc_md.hpte_insert = pSeries_hpte_insert;
ppc_md.hpte_remove = pSeries_hpte_remove;
ppc_md.make_pte = pSeries_make_pte;
/* Disable TLB batching on nighthawk */
root = find_path_device("/");
......
......@@ -403,67 +403,11 @@ int hvc_count(int *start_termno)
/*
* Create a pte - LPAR . Used during initialization only.
* We assume the PTE will fit in the primary PTEG.
*/
void pSeries_lpar_make_pte(HPTE *htab, unsigned long va, unsigned long pa,
int mode, unsigned long hash_mask, int large)
{
HPTE local_hpte;
unsigned long hash, slot, flags, lpar_rc, vpn;
unsigned long dummy1, dummy2;
if (large)
vpn = va >> LARGE_PAGE_SHIFT;
else
vpn = va >> PAGE_SHIFT;
hash = hpt_hash(vpn, large);
slot = ((hash & hash_mask)*HPTES_PER_GROUP);
local_hpte.dw1.dword1 = pa | mode;
local_hpte.dw0.dword0 = 0;
local_hpte.dw0.dw0.avpn = va >> 23;
local_hpte.dw0.dw0.bolted = 1; /* bolted */
if (large) {
local_hpte.dw0.dw0.l = 1; /* large page */
local_hpte.dw0.dw0.avpn &= ~0x1UL;
}
local_hpte.dw0.dw0.v = 1;
/* Set CEC cookie to 0 */
/* Zero page = 0 */
/* I-cache Invalidate = 0 */
/* I-cache synchronize = 0 */
/* Exact = 0 - modify any entry in group */
flags = 0;
lpar_rc = plpar_pte_enter(flags, slot, local_hpte.dw0.dword0,
local_hpte.dw1.dword1, &dummy1, &dummy2);
if (lpar_rc == H_PTEG_Full) {
while(1)
;
}
/*
* NOTE: we explicitly do not check return status here because it is
* "normal" for early boot code to map io regions for which a partition
* has no access. However, we will die if we actually fault on these
* "permission denied" pages.
*/
}
static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
unsigned long vpn, unsigned long prpn,
long pSeries_lpar_hpte_insert(unsigned long hpte_group,
unsigned long va, unsigned long prpn,
int secondary, unsigned long hpteflags,
int bolted, int large)
{
/* XXX fix for large page */
unsigned long avpn = vpn >> 11;
unsigned long arpn = physRpn_to_absRpn(prpn);
unsigned long lpar_rc;
unsigned long flags;
......@@ -476,13 +420,15 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
lhpte.dw1.flags.flags = hpteflags;
lhpte.dw0.dword0 = 0;
lhpte.dw0.dw0.avpn = avpn;
lhpte.dw0.dw0.avpn = va >> 23;
lhpte.dw0.dw0.h = secondary;
lhpte.dw0.dw0.bolted = bolted;
lhpte.dw0.dw0.v = 1;
if (large)
if (large) {
lhpte.dw0.dw0.l = 1;
lhpte.dw0.dw0.avpn &= ~0x1UL;
}
/* Now fill in the actual HPTE */
/* Set CEC cookie to 0 */
......@@ -559,12 +505,15 @@ static long pSeries_lpar_hpte_remove(unsigned long hpte_group)
* already zero. For now I am paranoid.
*/
static long pSeries_lpar_hpte_updatepp(unsigned long slot, unsigned long newpp,
unsigned long va, int large)
unsigned long va, int large, int local)
{
unsigned long lpar_rc;
unsigned long flags = (newpp & 7) | H_AVPN;
unsigned long avpn = va >> 23;
if (large)
avpn &= ~0x1UL;
lpar_rc = plpar_pte_protect(flags, slot, (avpn << 7));
if (lpar_rc == H_Not_Found) {
......@@ -662,6 +611,9 @@ static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long va,
unsigned long lpar_rc;
unsigned long dummy1, dummy2;
if (large)
avpn &= ~0x1UL;
lpar_rc = plpar_pte_remove(H_AVPN, slot, (avpn << 7), &dummy1,
&dummy2);
......@@ -700,6 +652,5 @@ void pSeries_lpar_mm_init(void)
ppc_md.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp;
ppc_md.hpte_insert = pSeries_lpar_hpte_insert;
ppc_md.hpte_remove = pSeries_lpar_hpte_remove;
ppc_md.make_pte = pSeries_lpar_make_pte;
ppc_md.flush_hash_range = pSeries_lpar_flush_hash_range;
}
......@@ -220,16 +220,26 @@ static void map_io_page(unsigned long ea, unsigned long pa, int flags)
set_pte(ptep, pfn_pte(pa >> PAGE_SHIFT, __pgprot(flags)));
spin_unlock(&ioremap_mm.page_table_lock);
} else {
/* If the mm subsystem is not fully up, we cannot create a
unsigned long va, vpn, hash, hpteg;
/*
* If the mm subsystem is not fully up, we cannot create a
* linux page table entry for this mapping. Simply bolt an
* entry in the hardware page table.
*/
vsid = get_kernel_vsid(ea);
ppc_md.make_pte(htab_data.htab,
(vsid << 28) | (ea & 0xFFFFFFF), // va (NOT the ea)
pa,
_PAGE_NO_CACHE | _PAGE_GUARDED | PP_RWXX,
htab_data.htab_hash_mask, 0);
va = (vsid << 28) | (ea & 0xFFFFFFF);
vpn = va >> PAGE_SHIFT;
hash = hpt_hash(vpn, 0);
hpteg = ((hash & htab_data.htab_hash_mask)*HPTES_PER_GROUP);
if (ppc_md.hpte_insert(hpteg, va, pa >> PAGE_SHIFT, 0,
_PAGE_NO_CACHE|_PAGE_GUARDED|PP_RWXX,
1, 0) == -1) {
panic("map_io_page: could not insert mapping");
}
}
}
......@@ -649,7 +659,7 @@ void flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
extern pte_t *find_linux_pte(pgd_t *pgdir, unsigned long ea);
int __hash_page(unsigned long ea, unsigned long access, unsigned long vsid,
pte_t *ptep, unsigned long trap);
pte_t *ptep, unsigned long trap, int local);
/*
* This is called at the end of handling a user page fault, when the
......@@ -665,6 +675,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long ea,
unsigned long vsid;
void *pgdir;
pte_t *ptep;
int local = 0;
/* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
if (!pte_young(pte))
......@@ -677,6 +688,9 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long ea,
ptep = find_linux_pte(pgdir, ea);
vsid = get_vsid(vma->vm_mm->context, ea);
if (vma->vm_mm->cpu_vm_mask == (1 << smp_processor_id()))
local = 1;
__hash_page(ea, pte_val(pte) & (_PAGE_USER|_PAGE_RW), vsid, ptep,
0x300);
0x300, local);
}
......@@ -37,11 +37,12 @@ struct machdep_calls {
long (*hpte_updatepp)(unsigned long slot,
unsigned long newpp,
unsigned long va,
int large);
int large,
int local);
void (*hpte_updateboltedpp)(unsigned long newpp,
unsigned long ea);
long (*hpte_insert)(unsigned long hpte_group,
unsigned long vpn,
unsigned long va,
unsigned long prpn,
int secondary,
unsigned long hpteflags,
......@@ -51,11 +52,6 @@ struct machdep_calls {
void (*flush_hash_range)(unsigned long context,
unsigned long number,
int local);
void (*make_pte)(void *htab, unsigned long va,
unsigned long pa,
int mode,
unsigned long hash_mask,
int large);
void (*tce_build)(struct TceTable * tbl,
long tcenum,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment