Commit 962734fc authored by Christian Borntraeger's avatar Christian Borntraeger

Merge branch 'tlb-flushing' of...

Merge branch 'tlb-flushing' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux  into kernelorgnext

base patches for tlb flushing optimizations
parents 520eccdf cd774b90
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
#define ESSA_SET_POT_VOLATILE 4 #define ESSA_SET_POT_VOLATILE 4
#define ESSA_SET_STABLE_RESIDENT 5 #define ESSA_SET_STABLE_RESIDENT 5
#define ESSA_SET_STABLE_IF_RESIDENT 6 #define ESSA_SET_STABLE_IF_RESIDENT 6
#define ESSA_SET_STABLE_NODAT 7
#define ESSA_MAX ESSA_SET_STABLE_IF_RESIDENT #define ESSA_MAX ESSA_SET_STABLE_IF_RESIDENT
......
...@@ -133,6 +133,9 @@ static inline int page_reset_referenced(unsigned long addr) ...@@ -133,6 +133,9 @@ static inline int page_reset_referenced(unsigned long addr)
struct page; struct page;
void arch_free_page(struct page *page, int order); void arch_free_page(struct page *page, int order);
void arch_alloc_page(struct page *page, int order); void arch_alloc_page(struct page *page, int order);
void arch_set_page_dat(struct page *page, int order);
void arch_set_page_nodat(struct page *page, int order);
int arch_test_page_nodat(struct page *page);
void arch_set_page_states(int make_stable); void arch_set_page_states(int make_stable);
static inline int devmem_is_allowed(unsigned long pfn) static inline int devmem_is_allowed(unsigned long pfn)
......
...@@ -376,6 +376,7 @@ static inline int is_module_addr(void *addr) ...@@ -376,6 +376,7 @@ static inline int is_module_addr(void *addr)
/* Guest Page State used for virtualization */ /* Guest Page State used for virtualization */
#define _PGSTE_GPS_ZERO 0x0000000080000000UL #define _PGSTE_GPS_ZERO 0x0000000080000000UL
#define _PGSTE_GPS_NODAT 0x0000000040000000UL
#define _PGSTE_GPS_USAGE_MASK 0x0000000003000000UL #define _PGSTE_GPS_USAGE_MASK 0x0000000003000000UL
#define _PGSTE_GPS_USAGE_STABLE 0x0000000000000000UL #define _PGSTE_GPS_USAGE_STABLE 0x0000000000000000UL
#define _PGSTE_GPS_USAGE_UNUSED 0x0000000001000000UL #define _PGSTE_GPS_USAGE_UNUSED 0x0000000001000000UL
...@@ -952,15 +953,30 @@ static inline pte_t pte_mkhuge(pte_t pte) ...@@ -952,15 +953,30 @@ static inline pte_t pte_mkhuge(pte_t pte)
#define IPTE_GLOBAL 0 #define IPTE_GLOBAL 0
#define IPTE_LOCAL 1 #define IPTE_LOCAL 1
static inline void __ptep_ipte(unsigned long address, pte_t *ptep, int local) #define IPTE_NODAT 0x400
#define IPTE_GUEST_ASCE 0x800
static inline void __ptep_ipte(unsigned long address, pte_t *ptep,
unsigned long opt, unsigned long asce,
int local)
{ {
unsigned long pto = (unsigned long) ptep; unsigned long pto = (unsigned long) ptep;
/* Invalidation + TLB flush for the pte */ if (__builtin_constant_p(opt) && opt == 0) {
/* Invalidation + TLB flush for the pte */
asm volatile(
" .insn rrf,0xb2210000,%[r1],%[r2],0,%[m4]"
: "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address),
[m4] "i" (local));
return;
}
/* Invalidate ptes with options + TLB flush of the ptes */
opt = opt | (asce & _ASCE_ORIGIN);
asm volatile( asm volatile(
" .insn rrf,0xb2210000,%[r1],%[r2],0,%[m4]" " .insn rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]"
: "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address), : [r2] "+a" (address), [r3] "+a" (opt)
[m4] "i" (local)); : [r1] "a" (pto), [m4] "i" (local) : "memory");
} }
static inline void __ptep_ipte_range(unsigned long address, int nr, static inline void __ptep_ipte_range(unsigned long address, int nr,
...@@ -1341,31 +1357,61 @@ static inline void __pmdp_csp(pmd_t *pmdp) ...@@ -1341,31 +1357,61 @@ static inline void __pmdp_csp(pmd_t *pmdp)
#define IDTE_GLOBAL 0 #define IDTE_GLOBAL 0
#define IDTE_LOCAL 1 #define IDTE_LOCAL 1
static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp, int local) #define IDTE_PTOA 0x0800
#define IDTE_NODAT 0x1000
#define IDTE_GUEST_ASCE 0x2000
static inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp,
unsigned long opt, unsigned long asce,
int local)
{ {
unsigned long sto; unsigned long sto;
sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t); sto = (unsigned long) pmdp - pmd_index(addr) * sizeof(pmd_t);
asm volatile( if (__builtin_constant_p(opt) && opt == 0) {
" .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]" /* flush without guest asce */
: "+m" (*pmdp) asm volatile(
: [r1] "a" (sto), [r2] "a" ((address & HPAGE_MASK)), " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
[m4] "i" (local) : "+m" (*pmdp)
: "cc" ); : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK)),
[m4] "i" (local)
: "cc" );
} else {
/* flush with guest asce */
asm volatile(
" .insn rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"
: "+m" (*pmdp)
: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
[r3] "a" (asce), [m4] "i" (local)
: "cc" );
}
} }
static inline void __pudp_idte(unsigned long address, pud_t *pudp, int local) static inline void __pudp_idte(unsigned long addr, pud_t *pudp,
unsigned long opt, unsigned long asce,
int local)
{ {
unsigned long r3o; unsigned long r3o;
r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t); r3o = (unsigned long) pudp - pud_index(addr) * sizeof(pud_t);
r3o |= _ASCE_TYPE_REGION3; r3o |= _ASCE_TYPE_REGION3;
asm volatile( if (__builtin_constant_p(opt) && opt == 0) {
" .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]" /* flush without guest asce */
: "+m" (*pudp) asm volatile(
: [r1] "a" (r3o), [r2] "a" ((address & PUD_MASK)), " .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
[m4] "i" (local) : "+m" (*pudp)
: "cc"); : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK)),
[m4] "i" (local)
: "cc");
} else {
/* flush with guest asce */
asm volatile(
" .insn rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"
: "+m" (*pudp)
: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
[r3] "a" (asce), [m4] "i" (local)
: "cc" );
}
} }
pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t); pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
......
...@@ -29,8 +29,9 @@ ...@@ -29,8 +29,9 @@
#define MACHINE_FLAG_TE _BITUL(11) #define MACHINE_FLAG_TE _BITUL(11)
#define MACHINE_FLAG_TLB_LC _BITUL(12) #define MACHINE_FLAG_TLB_LC _BITUL(12)
#define MACHINE_FLAG_VX _BITUL(13) #define MACHINE_FLAG_VX _BITUL(13)
#define MACHINE_FLAG_NX _BITUL(14) #define MACHINE_FLAG_TLB_GUEST _BITUL(14)
#define MACHINE_FLAG_GS _BITUL(15) #define MACHINE_FLAG_NX _BITUL(15)
#define MACHINE_FLAG_GS _BITUL(16)
#define LPP_MAGIC _BITUL(31) #define LPP_MAGIC _BITUL(31)
#define LPP_PFAULT_PID_MASK _AC(0xffffffff, UL) #define LPP_PFAULT_PID_MASK _AC(0xffffffff, UL)
...@@ -68,6 +69,7 @@ extern void detect_memory_memblock(void); ...@@ -68,6 +69,7 @@ extern void detect_memory_memblock(void);
#define MACHINE_HAS_TE (S390_lowcore.machine_flags & MACHINE_FLAG_TE) #define MACHINE_HAS_TE (S390_lowcore.machine_flags & MACHINE_FLAG_TE)
#define MACHINE_HAS_TLB_LC (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_LC) #define MACHINE_HAS_TLB_LC (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_LC)
#define MACHINE_HAS_VX (S390_lowcore.machine_flags & MACHINE_FLAG_VX) #define MACHINE_HAS_VX (S390_lowcore.machine_flags & MACHINE_FLAG_VX)
#define MACHINE_HAS_TLB_GUEST (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_GUEST)
#define MACHINE_HAS_NX (S390_lowcore.machine_flags & MACHINE_FLAG_NX) #define MACHINE_HAS_NX (S390_lowcore.machine_flags & MACHINE_FLAG_NX)
#define MACHINE_HAS_GS (S390_lowcore.machine_flags & MACHINE_FLAG_GS) #define MACHINE_HAS_GS (S390_lowcore.machine_flags & MACHINE_FLAG_GS)
...@@ -106,7 +108,8 @@ extern void pfault_fini(void); ...@@ -106,7 +108,8 @@ extern void pfault_fini(void);
void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault); void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault);
extern void cmma_init(void); void cmma_init(void);
void cmma_init_nodat(void);
extern void (*_machine_restart)(char *command); extern void (*_machine_restart)(char *command);
extern void (*_machine_halt)(void); extern void (*_machine_halt)(void);
......
...@@ -20,10 +20,15 @@ static inline void __tlb_flush_local(void) ...@@ -20,10 +20,15 @@ static inline void __tlb_flush_local(void)
*/ */
static inline void __tlb_flush_idte(unsigned long asce) static inline void __tlb_flush_idte(unsigned long asce)
{ {
unsigned long opt;
opt = IDTE_PTOA;
if (MACHINE_HAS_TLB_GUEST)
opt |= IDTE_GUEST_ASCE;
/* Global TLB flush for the mm */ /* Global TLB flush for the mm */
asm volatile( asm volatile(
" .insn rrf,0xb98e0000,0,%0,%1,0" " .insn rrf,0xb98e0000,0,%0,%1,0"
: : "a" (2048), "a" (asce) : "cc"); : : "a" (opt), "a" (asce) : "cc");
} }
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
......
...@@ -98,10 +98,16 @@ int page_key_alloc(unsigned long pages) ...@@ -98,10 +98,16 @@ int page_key_alloc(unsigned long pages)
*/ */
void page_key_read(unsigned long *pfn) void page_key_read(unsigned long *pfn)
{ {
struct page *page;
unsigned long addr; unsigned long addr;
unsigned char key;
addr = (unsigned long) page_address(pfn_to_page(*pfn));
*(unsigned char *) pfn = (unsigned char) page_get_storage_key(addr); page = pfn_to_page(*pfn);
addr = (unsigned long) page_address(page);
key = (unsigned char) page_get_storage_key(addr) & 0x7f;
if (arch_test_page_nodat(page))
key |= 0x80;
*(unsigned char *) pfn = key;
} }
/* /*
...@@ -126,8 +132,16 @@ void page_key_memorize(unsigned long *pfn) ...@@ -126,8 +132,16 @@ void page_key_memorize(unsigned long *pfn)
*/ */
void page_key_write(void *address) void page_key_write(void *address)
{ {
page_set_storage_key((unsigned long) address, struct page *page;
page_key_rp->data[page_key_rx], 0); unsigned char key;
key = page_key_rp->data[page_key_rx];
page_set_storage_key((unsigned long) address, key & 0x7f, 0);
page = virt_to_page(address);
if (key & 0x80)
arch_set_page_nodat(page, 0);
else
arch_set_page_dat(page, 0);
if (++page_key_rx >= PAGE_KEY_DATA_SIZE) if (++page_key_rx >= PAGE_KEY_DATA_SIZE)
return; return;
page_key_rp = page_key_rp->next; page_key_rp = page_key_rp->next;
......
...@@ -157,6 +157,8 @@ int vdso_alloc_per_cpu(struct lowcore *lowcore) ...@@ -157,6 +157,8 @@ int vdso_alloc_per_cpu(struct lowcore *lowcore)
page_frame = get_zeroed_page(GFP_KERNEL); page_frame = get_zeroed_page(GFP_KERNEL);
if (!segment_table || !page_table || !page_frame) if (!segment_table || !page_table || !page_frame)
goto out; goto out;
arch_set_page_dat(virt_to_page(segment_table), SEGMENT_ORDER);
arch_set_page_dat(virt_to_page(page_table), 0);
/* Initialize per-cpu vdso data page */ /* Initialize per-cpu vdso data page */
vd = (struct vdso_per_cpu_data *) page_frame; vd = (struct vdso_per_cpu_data *) page_frame;
......
...@@ -137,6 +137,8 @@ void __init mem_init(void) ...@@ -137,6 +137,8 @@ void __init mem_init(void)
free_all_bootmem(); free_all_bootmem();
setup_zero_pages(); /* Setup zeroed pages. */ setup_zero_pages(); /* Setup zeroed pages. */
cmma_init_nodat();
mem_init_print_info(NULL); mem_init_print_info(NULL);
} }
......
...@@ -10,9 +10,10 @@ ...@@ -10,9 +10,10 @@
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/memblock.h>
#include <linux/gfp.h> #include <linux/gfp.h>
#include <linux/init.h> #include <linux/init.h>
#include <asm/facility.h>
#include <asm/page-states.h> #include <asm/page-states.h>
static int cmma_flag = 1; static int cmma_flag = 1;
...@@ -36,14 +37,16 @@ __setup("cmma=", cmma); ...@@ -36,14 +37,16 @@ __setup("cmma=", cmma);
static inline int cmma_test_essa(void) static inline int cmma_test_essa(void)
{ {
register unsigned long tmp asm("0") = 0; register unsigned long tmp asm("0") = 0;
register int rc asm("1") = -EOPNOTSUPP; register int rc asm("1");
/* test ESSA_GET_STATE */
asm volatile( asm volatile(
" .insn rrf,0xb9ab0000,%1,%1,0,0\n" " .insn rrf,0xb9ab0000,%1,%1,%2,0\n"
"0: la %0,0\n" "0: la %0,0\n"
"1:\n" "1:\n"
EX_TABLE(0b,1b) EX_TABLE(0b,1b)
: "+&d" (rc), "+&d" (tmp)); : "=&d" (rc), "+&d" (tmp)
: "i" (ESSA_GET_STATE), "0" (-EOPNOTSUPP));
return rc; return rc;
} }
...@@ -51,11 +54,26 @@ void __init cmma_init(void) ...@@ -51,11 +54,26 @@ void __init cmma_init(void)
{ {
if (!cmma_flag) if (!cmma_flag)
return; return;
if (cmma_test_essa()) if (cmma_test_essa()) {
cmma_flag = 0; cmma_flag = 0;
return;
}
if (test_facility(147))
cmma_flag = 2;
} }
static inline void set_page_unstable(struct page *page, int order) static inline unsigned char get_page_state(struct page *page)
{
unsigned char state;
asm volatile(" .insn rrf,0xb9ab0000,%0,%1,%2,0"
: "=&d" (state)
: "a" (page_to_phys(page)),
"i" (ESSA_GET_STATE));
return state & 0x3f;
}
static inline void set_page_unused(struct page *page, int order)
{ {
int i, rc; int i, rc;
...@@ -66,14 +84,18 @@ static inline void set_page_unstable(struct page *page, int order) ...@@ -66,14 +84,18 @@ static inline void set_page_unstable(struct page *page, int order)
"i" (ESSA_SET_UNUSED)); "i" (ESSA_SET_UNUSED));
} }
void arch_free_page(struct page *page, int order) static inline void set_page_stable_dat(struct page *page, int order)
{ {
if (!cmma_flag) int i, rc;
return;
set_page_unstable(page, order); for (i = 0; i < (1 << order); i++)
asm volatile(".insn rrf,0xb9ab0000,%0,%1,%2,0"
: "=&d" (rc)
: "a" (page_to_phys(page + i)),
"i" (ESSA_SET_STABLE));
} }
static inline void set_page_stable(struct page *page, int order) static inline void set_page_stable_nodat(struct page *page, int order)
{ {
int i, rc; int i, rc;
...@@ -81,14 +103,154 @@ static inline void set_page_stable(struct page *page, int order) ...@@ -81,14 +103,154 @@ static inline void set_page_stable(struct page *page, int order)
asm volatile(".insn rrf,0xb9ab0000,%0,%1,%2,0" asm volatile(".insn rrf,0xb9ab0000,%0,%1,%2,0"
: "=&d" (rc) : "=&d" (rc)
: "a" (page_to_phys(page + i)), : "a" (page_to_phys(page + i)),
"i" (ESSA_SET_STABLE)); "i" (ESSA_SET_STABLE_NODAT));
}
static void mark_kernel_pmd(pud_t *pud, unsigned long addr, unsigned long end)
{
unsigned long next;
struct page *page;
pmd_t *pmd;
pmd = pmd_offset(pud, addr);
do {
next = pmd_addr_end(addr, end);
if (pmd_none(*pmd) || pmd_large(*pmd))
continue;
page = virt_to_page(pmd_val(*pmd));
set_bit(PG_arch_1, &page->flags);
} while (pmd++, addr = next, addr != end);
}
static void mark_kernel_pud(p4d_t *p4d, unsigned long addr, unsigned long end)
{
unsigned long next;
struct page *page;
pud_t *pud;
int i;
pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
if (pud_none(*pud) || pud_large(*pud))
continue;
if (!pud_folded(*pud)) {
page = virt_to_page(pud_val(*pud));
for (i = 0; i < 3; i++)
set_bit(PG_arch_1, &page[i].flags);
}
mark_kernel_pmd(pud, addr, next);
} while (pud++, addr = next, addr != end);
}
static void mark_kernel_p4d(pgd_t *pgd, unsigned long addr, unsigned long end)
{
unsigned long next;
struct page *page;
p4d_t *p4d;
int i;
p4d = p4d_offset(pgd, addr);
do {
next = p4d_addr_end(addr, end);
if (p4d_none(*p4d))
continue;
if (!p4d_folded(*p4d)) {
page = virt_to_page(p4d_val(*p4d));
for (i = 0; i < 3; i++)
set_bit(PG_arch_1, &page[i].flags);
}
mark_kernel_pud(p4d, addr, next);
} while (p4d++, addr = next, addr != end);
}
static void mark_kernel_pgd(void)
{
unsigned long addr, next;
struct page *page;
pgd_t *pgd;
int i;
addr = 0;
pgd = pgd_offset_k(addr);
do {
next = pgd_addr_end(addr, MODULES_END);
if (pgd_none(*pgd))
continue;
if (!pgd_folded(*pgd)) {
page = virt_to_page(pgd_val(*pgd));
for (i = 0; i < 3; i++)
set_bit(PG_arch_1, &page[i].flags);
}
mark_kernel_p4d(pgd, addr, next);
} while (pgd++, addr = next, addr != MODULES_END);
}
void __init cmma_init_nodat(void)
{
struct memblock_region *reg;
struct page *page;
unsigned long start, end, ix;
if (cmma_flag < 2)
return;
/* Mark pages used in kernel page tables */
mark_kernel_pgd();
/* Set all kernel pages not used for page tables to stable/no-dat */
for_each_memblock(memory, reg) {
start = memblock_region_memory_base_pfn(reg);
end = memblock_region_memory_end_pfn(reg);
page = pfn_to_page(start);
for (ix = start; ix < end; ix++, page++) {
if (__test_and_clear_bit(PG_arch_1, &page->flags))
continue; /* skip page table pages */
if (!list_empty(&page->lru))
continue; /* skip free pages */
set_page_stable_nodat(page, 0);
}
}
}
void arch_free_page(struct page *page, int order)
{
if (!cmma_flag)
return;
set_page_unused(page, order);
} }
void arch_alloc_page(struct page *page, int order) void arch_alloc_page(struct page *page, int order)
{ {
if (!cmma_flag) if (!cmma_flag)
return; return;
set_page_stable(page, order); if (cmma_flag < 2)
set_page_stable_dat(page, order);
else
set_page_stable_nodat(page, order);
}
void arch_set_page_dat(struct page *page, int order)
{
if (!cmma_flag)
return;
set_page_stable_dat(page, order);
}
void arch_set_page_nodat(struct page *page, int order)
{
if (cmma_flag < 2)
return;
set_page_stable_nodat(page, order);
}
int arch_test_page_nodat(struct page *page)
{
unsigned char state;
if (cmma_flag < 2)
return 0;
state = get_page_state(page);
return !!(state & 0x20);
} }
void arch_set_page_states(int make_stable) void arch_set_page_states(int make_stable)
...@@ -108,9 +270,9 @@ void arch_set_page_states(int make_stable) ...@@ -108,9 +270,9 @@ void arch_set_page_states(int make_stable)
list_for_each(l, &zone->free_area[order].free_list[t]) { list_for_each(l, &zone->free_area[order].free_list[t]) {
page = list_entry(l, struct page, lru); page = list_entry(l, struct page, lru);
if (make_stable) if (make_stable)
set_page_stable(page, order); set_page_stable_dat(page, 0);
else else
set_page_unstable(page, order); set_page_unused(page, order);
} }
} }
spin_unlock_irqrestore(&zone->lock, flags); spin_unlock_irqrestore(&zone->lock, flags);
......
...@@ -328,7 +328,7 @@ static void ipte_range(pte_t *pte, unsigned long address, int nr) ...@@ -328,7 +328,7 @@ static void ipte_range(pte_t *pte, unsigned long address, int nr)
return; return;
} }
for (i = 0; i < nr; i++) { for (i = 0; i < nr; i++) {
__ptep_ipte(address, pte, IPTE_GLOBAL); __ptep_ipte(address, pte, 0, 0, IPTE_GLOBAL);
address += PAGE_SIZE; address += PAGE_SIZE;
pte++; pte++;
} }
......
...@@ -57,6 +57,7 @@ unsigned long *crst_table_alloc(struct mm_struct *mm) ...@@ -57,6 +57,7 @@ unsigned long *crst_table_alloc(struct mm_struct *mm)
if (!page) if (!page)
return NULL; return NULL;
arch_set_page_dat(page, 2);
return (unsigned long *) page_to_phys(page); return (unsigned long *) page_to_phys(page);
} }
...@@ -214,6 +215,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm) ...@@ -214,6 +215,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
__free_page(page); __free_page(page);
return NULL; return NULL;
} }
arch_set_page_dat(page, 0);
/* Initialize page table */ /* Initialize page table */
table = (unsigned long *) page_to_phys(page); table = (unsigned long *) page_to_phys(page);
if (mm_alloc_pgste(mm)) { if (mm_alloc_pgste(mm)) {
......
...@@ -25,8 +25,49 @@ ...@@ -25,8 +25,49 @@
#include <asm/mmu_context.h> #include <asm/mmu_context.h>
#include <asm/page-states.h> #include <asm/page-states.h>
static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, int nodat)
{
unsigned long opt, asce;
if (MACHINE_HAS_TLB_GUEST) {
opt = 0;
asce = READ_ONCE(mm->context.gmap_asce);
if (asce == 0UL || nodat)
opt |= IPTE_NODAT;
if (asce != -1UL) {
asce = asce ? : mm->context.asce;
opt |= IPTE_GUEST_ASCE;
}
__ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);
} else {
__ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);
}
}
static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, int nodat)
{
unsigned long opt, asce;
if (MACHINE_HAS_TLB_GUEST) {
opt = 0;
asce = READ_ONCE(mm->context.gmap_asce);
if (asce == 0UL || nodat)
opt |= IPTE_NODAT;
if (asce != -1UL) {
asce = asce ? : mm->context.asce;
opt |= IPTE_GUEST_ASCE;
}
__ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);
} else {
__ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
}
}
static inline pte_t ptep_flush_direct(struct mm_struct *mm, static inline pte_t ptep_flush_direct(struct mm_struct *mm,
unsigned long addr, pte_t *ptep) unsigned long addr, pte_t *ptep,
int nodat)
{ {
pte_t old; pte_t old;
...@@ -36,15 +77,16 @@ static inline pte_t ptep_flush_direct(struct mm_struct *mm, ...@@ -36,15 +77,16 @@ static inline pte_t ptep_flush_direct(struct mm_struct *mm,
atomic_inc(&mm->context.flush_count); atomic_inc(&mm->context.flush_count);
if (MACHINE_HAS_TLB_LC && if (MACHINE_HAS_TLB_LC &&
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
__ptep_ipte(addr, ptep, IPTE_LOCAL); ptep_ipte_local(mm, addr, ptep, nodat);
else else
__ptep_ipte(addr, ptep, IPTE_GLOBAL); ptep_ipte_global(mm, addr, ptep, nodat);
atomic_dec(&mm->context.flush_count); atomic_dec(&mm->context.flush_count);
return old; return old;
} }
static inline pte_t ptep_flush_lazy(struct mm_struct *mm, static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
unsigned long addr, pte_t *ptep) unsigned long addr, pte_t *ptep,
int nodat)
{ {
pte_t old; pte_t old;
...@@ -57,7 +99,7 @@ static inline pte_t ptep_flush_lazy(struct mm_struct *mm, ...@@ -57,7 +99,7 @@ static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
pte_val(*ptep) |= _PAGE_INVALID; pte_val(*ptep) |= _PAGE_INVALID;
mm->context.flush_mm = 1; mm->context.flush_mm = 1;
} else } else
__ptep_ipte(addr, ptep, IPTE_GLOBAL); ptep_ipte_global(mm, addr, ptep, nodat);
atomic_dec(&mm->context.flush_count); atomic_dec(&mm->context.flush_count);
return old; return old;
} }
...@@ -229,10 +271,12 @@ pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, ...@@ -229,10 +271,12 @@ pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
{ {
pgste_t pgste; pgste_t pgste;
pte_t old; pte_t old;
int nodat;
preempt_disable(); preempt_disable();
pgste = ptep_xchg_start(mm, addr, ptep); pgste = ptep_xchg_start(mm, addr, ptep);
old = ptep_flush_direct(mm, addr, ptep); nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
old = ptep_flush_direct(mm, addr, ptep, nodat);
old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
preempt_enable(); preempt_enable();
return old; return old;
...@@ -244,10 +288,12 @@ pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, ...@@ -244,10 +288,12 @@ pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
{ {
pgste_t pgste; pgste_t pgste;
pte_t old; pte_t old;
int nodat;
preempt_disable(); preempt_disable();
pgste = ptep_xchg_start(mm, addr, ptep); pgste = ptep_xchg_start(mm, addr, ptep);
old = ptep_flush_lazy(mm, addr, ptep); nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
old = ptep_flush_lazy(mm, addr, ptep, nodat);
old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
preempt_enable(); preempt_enable();
return old; return old;
...@@ -259,10 +305,12 @@ pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, ...@@ -259,10 +305,12 @@ pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
{ {
pgste_t pgste; pgste_t pgste;
pte_t old; pte_t old;
int nodat;
preempt_disable(); preempt_disable();
pgste = ptep_xchg_start(mm, addr, ptep); pgste = ptep_xchg_start(mm, addr, ptep);
old = ptep_flush_lazy(mm, addr, ptep); nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
old = ptep_flush_lazy(mm, addr, ptep, nodat);
if (mm_has_pgste(mm)) { if (mm_has_pgste(mm)) {
pgste = pgste_update_all(old, pgste, mm); pgste = pgste_update_all(old, pgste, mm);
pgste_set(ptep, pgste); pgste_set(ptep, pgste);
...@@ -290,6 +338,28 @@ void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, ...@@ -290,6 +338,28 @@ void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
} }
EXPORT_SYMBOL(ptep_modify_prot_commit); EXPORT_SYMBOL(ptep_modify_prot_commit);
static inline void pmdp_idte_local(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp)
{
if (MACHINE_HAS_TLB_GUEST)
__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
mm->context.asce, IDTE_LOCAL);
else
__pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
}
static inline void pmdp_idte_global(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp)
{
if (MACHINE_HAS_TLB_GUEST)
__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
mm->context.asce, IDTE_GLOBAL);
else if (MACHINE_HAS_IDTE)
__pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
else
__pmdp_csp(pmdp);
}
static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp) unsigned long addr, pmd_t *pmdp)
{ {
...@@ -298,16 +368,12 @@ static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, ...@@ -298,16 +368,12 @@ static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
old = *pmdp; old = *pmdp;
if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
return old; return old;
if (!MACHINE_HAS_IDTE) {
__pmdp_csp(pmdp);
return old;
}
atomic_inc(&mm->context.flush_count); atomic_inc(&mm->context.flush_count);
if (MACHINE_HAS_TLB_LC && if (MACHINE_HAS_TLB_LC &&
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
__pmdp_idte(addr, pmdp, IDTE_LOCAL); pmdp_idte_local(mm, addr, pmdp);
else else
__pmdp_idte(addr, pmdp, IDTE_GLOBAL); pmdp_idte_global(mm, addr, pmdp);
atomic_dec(&mm->context.flush_count); atomic_dec(&mm->context.flush_count);
return old; return old;
} }
...@@ -325,10 +391,9 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, ...@@ -325,10 +391,9 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
cpumask_of(smp_processor_id()))) { cpumask_of(smp_processor_id()))) {
pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
mm->context.flush_mm = 1; mm->context.flush_mm = 1;
} else if (MACHINE_HAS_IDTE) } else {
__pmdp_idte(addr, pmdp, IDTE_GLOBAL); pmdp_idte_global(mm, addr, pmdp);
else }
__pmdp_csp(pmdp);
atomic_dec(&mm->context.flush_count); atomic_dec(&mm->context.flush_count);
return old; return old;
} }
...@@ -359,28 +424,46 @@ pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, ...@@ -359,28 +424,46 @@ pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
} }
EXPORT_SYMBOL(pmdp_xchg_lazy); EXPORT_SYMBOL(pmdp_xchg_lazy);
static inline pud_t pudp_flush_direct(struct mm_struct *mm, static inline void pudp_idte_local(struct mm_struct *mm,
unsigned long addr, pud_t *pudp) unsigned long addr, pud_t *pudp)
{ {
pud_t old; if (MACHINE_HAS_TLB_GUEST)
__pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
mm->context.asce, IDTE_LOCAL);
else
__pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL);
}
old = *pudp; static inline void pudp_idte_global(struct mm_struct *mm,
if (pud_val(old) & _REGION_ENTRY_INVALID) unsigned long addr, pud_t *pudp)
return old; {
if (!MACHINE_HAS_IDTE) { if (MACHINE_HAS_TLB_GUEST)
__pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
mm->context.asce, IDTE_GLOBAL);
else if (MACHINE_HAS_IDTE)
__pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL);
else
/* /*
* Invalid bit position is the same for pmd and pud, so we can * Invalid bit position is the same for pmd and pud, so we can
* re-use _pmd_csp() here * re-use _pmd_csp() here
*/ */
__pmdp_csp((pmd_t *) pudp); __pmdp_csp((pmd_t *) pudp);
}
static inline pud_t pudp_flush_direct(struct mm_struct *mm,
unsigned long addr, pud_t *pudp)
{
pud_t old;
old = *pudp;
if (pud_val(old) & _REGION_ENTRY_INVALID)
return old; return old;
}
atomic_inc(&mm->context.flush_count); atomic_inc(&mm->context.flush_count);
if (MACHINE_HAS_TLB_LC && if (MACHINE_HAS_TLB_LC &&
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
__pudp_idte(addr, pudp, IDTE_LOCAL); pudp_idte_local(mm, addr, pudp);
else else
__pudp_idte(addr, pudp, IDTE_GLOBAL); pudp_idte_global(mm, addr, pudp);
atomic_dec(&mm->context.flush_count); atomic_dec(&mm->context.flush_count);
return old; return old;
} }
...@@ -482,7 +565,7 @@ int ptep_force_prot(struct mm_struct *mm, unsigned long addr, ...@@ -482,7 +565,7 @@ int ptep_force_prot(struct mm_struct *mm, unsigned long addr,
{ {
pte_t entry; pte_t entry;
pgste_t pgste; pgste_t pgste;
int pte_i, pte_p; int pte_i, pte_p, nodat;
pgste = pgste_get_lock(ptep); pgste = pgste_get_lock(ptep);
entry = *ptep; entry = *ptep;
...@@ -495,13 +578,14 @@ int ptep_force_prot(struct mm_struct *mm, unsigned long addr, ...@@ -495,13 +578,14 @@ int ptep_force_prot(struct mm_struct *mm, unsigned long addr,
return -EAGAIN; return -EAGAIN;
} }
/* Change access rights and set pgste bit */ /* Change access rights and set pgste bit */
nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
if (prot == PROT_NONE && !pte_i) { if (prot == PROT_NONE && !pte_i) {
ptep_flush_direct(mm, addr, ptep); ptep_flush_direct(mm, addr, ptep, nodat);
pgste = pgste_update_all(entry, pgste, mm); pgste = pgste_update_all(entry, pgste, mm);
pte_val(entry) |= _PAGE_INVALID; pte_val(entry) |= _PAGE_INVALID;
} }
if (prot == PROT_READ && !pte_p) { if (prot == PROT_READ && !pte_p) {
ptep_flush_direct(mm, addr, ptep); ptep_flush_direct(mm, addr, ptep, nodat);
pte_val(entry) &= ~_PAGE_INVALID; pte_val(entry) &= ~_PAGE_INVALID;
pte_val(entry) |= _PAGE_PROTECT; pte_val(entry) |= _PAGE_PROTECT;
} }
...@@ -541,10 +625,12 @@ int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, ...@@ -541,10 +625,12 @@ int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)
{ {
pgste_t pgste; pgste_t pgste;
int nodat;
pgste = pgste_get_lock(ptep); pgste = pgste_get_lock(ptep);
/* notifier is called by the caller */ /* notifier is called by the caller */
ptep_flush_direct(mm, saddr, ptep); nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
ptep_flush_direct(mm, saddr, ptep, nodat);
/* don't touch the storage key - it belongs to parent pgste */ /* don't touch the storage key - it belongs to parent pgste */
pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID)); pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));
pgste_set_unlock(ptep, pgste); pgste_set_unlock(ptep, pgste);
...@@ -617,6 +703,7 @@ bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) ...@@ -617,6 +703,7 @@ bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
pte_t *ptep; pte_t *ptep;
pte_t pte; pte_t pte;
bool dirty; bool dirty;
int nodat;
pgd = pgd_offset(mm, addr); pgd = pgd_offset(mm, addr);
p4d = p4d_alloc(mm, pgd, addr); p4d = p4d_alloc(mm, pgd, addr);
...@@ -645,7 +732,8 @@ bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) ...@@ -645,7 +732,8 @@ bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
pte = *ptep; pte = *ptep;
if (dirty && (pte_val(pte) & _PAGE_PRESENT)) { if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
pgste = pgste_pte_notify(mm, addr, ptep, pgste); pgste = pgste_pte_notify(mm, addr, ptep, pgste);
__ptep_ipte(addr, ptep, IPTE_GLOBAL); nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
ptep_ipte_global(mm, addr, ptep, nodat);
if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE)) if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
pte_val(pte) |= _PAGE_PROTECT; pte_val(pte) |= _PAGE_PROTECT;
else else
......
...@@ -39,7 +39,7 @@ struct read_info_sccb { ...@@ -39,7 +39,7 @@ struct read_info_sccb {
u8 fac84; /* 84 */ u8 fac84; /* 84 */
u8 fac85; /* 85 */ u8 fac85; /* 85 */
u8 _pad_86[91 - 86]; /* 86-90 */ u8 _pad_86[91 - 86]; /* 86-90 */
u8 flags; /* 91 */ u8 fac91; /* 91 */
u8 _pad_92[98 - 92]; /* 92-97 */ u8 _pad_92[98 - 92]; /* 92-97 */
u8 fac98; /* 98 */ u8 fac98; /* 98 */
u8 hamaxpow; /* 99 */ u8 hamaxpow; /* 99 */
...@@ -103,6 +103,8 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb) ...@@ -103,6 +103,8 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb)
sclp.has_kss = !!(sccb->fac98 & 0x01); sclp.has_kss = !!(sccb->fac98 & 0x01);
if (sccb->fac85 & 0x02) if (sccb->fac85 & 0x02)
S390_lowcore.machine_flags |= MACHINE_FLAG_ESOP; S390_lowcore.machine_flags |= MACHINE_FLAG_ESOP;
if (sccb->fac91 & 0x40)
S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_GUEST;
sclp.rnmax = sccb->rnmax ? sccb->rnmax : sccb->rnmax2; sclp.rnmax = sccb->rnmax ? sccb->rnmax : sccb->rnmax2;
sclp.rzm = sccb->rnsize ? sccb->rnsize : sccb->rnsize2; sclp.rzm = sccb->rnsize ? sccb->rnsize : sccb->rnsize2;
sclp.rzm <<= 20; sclp.rzm <<= 20;
...@@ -139,7 +141,7 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb) ...@@ -139,7 +141,7 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb)
/* Save IPL information */ /* Save IPL information */
sclp_ipl_info.is_valid = 1; sclp_ipl_info.is_valid = 1;
if (sccb->flags & 0x2) if (sccb->fac91 & 0x2)
sclp_ipl_info.has_dump = 1; sclp_ipl_info.has_dump = 1;
memcpy(&sclp_ipl_info.loadparm, &sccb->loadparm, LOADPARM_LEN); memcpy(&sclp_ipl_info.loadparm, &sccb->loadparm, LOADPARM_LEN);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment