Commit 3bbf7157 authored by Catalin Marinas's avatar Catalin Marinas

arm64: Convert pte handling from inline asm to using (cmp)xchg

With the support for hardware updates of the access and dirty states,
the following pte handling functions had to be implemented using
exclusives: __ptep_test_and_clear_young(), ptep_get_and_clear(),
ptep_set_wrprotect() and ptep_set_access_flags(). To take advantage of
the LSE atomic instructions and also make the code cleaner, convert
these pte functions to use the more generic cmpxchg()/xchg().
Reviewed-by: default avatarWill Deacon <will.deacon@arm.com>
Acked-by: default avatarMark Rutland <mark.rutland@arm.com>
Acked-by: default avatarSteve Capper <steve.capper@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent a7ba38d6
...@@ -39,6 +39,7 @@ ...@@ -39,6 +39,7 @@
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <asm/cmpxchg.h>
#include <asm/fixmap.h> #include <asm/fixmap.h>
#include <linux/mmdebug.h> #include <linux/mmdebug.h>
...@@ -173,6 +174,11 @@ static inline pte_t pte_clear_rdonly(pte_t pte) ...@@ -173,6 +174,11 @@ static inline pte_t pte_clear_rdonly(pte_t pte)
return clear_pte_bit(pte, __pgprot(PTE_RDONLY)); return clear_pte_bit(pte, __pgprot(PTE_RDONLY));
} }
static inline pte_t pte_set_rdonly(pte_t pte)
{
return set_pte_bit(pte, __pgprot(PTE_RDONLY));
}
static inline pte_t pte_mkpresent(pte_t pte) static inline pte_t pte_mkpresent(pte_t pte)
{ {
return set_pte_bit(pte, __pgprot(PTE_VALID)); return set_pte_bit(pte, __pgprot(PTE_VALID));
...@@ -593,20 +599,17 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, ...@@ -593,20 +599,17 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
static inline int __ptep_test_and_clear_young(pte_t *ptep) static inline int __ptep_test_and_clear_young(pte_t *ptep)
{ {
pteval_t pteval; pte_t old_pte, pte;
unsigned int tmp, res;
asm volatile("// __ptep_test_and_clear_young\n" pte = READ_ONCE(*ptep);
" prfm pstl1strm, %2\n" do {
"1: ldxr %0, %2\n" old_pte = pte;
" ubfx %w3, %w0, %5, #1 // extract PTE_AF (young)\n" pte = pte_mkold(pte);
" and %0, %0, %4 // clear PTE_AF\n" pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
" stxr %w1, %0, %2\n" pte_val(old_pte), pte_val(pte));
" cbnz %w1, 1b\n" } while (pte_val(pte) != pte_val(old_pte));
: "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)), "=&r" (res)
: "L" (~PTE_AF), "I" (ilog2(PTE_AF)));
return res; return pte_young(pte);
} }
static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
...@@ -630,17 +633,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, ...@@ -630,17 +633,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
static inline pte_t ptep_get_and_clear(struct mm_struct *mm, static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
unsigned long address, pte_t *ptep) unsigned long address, pte_t *ptep)
{ {
pteval_t old_pteval; return __pte(xchg_relaxed(&pte_val(*ptep), 0));
unsigned int tmp;
asm volatile("// ptep_get_and_clear\n"
" prfm pstl1strm, %2\n"
"1: ldxr %0, %2\n"
" stxr %w1, xzr, %2\n"
" cbnz %w1, 1b\n"
: "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)));
return __pte(old_pteval);
} }
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
...@@ -659,21 +652,23 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, ...@@ -659,21 +652,23 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
#define __HAVE_ARCH_PTEP_SET_WRPROTECT #define __HAVE_ARCH_PTEP_SET_WRPROTECT
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
{ {
pteval_t pteval; pte_t old_pte, pte;
unsigned long tmp;
pte = READ_ONCE(*ptep);
asm volatile("// ptep_set_wrprotect\n" do {
" prfm pstl1strm, %2\n" old_pte = pte;
"1: ldxr %0, %2\n" /*
" tst %0, %4 // check for hw dirty (!PTE_RDONLY)\n" * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY
" csel %1, %3, xzr, eq // set PTE_DIRTY|PTE_RDONLY if dirty\n" * clear), set the PTE_DIRTY and PTE_RDONLY bits.
" orr %0, %0, %1 // if !dirty, PTE_RDONLY is already set\n" */
" and %0, %0, %5 // clear PTE_WRITE/PTE_DBM\n" if (pte_hw_dirty(pte)) {
" stxr %w1, %0, %2\n" pte = pte_mkdirty(pte);
" cbnz %w1, 1b\n" pte = pte_set_rdonly(pte);
: "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) }
: "r" (PTE_DIRTY|PTE_RDONLY), "L" (PTE_RDONLY), "L" (~PTE_WRITE) pte = pte_wrprotect(pte);
: "cc"); pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
pte_val(old_pte), pte_val(pte));
} while (pte_val(pte) != pte_val(old_pte));
} }
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
......
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
#include <linux/hugetlb.h> #include <linux/hugetlb.h>
#include <asm/bug.h> #include <asm/bug.h>
#include <asm/cmpxchg.h>
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#include <asm/exception.h> #include <asm/exception.h>
#include <asm/debug-monitors.h> #include <asm/debug-monitors.h>
...@@ -197,8 +198,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma, ...@@ -197,8 +198,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep, unsigned long address, pte_t *ptep,
pte_t entry, int dirty) pte_t entry, int dirty)
{ {
pteval_t old_pteval; pteval_t old_pteval, pteval;
unsigned int tmp;
if (pte_same(*ptep, entry)) if (pte_same(*ptep, entry))
return 0; return 0;
...@@ -208,7 +208,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma, ...@@ -208,7 +208,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
/* set PTE_RDONLY if actual read-only or clean PTE */ /* set PTE_RDONLY if actual read-only or clean PTE */
if (!pte_write(entry) || !pte_sw_dirty(entry)) if (!pte_write(entry) || !pte_sw_dirty(entry))
pte_val(entry) |= PTE_RDONLY; entry = pte_set_rdonly(entry);
/* /*
* Setting the flags must be done atomically to avoid racing with the * Setting the flags must be done atomically to avoid racing with the
...@@ -217,16 +217,14 @@ int ptep_set_access_flags(struct vm_area_struct *vma, ...@@ -217,16 +217,14 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
* (calculated as: a & b == ~(~a | ~b)). * (calculated as: a & b == ~(~a | ~b)).
*/ */
pte_val(entry) ^= PTE_RDONLY; pte_val(entry) ^= PTE_RDONLY;
asm volatile("// ptep_set_access_flags\n" pteval = READ_ONCE(pte_val(*ptep));
" prfm pstl1strm, %2\n" do {
"1: ldxr %0, %2\n" old_pteval = pteval;
" eor %0, %0, %3 // negate PTE_RDONLY in *ptep\n" pteval ^= PTE_RDONLY;
" orr %0, %0, %4 // set flags\n" pteval |= pte_val(entry);
" eor %0, %0, %3 // negate final PTE_RDONLY\n" pteval ^= PTE_RDONLY;
" stxr %w1, %0, %2\n" pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
" cbnz %w1, 1b\n" } while (pteval != old_pteval);
: "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
: "L" (PTE_RDONLY), "r" (pte_val(entry)));
flush_tlb_fix_spurious_fault(vma, address); flush_tlb_fix_spurious_fault(vma, address);
return 1; return 1;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment