Commit 2ce77f6d authored by Will Deacon's avatar Will Deacon Committed by Catalin Marinas

arm64: proc: Set PTE_NG for table entries to avoid traversing them twice

When KASAN is enabled, the swapper page table contains many identical
mappings of the zero page, which can lead to a stall during boot whilst
the G -> nG code continually walks the same page table entries looking
for global mappings.

This patch sets the nG bit (bit 11, which is IGNORED) in table entries
after processing the subtree so we can easily skip them if we see them
a second time.
Tested-by: default avatarMark Rutland <mark.rutland@arm.com>
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 16e574d7
...@@ -205,7 +205,8 @@ ENDPROC(idmap_cpu_replace_ttbr1) ...@@ -205,7 +205,8 @@ ENDPROC(idmap_cpu_replace_ttbr1)
dc cvac, cur_\()\type\()p // Ensure any existing dirty dc cvac, cur_\()\type\()p // Ensure any existing dirty
dmb sy // lines are written back before dmb sy // lines are written back before
ldr \type, [cur_\()\type\()p] // loading the entry ldr \type, [cur_\()\type\()p] // loading the entry
tbz \type, #0, next_\()\type // Skip invalid entries tbz \type, #0, skip_\()\type // Skip invalid and
tbnz \type, #11, skip_\()\type // non-global entries
.endm .endm
.macro __idmap_kpti_put_pgtable_ent_ng, type .macro __idmap_kpti_put_pgtable_ent_ng, type
...@@ -265,8 +266,9 @@ ENTRY(idmap_kpti_install_ng_mappings) ...@@ -265,8 +266,9 @@ ENTRY(idmap_kpti_install_ng_mappings)
add end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8) add end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8)
do_pgd: __idmap_kpti_get_pgtable_ent pgd do_pgd: __idmap_kpti_get_pgtable_ent pgd
tbnz pgd, #1, walk_puds tbnz pgd, #1, walk_puds
__idmap_kpti_put_pgtable_ent_ng pgd
next_pgd: next_pgd:
__idmap_kpti_put_pgtable_ent_ng pgd
skip_pgd:
add cur_pgdp, cur_pgdp, #8 add cur_pgdp, cur_pgdp, #8
cmp cur_pgdp, end_pgdp cmp cur_pgdp, end_pgdp
b.ne do_pgd b.ne do_pgd
...@@ -294,8 +296,9 @@ walk_puds: ...@@ -294,8 +296,9 @@ walk_puds:
add end_pudp, cur_pudp, #(PTRS_PER_PUD * 8) add end_pudp, cur_pudp, #(PTRS_PER_PUD * 8)
do_pud: __idmap_kpti_get_pgtable_ent pud do_pud: __idmap_kpti_get_pgtable_ent pud
tbnz pud, #1, walk_pmds tbnz pud, #1, walk_pmds
__idmap_kpti_put_pgtable_ent_ng pud
next_pud: next_pud:
__idmap_kpti_put_pgtable_ent_ng pud
skip_pud:
add cur_pudp, cur_pudp, 8 add cur_pudp, cur_pudp, 8
cmp cur_pudp, end_pudp cmp cur_pudp, end_pudp
b.ne do_pud b.ne do_pud
...@@ -314,8 +317,9 @@ walk_pmds: ...@@ -314,8 +317,9 @@ walk_pmds:
add end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8) add end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8)
do_pmd: __idmap_kpti_get_pgtable_ent pmd do_pmd: __idmap_kpti_get_pgtable_ent pmd
tbnz pmd, #1, walk_ptes tbnz pmd, #1, walk_ptes
__idmap_kpti_put_pgtable_ent_ng pmd
next_pmd: next_pmd:
__idmap_kpti_put_pgtable_ent_ng pmd
skip_pmd:
add cur_pmdp, cur_pmdp, #8 add cur_pmdp, cur_pmdp, #8
cmp cur_pmdp, end_pmdp cmp cur_pmdp, end_pmdp
b.ne do_pmd b.ne do_pmd
...@@ -333,7 +337,7 @@ walk_ptes: ...@@ -333,7 +337,7 @@ walk_ptes:
add end_ptep, cur_ptep, #(PTRS_PER_PTE * 8) add end_ptep, cur_ptep, #(PTRS_PER_PTE * 8)
do_pte: __idmap_kpti_get_pgtable_ent pte do_pte: __idmap_kpti_get_pgtable_ent pte
__idmap_kpti_put_pgtable_ent_ng pte __idmap_kpti_put_pgtable_ent_ng pte
next_pte: skip_pte:
add cur_ptep, cur_ptep, #8 add cur_ptep, cur_ptep, #8
cmp cur_ptep, end_ptep cmp cur_ptep, end_ptep
b.ne do_pte b.ne do_pte
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment