Commit f992b4df authored by Will Deacon's avatar Will Deacon Committed by Catalin Marinas

arm64: kpti: Add ->enable callback to remap swapper using nG mappings

Defaulting to global mappings for kernel space is generally good for
performance and appears to be necessary for Cavium ThunderX. If we
subsequently decide that we need to enable kpti, then we need to rewrite
our existing page table entries to be non-global. This is fiddly, and
made worse by the possible use of contiguous mappings, which require
a strict break-before-make sequence.

Since the enable callback runs on each online CPU from stop_machine
context, we can have all CPUs enter the idmap, where secondaries can
wait for the primary CPU to rewrite swapper with its MMU off. It's all
fairly horrible, but at least it only runs once.
Tested-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
Reviewed-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 4e602056
......@@ -523,6 +523,16 @@ alternative_endif
#endif
.endm
.macro pte_to_phys, phys, pte
#ifdef CONFIG_ARM64_PA_BITS_52
ubfiz \phys, \pte, #(48 - 16 - 12), #16
bfxil \phys, \pte, #16, #32
lsl \phys, \phys, #16
#else
and \phys, \pte, #PTE_ADDR_MASK
#endif
.endm
/**
* Errata workaround prior to disable MMU. Insert an ISB immediately prior
* to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0.
......
......@@ -880,6 +880,30 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
ID_AA64PFR0_CSV3_SHIFT);
}
static int kpti_install_ng_mappings(void *__unused)
{
typedef void (kpti_remap_fn)(int, int, phys_addr_t);
extern kpti_remap_fn idmap_kpti_install_ng_mappings;
kpti_remap_fn *remap_fn;
static bool kpti_applied = false;
int cpu = smp_processor_id();
if (kpti_applied)
return 0;
remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
cpu_install_idmap();
remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));
cpu_uninstall_idmap();
if (!cpu)
kpti_applied = true;
return 0;
}
static int __init parse_kpti(char *str)
{
bool enabled;
......@@ -1003,6 +1027,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.capability = ARM64_UNMAP_KERNEL_AT_EL0,
.def_scope = SCOPE_SYSTEM,
.matches = unmap_kernel_at_el0,
.enable = kpti_install_ng_mappings,
},
#endif
{
......
......@@ -166,6 +166,17 @@ ENTRY(cpu_do_switch_mm)
ENDPROC(cpu_do_switch_mm)
.pushsection ".idmap.text", "ax"
.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2
adrp \tmp1, empty_zero_page
phys_to_ttbr \tmp1, \tmp2
msr ttbr1_el1, \tmp2
isb
tlbi vmalle1
dsb nsh
isb
.endm
/*
* void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd)
*
......@@ -175,14 +186,7 @@ ENDPROC(cpu_do_switch_mm)
ENTRY(idmap_cpu_replace_ttbr1)
save_and_disable_daif flags=x2
adrp x1, empty_zero_page
phys_to_ttbr x1, x3
msr ttbr1_el1, x3
isb
tlbi vmalle1
dsb nsh
isb
__idmap_cpu_set_reserved_ttbr1 x1, x3
phys_to_ttbr x0, x3
msr ttbr1_el1, x3
......@@ -194,6 +198,190 @@ ENTRY(idmap_cpu_replace_ttbr1)
ENDPROC(idmap_cpu_replace_ttbr1)
.popsection
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
.pushsection ".idmap.text", "ax"
.macro __idmap_kpti_get_pgtable_ent, type
dc cvac, cur_\()\type\()p // Ensure any existing dirty
dmb sy // lines are written back before
ldr \type, [cur_\()\type\()p] // loading the entry
tbz \type, #0, next_\()\type // Skip invalid entries
.endm
.macro __idmap_kpti_put_pgtable_ent_ng, type
orr \type, \type, #PTE_NG // Same bit for blocks and pages
str \type, [cur_\()\type\()p] // Update the entry and ensure it
dc civac, cur_\()\type\()p // is visible to all CPUs.
.endm
/*
* void __kpti_install_ng_mappings(int cpu, int num_cpus, phys_addr_t swapper)
*
* Called exactly once from stop_machine context by each CPU found during boot.
*/
__idmap_kpti_flag:
.long 1
ENTRY(idmap_kpti_install_ng_mappings)
cpu .req w0
num_cpus .req w1
swapper_pa .req x2
swapper_ttb .req x3
flag_ptr .req x4
cur_pgdp .req x5
end_pgdp .req x6
pgd .req x7
cur_pudp .req x8
end_pudp .req x9
pud .req x10
cur_pmdp .req x11
end_pmdp .req x12
pmd .req x13
cur_ptep .req x14
end_ptep .req x15
pte .req x16
mrs swapper_ttb, ttbr1_el1
adr flag_ptr, __idmap_kpti_flag
cbnz cpu, __idmap_kpti_secondary
/* We're the boot CPU. Wait for the others to catch up */
sevl
1: wfe
ldaxr w18, [flag_ptr]
eor w18, w18, num_cpus
cbnz w18, 1b
/* We need to walk swapper, so turn off the MMU. */
pre_disable_mmu_workaround
mrs x18, sctlr_el1
bic x18, x18, #SCTLR_ELx_M
msr sctlr_el1, x18
isb
/* Everybody is enjoying the idmap, so we can rewrite swapper. */
/* PGD */
mov cur_pgdp, swapper_pa
add end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8)
do_pgd: __idmap_kpti_get_pgtable_ent pgd
tbnz pgd, #1, walk_puds
__idmap_kpti_put_pgtable_ent_ng pgd
next_pgd:
add cur_pgdp, cur_pgdp, #8
cmp cur_pgdp, end_pgdp
b.ne do_pgd
/* Publish the updated tables and nuke all the TLBs */
dsb sy
tlbi vmalle1is
dsb ish
isb
/* We're done: fire up the MMU again */
mrs x18, sctlr_el1
orr x18, x18, #SCTLR_ELx_M
msr sctlr_el1, x18
isb
/* Set the flag to zero to indicate that we're all done */
str wzr, [flag_ptr]
ret
/* PUD */
walk_puds:
.if CONFIG_PGTABLE_LEVELS > 3
pte_to_phys cur_pudp, pgd
add end_pudp, cur_pudp, #(PTRS_PER_PUD * 8)
do_pud: __idmap_kpti_get_pgtable_ent pud
tbnz pud, #1, walk_pmds
__idmap_kpti_put_pgtable_ent_ng pud
next_pud:
add cur_pudp, cur_pudp, 8
cmp cur_pudp, end_pudp
b.ne do_pud
b next_pgd
.else /* CONFIG_PGTABLE_LEVELS <= 3 */
mov pud, pgd
b walk_pmds
next_pud:
b next_pgd
.endif
/* PMD */
walk_pmds:
.if CONFIG_PGTABLE_LEVELS > 2
pte_to_phys cur_pmdp, pud
add end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8)
do_pmd: __idmap_kpti_get_pgtable_ent pmd
tbnz pmd, #1, walk_ptes
__idmap_kpti_put_pgtable_ent_ng pmd
next_pmd:
add cur_pmdp, cur_pmdp, #8
cmp cur_pmdp, end_pmdp
b.ne do_pmd
b next_pud
.else /* CONFIG_PGTABLE_LEVELS <= 2 */
mov pmd, pud
b walk_ptes
next_pmd:
b next_pud
.endif
/* PTE */
walk_ptes:
pte_to_phys cur_ptep, pmd
add end_ptep, cur_ptep, #(PTRS_PER_PTE * 8)
do_pte: __idmap_kpti_get_pgtable_ent pte
__idmap_kpti_put_pgtable_ent_ng pte
next_pte:
add cur_ptep, cur_ptep, #8
cmp cur_ptep, end_ptep
b.ne do_pte
b next_pmd
/* Secondary CPUs end up here */
__idmap_kpti_secondary:
/* Uninstall swapper before surgery begins */
__idmap_cpu_set_reserved_ttbr1 x18, x17
/* Increment the flag to let the boot CPU we're ready */
1: ldxr w18, [flag_ptr]
add w18, w18, #1
stxr w17, w18, [flag_ptr]
cbnz w17, 1b
/* Wait for the boot CPU to finish messing around with swapper */
sevl
1: wfe
ldxr w18, [flag_ptr]
cbnz w18, 1b
/* All done, act like nothing happened */
msr ttbr1_el1, swapper_ttb
isb
ret
.unreq cpu
.unreq num_cpus
.unreq swapper_pa
.unreq swapper_ttb
.unreq flag_ptr
.unreq cur_pgdp
.unreq end_pgdp
.unreq pgd
.unreq cur_pudp
.unreq end_pudp
.unreq pud
.unreq cur_pmdp
.unreq end_pmdp
.unreq pmd
.unreq cur_ptep
.unreq end_ptep
.unreq pte
ENDPROC(idmap_kpti_install_ng_mappings)
.popsection
#endif
/*
* __cpu_setup
*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment