Commit c0136321 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull more arm64 updates from Catalin Marinas:
 "As I mentioned in the last pull request, there's a second batch of
  security updates for arm64 with mitigations for Spectre/v1 and an
  improved one for Spectre/v2 (via a newly defined firmware interface
  API).

  Spectre v1 mitigation:

   - back-end version of array_index_mask_nospec()

   - masking of the syscall number to restrict speculation through the
     syscall table

   - masking of __user pointers prior to deference in uaccess routines

  Spectre v2 mitigation update:

   - using the new firmware SMC calling convention specification update

   - removing the current PSCI GET_VERSION firmware call mitigation as
     vendors are deploying new SMCCC-capable firmware

   - additional branch predictor hardening for synchronous exceptions
     and interrupts while in user mode

  Meltdown v3 mitigation update:

    - Cavium Thunder X is unaffected but a hardware erratum gets in the
      way. The kernel now starts with the page tables mapped as global
      and switches to non-global if kpti needs to be enabled.

  Other:

   - Theoretical trylock bug fixed"

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (38 commits)
  arm64: Kill PSCI_GET_VERSION as a variant-2 workaround
  arm64: Add ARM_SMCCC_ARCH_WORKAROUND_1 BP hardening support
  arm/arm64: smccc: Implement SMCCC v1.1 inline primitive
  arm/arm64: smccc: Make function identifiers an unsigned quantity
  firmware/psci: Expose SMCCC version through psci_ops
  firmware/psci: Expose PSCI conduit
  arm64: KVM: Add SMCCC_ARCH_WORKAROUND_1 fast handling
  arm64: KVM: Report SMCCC_ARCH_WORKAROUND_1 BP hardening support
  arm/arm64: KVM: Turn kvm_psci_version into a static inline
  arm/arm64: KVM: Advertise SMCCC v1.1
  arm/arm64: KVM: Implement PSCI 1.0 support
  arm/arm64: KVM: Add smccc accessors to PSCI code
  arm/arm64: KVM: Add PSCI_VERSION helper
  arm/arm64: KVM: Consolidate the PSCI include files
  arm64: KVM: Increment PC after handling an SMC trap
  arm: KVM: Fix SMCCC handling of unimplemented SMC/HVC calls
  arm64: KVM: Fix SMCCC handling of unimplemented SMC/HVC calls
  arm64: entry: Apply BP hardening for suspicious interrupts from EL0
  arm64: entry: Apply BP hardening for high-priority synchronous exceptions
  arm64: futex: Mask __user pointers prior to dereference
  ...
parents 846ade7d 3a0a397f
...@@ -306,4 +306,11 @@ static inline void kvm_fpsimd_flush_cpu_state(void) {} ...@@ -306,4 +306,11 @@ static inline void kvm_fpsimd_flush_cpu_state(void) {}
static inline void kvm_arm_vhe_guest_enter(void) {} static inline void kvm_arm_vhe_guest_enter(void) {}
static inline void kvm_arm_vhe_guest_exit(void) {} static inline void kvm_arm_vhe_guest_exit(void) {}
static inline bool kvm_arm_harden_branch_predictor(void)
{
/* No way to detect it yet, pretend it is not there. */
return false;
}
#endif /* __ARM_KVM_HOST_H__ */ #endif /* __ARM_KVM_HOST_H__ */
/*
* Copyright (C) 2012 - ARM Ltd
* Author: Marc Zyngier <marc.zyngier@arm.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ARM_KVM_PSCI_H__
#define __ARM_KVM_PSCI_H__
#define KVM_ARM_PSCI_0_1 1
#define KVM_ARM_PSCI_0_2 2
int kvm_psci_version(struct kvm_vcpu *vcpu);
int kvm_psci_call(struct kvm_vcpu *vcpu);
#endif /* __ARM_KVM_PSCI_H__ */
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
#include <asm/kvm_emulate.h> #include <asm/kvm_emulate.h>
#include <asm/kvm_coproc.h> #include <asm/kvm_coproc.h>
#include <asm/kvm_mmu.h> #include <asm/kvm_mmu.h>
#include <asm/kvm_psci.h> #include <kvm/arm_psci.h>
#include <trace/events/kvm.h> #include <trace/events/kvm.h>
#include "trace.h" #include "trace.h"
...@@ -36,9 +36,9 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) ...@@ -36,9 +36,9 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
kvm_vcpu_hvc_get_imm(vcpu)); kvm_vcpu_hvc_get_imm(vcpu));
vcpu->stat.hvc_exit_stat++; vcpu->stat.hvc_exit_stat++;
ret = kvm_psci_call(vcpu); ret = kvm_hvc_call_handler(vcpu);
if (ret < 0) { if (ret < 0) {
kvm_inject_undefined(vcpu); vcpu_set_reg(vcpu, 0, ~0UL);
return 1; return 1;
} }
...@@ -47,7 +47,16 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) ...@@ -47,7 +47,16 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run) static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
{ {
kvm_inject_undefined(vcpu); /*
* "If an SMC instruction executed at Non-secure EL1 is
* trapped to EL2 because HCR_EL2.TSC is 1, the exception is a
* Trap exception, not a Secure Monitor Call exception [...]"
*
* We need to advance the PC after the trap, as it would
* otherwise return to the same address...
*/
vcpu_set_reg(vcpu, 0, ~0UL);
kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
return 1; return 1;
} }
......
...@@ -115,6 +115,24 @@ ...@@ -115,6 +115,24 @@
hint #16 hint #16
.endm .endm
/*
* Value prediction barrier
*/
.macro csdb
hint #20
.endm
/*
* Sanitise a 64-bit bounded index wrt speculation, returning zero if out
* of bounds.
*/
.macro mask_nospec64, idx, limit, tmp
sub \tmp, \idx, \limit
bic \tmp, \tmp, \idx
and \idx, \idx, \tmp, asr #63
csdb
.endm
/* /*
* NOP sequence * NOP sequence
*/ */
...@@ -514,7 +532,7 @@ alternative_endif ...@@ -514,7 +532,7 @@ alternative_endif
* phys: physical address, preserved * phys: physical address, preserved
* ttbr: returns the TTBR value * ttbr: returns the TTBR value
*/ */
.macro phys_to_ttbr, phys, ttbr .macro phys_to_ttbr, ttbr, phys
#ifdef CONFIG_ARM64_PA_BITS_52 #ifdef CONFIG_ARM64_PA_BITS_52
orr \ttbr, \phys, \phys, lsr #46 orr \ttbr, \phys, \phys, lsr #46
and \ttbr, \ttbr, #TTBR_BADDR_MASK_52 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52
...@@ -523,6 +541,29 @@ alternative_endif ...@@ -523,6 +541,29 @@ alternative_endif
#endif #endif
.endm .endm
.macro phys_to_pte, pte, phys
#ifdef CONFIG_ARM64_PA_BITS_52
/*
* We assume \phys is 64K aligned and this is guaranteed by only
* supporting this configuration with 64K pages.
*/
orr \pte, \phys, \phys, lsr #36
and \pte, \pte, #PTE_ADDR_MASK
#else
mov \pte, \phys
#endif
.endm
.macro pte_to_phys, phys, pte
#ifdef CONFIG_ARM64_PA_BITS_52
ubfiz \phys, \pte, #(48 - 16 - 12), #16
bfxil \phys, \pte, #16, #32
lsl \phys, \phys, #16
#else
and \phys, \pte, #PTE_ADDR_MASK
#endif
.endm
/** /**
* Errata workaround prior to disable MMU. Insert an ISB immediately prior * Errata workaround prior to disable MMU. Insert an ISB immediately prior
* to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0. * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0.
......
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
#define dsb(opt) asm volatile("dsb " #opt : : : "memory") #define dsb(opt) asm volatile("dsb " #opt : : : "memory")
#define psb_csync() asm volatile("hint #17" : : : "memory") #define psb_csync() asm volatile("hint #17" : : : "memory")
#define csdb() asm volatile("hint #20" : : : "memory")
#define mb() dsb(sy) #define mb() dsb(sy)
#define rmb() dsb(ld) #define rmb() dsb(ld)
...@@ -40,6 +41,27 @@ ...@@ -40,6 +41,27 @@
#define dma_rmb() dmb(oshld) #define dma_rmb() dmb(oshld)
#define dma_wmb() dmb(oshst) #define dma_wmb() dmb(oshst)
/*
* Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz
* and 0 otherwise.
*/
#define array_index_mask_nospec array_index_mask_nospec
static inline unsigned long array_index_mask_nospec(unsigned long idx,
unsigned long sz)
{
unsigned long mask;
asm volatile(
" cmp %1, %2\n"
" sbc %0, xzr, xzr\n"
: "=r" (mask)
: "r" (idx), "Ir" (sz)
: "cc");
csdb();
return mask;
}
#define __smp_mb() dmb(ish) #define __smp_mb() dmb(ish)
#define __smp_rmb() dmb(ishld) #define __smp_rmb() dmb(ishld)
#define __smp_wmb() dmb(ishst) #define __smp_wmb() dmb(ishst)
......
...@@ -48,9 +48,10 @@ do { \ ...@@ -48,9 +48,10 @@ do { \
} while (0) } while (0)
static inline int static inline int
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)
{ {
int oldval = 0, ret, tmp; int oldval = 0, ret, tmp;
u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);
pagefault_disable(); pagefault_disable();
...@@ -88,15 +89,17 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) ...@@ -88,15 +89,17 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
} }
static inline int static inline int
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
u32 oldval, u32 newval) u32 oldval, u32 newval)
{ {
int ret = 0; int ret = 0;
u32 val, tmp; u32 val, tmp;
u32 __user *uaddr;
if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) if (!access_ok(VERIFY_WRITE, _uaddr, sizeof(u32)))
return -EFAULT; return -EFAULT;
uaddr = __uaccess_mask_ptr(_uaddr);
uaccess_enable(); uaccess_enable();
asm volatile("// futex_atomic_cmpxchg_inatomic\n" asm volatile("// futex_atomic_cmpxchg_inatomic\n"
" prfm pstl1strm, %2\n" " prfm pstl1strm, %2\n"
......
...@@ -123,16 +123,8 @@ ...@@ -123,16 +123,8 @@
/* /*
* Initial memory map attributes. * Initial memory map attributes.
*/ */
#define _SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) #define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
#define _SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) #define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
#define SWAPPER_PTE_FLAGS (_SWAPPER_PTE_FLAGS | PTE_NG)
#define SWAPPER_PMD_FLAGS (_SWAPPER_PMD_FLAGS | PMD_SECT_NG)
#else
#define SWAPPER_PTE_FLAGS _SWAPPER_PTE_FLAGS
#define SWAPPER_PMD_FLAGS _SWAPPER_PMD_FLAGS
#endif
#if ARM64_SWAPPER_USES_SECTION_MAPS #if ARM64_SWAPPER_USES_SECTION_MAPS
#define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS) #define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
......
...@@ -415,4 +415,10 @@ static inline void kvm_arm_vhe_guest_exit(void) ...@@ -415,4 +415,10 @@ static inline void kvm_arm_vhe_guest_exit(void)
{ {
local_daif_restore(DAIF_PROCCTX_NOIRQ); local_daif_restore(DAIF_PROCCTX_NOIRQ);
} }
static inline bool kvm_arm_harden_branch_predictor(void)
{
return cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR);
}
#endif /* __ARM64_KVM_HOST_H__ */ #endif /* __ARM64_KVM_HOST_H__ */
...@@ -37,13 +37,11 @@ ...@@ -37,13 +37,11 @@
#define _PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) #define _PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
#define _PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) #define _PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 #define PTE_MAYBE_NG (arm64_kernel_unmapped_at_el0() ? PTE_NG : 0)
#define PROT_DEFAULT (_PROT_DEFAULT | PTE_NG) #define PMD_MAYBE_NG (arm64_kernel_unmapped_at_el0() ? PMD_SECT_NG : 0)
#define PROT_SECT_DEFAULT (_PROT_SECT_DEFAULT | PMD_SECT_NG)
#else #define PROT_DEFAULT (_PROT_DEFAULT | PTE_MAYBE_NG)
#define PROT_DEFAULT _PROT_DEFAULT #define PROT_SECT_DEFAULT (_PROT_SECT_DEFAULT | PMD_MAYBE_NG)
#define PROT_SECT_DEFAULT _PROT_SECT_DEFAULT
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
#define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE)) #define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE))
#define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE)) #define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE))
...@@ -55,22 +53,22 @@ ...@@ -55,22 +53,22 @@
#define PROT_SECT_NORMAL (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL)) #define PROT_SECT_NORMAL (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
#define PROT_SECT_NORMAL_EXEC (PROT_SECT_DEFAULT | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL)) #define PROT_SECT_NORMAL_EXEC (PROT_SECT_DEFAULT | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
#define _PAGE_DEFAULT (PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL)) #define _PAGE_DEFAULT (_PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL))
#define _HYP_PAGE_DEFAULT (_PAGE_DEFAULT & ~PTE_NG) #define _HYP_PAGE_DEFAULT _PAGE_DEFAULT
#define PAGE_KERNEL __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE) #define PAGE_KERNEL __pgprot(PROT_NORMAL)
#define PAGE_KERNEL_RO __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_RDONLY) #define PAGE_KERNEL_RO __pgprot((PROT_NORMAL & ~PTE_WRITE) | PTE_RDONLY)
#define PAGE_KERNEL_ROX __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_RDONLY) #define PAGE_KERNEL_ROX __pgprot((PROT_NORMAL & ~(PTE_WRITE | PTE_PXN)) | PTE_RDONLY)
#define PAGE_KERNEL_EXEC __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE) #define PAGE_KERNEL_EXEC __pgprot(PROT_NORMAL & ~PTE_PXN)
#define PAGE_KERNEL_EXEC_CONT __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_CONT) #define PAGE_KERNEL_EXEC_CONT __pgprot((PROT_NORMAL & ~PTE_PXN) | PTE_CONT)
#define PAGE_HYP __pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_HYP_XN) #define PAGE_HYP __pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_HYP_XN)
#define PAGE_HYP_EXEC __pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY) #define PAGE_HYP_EXEC __pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY)
#define PAGE_HYP_RO __pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY | PTE_HYP_XN) #define PAGE_HYP_RO __pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY | PTE_HYP_XN)
#define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP) #define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP)
#define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY) #define PAGE_S2 __pgprot(_PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)
#define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN) #define PAGE_S2_DEVICE __pgprot(_PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN)
#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN) #define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN)
#define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE) #define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)
......
...@@ -21,6 +21,9 @@ ...@@ -21,6 +21,9 @@
#define TASK_SIZE_64 (UL(1) << VA_BITS) #define TASK_SIZE_64 (UL(1) << VA_BITS)
#define KERNEL_DS UL(-1)
#define USER_DS (TASK_SIZE_64 - 1)
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
/* /*
......
...@@ -87,8 +87,8 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock) ...@@ -87,8 +87,8 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock)
" cbnz %w1, 1f\n" " cbnz %w1, 1f\n"
" add %w1, %w0, %3\n" " add %w1, %w0, %3\n"
" casa %w0, %w1, %2\n" " casa %w0, %w1, %2\n"
" and %w1, %w1, #0xffff\n" " sub %w1, %w1, %3\n"
" eor %w1, %w1, %w0, lsr #16\n" " eor %w1, %w1, %w0\n"
"1:") "1:")
: "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
: "I" (1 << TICKET_SHIFT) : "I" (1 << TICKET_SHIFT)
......
...@@ -35,16 +35,20 @@ ...@@ -35,16 +35,20 @@
#include <asm/compiler.h> #include <asm/compiler.h>
#include <asm/extable.h> #include <asm/extable.h>
#define KERNEL_DS (-1UL)
#define get_ds() (KERNEL_DS) #define get_ds() (KERNEL_DS)
#define USER_DS TASK_SIZE_64
#define get_fs() (current_thread_info()->addr_limit) #define get_fs() (current_thread_info()->addr_limit)
static inline void set_fs(mm_segment_t fs) static inline void set_fs(mm_segment_t fs)
{ {
current_thread_info()->addr_limit = fs; current_thread_info()->addr_limit = fs;
/*
* Prevent a mispredicted conditional call to set_fs from forwarding
* the wrong address limit to access_ok under speculation.
*/
dsb(nsh);
isb();
/* On user-mode return, check fs is correct */ /* On user-mode return, check fs is correct */
set_thread_flag(TIF_FSCHECK); set_thread_flag(TIF_FSCHECK);
...@@ -66,22 +70,32 @@ static inline void set_fs(mm_segment_t fs) ...@@ -66,22 +70,32 @@ static inline void set_fs(mm_segment_t fs)
* Returns 1 if the range is valid, 0 otherwise. * Returns 1 if the range is valid, 0 otherwise.
* *
* This is equivalent to the following test: * This is equivalent to the following test:
* (u65)addr + (u65)size <= current->addr_limit * (u65)addr + (u65)size <= (u65)current->addr_limit + 1
*
* This needs 65-bit arithmetic.
*/ */
#define __range_ok(addr, size) \ static inline unsigned long __range_ok(unsigned long addr, unsigned long size)
({ \ {
unsigned long __addr = (unsigned long)(addr); \ unsigned long limit = current_thread_info()->addr_limit;
unsigned long flag, roksum; \
__chk_user_ptr(addr); \ __chk_user_ptr(addr);
asm("adds %1, %1, %3; ccmp %1, %4, #2, cc; cset %0, ls" \ asm volatile(
: "=&r" (flag), "=&r" (roksum) \ // A + B <= C + 1 for all A,B,C, in four easy steps:
: "1" (__addr), "Ir" (size), \ // 1: X = A + B; X' = X % 2^64
"r" (current_thread_info()->addr_limit) \ " adds %0, %0, %2\n"
: "cc"); \ // 2: Set C = 0 if X > 2^64, to guarantee X' > C in step 4
flag; \ " csel %1, xzr, %1, hi\n"
}) // 3: Set X' = ~0 if X >= 2^64. For X == 2^64, this decrements X'
// to compensate for the carry flag being set in step 4. For
// X > 2^64, X' merely has to remain nonzero, which it does.
" csinv %0, %0, xzr, cc\n"
// 4: For X < 2^64, this gives us X' - C - 1 <= 0, where the -1
// comes from the carry in being clear. Otherwise, we are
// testing X' - C == 0, subject to the previous adjustments.
" sbcs xzr, %0, %1\n"
" cset %0, ls\n"
: "+r" (addr), "+r" (limit) : "Ir" (size) : "cc");
return addr;
}
/* /*
* When dealing with data aborts, watchpoints, or instruction traps we may end * When dealing with data aborts, watchpoints, or instruction traps we may end
...@@ -90,7 +104,7 @@ static inline void set_fs(mm_segment_t fs) ...@@ -90,7 +104,7 @@ static inline void set_fs(mm_segment_t fs)
*/ */
#define untagged_addr(addr) sign_extend64(addr, 55) #define untagged_addr(addr) sign_extend64(addr, 55)
#define access_ok(type, addr, size) __range_ok(addr, size) #define access_ok(type, addr, size) __range_ok((unsigned long)(addr), size)
#define user_addr_max get_fs #define user_addr_max get_fs
#define _ASM_EXTABLE(from, to) \ #define _ASM_EXTABLE(from, to) \
...@@ -220,6 +234,26 @@ static inline void uaccess_enable_not_uao(void) ...@@ -220,6 +234,26 @@ static inline void uaccess_enable_not_uao(void)
__uaccess_enable(ARM64_ALT_PAN_NOT_UAO); __uaccess_enable(ARM64_ALT_PAN_NOT_UAO);
} }
/*
* Sanitise a uaccess pointer such that it becomes NULL if above the
* current addr_limit.
*/
#define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr)
static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
{
void __user *safe_ptr;
asm volatile(
" bics xzr, %1, %2\n"
" csel %0, %1, xzr, eq\n"
: "=&r" (safe_ptr)
: "r" (ptr), "r" (current_thread_info()->addr_limit)
: "cc");
csdb();
return safe_ptr;
}
/* /*
* The "__xxx" versions of the user access functions do not verify the address * The "__xxx" versions of the user access functions do not verify the address
* space - it must have been done previously with a separate "access_ok()" * space - it must have been done previously with a separate "access_ok()"
...@@ -272,28 +306,33 @@ do { \ ...@@ -272,28 +306,33 @@ do { \
(x) = (__force __typeof__(*(ptr)))__gu_val; \ (x) = (__force __typeof__(*(ptr)))__gu_val; \
} while (0) } while (0)
#define __get_user(x, ptr) \ #define __get_user_check(x, ptr, err) \
({ \ ({ \
int __gu_err = 0; \ __typeof__(*(ptr)) __user *__p = (ptr); \
__get_user_err((x), (ptr), __gu_err); \ might_fault(); \
__gu_err; \ if (access_ok(VERIFY_READ, __p, sizeof(*__p))) { \
__p = uaccess_mask_ptr(__p); \
__get_user_err((x), __p, (err)); \
} else { \
(x) = 0; (err) = -EFAULT; \
} \
}) })
#define __get_user_error(x, ptr, err) \ #define __get_user_error(x, ptr, err) \
({ \ ({ \
__get_user_err((x), (ptr), (err)); \ __get_user_check((x), (ptr), (err)); \
(void)0; \ (void)0; \
}) })
#define get_user(x, ptr) \ #define __get_user(x, ptr) \
({ \ ({ \
__typeof__(*(ptr)) __user *__p = (ptr); \ int __gu_err = 0; \
might_fault(); \ __get_user_check((x), (ptr), __gu_err); \
access_ok(VERIFY_READ, __p, sizeof(*__p)) ? \ __gu_err; \
__get_user((x), __p) : \
((x) = 0, -EFAULT); \
}) })
#define get_user __get_user
#define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature) \ #define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature) \
asm volatile( \ asm volatile( \
"1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \ "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \
...@@ -336,43 +375,63 @@ do { \ ...@@ -336,43 +375,63 @@ do { \
uaccess_disable_not_uao(); \ uaccess_disable_not_uao(); \
} while (0) } while (0)
#define __put_user(x, ptr) \ #define __put_user_check(x, ptr, err) \
({ \ ({ \
int __pu_err = 0; \ __typeof__(*(ptr)) __user *__p = (ptr); \
__put_user_err((x), (ptr), __pu_err); \ might_fault(); \
__pu_err; \ if (access_ok(VERIFY_WRITE, __p, sizeof(*__p))) { \
__p = uaccess_mask_ptr(__p); \
__put_user_err((x), __p, (err)); \
} else { \
(err) = -EFAULT; \
} \
}) })
#define __put_user_error(x, ptr, err) \ #define __put_user_error(x, ptr, err) \
({ \ ({ \
__put_user_err((x), (ptr), (err)); \ __put_user_check((x), (ptr), (err)); \
(void)0; \ (void)0; \
}) })
#define put_user(x, ptr) \ #define __put_user(x, ptr) \
({ \ ({ \
__typeof__(*(ptr)) __user *__p = (ptr); \ int __pu_err = 0; \
might_fault(); \ __put_user_check((x), (ptr), __pu_err); \
access_ok(VERIFY_WRITE, __p, sizeof(*__p)) ? \ __pu_err; \
__put_user((x), __p) : \
-EFAULT; \
}) })
#define put_user __put_user
extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
#define raw_copy_from_user __arch_copy_from_user #define raw_copy_from_user(to, from, n) \
({ \
__arch_copy_from_user((to), __uaccess_mask_ptr(from), (n)); \
})
extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n); extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
#define raw_copy_to_user __arch_copy_to_user #define raw_copy_to_user(to, from, n) \
extern unsigned long __must_check raw_copy_in_user(void __user *to, const void __user *from, unsigned long n); ({ \
extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n); __arch_copy_to_user(__uaccess_mask_ptr(to), (from), (n)); \
})
extern unsigned long __must_check __arch_copy_in_user(void __user *to, const void __user *from, unsigned long n);
#define raw_copy_in_user(to, from, n) \
({ \
__arch_copy_in_user(__uaccess_mask_ptr(to), \
__uaccess_mask_ptr(from), (n)); \
})
#define INLINE_COPY_TO_USER #define INLINE_COPY_TO_USER
#define INLINE_COPY_FROM_USER #define INLINE_COPY_FROM_USER
static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n);
static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n)
{ {
if (access_ok(VERIFY_WRITE, to, n)) if (access_ok(VERIFY_WRITE, to, n))
n = __clear_user(to, n); n = __arch_clear_user(__uaccess_mask_ptr(to), n);
return n; return n;
} }
#define clear_user __clear_user
extern long strncpy_from_user(char *dest, const char __user *src, long count); extern long strncpy_from_user(char *dest, const char __user *src, long count);
...@@ -386,7 +445,7 @@ extern unsigned long __must_check __copy_user_flushcache(void *to, const void __ ...@@ -386,7 +445,7 @@ extern unsigned long __must_check __copy_user_flushcache(void *to, const void __
static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
{ {
kasan_check_write(dst, size); kasan_check_write(dst, size);
return __copy_user_flushcache(dst, src, size); return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
} }
#endif #endif
......
...@@ -37,8 +37,8 @@ EXPORT_SYMBOL(clear_page); ...@@ -37,8 +37,8 @@ EXPORT_SYMBOL(clear_page);
/* user mem (segment) */ /* user mem (segment) */
EXPORT_SYMBOL(__arch_copy_from_user); EXPORT_SYMBOL(__arch_copy_from_user);
EXPORT_SYMBOL(__arch_copy_to_user); EXPORT_SYMBOL(__arch_copy_to_user);
EXPORT_SYMBOL(__clear_user); EXPORT_SYMBOL(__arch_clear_user);
EXPORT_SYMBOL(raw_copy_in_user); EXPORT_SYMBOL(__arch_copy_in_user);
/* physical memory */ /* physical memory */
EXPORT_SYMBOL(memstart_addr); EXPORT_SYMBOL(memstart_addr);
......
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
*/ */
#include <linux/linkage.h> #include <linux/linkage.h>
#include <linux/arm-smccc.h>
.macro ventry target .macro ventry target
.rept 31 .rept 31
...@@ -53,30 +54,6 @@ ENTRY(__bp_harden_hyp_vecs_start) ...@@ -53,30 +54,6 @@ ENTRY(__bp_harden_hyp_vecs_start)
vectors __kvm_hyp_vector vectors __kvm_hyp_vector
.endr .endr
ENTRY(__bp_harden_hyp_vecs_end) ENTRY(__bp_harden_hyp_vecs_end)
ENTRY(__psci_hyp_bp_inval_start)
sub sp, sp, #(8 * 18)
stp x16, x17, [sp, #(16 * 0)]
stp x14, x15, [sp, #(16 * 1)]
stp x12, x13, [sp, #(16 * 2)]
stp x10, x11, [sp, #(16 * 3)]
stp x8, x9, [sp, #(16 * 4)]
stp x6, x7, [sp, #(16 * 5)]
stp x4, x5, [sp, #(16 * 6)]
stp x2, x3, [sp, #(16 * 7)]
stp x0, x1, [sp, #(16 * 8)]
mov x0, #0x84000000
smc #0
ldp x16, x17, [sp, #(16 * 0)]
ldp x14, x15, [sp, #(16 * 1)]
ldp x12, x13, [sp, #(16 * 2)]
ldp x10, x11, [sp, #(16 * 3)]
ldp x8, x9, [sp, #(16 * 4)]
ldp x6, x7, [sp, #(16 * 5)]
ldp x4, x5, [sp, #(16 * 6)]
ldp x2, x3, [sp, #(16 * 7)]
ldp x0, x1, [sp, #(16 * 8)]
add sp, sp, #(8 * 18)
ENTRY(__psci_hyp_bp_inval_end)
ENTRY(__qcom_hyp_sanitize_link_stack_start) ENTRY(__qcom_hyp_sanitize_link_stack_start)
stp x29, x30, [sp, #-16]! stp x29, x30, [sp, #-16]!
...@@ -85,3 +62,22 @@ ENTRY(__qcom_hyp_sanitize_link_stack_start) ...@@ -85,3 +62,22 @@ ENTRY(__qcom_hyp_sanitize_link_stack_start)
.endr .endr
ldp x29, x30, [sp], #16 ldp x29, x30, [sp], #16
ENTRY(__qcom_hyp_sanitize_link_stack_end) ENTRY(__qcom_hyp_sanitize_link_stack_end)
.macro smccc_workaround_1 inst
sub sp, sp, #(8 * 4)
stp x2, x3, [sp, #(8 * 0)]
stp x0, x1, [sp, #(8 * 2)]
mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1
\inst #0
ldp x2, x3, [sp, #(8 * 0)]
ldp x0, x1, [sp, #(8 * 2)]
add sp, sp, #(8 * 4)
.endm
ENTRY(__smccc_workaround_1_smc_start)
smccc_workaround_1 smc
ENTRY(__smccc_workaround_1_smc_end)
ENTRY(__smccc_workaround_1_hvc_start)
smccc_workaround_1 hvc
ENTRY(__smccc_workaround_1_hvc_end)
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
#include <asm/virt.h> #include <asm/virt.h>
.text .text
.pushsection .idmap.text, "ax" .pushsection .idmap.text, "awx"
/* /*
* __cpu_soft_restart(el2_switch, entry, arg0, arg1, arg2) - Helper for * __cpu_soft_restart(el2_switch, entry, arg0, arg1, arg2) - Helper for
......
...@@ -67,9 +67,12 @@ static int cpu_enable_trap_ctr_access(void *__unused) ...@@ -67,9 +67,12 @@ static int cpu_enable_trap_ctr_access(void *__unused)
DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data); DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
#ifdef CONFIG_KVM #ifdef CONFIG_KVM
extern char __psci_hyp_bp_inval_start[], __psci_hyp_bp_inval_end[];
extern char __qcom_hyp_sanitize_link_stack_start[]; extern char __qcom_hyp_sanitize_link_stack_start[];
extern char __qcom_hyp_sanitize_link_stack_end[]; extern char __qcom_hyp_sanitize_link_stack_end[];
extern char __smccc_workaround_1_smc_start[];
extern char __smccc_workaround_1_smc_end[];
extern char __smccc_workaround_1_hvc_start[];
extern char __smccc_workaround_1_hvc_end[];
static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start, static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
const char *hyp_vecs_end) const char *hyp_vecs_end)
...@@ -112,10 +115,12 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn, ...@@ -112,10 +115,12 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
spin_unlock(&bp_lock); spin_unlock(&bp_lock);
} }
#else #else
#define __psci_hyp_bp_inval_start NULL
#define __psci_hyp_bp_inval_end NULL
#define __qcom_hyp_sanitize_link_stack_start NULL #define __qcom_hyp_sanitize_link_stack_start NULL
#define __qcom_hyp_sanitize_link_stack_end NULL #define __qcom_hyp_sanitize_link_stack_end NULL
#define __smccc_workaround_1_smc_start NULL
#define __smccc_workaround_1_smc_end NULL
#define __smccc_workaround_1_hvc_start NULL
#define __smccc_workaround_1_hvc_end NULL
static void __install_bp_hardening_cb(bp_hardening_cb_t fn, static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
const char *hyp_vecs_start, const char *hyp_vecs_start,
...@@ -142,17 +147,59 @@ static void install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry, ...@@ -142,17 +147,59 @@ static void install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry,
__install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end); __install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end);
} }
#include <uapi/linux/psci.h>
#include <linux/arm-smccc.h>
#include <linux/psci.h> #include <linux/psci.h>
static int enable_psci_bp_hardening(void *data) static void call_smc_arch_workaround_1(void)
{
arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
}
static void call_hvc_arch_workaround_1(void)
{
arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
}
static int enable_smccc_arch_workaround_1(void *data)
{ {
const struct arm64_cpu_capabilities *entry = data; const struct arm64_cpu_capabilities *entry = data;
bp_hardening_cb_t cb;
void *smccc_start, *smccc_end;
struct arm_smccc_res res;
if (!entry->matches(entry, SCOPE_LOCAL_CPU))
return 0;
if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
return 0;
switch (psci_ops.conduit) {
case PSCI_CONDUIT_HVC:
arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
ARM_SMCCC_ARCH_WORKAROUND_1, &res);
if (res.a0)
return 0;
cb = call_hvc_arch_workaround_1;
smccc_start = __smccc_workaround_1_hvc_start;
smccc_end = __smccc_workaround_1_hvc_end;
break;
case PSCI_CONDUIT_SMC:
arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
ARM_SMCCC_ARCH_WORKAROUND_1, &res);
if (res.a0)
return 0;
cb = call_smc_arch_workaround_1;
smccc_start = __smccc_workaround_1_smc_start;
smccc_end = __smccc_workaround_1_smc_end;
break;
default:
return 0;
}
if (psci_ops.get_version) install_bp_hardening_cb(entry, cb, smccc_start, smccc_end);
install_bp_hardening_cb(entry,
(bp_hardening_cb_t)psci_ops.get_version,
__psci_hyp_bp_inval_start,
__psci_hyp_bp_inval_end);
return 0; return 0;
} }
...@@ -333,22 +380,22 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ...@@ -333,22 +380,22 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
{ {
.capability = ARM64_HARDEN_BRANCH_PREDICTOR, .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
.enable = enable_psci_bp_hardening, .enable = enable_smccc_arch_workaround_1,
}, },
{ {
.capability = ARM64_HARDEN_BRANCH_PREDICTOR, .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
.enable = enable_psci_bp_hardening, .enable = enable_smccc_arch_workaround_1,
}, },
{ {
.capability = ARM64_HARDEN_BRANCH_PREDICTOR, .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
.enable = enable_psci_bp_hardening, .enable = enable_smccc_arch_workaround_1,
}, },
{ {
.capability = ARM64_HARDEN_BRANCH_PREDICTOR, .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_CORTEX_A75), MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
.enable = enable_psci_bp_hardening, .enable = enable_smccc_arch_workaround_1,
}, },
{ {
.capability = ARM64_HARDEN_BRANCH_PREDICTOR, .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
...@@ -362,12 +409,12 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ...@@ -362,12 +409,12 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
{ {
.capability = ARM64_HARDEN_BRANCH_PREDICTOR, .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN), MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
.enable = enable_psci_bp_hardening, .enable = enable_smccc_arch_workaround_1,
}, },
{ {
.capability = ARM64_HARDEN_BRANCH_PREDICTOR, .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2), MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
.enable = enable_psci_bp_hardening, .enable = enable_smccc_arch_workaround_1,
}, },
#endif #endif
{ {
......
...@@ -856,12 +856,23 @@ static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */ ...@@ -856,12 +856,23 @@ static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry, static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
int __unused) int __unused)
{ {
char const *str = "command line option";
u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
/* Forced on command line? */ /*
* For reasons that aren't entirely clear, enabling KPTI on Cavium
* ThunderX leads to apparent I-cache corruption of kernel text, which
* ends as well as you might imagine. Don't even try.
*/
if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_27456)) {
str = "ARM64_WORKAROUND_CAVIUM_27456";
__kpti_forced = -1;
}
/* Forced? */
if (__kpti_forced) { if (__kpti_forced) {
pr_info_once("kernel page table isolation forced %s by command line option\n", pr_info_once("kernel page table isolation forced %s by %s\n",
__kpti_forced > 0 ? "ON" : "OFF"); __kpti_forced > 0 ? "ON" : "OFF", str);
return __kpti_forced > 0; return __kpti_forced > 0;
} }
...@@ -881,6 +892,30 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry, ...@@ -881,6 +892,30 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
ID_AA64PFR0_CSV3_SHIFT); ID_AA64PFR0_CSV3_SHIFT);
} }
static int kpti_install_ng_mappings(void *__unused)
{
typedef void (kpti_remap_fn)(int, int, phys_addr_t);
extern kpti_remap_fn idmap_kpti_install_ng_mappings;
kpti_remap_fn *remap_fn;
static bool kpti_applied = false;
int cpu = smp_processor_id();
if (kpti_applied)
return 0;
remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
cpu_install_idmap();
remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));
cpu_uninstall_idmap();
if (!cpu)
kpti_applied = true;
return 0;
}
static int __init parse_kpti(char *str) static int __init parse_kpti(char *str)
{ {
bool enabled; bool enabled;
...@@ -1004,6 +1039,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { ...@@ -1004,6 +1039,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.capability = ARM64_UNMAP_KERNEL_AT_EL0, .capability = ARM64_UNMAP_KERNEL_AT_EL0,
.def_scope = SCOPE_SYSTEM, .def_scope = SCOPE_SYSTEM,
.matches = unmap_kernel_at_el0, .matches = unmap_kernel_at_el0,
.enable = kpti_install_ng_mappings,
}, },
#endif #endif
{ {
......
...@@ -167,10 +167,10 @@ alternative_else_nop_endif ...@@ -167,10 +167,10 @@ alternative_else_nop_endif
.else .else
add x21, sp, #S_FRAME_SIZE add x21, sp, #S_FRAME_SIZE
get_thread_info tsk get_thread_info tsk
/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */ /* Save the task's original addr_limit and set USER_DS */
ldr x20, [tsk, #TSK_TI_ADDR_LIMIT] ldr x20, [tsk, #TSK_TI_ADDR_LIMIT]
str x20, [sp, #S_ORIG_ADDR_LIMIT] str x20, [sp, #S_ORIG_ADDR_LIMIT]
mov x20, #TASK_SIZE_64 mov x20, #USER_DS
str x20, [tsk, #TSK_TI_ADDR_LIMIT] str x20, [tsk, #TSK_TI_ADDR_LIMIT]
/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */ /* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
.endif /* \el == 0 */ .endif /* \el == 0 */
...@@ -382,6 +382,7 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 ...@@ -382,6 +382,7 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
* x7 is reserved for the system call number in 32-bit mode. * x7 is reserved for the system call number in 32-bit mode.
*/ */
wsc_nr .req w25 // number of system calls wsc_nr .req w25 // number of system calls
xsc_nr .req x25 // number of system calls (zero-extended)
wscno .req w26 // syscall number wscno .req w26 // syscall number
xscno .req x26 // syscall number (zero-extended) xscno .req x26 // syscall number (zero-extended)
stbl .req x27 // syscall table pointer stbl .req x27 // syscall table pointer
...@@ -770,7 +771,10 @@ el0_sp_pc: ...@@ -770,7 +771,10 @@ el0_sp_pc:
* Stack or PC alignment exception handling * Stack or PC alignment exception handling
*/ */
mrs x26, far_el1 mrs x26, far_el1
enable_daif enable_da_f
#ifdef CONFIG_TRACE_IRQFLAGS
bl trace_hardirqs_off
#endif
ct_user_exit ct_user_exit
mov x0, x26 mov x0, x26
mov x1, x25 mov x1, x25
...@@ -828,6 +832,11 @@ el0_irq_naked: ...@@ -828,6 +832,11 @@ el0_irq_naked:
#endif #endif
ct_user_exit ct_user_exit
#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
tbz x22, #55, 1f
bl do_el0_irq_bp_hardening
1:
#endif
irq_handler irq_handler
#ifdef CONFIG_TRACE_IRQFLAGS #ifdef CONFIG_TRACE_IRQFLAGS
...@@ -939,6 +948,7 @@ el0_svc_naked: // compat entry point ...@@ -939,6 +948,7 @@ el0_svc_naked: // compat entry point
b.ne __sys_trace b.ne __sys_trace
cmp wscno, wsc_nr // check upper syscall limit cmp wscno, wsc_nr // check upper syscall limit
b.hs ni_sys b.hs ni_sys
mask_nospec64 xscno, xsc_nr, x19 // enforce bounds for syscall number
ldr x16, [stbl, xscno, lsl #3] // address in the syscall table ldr x16, [stbl, xscno, lsl #3] // address in the syscall table
blr x16 // call sys_* routine blr x16 // call sys_* routine
b ret_fast_syscall b ret_fast_syscall
...@@ -1017,16 +1027,9 @@ alternative_else_nop_endif ...@@ -1017,16 +1027,9 @@ alternative_else_nop_endif
orr \tmp, \tmp, #USER_ASID_FLAG orr \tmp, \tmp, #USER_ASID_FLAG
msr ttbr1_el1, \tmp msr ttbr1_el1, \tmp
/* /*
* We avoid running the post_ttbr_update_workaround here because the * We avoid running the post_ttbr_update_workaround here because
* user and kernel ASIDs don't have conflicting mappings, so any * it's only needed by Cavium ThunderX, which requires KPTI to be
* "blessing" as described in: * disabled.
*
* http://lkml.kernel.org/r/56BB848A.6060603@caviumnetworks.com
*
* will not hurt correctness. Whilst this may partially defeat the
* point of using split ASIDs in the first place, it avoids
* the hit of invalidating the entire I-cache on every return to
* userspace.
*/ */
.endm .endm
......
...@@ -147,26 +147,6 @@ preserve_boot_args: ...@@ -147,26 +147,6 @@ preserve_boot_args:
b __inval_dcache_area // tail call b __inval_dcache_area // tail call
ENDPROC(preserve_boot_args) ENDPROC(preserve_boot_args)
/*
* Macro to arrange a physical address in a page table entry, taking care of
* 52-bit addresses.
*
* Preserves: phys
* Returns: pte
*/
.macro phys_to_pte, phys, pte
#ifdef CONFIG_ARM64_PA_BITS_52
/*
* We assume \phys is 64K aligned and this is guaranteed by only
* supporting this configuration with 64K pages.
*/
orr \pte, \phys, \phys, lsr #36
and \pte, \pte, #PTE_ADDR_MASK
#else
mov \pte, \phys
#endif
.endm
/* /*
* Macro to create a table entry to the next page. * Macro to create a table entry to the next page.
* *
...@@ -181,7 +161,7 @@ ENDPROC(preserve_boot_args) ...@@ -181,7 +161,7 @@ ENDPROC(preserve_boot_args)
*/ */
.macro create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2 .macro create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2
add \tmp1, \tbl, #PAGE_SIZE add \tmp1, \tbl, #PAGE_SIZE
phys_to_pte \tmp1, \tmp2 phys_to_pte \tmp2, \tmp1
orr \tmp2, \tmp2, #PMD_TYPE_TABLE // address of next table and entry type orr \tmp2, \tmp2, #PMD_TYPE_TABLE // address of next table and entry type
lsr \tmp1, \virt, #\shift lsr \tmp1, \virt, #\shift
sub \ptrs, \ptrs, #1 sub \ptrs, \ptrs, #1
...@@ -207,7 +187,7 @@ ENDPROC(preserve_boot_args) ...@@ -207,7 +187,7 @@ ENDPROC(preserve_boot_args)
* Returns: rtbl * Returns: rtbl
*/ */
.macro populate_entries, tbl, rtbl, index, eindex, flags, inc, tmp1 .macro populate_entries, tbl, rtbl, index, eindex, flags, inc, tmp1
.Lpe\@: phys_to_pte \rtbl, \tmp1 .Lpe\@: phys_to_pte \tmp1, \rtbl
orr \tmp1, \tmp1, \flags // tmp1 = table entry orr \tmp1, \tmp1, \flags // tmp1 = table entry
str \tmp1, [\tbl, \index, lsl #3] str \tmp1, [\tbl, \index, lsl #3]
add \rtbl, \rtbl, \inc // rtbl = pa next level add \rtbl, \rtbl, \inc // rtbl = pa next level
...@@ -475,7 +455,7 @@ ENDPROC(__primary_switched) ...@@ -475,7 +455,7 @@ ENDPROC(__primary_switched)
* end early head section, begin head code that is also used for * end early head section, begin head code that is also used for
* hotplug and needs to have the same protections as the text region * hotplug and needs to have the same protections as the text region
*/ */
.section ".idmap.text","ax" .section ".idmap.text","awx"
ENTRY(kimage_vaddr) ENTRY(kimage_vaddr)
.quad _text - TEXT_OFFSET .quad _text - TEXT_OFFSET
...@@ -776,8 +756,8 @@ ENTRY(__enable_mmu) ...@@ -776,8 +756,8 @@ ENTRY(__enable_mmu)
update_early_cpu_boot_status 0, x1, x2 update_early_cpu_boot_status 0, x1, x2
adrp x1, idmap_pg_dir adrp x1, idmap_pg_dir
adrp x2, swapper_pg_dir adrp x2, swapper_pg_dir
phys_to_ttbr x1, x3 phys_to_ttbr x3, x1
phys_to_ttbr x2, x4 phys_to_ttbr x4, x2
msr ttbr0_el1, x3 // load TTBR0 msr ttbr0_el1, x3 // load TTBR0
msr ttbr1_el1, x4 // load TTBR1 msr ttbr1_el1, x4 // load TTBR1
isb isb
......
...@@ -34,12 +34,12 @@ ...@@ -34,12 +34,12 @@
* each stage of the walk. * each stage of the walk.
*/ */
.macro break_before_make_ttbr_switch zero_page, page_table, tmp .macro break_before_make_ttbr_switch zero_page, page_table, tmp
phys_to_ttbr \zero_page, \tmp phys_to_ttbr \tmp, \zero_page
msr ttbr1_el1, \tmp msr ttbr1_el1, \tmp
isb isb
tlbi vmalle1 tlbi vmalle1
dsb nsh dsb nsh
phys_to_ttbr \page_table, \tmp phys_to_ttbr \tmp, \page_table
msr ttbr1_el1, \tmp msr ttbr1_el1, \tmp
isb isb
.endm .endm
......
...@@ -96,7 +96,7 @@ ENTRY(__cpu_suspend_enter) ...@@ -96,7 +96,7 @@ ENTRY(__cpu_suspend_enter)
ret ret
ENDPROC(__cpu_suspend_enter) ENDPROC(__cpu_suspend_enter)
.pushsection ".idmap.text", "ax" .pushsection ".idmap.text", "awx"
ENTRY(cpu_resume) ENTRY(cpu_resume)
bl el2_setup // if in EL2 drop to EL1 cleanly bl el2_setup // if in EL2 drop to EL1 cleanly
bl __cpu_setup bl __cpu_setup
......
...@@ -22,13 +22,14 @@ ...@@ -22,13 +22,14 @@
#include <linux/kvm.h> #include <linux/kvm.h>
#include <linux/kvm_host.h> #include <linux/kvm_host.h>
#include <kvm/arm_psci.h>
#include <asm/esr.h> #include <asm/esr.h>
#include <asm/exception.h> #include <asm/exception.h>
#include <asm/kvm_asm.h> #include <asm/kvm_asm.h>
#include <asm/kvm_coproc.h> #include <asm/kvm_coproc.h>
#include <asm/kvm_emulate.h> #include <asm/kvm_emulate.h>
#include <asm/kvm_mmu.h> #include <asm/kvm_mmu.h>
#include <asm/kvm_psci.h>
#include <asm/debug-monitors.h> #include <asm/debug-monitors.h>
#include <asm/traps.h> #include <asm/traps.h>
...@@ -51,7 +52,7 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) ...@@ -51,7 +52,7 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
kvm_vcpu_hvc_get_imm(vcpu)); kvm_vcpu_hvc_get_imm(vcpu));
vcpu->stat.hvc_exit_stat++; vcpu->stat.hvc_exit_stat++;
ret = kvm_psci_call(vcpu); ret = kvm_hvc_call_handler(vcpu);
if (ret < 0) { if (ret < 0) {
vcpu_set_reg(vcpu, 0, ~0UL); vcpu_set_reg(vcpu, 0, ~0UL);
return 1; return 1;
...@@ -62,7 +63,16 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) ...@@ -62,7 +63,16 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run) static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
{ {
/*
* "If an SMC instruction executed at Non-secure EL1 is
* trapped to EL2 because HCR_EL2.TSC is 1, the exception is a
* Trap exception, not a Secure Monitor Call exception [...]"
*
* We need to advance the PC after the trap, as it would
* otherwise return to the same address...
*/
vcpu_set_reg(vcpu, 0, ~0UL); vcpu_set_reg(vcpu, 0, ~0UL);
kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
return 1; return 1;
} }
......
...@@ -63,7 +63,7 @@ __do_hyp_init: ...@@ -63,7 +63,7 @@ __do_hyp_init:
cmp x0, #HVC_STUB_HCALL_NR cmp x0, #HVC_STUB_HCALL_NR
b.lo __kvm_handle_stub_hvc b.lo __kvm_handle_stub_hvc
phys_to_ttbr x0, x4 phys_to_ttbr x4, x0
msr ttbr0_el2, x4 msr ttbr0_el2, x4
mrs x4, tcr_el1 mrs x4, tcr_el1
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <linux/arm-smccc.h>
#include <linux/linkage.h> #include <linux/linkage.h>
#include <asm/alternative.h> #include <asm/alternative.h>
...@@ -64,10 +65,11 @@ alternative_endif ...@@ -64,10 +65,11 @@ alternative_endif
lsr x0, x1, #ESR_ELx_EC_SHIFT lsr x0, x1, #ESR_ELx_EC_SHIFT
cmp x0, #ESR_ELx_EC_HVC64 cmp x0, #ESR_ELx_EC_HVC64
ccmp x0, #ESR_ELx_EC_HVC32, #4, ne
b.ne el1_trap b.ne el1_trap
mrs x1, vttbr_el2 // If vttbr is valid, the 64bit guest mrs x1, vttbr_el2 // If vttbr is valid, the guest
cbnz x1, el1_trap // called HVC cbnz x1, el1_hvc_guest // called HVC
/* Here, we're pretty sure the host called HVC. */ /* Here, we're pretty sure the host called HVC. */
ldp x0, x1, [sp], #16 ldp x0, x1, [sp], #16
...@@ -100,6 +102,20 @@ alternative_endif ...@@ -100,6 +102,20 @@ alternative_endif
eret eret
el1_hvc_guest:
/*
* Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1.
* The workaround has already been applied on the host,
* so let's quickly get back to the guest. We don't bother
* restoring x1, as it can be clobbered anyway.
*/
ldr x1, [sp] // Guest's x0
eor w1, w1, #ARM_SMCCC_ARCH_WORKAROUND_1
cbnz w1, el1_trap
mov x0, x1
add sp, sp, #16
eret
el1_trap: el1_trap:
/* /*
* x0: ESR_EC * x0: ESR_EC
......
...@@ -19,6 +19,8 @@ ...@@ -19,6 +19,8 @@
#include <linux/jump_label.h> #include <linux/jump_label.h>
#include <uapi/linux/psci.h> #include <uapi/linux/psci.h>
#include <kvm/arm_psci.h>
#include <asm/kvm_asm.h> #include <asm/kvm_asm.h>
#include <asm/kvm_emulate.h> #include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h> #include <asm/kvm_hyp.h>
...@@ -348,18 +350,6 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu) ...@@ -348,18 +350,6 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
if (exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu)) if (exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
goto again; goto again;
if (exit_code == ARM_EXCEPTION_TRAP &&
(kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_HVC64 ||
kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_HVC32) &&
vcpu_get_reg(vcpu, 0) == PSCI_0_2_FN_PSCI_VERSION) {
u64 val = PSCI_RET_NOT_SUPPORTED;
if (test_bit(KVM_ARM_VCPU_PSCI_0_2, vcpu->arch.features))
val = 2;
vcpu_set_reg(vcpu, 0, val);
goto again;
}
if (static_branch_unlikely(&vgic_v2_cpuif_trap) && if (static_branch_unlikely(&vgic_v2_cpuif_trap) &&
exit_code == ARM_EXCEPTION_TRAP) { exit_code == ARM_EXCEPTION_TRAP) {
bool valid; bool valid;
......
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
.text .text
/* Prototype: int __clear_user(void *addr, size_t sz) /* Prototype: int __arch_clear_user(void *addr, size_t sz)
* Purpose : clear some user memory * Purpose : clear some user memory
* Params : addr - user memory address to clear * Params : addr - user memory address to clear
* : sz - number of bytes to clear * : sz - number of bytes to clear
...@@ -29,7 +29,7 @@ ...@@ -29,7 +29,7 @@
* *
* Alignment fixed up by hardware. * Alignment fixed up by hardware.
*/ */
ENTRY(__clear_user) ENTRY(__arch_clear_user)
uaccess_enable_not_uao x2, x3, x4 uaccess_enable_not_uao x2, x3, x4
mov x2, x1 // save the size for fixup return mov x2, x1 // save the size for fixup return
subs x1, x1, #8 subs x1, x1, #8
...@@ -52,7 +52,7 @@ uao_user_alternative 9f, strb, sttrb, wzr, x0, 0 ...@@ -52,7 +52,7 @@ uao_user_alternative 9f, strb, sttrb, wzr, x0, 0
5: mov x0, #0 5: mov x0, #0
uaccess_disable_not_uao x2, x3 uaccess_disable_not_uao x2, x3
ret ret
ENDPROC(__clear_user) ENDPROC(__arch_clear_user)
.section .fixup,"ax" .section .fixup,"ax"
.align 2 .align 2
......
...@@ -64,14 +64,15 @@ ...@@ -64,14 +64,15 @@
.endm .endm
end .req x5 end .req x5
ENTRY(raw_copy_in_user)
ENTRY(__arch_copy_in_user)
uaccess_enable_not_uao x3, x4, x5 uaccess_enable_not_uao x3, x4, x5
add end, x0, x2 add end, x0, x2
#include "copy_template.S" #include "copy_template.S"
uaccess_disable_not_uao x3, x4 uaccess_disable_not_uao x3, x4
mov x0, #0 mov x0, #0
ret ret
ENDPROC(raw_copy_in_user) ENDPROC(__arch_copy_in_user)
.section .fixup,"ax" .section .fixup,"ax"
.align 2 .align 2
......
...@@ -240,7 +240,7 @@ static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs, ...@@ -240,7 +240,7 @@ static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs,
if (fsc_type == ESR_ELx_FSC_PERM) if (fsc_type == ESR_ELx_FSC_PERM)
return true; return true;
if (addr < USER_DS && system_uses_ttbr0_pan()) if (addr < TASK_SIZE && system_uses_ttbr0_pan())
return fsc_type == ESR_ELx_FSC_FAULT && return fsc_type == ESR_ELx_FSC_FAULT &&
(regs->pstate & PSR_PAN_BIT); (regs->pstate & PSR_PAN_BIT);
...@@ -414,7 +414,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, ...@@ -414,7 +414,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
mm_flags |= FAULT_FLAG_WRITE; mm_flags |= FAULT_FLAG_WRITE;
} }
if (addr < USER_DS && is_permission_fault(esr, regs, addr)) { if (addr < TASK_SIZE && is_permission_fault(esr, regs, addr)) {
/* regs->orig_addr_limit may be 0 if we entered from EL0 */ /* regs->orig_addr_limit may be 0 if we entered from EL0 */
if (regs->orig_addr_limit == KERNEL_DS) if (regs->orig_addr_limit == KERNEL_DS)
die("Accessing user space memory with fs=KERNEL_DS", regs, esr); die("Accessing user space memory with fs=KERNEL_DS", regs, esr);
...@@ -707,6 +707,12 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, ...@@ -707,6 +707,12 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
arm64_notify_die("", regs, &info, esr); arm64_notify_die("", regs, &info, esr);
} }
asmlinkage void __exception do_el0_irq_bp_hardening(void)
{
/* PC has already been checked in entry.S */
arm64_apply_bp_hardening();
}
asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr, asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr,
unsigned int esr, unsigned int esr,
struct pt_regs *regs) struct pt_regs *regs)
...@@ -731,6 +737,12 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr, ...@@ -731,6 +737,12 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
struct siginfo info; struct siginfo info;
struct task_struct *tsk = current; struct task_struct *tsk = current;
if (user_mode(regs)) {
if (instruction_pointer(regs) > TASK_SIZE)
arm64_apply_bp_hardening();
local_irq_enable();
}
if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS)) if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS))
pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n", pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n",
tsk->comm, task_pid_nr(tsk), tsk->comm, task_pid_nr(tsk),
...@@ -790,6 +802,9 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, ...@@ -790,6 +802,9 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
if (interrupts_enabled(regs)) if (interrupts_enabled(regs))
trace_hardirqs_off(); trace_hardirqs_off();
if (user_mode(regs) && instruction_pointer(regs) > TASK_SIZE)
arm64_apply_bp_hardening();
if (!inf->fn(addr, esr, regs)) { if (!inf->fn(addr, esr, regs)) {
rv = 1; rv = 1;
} else { } else {
......
...@@ -118,6 +118,10 @@ static bool pgattr_change_is_safe(u64 old, u64 new) ...@@ -118,6 +118,10 @@ static bool pgattr_change_is_safe(u64 old, u64 new)
if ((old | new) & PTE_CONT) if ((old | new) & PTE_CONT)
return false; return false;
/* Transitioning from Global to Non-Global is safe */
if (((old ^ new) == PTE_NG) && (new & PTE_NG))
return true;
return ((old ^ new) & ~mask) == 0; return ((old ^ new) & ~mask) == 0;
} }
......
...@@ -90,7 +90,7 @@ ENDPROC(cpu_do_suspend) ...@@ -90,7 +90,7 @@ ENDPROC(cpu_do_suspend)
* *
* x0: Address of context pointer * x0: Address of context pointer
*/ */
.pushsection ".idmap.text", "ax" .pushsection ".idmap.text", "awx"
ENTRY(cpu_do_resume) ENTRY(cpu_do_resume)
ldp x2, x3, [x0] ldp x2, x3, [x0]
ldp x4, x5, [x0, #16] ldp x4, x5, [x0, #16]
...@@ -153,7 +153,7 @@ ENDPROC(cpu_do_resume) ...@@ -153,7 +153,7 @@ ENDPROC(cpu_do_resume)
ENTRY(cpu_do_switch_mm) ENTRY(cpu_do_switch_mm)
mrs x2, ttbr1_el1 mrs x2, ttbr1_el1
mmid x1, x1 // get mm->context.id mmid x1, x1 // get mm->context.id
phys_to_ttbr x0, x3 phys_to_ttbr x3, x0
#ifdef CONFIG_ARM64_SW_TTBR0_PAN #ifdef CONFIG_ARM64_SW_TTBR0_PAN
bfi x3, x1, #48, #16 // set the ASID field in TTBR0 bfi x3, x1, #48, #16 // set the ASID field in TTBR0
#endif #endif
...@@ -165,7 +165,18 @@ ENTRY(cpu_do_switch_mm) ...@@ -165,7 +165,18 @@ ENTRY(cpu_do_switch_mm)
b post_ttbr_update_workaround // Back to C code... b post_ttbr_update_workaround // Back to C code...
ENDPROC(cpu_do_switch_mm) ENDPROC(cpu_do_switch_mm)
.pushsection ".idmap.text", "ax" .pushsection ".idmap.text", "awx"
.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2
adrp \tmp1, empty_zero_page
phys_to_ttbr \tmp2, \tmp1
msr ttbr1_el1, \tmp2
isb
tlbi vmalle1
dsb nsh
isb
.endm
/* /*
* void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd) * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd)
* *
...@@ -175,24 +186,201 @@ ENDPROC(cpu_do_switch_mm) ...@@ -175,24 +186,201 @@ ENDPROC(cpu_do_switch_mm)
ENTRY(idmap_cpu_replace_ttbr1) ENTRY(idmap_cpu_replace_ttbr1)
save_and_disable_daif flags=x2 save_and_disable_daif flags=x2
adrp x1, empty_zero_page __idmap_cpu_set_reserved_ttbr1 x1, x3
phys_to_ttbr x1, x3
phys_to_ttbr x3, x0
msr ttbr1_el1, x3 msr ttbr1_el1, x3
isb isb
tlbi vmalle1 restore_daif x2
dsb nsh
ret
ENDPROC(idmap_cpu_replace_ttbr1)
.popsection
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
.pushsection ".idmap.text", "awx"
.macro __idmap_kpti_get_pgtable_ent, type
dc cvac, cur_\()\type\()p // Ensure any existing dirty
dmb sy // lines are written back before
ldr \type, [cur_\()\type\()p] // loading the entry
tbz \type, #0, next_\()\type // Skip invalid entries
.endm
.macro __idmap_kpti_put_pgtable_ent_ng, type
orr \type, \type, #PTE_NG // Same bit for blocks and pages
str \type, [cur_\()\type\()p] // Update the entry and ensure it
dc civac, cur_\()\type\()p // is visible to all CPUs.
.endm
/*
* void __kpti_install_ng_mappings(int cpu, int num_cpus, phys_addr_t swapper)
*
* Called exactly once from stop_machine context by each CPU found during boot.
*/
__idmap_kpti_flag:
.long 1
ENTRY(idmap_kpti_install_ng_mappings)
cpu .req w0
num_cpus .req w1
swapper_pa .req x2
swapper_ttb .req x3
flag_ptr .req x4
cur_pgdp .req x5
end_pgdp .req x6
pgd .req x7
cur_pudp .req x8
end_pudp .req x9
pud .req x10
cur_pmdp .req x11
end_pmdp .req x12
pmd .req x13
cur_ptep .req x14
end_ptep .req x15
pte .req x16
mrs swapper_ttb, ttbr1_el1
adr flag_ptr, __idmap_kpti_flag
cbnz cpu, __idmap_kpti_secondary
/* We're the boot CPU. Wait for the others to catch up */
sevl
1: wfe
ldaxr w18, [flag_ptr]
eor w18, w18, num_cpus
cbnz w18, 1b
/* We need to walk swapper, so turn off the MMU. */
pre_disable_mmu_workaround
mrs x18, sctlr_el1
bic x18, x18, #SCTLR_ELx_M
msr sctlr_el1, x18
isb isb
phys_to_ttbr x0, x3 /* Everybody is enjoying the idmap, so we can rewrite swapper. */
msr ttbr1_el1, x3 /* PGD */
mov cur_pgdp, swapper_pa
add end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8)
do_pgd: __idmap_kpti_get_pgtable_ent pgd
tbnz pgd, #1, walk_puds
__idmap_kpti_put_pgtable_ent_ng pgd
next_pgd:
add cur_pgdp, cur_pgdp, #8
cmp cur_pgdp, end_pgdp
b.ne do_pgd
/* Publish the updated tables and nuke all the TLBs */
dsb sy
tlbi vmalle1is
dsb ish
isb isb
restore_daif x2 /* We're done: fire up the MMU again */
mrs x18, sctlr_el1
orr x18, x18, #SCTLR_ELx_M
msr sctlr_el1, x18
isb
/* Set the flag to zero to indicate that we're all done */
str wzr, [flag_ptr]
ret ret
ENDPROC(idmap_cpu_replace_ttbr1)
/* PUD */
walk_puds:
.if CONFIG_PGTABLE_LEVELS > 3
pte_to_phys cur_pudp, pgd
add end_pudp, cur_pudp, #(PTRS_PER_PUD * 8)
do_pud: __idmap_kpti_get_pgtable_ent pud
tbnz pud, #1, walk_pmds
__idmap_kpti_put_pgtable_ent_ng pud
next_pud:
add cur_pudp, cur_pudp, 8
cmp cur_pudp, end_pudp
b.ne do_pud
b next_pgd
.else /* CONFIG_PGTABLE_LEVELS <= 3 */
mov pud, pgd
b walk_pmds
next_pud:
b next_pgd
.endif
/* PMD */
walk_pmds:
.if CONFIG_PGTABLE_LEVELS > 2
pte_to_phys cur_pmdp, pud
add end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8)
do_pmd: __idmap_kpti_get_pgtable_ent pmd
tbnz pmd, #1, walk_ptes
__idmap_kpti_put_pgtable_ent_ng pmd
next_pmd:
add cur_pmdp, cur_pmdp, #8
cmp cur_pmdp, end_pmdp
b.ne do_pmd
b next_pud
.else /* CONFIG_PGTABLE_LEVELS <= 2 */
mov pmd, pud
b walk_ptes
next_pmd:
b next_pud
.endif
/* PTE */
walk_ptes:
pte_to_phys cur_ptep, pmd
add end_ptep, cur_ptep, #(PTRS_PER_PTE * 8)
do_pte: __idmap_kpti_get_pgtable_ent pte
__idmap_kpti_put_pgtable_ent_ng pte
next_pte:
add cur_ptep, cur_ptep, #8
cmp cur_ptep, end_ptep
b.ne do_pte
b next_pmd
/* Secondary CPUs end up here */
__idmap_kpti_secondary:
/* Uninstall swapper before surgery begins */
__idmap_cpu_set_reserved_ttbr1 x18, x17
/* Increment the flag to let the boot CPU we're ready */
1: ldxr w18, [flag_ptr]
add w18, w18, #1
stxr w17, w18, [flag_ptr]
cbnz w17, 1b
/* Wait for the boot CPU to finish messing around with swapper */
sevl
1: wfe
ldxr w18, [flag_ptr]
cbnz w18, 1b
/* All done, act like nothing happened */
msr ttbr1_el1, swapper_ttb
isb
ret
.unreq cpu
.unreq num_cpus
.unreq swapper_pa
.unreq swapper_ttb
.unreq flag_ptr
.unreq cur_pgdp
.unreq end_pgdp
.unreq pgd
.unreq cur_pudp
.unreq end_pudp
.unreq pud
.unreq cur_pmdp
.unreq end_pmdp
.unreq pmd
.unreq cur_ptep
.unreq end_ptep
.unreq pte
ENDPROC(idmap_kpti_install_ng_mappings)
.popsection .popsection
#endif
/* /*
* __cpu_setup * __cpu_setup
...@@ -200,7 +388,7 @@ ENDPROC(idmap_cpu_replace_ttbr1) ...@@ -200,7 +388,7 @@ ENDPROC(idmap_cpu_replace_ttbr1)
* Initialise the processor for turning the MMU on. Return in x0 the * Initialise the processor for turning the MMU on. Return in x0 the
* value of the SCTLR_EL1 register. * value of the SCTLR_EL1 register.
*/ */
.pushsection ".idmap.text", "ax" .pushsection ".idmap.text", "awx"
ENTRY(__cpu_setup) ENTRY(__cpu_setup)
tlbi vmalle1 // Invalidate local TLB tlbi vmalle1 // Invalidate local TLB
dsb nsh dsb nsh
......
...@@ -59,7 +59,10 @@ bool psci_tos_resident_on(int cpu) ...@@ -59,7 +59,10 @@ bool psci_tos_resident_on(int cpu)
return cpu == resident_cpu; return cpu == resident_cpu;
} }
struct psci_operations psci_ops; struct psci_operations psci_ops = {
.conduit = PSCI_CONDUIT_NONE,
.smccc_version = SMCCC_VERSION_1_0,
};
typedef unsigned long (psci_fn)(unsigned long, unsigned long, typedef unsigned long (psci_fn)(unsigned long, unsigned long,
unsigned long, unsigned long); unsigned long, unsigned long);
...@@ -210,6 +213,22 @@ static unsigned long psci_migrate_info_up_cpu(void) ...@@ -210,6 +213,22 @@ static unsigned long psci_migrate_info_up_cpu(void)
0, 0, 0); 0, 0, 0);
} }
static void set_conduit(enum psci_conduit conduit)
{
switch (conduit) {
case PSCI_CONDUIT_HVC:
invoke_psci_fn = __invoke_psci_fn_hvc;
break;
case PSCI_CONDUIT_SMC:
invoke_psci_fn = __invoke_psci_fn_smc;
break;
default:
WARN(1, "Unexpected PSCI conduit %d\n", conduit);
}
psci_ops.conduit = conduit;
}
static int get_set_conduit_method(struct device_node *np) static int get_set_conduit_method(struct device_node *np)
{ {
const char *method; const char *method;
...@@ -222,9 +241,9 @@ static int get_set_conduit_method(struct device_node *np) ...@@ -222,9 +241,9 @@ static int get_set_conduit_method(struct device_node *np)
} }
if (!strcmp("hvc", method)) { if (!strcmp("hvc", method)) {
invoke_psci_fn = __invoke_psci_fn_hvc; set_conduit(PSCI_CONDUIT_HVC);
} else if (!strcmp("smc", method)) { } else if (!strcmp("smc", method)) {
invoke_psci_fn = __invoke_psci_fn_smc; set_conduit(PSCI_CONDUIT_SMC);
} else { } else {
pr_warn("invalid \"method\" property: %s\n", method); pr_warn("invalid \"method\" property: %s\n", method);
return -EINVAL; return -EINVAL;
...@@ -493,6 +512,31 @@ static void __init psci_init_migrate(void) ...@@ -493,6 +512,31 @@ static void __init psci_init_migrate(void)
pr_info("Trusted OS resident on physical CPU 0x%lx\n", cpuid); pr_info("Trusted OS resident on physical CPU 0x%lx\n", cpuid);
} }
static void __init psci_init_smccc(void)
{
u32 ver = ARM_SMCCC_VERSION_1_0;
int feature;
feature = psci_features(ARM_SMCCC_VERSION_FUNC_ID);
if (feature != PSCI_RET_NOT_SUPPORTED) {
u32 ret;
ret = invoke_psci_fn(ARM_SMCCC_VERSION_FUNC_ID, 0, 0, 0);
if (ret == ARM_SMCCC_VERSION_1_1) {
psci_ops.smccc_version = SMCCC_VERSION_1_1;
ver = ret;
}
}
/*
* Conveniently, the SMCCC and PSCI versions are encoded the
* same way. No, this isn't accidental.
*/
pr_info("SMC Calling Convention v%d.%d\n",
PSCI_VERSION_MAJOR(ver), PSCI_VERSION_MINOR(ver));
}
static void __init psci_0_2_set_functions(void) static void __init psci_0_2_set_functions(void)
{ {
pr_info("Using standard PSCI v0.2 function IDs\n"); pr_info("Using standard PSCI v0.2 function IDs\n");
...@@ -541,6 +585,7 @@ static int __init psci_probe(void) ...@@ -541,6 +585,7 @@ static int __init psci_probe(void)
psci_init_migrate(); psci_init_migrate();
if (PSCI_VERSION_MAJOR(ver) >= 1) { if (PSCI_VERSION_MAJOR(ver) >= 1) {
psci_init_smccc();
psci_init_cpu_suspend(); psci_init_cpu_suspend();
psci_init_system_suspend(); psci_init_system_suspend();
} }
...@@ -654,9 +699,9 @@ int __init psci_acpi_init(void) ...@@ -654,9 +699,9 @@ int __init psci_acpi_init(void)
pr_info("probing for conduit method from ACPI.\n"); pr_info("probing for conduit method from ACPI.\n");
if (acpi_psci_use_hvc()) if (acpi_psci_use_hvc())
invoke_psci_fn = __invoke_psci_fn_hvc; set_conduit(PSCI_CONDUIT_HVC);
else else
invoke_psci_fn = __invoke_psci_fn_smc; set_conduit(PSCI_CONDUIT_SMC);
return psci_probe(); return psci_probe();
} }
......
...@@ -15,13 +15,37 @@ ...@@ -15,13 +15,37 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef __ARM64_KVM_PSCI_H__ #ifndef __KVM_ARM_PSCI_H__
#define __ARM64_KVM_PSCI_H__ #define __KVM_ARM_PSCI_H__
#define KVM_ARM_PSCI_0_1 1 #include <linux/kvm_host.h>
#define KVM_ARM_PSCI_0_2 2 #include <uapi/linux/psci.h>
int kvm_psci_version(struct kvm_vcpu *vcpu); #define KVM_ARM_PSCI_0_1 PSCI_VERSION(0, 1)
int kvm_psci_call(struct kvm_vcpu *vcpu); #define KVM_ARM_PSCI_0_2 PSCI_VERSION(0, 2)
#define KVM_ARM_PSCI_1_0 PSCI_VERSION(1, 0)
#endif /* __ARM64_KVM_PSCI_H__ */ #define KVM_ARM_PSCI_LATEST KVM_ARM_PSCI_1_0
/*
* We need the KVM pointer independently from the vcpu as we can call
* this from HYP, and need to apply kern_hyp_va on it...
*/
static inline int kvm_psci_version(struct kvm_vcpu *vcpu, struct kvm *kvm)
{
/*
* Our PSCI implementation stays the same across versions from
* v0.2 onward, only adding the few mandatory functions (such
* as FEATURES with 1.0) that are required by newer
* revisions. It is thus safe to return the latest.
*/
if (test_bit(KVM_ARM_VCPU_PSCI_0_2, vcpu->arch.features))
return KVM_ARM_PSCI_LATEST;
return KVM_ARM_PSCI_0_1;
}
int kvm_hvc_call_handler(struct kvm_vcpu *vcpu);
#endif /* __KVM_ARM_PSCI_H__ */
...@@ -14,14 +14,16 @@ ...@@ -14,14 +14,16 @@
#ifndef __LINUX_ARM_SMCCC_H #ifndef __LINUX_ARM_SMCCC_H
#define __LINUX_ARM_SMCCC_H #define __LINUX_ARM_SMCCC_H
#include <uapi/linux/const.h>
/* /*
* This file provides common defines for ARM SMC Calling Convention as * This file provides common defines for ARM SMC Calling Convention as
* specified in * specified in
* http://infocenter.arm.com/help/topic/com.arm.doc.den0028a/index.html * http://infocenter.arm.com/help/topic/com.arm.doc.den0028a/index.html
*/ */
#define ARM_SMCCC_STD_CALL 0 #define ARM_SMCCC_STD_CALL _AC(0,U)
#define ARM_SMCCC_FAST_CALL 1 #define ARM_SMCCC_FAST_CALL _AC(1,U)
#define ARM_SMCCC_TYPE_SHIFT 31 #define ARM_SMCCC_TYPE_SHIFT 31
#define ARM_SMCCC_SMC_32 0 #define ARM_SMCCC_SMC_32 0
...@@ -60,6 +62,24 @@ ...@@ -60,6 +62,24 @@
#define ARM_SMCCC_QUIRK_NONE 0 #define ARM_SMCCC_QUIRK_NONE 0
#define ARM_SMCCC_QUIRK_QCOM_A6 1 /* Save/restore register a6 */ #define ARM_SMCCC_QUIRK_QCOM_A6 1 /* Save/restore register a6 */
#define ARM_SMCCC_VERSION_1_0 0x10000
#define ARM_SMCCC_VERSION_1_1 0x10001
#define ARM_SMCCC_VERSION_FUNC_ID \
ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
ARM_SMCCC_SMC_32, \
0, 0)
#define ARM_SMCCC_ARCH_FEATURES_FUNC_ID \
ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
ARM_SMCCC_SMC_32, \
0, 1)
#define ARM_SMCCC_ARCH_WORKAROUND_1 \
ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
ARM_SMCCC_SMC_32, \
0, 0x8000)
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/linkage.h> #include <linux/linkage.h>
...@@ -130,5 +150,146 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, ...@@ -130,5 +150,146 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1,
#define arm_smccc_hvc_quirk(...) __arm_smccc_hvc(__VA_ARGS__) #define arm_smccc_hvc_quirk(...) __arm_smccc_hvc(__VA_ARGS__)
/* SMCCC v1.1 implementation madness follows */
#ifdef CONFIG_ARM64
#define SMCCC_SMC_INST "smc #0"
#define SMCCC_HVC_INST "hvc #0"
#elif defined(CONFIG_ARM)
#include <asm/opcodes-sec.h>
#include <asm/opcodes-virt.h>
#define SMCCC_SMC_INST __SMC(0)
#define SMCCC_HVC_INST __HVC(0)
#endif
#define ___count_args(_0, _1, _2, _3, _4, _5, _6, _7, _8, x, ...) x
#define __count_args(...) \
___count_args(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0)
#define __constraint_write_0 \
"+r" (r0), "=&r" (r1), "=&r" (r2), "=&r" (r3)
#define __constraint_write_1 \
"+r" (r0), "+r" (r1), "=&r" (r2), "=&r" (r3)
#define __constraint_write_2 \
"+r" (r0), "+r" (r1), "+r" (r2), "=&r" (r3)
#define __constraint_write_3 \
"+r" (r0), "+r" (r1), "+r" (r2), "+r" (r3)
#define __constraint_write_4 __constraint_write_3
#define __constraint_write_5 __constraint_write_4
#define __constraint_write_6 __constraint_write_5
#define __constraint_write_7 __constraint_write_6
#define __constraint_read_0
#define __constraint_read_1
#define __constraint_read_2
#define __constraint_read_3
#define __constraint_read_4 "r" (r4)
#define __constraint_read_5 __constraint_read_4, "r" (r5)
#define __constraint_read_6 __constraint_read_5, "r" (r6)
#define __constraint_read_7 __constraint_read_6, "r" (r7)
#define __declare_arg_0(a0, res) \
struct arm_smccc_res *___res = res; \
register u32 r0 asm("r0") = a0; \
register unsigned long r1 asm("r1"); \
register unsigned long r2 asm("r2"); \
register unsigned long r3 asm("r3")
#define __declare_arg_1(a0, a1, res) \
struct arm_smccc_res *___res = res; \
register u32 r0 asm("r0") = a0; \
register typeof(a1) r1 asm("r1") = a1; \
register unsigned long r2 asm("r2"); \
register unsigned long r3 asm("r3")
#define __declare_arg_2(a0, a1, a2, res) \
struct arm_smccc_res *___res = res; \
register u32 r0 asm("r0") = a0; \
register typeof(a1) r1 asm("r1") = a1; \
register typeof(a2) r2 asm("r2") = a2; \
register unsigned long r3 asm("r3")
#define __declare_arg_3(a0, a1, a2, a3, res) \
struct arm_smccc_res *___res = res; \
register u32 r0 asm("r0") = a0; \
register typeof(a1) r1 asm("r1") = a1; \
register typeof(a2) r2 asm("r2") = a2; \
register typeof(a3) r3 asm("r3") = a3
#define __declare_arg_4(a0, a1, a2, a3, a4, res) \
__declare_arg_3(a0, a1, a2, a3, res); \
register typeof(a4) r4 asm("r4") = a4
#define __declare_arg_5(a0, a1, a2, a3, a4, a5, res) \
__declare_arg_4(a0, a1, a2, a3, a4, res); \
register typeof(a5) r5 asm("r5") = a5
#define __declare_arg_6(a0, a1, a2, a3, a4, a5, a6, res) \
__declare_arg_5(a0, a1, a2, a3, a4, a5, res); \
register typeof(a6) r6 asm("r6") = a6
#define __declare_arg_7(a0, a1, a2, a3, a4, a5, a6, a7, res) \
__declare_arg_6(a0, a1, a2, a3, a4, a5, a6, res); \
register typeof(a7) r7 asm("r7") = a7
#define ___declare_args(count, ...) __declare_arg_ ## count(__VA_ARGS__)
#define __declare_args(count, ...) ___declare_args(count, __VA_ARGS__)
#define ___constraints(count) \
: __constraint_write_ ## count \
: __constraint_read_ ## count \
: "memory"
#define __constraints(count) ___constraints(count)
/*
* We have an output list that is not necessarily used, and GCC feels
* entitled to optimise the whole sequence away. "volatile" is what
* makes it stick.
*/
#define __arm_smccc_1_1(inst, ...) \
do { \
__declare_args(__count_args(__VA_ARGS__), __VA_ARGS__); \
asm volatile(inst "\n" \
__constraints(__count_args(__VA_ARGS__))); \
if (___res) \
*___res = (typeof(*___res)){r0, r1, r2, r3}; \
} while (0)
/*
* arm_smccc_1_1_smc() - make an SMCCC v1.1 compliant SMC call
*
* This is a variadic macro taking one to eight source arguments, and
* an optional return structure.
*
* @a0-a7: arguments passed in registers 0 to 7
* @res: result values from registers 0 to 3
*
* This macro is used to make SMC calls following SMC Calling Convention v1.1.
* The content of the supplied param are copied to registers 0 to 7 prior
* to the SMC instruction. The return values are updated with the content
* from register 0 to 3 on return from the SMC instruction if not NULL.
*/
#define arm_smccc_1_1_smc(...) __arm_smccc_1_1(SMCCC_SMC_INST, __VA_ARGS__)
/*
* arm_smccc_1_1_hvc() - make an SMCCC v1.1 compliant HVC call
*
* This is a variadic macro taking one to eight source arguments, and
* an optional return structure.
*
* @a0-a7: arguments passed in registers 0 to 7
* @res: result values from registers 0 to 3
*
* This macro is used to make HVC calls following SMC Calling Convention v1.1.
* The content of the supplied param are copied to registers 0 to 7 prior
* to the HVC instruction. The return values are updated with the content
* from register 0 to 3 on return from the HVC instruction if not NULL.
*/
#define arm_smccc_1_1_hvc(...) __arm_smccc_1_1(SMCCC_HVC_INST, __VA_ARGS__)
#endif /*__ASSEMBLY__*/ #endif /*__ASSEMBLY__*/
#endif /*__LINUX_ARM_SMCCC_H*/ #endif /*__LINUX_ARM_SMCCC_H*/
...@@ -25,6 +25,17 @@ bool psci_tos_resident_on(int cpu); ...@@ -25,6 +25,17 @@ bool psci_tos_resident_on(int cpu);
int psci_cpu_init_idle(unsigned int cpu); int psci_cpu_init_idle(unsigned int cpu);
int psci_cpu_suspend_enter(unsigned long index); int psci_cpu_suspend_enter(unsigned long index);
enum psci_conduit {
PSCI_CONDUIT_NONE,
PSCI_CONDUIT_SMC,
PSCI_CONDUIT_HVC,
};
enum smccc_version {
SMCCC_VERSION_1_0,
SMCCC_VERSION_1_1,
};
struct psci_operations { struct psci_operations {
u32 (*get_version)(void); u32 (*get_version)(void);
int (*cpu_suspend)(u32 state, unsigned long entry_point); int (*cpu_suspend)(u32 state, unsigned long entry_point);
...@@ -34,6 +45,8 @@ struct psci_operations { ...@@ -34,6 +45,8 @@ struct psci_operations {
int (*affinity_info)(unsigned long target_affinity, int (*affinity_info)(unsigned long target_affinity,
unsigned long lowest_affinity_level); unsigned long lowest_affinity_level);
int (*migrate_info_type)(void); int (*migrate_info_type)(void);
enum psci_conduit conduit;
enum smccc_version smccc_version;
}; };
extern struct psci_operations psci_ops; extern struct psci_operations psci_ops;
......
...@@ -88,6 +88,9 @@ ...@@ -88,6 +88,9 @@
(((ver) & PSCI_VERSION_MAJOR_MASK) >> PSCI_VERSION_MAJOR_SHIFT) (((ver) & PSCI_VERSION_MAJOR_MASK) >> PSCI_VERSION_MAJOR_SHIFT)
#define PSCI_VERSION_MINOR(ver) \ #define PSCI_VERSION_MINOR(ver) \
((ver) & PSCI_VERSION_MINOR_MASK) ((ver) & PSCI_VERSION_MINOR_MASK)
#define PSCI_VERSION(maj, min) \
((((maj) << PSCI_VERSION_MAJOR_SHIFT) & PSCI_VERSION_MAJOR_MASK) | \
((min) & PSCI_VERSION_MINOR_MASK))
/* PSCI features decoding (>=1.0) */ /* PSCI features decoding (>=1.0) */
#define PSCI_1_0_FEATURES_CPU_SUSPEND_PF_SHIFT 1 #define PSCI_1_0_FEATURES_CPU_SUSPEND_PF_SHIFT 1
......
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include <linux/irqbypass.h> #include <linux/irqbypass.h>
#include <trace/events/kvm.h> #include <trace/events/kvm.h>
#include <kvm/arm_pmu.h> #include <kvm/arm_pmu.h>
#include <kvm/arm_psci.h>
#define CREATE_TRACE_POINTS #define CREATE_TRACE_POINTS
#include "trace.h" #include "trace.h"
...@@ -46,7 +47,6 @@ ...@@ -46,7 +47,6 @@
#include <asm/kvm_mmu.h> #include <asm/kvm_mmu.h>
#include <asm/kvm_emulate.h> #include <asm/kvm_emulate.h>
#include <asm/kvm_coproc.h> #include <asm/kvm_coproc.h>
#include <asm/kvm_psci.h>
#include <asm/sections.h> #include <asm/sections.h>
#ifdef REQUIRES_VIRT #ifdef REQUIRES_VIRT
......
...@@ -15,16 +15,16 @@ ...@@ -15,16 +15,16 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <linux/arm-smccc.h>
#include <linux/preempt.h> #include <linux/preempt.h>
#include <linux/kvm_host.h> #include <linux/kvm_host.h>
#include <linux/wait.h> #include <linux/wait.h>
#include <asm/cputype.h> #include <asm/cputype.h>
#include <asm/kvm_emulate.h> #include <asm/kvm_emulate.h>
#include <asm/kvm_psci.h>
#include <asm/kvm_host.h> #include <asm/kvm_host.h>
#include <uapi/linux/psci.h> #include <kvm/arm_psci.h>
/* /*
* This is an implementation of the Power State Coordination Interface * This is an implementation of the Power State Coordination Interface
...@@ -33,6 +33,38 @@ ...@@ -33,6 +33,38 @@
#define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1) #define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1)
static u32 smccc_get_function(struct kvm_vcpu *vcpu)
{
return vcpu_get_reg(vcpu, 0);
}
static unsigned long smccc_get_arg1(struct kvm_vcpu *vcpu)
{
return vcpu_get_reg(vcpu, 1);
}
static unsigned long smccc_get_arg2(struct kvm_vcpu *vcpu)
{
return vcpu_get_reg(vcpu, 2);
}
static unsigned long smccc_get_arg3(struct kvm_vcpu *vcpu)
{
return vcpu_get_reg(vcpu, 3);
}
static void smccc_set_retval(struct kvm_vcpu *vcpu,
unsigned long a0,
unsigned long a1,
unsigned long a2,
unsigned long a3)
{
vcpu_set_reg(vcpu, 0, a0);
vcpu_set_reg(vcpu, 1, a1);
vcpu_set_reg(vcpu, 2, a2);
vcpu_set_reg(vcpu, 3, a3);
}
static unsigned long psci_affinity_mask(unsigned long affinity_level) static unsigned long psci_affinity_mask(unsigned long affinity_level)
{ {
if (affinity_level <= 3) if (affinity_level <= 3)
...@@ -78,7 +110,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) ...@@ -78,7 +110,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
unsigned long context_id; unsigned long context_id;
phys_addr_t target_pc; phys_addr_t target_pc;
cpu_id = vcpu_get_reg(source_vcpu, 1) & MPIDR_HWID_BITMASK; cpu_id = smccc_get_arg1(source_vcpu) & MPIDR_HWID_BITMASK;
if (vcpu_mode_is_32bit(source_vcpu)) if (vcpu_mode_is_32bit(source_vcpu))
cpu_id &= ~((u32) 0); cpu_id &= ~((u32) 0);
...@@ -91,14 +123,14 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) ...@@ -91,14 +123,14 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
if (!vcpu) if (!vcpu)
return PSCI_RET_INVALID_PARAMS; return PSCI_RET_INVALID_PARAMS;
if (!vcpu->arch.power_off) { if (!vcpu->arch.power_off) {
if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1) if (kvm_psci_version(source_vcpu, kvm) != KVM_ARM_PSCI_0_1)
return PSCI_RET_ALREADY_ON; return PSCI_RET_ALREADY_ON;
else else
return PSCI_RET_INVALID_PARAMS; return PSCI_RET_INVALID_PARAMS;
} }
target_pc = vcpu_get_reg(source_vcpu, 2); target_pc = smccc_get_arg2(source_vcpu);
context_id = vcpu_get_reg(source_vcpu, 3); context_id = smccc_get_arg3(source_vcpu);
kvm_reset_vcpu(vcpu); kvm_reset_vcpu(vcpu);
...@@ -117,7 +149,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) ...@@ -117,7 +149,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
* NOTE: We always update r0 (or x0) because for PSCI v0.1 * NOTE: We always update r0 (or x0) because for PSCI v0.1
* the general puspose registers are undefined upon CPU_ON. * the general puspose registers are undefined upon CPU_ON.
*/ */
vcpu_set_reg(vcpu, 0, context_id); smccc_set_retval(vcpu, context_id, 0, 0, 0);
vcpu->arch.power_off = false; vcpu->arch.power_off = false;
smp_mb(); /* Make sure the above is visible */ smp_mb(); /* Make sure the above is visible */
...@@ -137,8 +169,8 @@ static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) ...@@ -137,8 +169,8 @@ static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu)
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
struct kvm_vcpu *tmp; struct kvm_vcpu *tmp;
target_affinity = vcpu_get_reg(vcpu, 1); target_affinity = smccc_get_arg1(vcpu);
lowest_affinity_level = vcpu_get_reg(vcpu, 2); lowest_affinity_level = smccc_get_arg2(vcpu);
/* Determine target affinity mask */ /* Determine target affinity mask */
target_affinity_mask = psci_affinity_mask(lowest_affinity_level); target_affinity_mask = psci_affinity_mask(lowest_affinity_level);
...@@ -200,18 +232,10 @@ static void kvm_psci_system_reset(struct kvm_vcpu *vcpu) ...@@ -200,18 +232,10 @@ static void kvm_psci_system_reset(struct kvm_vcpu *vcpu)
kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET); kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET);
} }
int kvm_psci_version(struct kvm_vcpu *vcpu)
{
if (test_bit(KVM_ARM_VCPU_PSCI_0_2, vcpu->arch.features))
return KVM_ARM_PSCI_0_2;
return KVM_ARM_PSCI_0_1;
}
static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu)
{ {
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
unsigned long psci_fn = vcpu_get_reg(vcpu, 0) & ~((u32) 0); u32 psci_fn = smccc_get_function(vcpu);
unsigned long val; unsigned long val;
int ret = 1; int ret = 1;
...@@ -221,7 +245,7 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) ...@@ -221,7 +245,7 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu)
* Bits[31:16] = Major Version = 0 * Bits[31:16] = Major Version = 0
* Bits[15:0] = Minor Version = 2 * Bits[15:0] = Minor Version = 2
*/ */
val = 2; val = KVM_ARM_PSCI_0_2;
break; break;
case PSCI_0_2_FN_CPU_SUSPEND: case PSCI_0_2_FN_CPU_SUSPEND:
case PSCI_0_2_FN64_CPU_SUSPEND: case PSCI_0_2_FN64_CPU_SUSPEND:
...@@ -278,14 +302,56 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) ...@@ -278,14 +302,56 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu)
break; break;
} }
vcpu_set_reg(vcpu, 0, val); smccc_set_retval(vcpu, val, 0, 0, 0);
return ret;
}
static int kvm_psci_1_0_call(struct kvm_vcpu *vcpu)
{
u32 psci_fn = smccc_get_function(vcpu);
u32 feature;
unsigned long val;
int ret = 1;
switch(psci_fn) {
case PSCI_0_2_FN_PSCI_VERSION:
val = KVM_ARM_PSCI_1_0;
break;
case PSCI_1_0_FN_PSCI_FEATURES:
feature = smccc_get_arg1(vcpu);
switch(feature) {
case PSCI_0_2_FN_PSCI_VERSION:
case PSCI_0_2_FN_CPU_SUSPEND:
case PSCI_0_2_FN64_CPU_SUSPEND:
case PSCI_0_2_FN_CPU_OFF:
case PSCI_0_2_FN_CPU_ON:
case PSCI_0_2_FN64_CPU_ON:
case PSCI_0_2_FN_AFFINITY_INFO:
case PSCI_0_2_FN64_AFFINITY_INFO:
case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
case PSCI_0_2_FN_SYSTEM_OFF:
case PSCI_0_2_FN_SYSTEM_RESET:
case PSCI_1_0_FN_PSCI_FEATURES:
case ARM_SMCCC_VERSION_FUNC_ID:
val = 0;
break;
default:
val = PSCI_RET_NOT_SUPPORTED;
break;
}
break;
default:
return kvm_psci_0_2_call(vcpu);
}
smccc_set_retval(vcpu, val, 0, 0, 0);
return ret; return ret;
} }
static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu) static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu)
{ {
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
unsigned long psci_fn = vcpu_get_reg(vcpu, 0) & ~((u32) 0); u32 psci_fn = smccc_get_function(vcpu);
unsigned long val; unsigned long val;
switch (psci_fn) { switch (psci_fn) {
...@@ -303,7 +369,7 @@ static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu) ...@@ -303,7 +369,7 @@ static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu)
break; break;
} }
vcpu_set_reg(vcpu, 0, val); smccc_set_retval(vcpu, val, 0, 0, 0);
return 1; return 1;
} }
...@@ -321,9 +387,11 @@ static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu) ...@@ -321,9 +387,11 @@ static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu)
* Errors: * Errors:
* -EINVAL: Unrecognized PSCI function * -EINVAL: Unrecognized PSCI function
*/ */
int kvm_psci_call(struct kvm_vcpu *vcpu) static int kvm_psci_call(struct kvm_vcpu *vcpu)
{ {
switch (kvm_psci_version(vcpu)) { switch (kvm_psci_version(vcpu, vcpu->kvm)) {
case KVM_ARM_PSCI_1_0:
return kvm_psci_1_0_call(vcpu);
case KVM_ARM_PSCI_0_2: case KVM_ARM_PSCI_0_2:
return kvm_psci_0_2_call(vcpu); return kvm_psci_0_2_call(vcpu);
case KVM_ARM_PSCI_0_1: case KVM_ARM_PSCI_0_1:
...@@ -332,3 +400,30 @@ int kvm_psci_call(struct kvm_vcpu *vcpu) ...@@ -332,3 +400,30 @@ int kvm_psci_call(struct kvm_vcpu *vcpu)
return -EINVAL; return -EINVAL;
}; };
} }
int kvm_hvc_call_handler(struct kvm_vcpu *vcpu)
{
u32 func_id = smccc_get_function(vcpu);
u32 val = PSCI_RET_NOT_SUPPORTED;
u32 feature;
switch (func_id) {
case ARM_SMCCC_VERSION_FUNC_ID:
val = ARM_SMCCC_VERSION_1_1;
break;
case ARM_SMCCC_ARCH_FEATURES_FUNC_ID:
feature = smccc_get_arg1(vcpu);
switch(feature) {
case ARM_SMCCC_ARCH_WORKAROUND_1:
if (kvm_arm_harden_branch_predictor())
val = 0;
break;
}
break;
default:
return kvm_psci_call(vcpu);
}
smccc_set_retval(vcpu, val, 0, 0, 0);
return 1;
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment