Commit 6b00f7ef authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Catalin Marinas:
 "arm64 updates for 3.20:

   - reimplementation of the virtual remapping of UEFI Runtime Services
     in a way that is stable across kexec
   - emulation of the "setend" instruction for 32-bit tasks (user
     endianness switching trapped in the kernel, SCTLR_EL1.E0E bit set
     accordingly)
   - compat_sys_call_table implemented in C (from asm) and made it a
     constant array together with sys_call_table
   - export CPU cache information via /sys (like other architectures)
   - DMA API implementation clean-up in preparation for IOMMU support
   - macros clean-up for KVM
   - dropped some unnecessary cache+tlb maintenance
   - CONFIG_ARM64_CPU_SUSPEND clean-up
   - defconfig update (CPU_IDLE)

  The EFI changes going via the arm64 tree have been acked by Matt
  Fleming.  There is also a patch adding sys_*stat64 prototypes to
  include/linux/syscalls.h, acked by Andrew Morton"

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (47 commits)
  arm64: compat: Remove incorrect comment in compat_siginfo
  arm64: Fix section mismatch on alloc_init_p[mu]d()
  arm64: Avoid breakage caused by .altmacro in fpsimd save/restore macros
  arm64: mm: use *_sect to check for section maps
  arm64: drop unnecessary cache+tlb maintenance
  arm64:mm: free the useless initial page table
  arm64: Enable CPU_IDLE in defconfig
  arm64: kernel: remove ARM64_CPU_SUSPEND config option
  arm64: make sys_call_table const
  arm64: Remove asm/syscalls.h
  arm64: Implement the compat_sys_call_table in C
  syscalls: Declare sys_*stat64 prototypes if __ARCH_WANT_(COMPAT_)STAT64
  compat: Declare compat_sys_sigpending and compat_sys_sigprocmask prototypes
  arm64: uapi: expose our struct ucontext to the uapi headers
  smp, ARM64: Kill SMP single function call interrupt
  arm64: Emulate SETEND for AArch32 tasks
  arm64: Consolidate hotplug notifier for instruction emulation
  arm64: Track system support for mixed endian EL0
  arm64: implement generic IOMMU configuration
  arm64: Combine coherent and non-coherent swiotlb dma_ops
  ...
parents b3d6524f d476d94f
...@@ -32,6 +32,9 @@ The default mode depends on the status of the instruction in the ...@@ -32,6 +32,9 @@ The default mode depends on the status of the instruction in the
architecture. Deprecated instructions should default to emulation architecture. Deprecated instructions should default to emulation
while obsolete instructions must be undefined by default. while obsolete instructions must be undefined by default.
Note: Instruction emulation may not be possible in all cases. See
individual instruction notes for further information.
Supported legacy instructions Supported legacy instructions
----------------------------- -----------------------------
* SWP{B} * SWP{B}
...@@ -43,3 +46,12 @@ Default: Undef (0) ...@@ -43,3 +46,12 @@ Default: Undef (0)
Node: /proc/sys/abi/cp15_barrier Node: /proc/sys/abi/cp15_barrier
Status: Deprecated Status: Deprecated
Default: Emulate (1) Default: Emulate (1)
* SETEND
Node: /proc/sys/abi/setend
Status: Deprecated
Default: Emulate (1)*
Note: All the cpus on the system must have mixed endian support at EL0
for this feature to be enabled. If a new CPU - which doesn't support mixed
endian - is hotplugged in after this feature has been enabled, there could
be unexpected results in the application.
...@@ -540,6 +540,21 @@ config CP15_BARRIER_EMULATION ...@@ -540,6 +540,21 @@ config CP15_BARRIER_EMULATION
If unsure, say Y If unsure, say Y
config SETEND_EMULATION
bool "Emulate SETEND instruction"
help
The SETEND instruction alters the data-endianness of the
AArch32 EL0, and is deprecated in ARMv8.
Say Y here to enable software emulation of the instruction
for AArch32 userspace code.
Note: All the cpus on the system must have mixed endian support at EL0
for this feature to be enabled. If a new CPU - which doesn't support mixed
endian - is hotplugged in after this feature has been enabled, there could
be unexpected results in the applications.
If unsure, say Y
endif endif
endmenu endmenu
...@@ -627,9 +642,6 @@ source "kernel/power/Kconfig" ...@@ -627,9 +642,6 @@ source "kernel/power/Kconfig"
config ARCH_SUSPEND_POSSIBLE config ARCH_SUSPEND_POSSIBLE
def_bool y def_bool y
config ARM64_CPU_SUSPEND
def_bool PM_SLEEP
endmenu endmenu
menu "CPU Power Management" menu "CPU Power Management"
......
...@@ -66,4 +66,27 @@ config DEBUG_SET_MODULE_RONX ...@@ -66,4 +66,27 @@ config DEBUG_SET_MODULE_RONX
against certain classes of kernel exploits. against certain classes of kernel exploits.
If in doubt, say "N". If in doubt, say "N".
config DEBUG_RODATA
bool "Make kernel text and rodata read-only"
help
If this is set, kernel text and rodata will be made read-only. This
is to help catch accidental or malicious attempts to change the
kernel's executable code. Additionally splits rodata from kernel
text so it can be made explicitly non-executable.
If in doubt, say Y
config DEBUG_ALIGN_RODATA
depends on DEBUG_RODATA && !ARM64_64K_PAGES
bool "Align linker sections up to SECTION_SIZE"
help
If this option is enabled, sections that may potentially be marked as
read only or non-executable will be aligned up to the section size of
the kernel. This prevents sections from being split into pages and
avoids a potential TLB penalty. The downside is an increase in
alignment and potentially wasted space. Turn on this option if
performance is more important than memory pressure.
If in doubt, say N
endmenu endmenu
...@@ -15,8 +15,6 @@ CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET) ...@@ -15,8 +15,6 @@ CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET)
OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S
GZFLAGS :=-9 GZFLAGS :=-9
LIBGCC := $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name)
KBUILD_DEFCONFIG := defconfig KBUILD_DEFCONFIG := defconfig
KBUILD_CFLAGS += -mgeneral-regs-only KBUILD_CFLAGS += -mgeneral-regs-only
...@@ -50,7 +48,6 @@ core-$(CONFIG_KVM) += arch/arm64/kvm/ ...@@ -50,7 +48,6 @@ core-$(CONFIG_KVM) += arch/arm64/kvm/
core-$(CONFIG_XEN) += arch/arm64/xen/ core-$(CONFIG_XEN) += arch/arm64/xen/
core-$(CONFIG_CRYPTO) += arch/arm64/crypto/ core-$(CONFIG_CRYPTO) += arch/arm64/crypto/
libs-y := arch/arm64/lib/ $(libs-y) libs-y := arch/arm64/lib/ $(libs-y)
libs-y += $(LIBGCC)
libs-$(CONFIG_EFI_STUB) += drivers/firmware/efi/libstub/ libs-$(CONFIG_EFI_STUB) += drivers/firmware/efi/libstub/
# Default target when executing plain make # Default target when executing plain make
......
...@@ -45,6 +45,8 @@ CONFIG_CMA=y ...@@ -45,6 +45,8 @@ CONFIG_CMA=y
CONFIG_CMDLINE="console=ttyAMA0" CONFIG_CMDLINE="console=ttyAMA0"
# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
CONFIG_COMPAT=y CONFIG_COMPAT=y
CONFIG_CPU_IDLE=y
CONFIG_ARM64_CPUIDLE=y
CONFIG_NET=y CONFIG_NET=y
CONFIG_PACKET=y CONFIG_PACKET=y
CONFIG_UNIX=y CONFIG_UNIX=y
......
...@@ -152,4 +152,9 @@ int set_memory_ro(unsigned long addr, int numpages); ...@@ -152,4 +152,9 @@ int set_memory_ro(unsigned long addr, int numpages);
int set_memory_rw(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages);
int set_memory_x(unsigned long addr, int numpages); int set_memory_x(unsigned long addr, int numpages);
int set_memory_nx(unsigned long addr, int numpages); int set_memory_nx(unsigned long addr, int numpages);
#ifdef CONFIG_DEBUG_RODATA
void mark_rodata_ro(void);
#endif
#endif #endif
...@@ -39,24 +39,41 @@ ...@@ -39,24 +39,41 @@
extern unsigned long __icache_flags; extern unsigned long __icache_flags;
/*
* NumSets, bits[27:13] - (Number of sets in cache) - 1
* Associativity, bits[12:3] - (Associativity of cache) - 1
* LineSize, bits[2:0] - (Log2(Number of words in cache line)) - 2
*/
#define CCSIDR_EL1_WRITE_THROUGH BIT(31)
#define CCSIDR_EL1_WRITE_BACK BIT(30)
#define CCSIDR_EL1_READ_ALLOCATE BIT(29)
#define CCSIDR_EL1_WRITE_ALLOCATE BIT(28)
#define CCSIDR_EL1_LINESIZE_MASK 0x7 #define CCSIDR_EL1_LINESIZE_MASK 0x7
#define CCSIDR_EL1_LINESIZE(x) ((x) & CCSIDR_EL1_LINESIZE_MASK) #define CCSIDR_EL1_LINESIZE(x) ((x) & CCSIDR_EL1_LINESIZE_MASK)
#define CCSIDR_EL1_ASSOCIATIVITY_SHIFT 3
#define CCSIDR_EL1_ASSOCIATIVITY_MASK 0x3ff
#define CCSIDR_EL1_ASSOCIATIVITY(x) \
(((x) >> CCSIDR_EL1_ASSOCIATIVITY_SHIFT) & CCSIDR_EL1_ASSOCIATIVITY_MASK)
#define CCSIDR_EL1_NUMSETS_SHIFT 13 #define CCSIDR_EL1_NUMSETS_SHIFT 13
#define CCSIDR_EL1_NUMSETS_MASK (0x7fff << CCSIDR_EL1_NUMSETS_SHIFT) #define CCSIDR_EL1_NUMSETS_MASK 0x7fff
#define CCSIDR_EL1_NUMSETS(x) \ #define CCSIDR_EL1_NUMSETS(x) \
(((x) & CCSIDR_EL1_NUMSETS_MASK) >> CCSIDR_EL1_NUMSETS_SHIFT) (((x) >> CCSIDR_EL1_NUMSETS_SHIFT) & CCSIDR_EL1_NUMSETS_MASK)
#define CACHE_LINESIZE(x) (16 << CCSIDR_EL1_LINESIZE(x))
#define CACHE_NUMSETS(x) (CCSIDR_EL1_NUMSETS(x) + 1)
#define CACHE_ASSOCIATIVITY(x) (CCSIDR_EL1_ASSOCIATIVITY(x) + 1)
extern u64 __attribute_const__ icache_get_ccsidr(void); extern u64 __attribute_const__ cache_get_ccsidr(u64 csselr);
/* Helpers for Level 1 Instruction cache csselr = 1L */
static inline int icache_get_linesize(void) static inline int icache_get_linesize(void)
{ {
return 16 << CCSIDR_EL1_LINESIZE(icache_get_ccsidr()); return CACHE_LINESIZE(cache_get_ccsidr(1L));
} }
static inline int icache_get_numsets(void) static inline int icache_get_numsets(void)
{ {
return 1 + CCSIDR_EL1_NUMSETS(icache_get_ccsidr()); return CACHE_NUMSETS(cache_get_ccsidr(1L));
} }
/* /*
......
...@@ -161,7 +161,6 @@ typedef struct compat_siginfo { ...@@ -161,7 +161,6 @@ typedef struct compat_siginfo {
int si_code; int si_code;
union { union {
/* The padding is the same size as AArch64. */
int _pad[128/sizeof(int) - 3]; int _pad[128/sizeof(int) - 3];
/* kill() */ /* kill() */
......
...@@ -28,8 +28,6 @@ struct device_node; ...@@ -28,8 +28,6 @@ struct device_node;
* enable-method property. * enable-method property.
* @cpu_init: Reads any data necessary for a specific enable-method from the * @cpu_init: Reads any data necessary for a specific enable-method from the
* devicetree, for a given cpu node and proposed logical id. * devicetree, for a given cpu node and proposed logical id.
* @cpu_init_idle: Reads any data necessary to initialize CPU idle states from
* devicetree, for a given cpu node and proposed logical id.
* @cpu_prepare: Early one-time preparation step for a cpu. If there is a * @cpu_prepare: Early one-time preparation step for a cpu. If there is a
* mechanism for doing so, tests whether it is possible to boot * mechanism for doing so, tests whether it is possible to boot
* the given CPU. * the given CPU.
...@@ -42,6 +40,8 @@ struct device_node; ...@@ -42,6 +40,8 @@ struct device_node;
* @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the * @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the
* cpu being killed. * cpu being killed.
* @cpu_kill: Ensures a cpu has left the kernel. Called from another cpu. * @cpu_kill: Ensures a cpu has left the kernel. Called from another cpu.
* @cpu_init_idle: Reads any data necessary to initialize CPU idle states from
* devicetree, for a given cpu node and proposed logical id.
* @cpu_suspend: Suspends a cpu and saves the required context. May fail owing * @cpu_suspend: Suspends a cpu and saves the required context. May fail owing
* to wrong parameters or error conditions. Called from the * to wrong parameters or error conditions. Called from the
* CPU being suspended. Must be called with IRQs disabled. * CPU being suspended. Must be called with IRQs disabled.
...@@ -49,7 +49,6 @@ struct device_node; ...@@ -49,7 +49,6 @@ struct device_node;
struct cpu_operations { struct cpu_operations {
const char *name; const char *name;
int (*cpu_init)(struct device_node *, unsigned int); int (*cpu_init)(struct device_node *, unsigned int);
int (*cpu_init_idle)(struct device_node *, unsigned int);
int (*cpu_prepare)(unsigned int); int (*cpu_prepare)(unsigned int);
int (*cpu_boot)(unsigned int); int (*cpu_boot)(unsigned int);
void (*cpu_postboot)(void); void (*cpu_postboot)(void);
...@@ -58,7 +57,8 @@ struct cpu_operations { ...@@ -58,7 +57,8 @@ struct cpu_operations {
void (*cpu_die)(unsigned int cpu); void (*cpu_die)(unsigned int cpu);
int (*cpu_kill)(unsigned int cpu); int (*cpu_kill)(unsigned int cpu);
#endif #endif
#ifdef CONFIG_ARM64_CPU_SUSPEND #ifdef CONFIG_CPU_IDLE
int (*cpu_init_idle)(struct device_node *, unsigned int);
int (*cpu_suspend)(unsigned long); int (*cpu_suspend)(unsigned long);
#endif #endif
}; };
......
...@@ -52,6 +52,8 @@ static inline void cpus_set_cap(unsigned int num) ...@@ -52,6 +52,8 @@ static inline void cpus_set_cap(unsigned int num)
} }
void check_local_cpu_errata(void); void check_local_cpu_errata(void);
bool cpu_supports_mixed_endian_el0(void);
bool system_supports_mixed_endian_el0(void);
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
......
...@@ -3,11 +3,17 @@ ...@@ -3,11 +3,17 @@
#ifdef CONFIG_CPU_IDLE #ifdef CONFIG_CPU_IDLE
extern int cpu_init_idle(unsigned int cpu); extern int cpu_init_idle(unsigned int cpu);
extern int cpu_suspend(unsigned long arg);
#else #else
static inline int cpu_init_idle(unsigned int cpu) static inline int cpu_init_idle(unsigned int cpu)
{ {
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
static inline int cpu_suspend(unsigned long arg)
{
return -EOPNOTSUPP;
}
#endif #endif
#endif #endif
...@@ -72,6 +72,18 @@ ...@@ -72,6 +72,18 @@
#define APM_CPU_PART_POTENZA 0x000 #define APM_CPU_PART_POTENZA 0x000
#define ID_AA64MMFR0_BIGENDEL0_SHIFT 16
#define ID_AA64MMFR0_BIGENDEL0_MASK (0xf << ID_AA64MMFR0_BIGENDEL0_SHIFT)
#define ID_AA64MMFR0_BIGENDEL0(mmfr0) \
(((mmfr0) & ID_AA64MMFR0_BIGENDEL0_MASK) >> ID_AA64MMFR0_BIGENDEL0_SHIFT)
#define ID_AA64MMFR0_BIGEND_SHIFT 8
#define ID_AA64MMFR0_BIGEND_MASK (0xf << ID_AA64MMFR0_BIGEND_SHIFT)
#define ID_AA64MMFR0_BIGEND(mmfr0) \
(((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT)
#define SCTLR_EL1_CP15BEN (0x1 << 5)
#define SCTLR_EL1_SED (0x1 << 8)
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
/* /*
...@@ -104,6 +116,11 @@ static inline u32 __attribute_const__ read_cpuid_cachetype(void) ...@@ -104,6 +116,11 @@ static inline u32 __attribute_const__ read_cpuid_cachetype(void)
return read_cpuid(CTR_EL0); return read_cpuid(CTR_EL0);
} }
static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
{
return (ID_AA64MMFR0_BIGEND(mmfr0) == 0x1) ||
(ID_AA64MMFR0_BIGENDEL0(mmfr0) == 0x1);
}
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
#endif #endif
...@@ -28,8 +28,6 @@ ...@@ -28,8 +28,6 @@
#define DMA_ERROR_CODE (~(dma_addr_t)0) #define DMA_ERROR_CODE (~(dma_addr_t)0)
extern struct dma_map_ops *dma_ops; extern struct dma_map_ops *dma_ops;
extern struct dma_map_ops coherent_swiotlb_dma_ops;
extern struct dma_map_ops noncoherent_swiotlb_dma_ops;
static inline struct dma_map_ops *__generic_dma_ops(struct device *dev) static inline struct dma_map_ops *__generic_dma_ops(struct device *dev)
{ {
...@@ -47,23 +45,18 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) ...@@ -47,23 +45,18 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev)
return __generic_dma_ops(dev); return __generic_dma_ops(dev);
} }
static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops)
{
dev->archdata.dma_ops = ops;
}
static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
struct iommu_ops *iommu, bool coherent) struct iommu_ops *iommu, bool coherent)
{ {
dev->archdata.dma_coherent = coherent; dev->archdata.dma_coherent = coherent;
if (coherent)
set_dma_ops(dev, &coherent_swiotlb_dma_ops);
} }
#define arch_setup_dma_ops arch_setup_dma_ops #define arch_setup_dma_ops arch_setup_dma_ops
/* do not use this function in a driver */ /* do not use this function in a driver */
static inline bool is_device_dma_coherent(struct device *dev) static inline bool is_device_dma_coherent(struct device *dev)
{ {
if (!dev)
return false;
return dev->archdata.dma_coherent; return dev->archdata.dma_coherent;
} }
......
...@@ -6,29 +6,33 @@ ...@@ -6,29 +6,33 @@
#ifdef CONFIG_EFI #ifdef CONFIG_EFI
extern void efi_init(void); extern void efi_init(void);
extern void efi_idmap_init(void);
#else #else
#define efi_init() #define efi_init()
#define efi_idmap_init()
#endif #endif
#define efi_call_virt(f, ...) \ #define efi_call_virt(f, ...) \
({ \ ({ \
efi_##f##_t *__f = efi.systab->runtime->f; \ efi_##f##_t *__f; \
efi_status_t __s; \ efi_status_t __s; \
\ \
kernel_neon_begin(); \ kernel_neon_begin(); \
efi_virtmap_load(); \
__f = efi.systab->runtime->f; \
__s = __f(__VA_ARGS__); \ __s = __f(__VA_ARGS__); \
efi_virtmap_unload(); \
kernel_neon_end(); \ kernel_neon_end(); \
__s; \ __s; \
}) })
#define __efi_call_virt(f, ...) \ #define __efi_call_virt(f, ...) \
({ \ ({ \
efi_##f##_t *__f = efi.systab->runtime->f; \ efi_##f##_t *__f; \
\ \
kernel_neon_begin(); \ kernel_neon_begin(); \
efi_virtmap_load(); \
__f = efi.systab->runtime->f; \
__f(__VA_ARGS__); \ __f(__VA_ARGS__); \
efi_virtmap_unload(); \
kernel_neon_end(); \ kernel_neon_end(); \
}) })
...@@ -44,4 +48,22 @@ extern void efi_idmap_init(void); ...@@ -44,4 +48,22 @@ extern void efi_idmap_init(void);
#define efi_call_early(f, ...) sys_table_arg->boottime->f(__VA_ARGS__) #define efi_call_early(f, ...) sys_table_arg->boottime->f(__VA_ARGS__)
#define EFI_ALLOC_ALIGN SZ_64K
/*
* On ARM systems, virtually remapped UEFI runtime services are set up in two
* distinct stages:
* - The stub retrieves the final version of the memory map from UEFI, populates
* the virt_addr fields and calls the SetVirtualAddressMap() [SVAM] runtime
* service to communicate the new mapping to the firmware (Note that the new
* mapping is not live at this time)
* - During an early initcall(), the EFI system table is permanently remapped
* and the virtual remapping of the UEFI Runtime Services regions is loaded
* into a private set of page tables. If this all succeeds, the Runtime
* Services are enabled and the EFI_RUNTIME_SERVICES bit set.
*/
void efi_virtmap_load(void);
void efi_virtmap_unload(void);
#endif /* _ASM_EFI_H */ #endif /* _ASM_EFI_H */
...@@ -18,40 +18,89 @@ ...@@ -18,40 +18,89 @@
#ifndef __ASM_ESR_H #ifndef __ASM_ESR_H
#define __ASM_ESR_H #define __ASM_ESR_H
#define ESR_EL1_WRITE (1 << 6) #define ESR_ELx_EC_UNKNOWN (0x00)
#define ESR_EL1_CM (1 << 8) #define ESR_ELx_EC_WFx (0x01)
#define ESR_EL1_IL (1 << 25) /* Unallocated EC: 0x02 */
#define ESR_ELx_EC_CP15_32 (0x03)
#define ESR_ELx_EC_CP15_64 (0x04)
#define ESR_ELx_EC_CP14_MR (0x05)
#define ESR_ELx_EC_CP14_LS (0x06)
#define ESR_ELx_EC_FP_ASIMD (0x07)
#define ESR_ELx_EC_CP10_ID (0x08)
/* Unallocated EC: 0x09 - 0x0B */
#define ESR_ELx_EC_CP14_64 (0x0C)
/* Unallocated EC: 0x0d */
#define ESR_ELx_EC_ILL (0x0E)
/* Unallocated EC: 0x0F - 0x10 */
#define ESR_ELx_EC_SVC32 (0x11)
#define ESR_ELx_EC_HVC32 (0x12)
#define ESR_ELx_EC_SMC32 (0x13)
/* Unallocated EC: 0x14 */
#define ESR_ELx_EC_SVC64 (0x15)
#define ESR_ELx_EC_HVC64 (0x16)
#define ESR_ELx_EC_SMC64 (0x17)
#define ESR_ELx_EC_SYS64 (0x18)
/* Unallocated EC: 0x19 - 0x1E */
#define ESR_ELx_EC_IMP_DEF (0x1f)
#define ESR_ELx_EC_IABT_LOW (0x20)
#define ESR_ELx_EC_IABT_CUR (0x21)
#define ESR_ELx_EC_PC_ALIGN (0x22)
/* Unallocated EC: 0x23 */
#define ESR_ELx_EC_DABT_LOW (0x24)
#define ESR_ELx_EC_DABT_CUR (0x25)
#define ESR_ELx_EC_SP_ALIGN (0x26)
/* Unallocated EC: 0x27 */
#define ESR_ELx_EC_FP_EXC32 (0x28)
/* Unallocated EC: 0x29 - 0x2B */
#define ESR_ELx_EC_FP_EXC64 (0x2C)
/* Unallocated EC: 0x2D - 0x2E */
#define ESR_ELx_EC_SERROR (0x2F)
#define ESR_ELx_EC_BREAKPT_LOW (0x30)
#define ESR_ELx_EC_BREAKPT_CUR (0x31)
#define ESR_ELx_EC_SOFTSTP_LOW (0x32)
#define ESR_ELx_EC_SOFTSTP_CUR (0x33)
#define ESR_ELx_EC_WATCHPT_LOW (0x34)
#define ESR_ELx_EC_WATCHPT_CUR (0x35)
/* Unallocated EC: 0x36 - 0x37 */
#define ESR_ELx_EC_BKPT32 (0x38)
/* Unallocated EC: 0x39 */
#define ESR_ELx_EC_VECTOR32 (0x3A)
/* Unallocted EC: 0x3B */
#define ESR_ELx_EC_BRK64 (0x3C)
/* Unallocated EC: 0x3D - 0x3F */
#define ESR_ELx_EC_MAX (0x3F)
#define ESR_EL1_EC_SHIFT (26) #define ESR_ELx_EC_SHIFT (26)
#define ESR_EL1_EC_UNKNOWN (0x00) #define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT)
#define ESR_EL1_EC_WFI (0x01)
#define ESR_EL1_EC_CP15_32 (0x03) #define ESR_ELx_IL (UL(1) << 25)
#define ESR_EL1_EC_CP15_64 (0x04) #define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1)
#define ESR_EL1_EC_CP14_MR (0x05) #define ESR_ELx_ISV (UL(1) << 24)
#define ESR_EL1_EC_CP14_LS (0x06) #define ESR_ELx_SAS_SHIFT (22)
#define ESR_EL1_EC_FP_ASIMD (0x07) #define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT)
#define ESR_EL1_EC_CP10_ID (0x08) #define ESR_ELx_SSE (UL(1) << 21)
#define ESR_EL1_EC_CP14_64 (0x0C) #define ESR_ELx_SRT_SHIFT (16)
#define ESR_EL1_EC_ILL_ISS (0x0E) #define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT)
#define ESR_EL1_EC_SVC32 (0x11) #define ESR_ELx_SF (UL(1) << 15)
#define ESR_EL1_EC_SVC64 (0x15) #define ESR_ELx_AR (UL(1) << 14)
#define ESR_EL1_EC_SYS64 (0x18) #define ESR_ELx_EA (UL(1) << 9)
#define ESR_EL1_EC_IABT_EL0 (0x20) #define ESR_ELx_CM (UL(1) << 8)
#define ESR_EL1_EC_IABT_EL1 (0x21) #define ESR_ELx_S1PTW (UL(1) << 7)
#define ESR_EL1_EC_PC_ALIGN (0x22) #define ESR_ELx_WNR (UL(1) << 6)
#define ESR_EL1_EC_DABT_EL0 (0x24) #define ESR_ELx_FSC (0x3F)
#define ESR_EL1_EC_DABT_EL1 (0x25) #define ESR_ELx_FSC_TYPE (0x3C)
#define ESR_EL1_EC_SP_ALIGN (0x26) #define ESR_ELx_FSC_EXTABT (0x10)
#define ESR_EL1_EC_FP_EXC32 (0x28) #define ESR_ELx_FSC_FAULT (0x04)
#define ESR_EL1_EC_FP_EXC64 (0x2C) #define ESR_ELx_FSC_PERM (0x0C)
#define ESR_EL1_EC_SERROR (0x2F) #define ESR_ELx_CV (UL(1) << 24)
#define ESR_EL1_EC_BREAKPT_EL0 (0x30) #define ESR_ELx_COND_SHIFT (20)
#define ESR_EL1_EC_BREAKPT_EL1 (0x31) #define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT)
#define ESR_EL1_EC_SOFTSTP_EL0 (0x32) #define ESR_ELx_WFx_ISS_WFE (UL(1) << 0)
#define ESR_EL1_EC_SOFTSTP_EL1 (0x33)
#define ESR_EL1_EC_WATCHPT_EL0 (0x34) #ifndef __ASSEMBLY__
#define ESR_EL1_EC_WATCHPT_EL1 (0x35) #include <asm/types.h>
#define ESR_EL1_EC_BKPT32 (0x38)
#define ESR_EL1_EC_BRK64 (0x3C) const char *esr_get_class_string(u32 esr);
#endif /* __ASSEMBLY */
#endif /* __ASM_ESR_H */ #endif /* __ASM_ESR_H */
...@@ -49,6 +49,7 @@ enum fixed_addresses { ...@@ -49,6 +49,7 @@ enum fixed_addresses {
FIX_BTMAP_END = __end_of_permanent_fixed_addresses, FIX_BTMAP_END = __end_of_permanent_fixed_addresses,
FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1, FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1,
FIX_TEXT_POKE0,
__end_of_fixed_addresses __end_of_fixed_addresses
}; };
......
...@@ -76,7 +76,6 @@ ...@@ -76,7 +76,6 @@
fpsimd_restore_fpcr x\tmpnr, \state fpsimd_restore_fpcr x\tmpnr, \state
.endm .endm
.altmacro
.macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2 .macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2
mrs x\tmpnr1, fpsr mrs x\tmpnr1, fpsr
str w\numnr, [\state, #8] str w\numnr, [\state, #8]
...@@ -86,11 +85,22 @@ ...@@ -86,11 +85,22 @@
add \state, \state, x\numnr, lsl #4 add \state, \state, x\numnr, lsl #4
sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1 sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1
br x\tmpnr1 br x\tmpnr1
.irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0 stp q30, q31, [\state, #-16 * 30 - 16]
.irp qb, %(qa + 1) stp q28, q29, [\state, #-16 * 28 - 16]
stp q\qa, q\qb, [\state, # -16 * \qa - 16] stp q26, q27, [\state, #-16 * 26 - 16]
.endr stp q24, q25, [\state, #-16 * 24 - 16]
.endr stp q22, q23, [\state, #-16 * 22 - 16]
stp q20, q21, [\state, #-16 * 20 - 16]
stp q18, q19, [\state, #-16 * 18 - 16]
stp q16, q17, [\state, #-16 * 16 - 16]
stp q14, q15, [\state, #-16 * 14 - 16]
stp q12, q13, [\state, #-16 * 12 - 16]
stp q10, q11, [\state, #-16 * 10 - 16]
stp q8, q9, [\state, #-16 * 8 - 16]
stp q6, q7, [\state, #-16 * 6 - 16]
stp q4, q5, [\state, #-16 * 4 - 16]
stp q2, q3, [\state, #-16 * 2 - 16]
stp q0, q1, [\state, #-16 * 0 - 16]
0: 0:
.endm .endm
...@@ -103,10 +113,21 @@ ...@@ -103,10 +113,21 @@
add \state, \state, x\tmpnr2, lsl #4 add \state, \state, x\tmpnr2, lsl #4
sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1 sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1
br x\tmpnr1 br x\tmpnr1
.irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0 ldp q30, q31, [\state, #-16 * 30 - 16]
.irp qb, %(qa + 1) ldp q28, q29, [\state, #-16 * 28 - 16]
ldp q\qa, q\qb, [\state, # -16 * \qa - 16] ldp q26, q27, [\state, #-16 * 26 - 16]
.endr ldp q24, q25, [\state, #-16 * 24 - 16]
.endr ldp q22, q23, [\state, #-16 * 22 - 16]
ldp q20, q21, [\state, #-16 * 20 - 16]
ldp q18, q19, [\state, #-16 * 18 - 16]
ldp q16, q17, [\state, #-16 * 16 - 16]
ldp q14, q15, [\state, #-16 * 14 - 16]
ldp q12, q13, [\state, #-16 * 12 - 16]
ldp q10, q11, [\state, #-16 * 10 - 16]
ldp q8, q9, [\state, #-16 * 8 - 16]
ldp q6, q7, [\state, #-16 * 6 - 16]
ldp q4, q5, [\state, #-16 * 4 - 16]
ldp q2, q3, [\state, #-16 * 2 - 16]
ldp q0, q1, [\state, #-16 * 0 - 16]
0: 0:
.endm .endm
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
#include <linux/threads.h> #include <linux/threads.h>
#include <asm/irq.h> #include <asm/irq.h>
#define NR_IPI 6 #define NR_IPI 5
typedef struct { typedef struct {
unsigned int __softirq_pending; unsigned int __softirq_pending;
......
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
#include <asm/byteorder.h> #include <asm/byteorder.h>
#include <asm/barrier.h> #include <asm/barrier.h>
#include <asm/memory.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/early_ioremap.h> #include <asm/early_ioremap.h>
#include <asm/alternative.h> #include <asm/alternative.h>
...@@ -145,8 +146,8 @@ static inline u64 __raw_readq(const volatile void __iomem *addr) ...@@ -145,8 +146,8 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
* I/O port access primitives. * I/O port access primitives.
*/ */
#define arch_has_dev_port() (1) #define arch_has_dev_port() (1)
#define IO_SPACE_LIMIT (SZ_32M - 1) #define IO_SPACE_LIMIT (PCI_IO_SIZE - 1)
#define PCI_IOBASE ((void __iomem *)(MODULES_VADDR - SZ_32M)) #define PCI_IOBASE ((void __iomem *)PCI_IO_START)
/* /*
* String version of I/O memory access operations. * String version of I/O memory access operations.
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#ifndef __ARM64_KVM_ARM_H__ #ifndef __ARM64_KVM_ARM_H__
#define __ARM64_KVM_ARM_H__ #define __ARM64_KVM_ARM_H__
#include <asm/esr.h>
#include <asm/memory.h> #include <asm/memory.h>
#include <asm/types.h> #include <asm/types.h>
...@@ -184,77 +185,11 @@ ...@@ -184,77 +185,11 @@
#define MDCR_EL2_TPMCR (1 << 5) #define MDCR_EL2_TPMCR (1 << 5)
#define MDCR_EL2_HPMN_MASK (0x1F) #define MDCR_EL2_HPMN_MASK (0x1F)
/* Exception Syndrome Register (ESR) bits */ /* For compatibility with fault code shared with 32-bit */
#define ESR_EL2_EC_SHIFT (26) #define FSC_FAULT ESR_ELx_FSC_FAULT
#define ESR_EL2_EC (UL(0x3f) << ESR_EL2_EC_SHIFT) #define FSC_PERM ESR_ELx_FSC_PERM
#define ESR_EL2_IL (UL(1) << 25)
#define ESR_EL2_ISS (ESR_EL2_IL - 1)
#define ESR_EL2_ISV_SHIFT (24)
#define ESR_EL2_ISV (UL(1) << ESR_EL2_ISV_SHIFT)
#define ESR_EL2_SAS_SHIFT (22)
#define ESR_EL2_SAS (UL(3) << ESR_EL2_SAS_SHIFT)
#define ESR_EL2_SSE (1 << 21)
#define ESR_EL2_SRT_SHIFT (16)
#define ESR_EL2_SRT_MASK (0x1f << ESR_EL2_SRT_SHIFT)
#define ESR_EL2_SF (1 << 15)
#define ESR_EL2_AR (1 << 14)
#define ESR_EL2_EA (1 << 9)
#define ESR_EL2_CM (1 << 8)
#define ESR_EL2_S1PTW (1 << 7)
#define ESR_EL2_WNR (1 << 6)
#define ESR_EL2_FSC (0x3f)
#define ESR_EL2_FSC_TYPE (0x3c)
#define ESR_EL2_CV_SHIFT (24)
#define ESR_EL2_CV (UL(1) << ESR_EL2_CV_SHIFT)
#define ESR_EL2_COND_SHIFT (20)
#define ESR_EL2_COND (UL(0xf) << ESR_EL2_COND_SHIFT)
#define FSC_FAULT (0x04)
#define FSC_PERM (0x0c)
/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
#define HPFAR_MASK (~UL(0xf)) #define HPFAR_MASK (~UL(0xf))
#define ESR_EL2_EC_UNKNOWN (0x00)
#define ESR_EL2_EC_WFI (0x01)
#define ESR_EL2_EC_CP15_32 (0x03)
#define ESR_EL2_EC_CP15_64 (0x04)
#define ESR_EL2_EC_CP14_MR (0x05)
#define ESR_EL2_EC_CP14_LS (0x06)
#define ESR_EL2_EC_FP_ASIMD (0x07)
#define ESR_EL2_EC_CP10_ID (0x08)
#define ESR_EL2_EC_CP14_64 (0x0C)
#define ESR_EL2_EC_ILL_ISS (0x0E)
#define ESR_EL2_EC_SVC32 (0x11)
#define ESR_EL2_EC_HVC32 (0x12)
#define ESR_EL2_EC_SMC32 (0x13)
#define ESR_EL2_EC_SVC64 (0x15)
#define ESR_EL2_EC_HVC64 (0x16)
#define ESR_EL2_EC_SMC64 (0x17)
#define ESR_EL2_EC_SYS64 (0x18)
#define ESR_EL2_EC_IABT (0x20)
#define ESR_EL2_EC_IABT_HYP (0x21)
#define ESR_EL2_EC_PC_ALIGN (0x22)
#define ESR_EL2_EC_DABT (0x24)
#define ESR_EL2_EC_DABT_HYP (0x25)
#define ESR_EL2_EC_SP_ALIGN (0x26)
#define ESR_EL2_EC_FP_EXC32 (0x28)
#define ESR_EL2_EC_FP_EXC64 (0x2C)
#define ESR_EL2_EC_SERROR (0x2F)
#define ESR_EL2_EC_BREAKPT (0x30)
#define ESR_EL2_EC_BREAKPT_HYP (0x31)
#define ESR_EL2_EC_SOFTSTP (0x32)
#define ESR_EL2_EC_SOFTSTP_HYP (0x33)
#define ESR_EL2_EC_WATCHPT (0x34)
#define ESR_EL2_EC_WATCHPT_HYP (0x35)
#define ESR_EL2_EC_BKPT32 (0x38)
#define ESR_EL2_EC_VECTOR32 (0x3A)
#define ESR_EL2_EC_BRK64 (0x3C)
#define ESR_EL2_EC_xABT_xFSR_EXTABT 0x10
#define ESR_EL2_EC_WFI_ISS_WFE (1 << 0)
#endif /* __ARM64_KVM_ARM_H__ */ #endif /* __ARM64_KVM_ARM_H__ */
...@@ -23,8 +23,10 @@ ...@@ -23,8 +23,10 @@
#define __ARM64_KVM_EMULATE_H__ #define __ARM64_KVM_EMULATE_H__
#include <linux/kvm_host.h> #include <linux/kvm_host.h>
#include <asm/kvm_asm.h>
#include <asm/esr.h>
#include <asm/kvm_arm.h> #include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_mmio.h> #include <asm/kvm_mmio.h>
#include <asm/ptrace.h> #include <asm/ptrace.h>
...@@ -140,63 +142,63 @@ static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu) ...@@ -140,63 +142,63 @@ static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
{ {
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_ISV); return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_ISV);
} }
static inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
{ {
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_WNR); return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WNR);
} }
static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu)
{ {
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SSE); return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SSE);
} }
static inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu) static inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
{ {
return (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SRT_MASK) >> ESR_EL2_SRT_SHIFT; return (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
} }
static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu)
{ {
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EA); return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_EA);
} }
static inline bool kvm_vcpu_dabt_iss1tw(const struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_dabt_iss1tw(const struct kvm_vcpu *vcpu)
{ {
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_S1PTW); return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_S1PTW);
} }
static inline int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu) static inline int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
{ {
return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SAS) >> ESR_EL2_SAS_SHIFT); return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
} }
/* This one is not specific to Data Abort */ /* This one is not specific to Data Abort */
static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu)
{ {
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_IL); return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_IL);
} }
static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu) static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
{ {
return kvm_vcpu_get_hsr(vcpu) >> ESR_EL2_EC_SHIFT; return kvm_vcpu_get_hsr(vcpu) >> ESR_ELx_EC_SHIFT;
} }
static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
{ {
return kvm_vcpu_trap_get_class(vcpu) == ESR_EL2_EC_IABT; return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
} }
static inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) static inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
{ {
return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC; return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC;
} }
static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu) static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
{ {
return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC_TYPE; return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC_TYPE;
} }
static inline unsigned long kvm_vcpu_get_mpidr(struct kvm_vcpu *vcpu) static inline unsigned long kvm_vcpu_get_mpidr(struct kvm_vcpu *vcpu)
......
...@@ -32,6 +32,12 @@ ...@@ -32,6 +32,12 @@
*/ */
#define UL(x) _AC(x, UL) #define UL(x) _AC(x, UL)
/*
* Size of the PCI I/O space. This must remain a power of two so that
* IO_SPACE_LIMIT acts as a mask for the low bits of I/O addresses.
*/
#define PCI_IO_SIZE SZ_16M
/* /*
* PAGE_OFFSET - the virtual address of the start of the kernel image (top * PAGE_OFFSET - the virtual address of the start of the kernel image (top
* (VA_BITS - 1)) * (VA_BITS - 1))
...@@ -45,7 +51,9 @@ ...@@ -45,7 +51,9 @@
#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1)) #define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1))
#define MODULES_END (PAGE_OFFSET) #define MODULES_END (PAGE_OFFSET)
#define MODULES_VADDR (MODULES_END - SZ_64M) #define MODULES_VADDR (MODULES_END - SZ_64M)
#define FIXADDR_TOP (MODULES_VADDR - SZ_2M - PAGE_SIZE) #define PCI_IO_END (MODULES_VADDR - SZ_2M)
#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
#define FIXADDR_TOP (PCI_IO_START - SZ_2M)
#define TASK_SIZE_64 (UL(1) << VA_BITS) #define TASK_SIZE_64 (UL(1) << VA_BITS)
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
......
...@@ -31,7 +31,8 @@ extern void paging_init(void); ...@@ -31,7 +31,8 @@ extern void paging_init(void);
extern void setup_mm_for_reboot(void); extern void setup_mm_for_reboot(void);
extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt); extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
extern void init_mem_pgprot(void); extern void init_mem_pgprot(void);
/* create an identity mapping for memory (or io if map_io is true) */ extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
extern void create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io); unsigned long virt, phys_addr_t size,
pgprot_t prot);
#endif #endif
...@@ -263,6 +263,11 @@ static inline pmd_t pte_pmd(pte_t pte) ...@@ -263,6 +263,11 @@ static inline pmd_t pte_pmd(pte_t pte)
return __pmd(pte_val(pte)); return __pmd(pte_val(pte));
} }
static inline pgprot_t mk_sect_prot(pgprot_t prot)
{
return __pgprot(pgprot_val(prot) & ~PTE_TABLE_BIT);
}
/* /*
* THP definitions. * THP definitions.
*/ */
...@@ -336,9 +341,12 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, ...@@ -336,9 +341,12 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
#ifdef CONFIG_ARM64_64K_PAGES #ifdef CONFIG_ARM64_64K_PAGES
#define pud_sect(pud) (0) #define pud_sect(pud) (0)
#define pud_table(pud) (1)
#else #else
#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ #define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
PUD_TYPE_SECT) PUD_TYPE_SECT)
#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
PUD_TYPE_TABLE)
#endif #endif
static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
......
...@@ -58,6 +58,13 @@ ...@@ -58,6 +58,13 @@
#define COMPAT_PSR_Z_BIT 0x40000000 #define COMPAT_PSR_Z_BIT 0x40000000
#define COMPAT_PSR_N_BIT 0x80000000 #define COMPAT_PSR_N_BIT 0x80000000
#define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */ #define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */
#ifdef CONFIG_CPU_BIG_ENDIAN
#define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT
#else
#define COMPAT_PSR_ENDSTATE 0
#endif
/* /*
* These are 'magic' values for PTRACE_PEEKUSR that return info about where a * These are 'magic' values for PTRACE_PEEKUSR that return info about where a
* process is located in memory. * process is located in memory.
......
...@@ -23,6 +23,4 @@ struct sleep_save_sp { ...@@ -23,6 +23,4 @@ struct sleep_save_sp {
extern int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)); extern int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long));
extern void cpu_resume(void); extern void cpu_resume(void);
extern int cpu_suspend(unsigned long);
#endif #endif
/*
* Copyright (C) 2012 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_SYSCALLS_H
#define __ASM_SYSCALLS_H
#include <linux/linkage.h>
#include <linux/compiler.h>
#include <linux/signal.h>
/*
* System call wrappers implemented in kernel/entry.S.
*/
asmlinkage long sys_rt_sigreturn_wrapper(void);
#include <asm-generic/syscalls.h>
#endif /* __ASM_SYSCALLS_H */
...@@ -48,6 +48,9 @@ ...@@ -48,6 +48,9 @@
#endif #endif
#define __ARCH_WANT_SYS_CLONE #define __ARCH_WANT_SYS_CLONE
#ifndef __COMPAT_SYSCALL_NR
#include <uapi/asm/unistd.h> #include <uapi/asm/unistd.h>
#endif
#define NR_syscalls (__NR_syscalls) #define NR_syscalls (__NR_syscalls)
...@@ -18,4 +18,5 @@ header-y += siginfo.h ...@@ -18,4 +18,5 @@ header-y += siginfo.h
header-y += signal.h header-y += signal.h
header-y += stat.h header-y += stat.h
header-y += statfs.h header-y += statfs.h
header-y += ucontext.h
header-y += unistd.h header-y += unistd.h
...@@ -13,8 +13,10 @@ ...@@ -13,8 +13,10 @@
* You should have received a copy of the GNU General Public License * You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef __ASM_UCONTEXT_H #ifndef _UAPI__ASM_UCONTEXT_H
#define __ASM_UCONTEXT_H #define _UAPI__ASM_UCONTEXT_H
#include <linux/types.h>
struct ucontext { struct ucontext {
unsigned long uc_flags; unsigned long uc_flags;
...@@ -27,4 +29,4 @@ struct ucontext { ...@@ -27,4 +29,4 @@ struct ucontext {
struct sigcontext uc_mcontext; struct sigcontext uc_mcontext;
}; };
#endif /* __ASM_UCONTEXT_H */ #endif /* _UAPI__ASM_UCONTEXT_H */
...@@ -16,10 +16,10 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ ...@@ -16,10 +16,10 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \
entry-fpsimd.o process.o ptrace.o setup.o signal.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \
sys.o stacktrace.o time.o traps.o io.o vdso.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \
hyp-stub.o psci.o cpu_ops.o insn.o return_address.o \ hyp-stub.o psci.o cpu_ops.o insn.o return_address.o \
cpuinfo.o cpu_errata.o alternative.o cpuinfo.o cpu_errata.o alternative.o cacheinfo.o
arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
sys_compat.o \ sys_compat.o entry32.o \
../../arm/kernel/opcodes.o ../../arm/kernel/opcodes.o
arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o
arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
...@@ -27,7 +27,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o topology.o ...@@ -27,7 +27,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o topology.o
arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o
arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o arm64-obj-$(CONFIG_CPU_PM) += sleep.o suspend.o
arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o
arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o
arm64-obj-$(CONFIG_KGDB) += kgdb.o arm64-obj-$(CONFIG_KGDB) += kgdb.o
......
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
#include <asm/system_misc.h> #include <asm/system_misc.h>
#include <asm/traps.h> #include <asm/traps.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include <asm/cpufeature.h>
#define CREATE_TRACE_POINTS #define CREATE_TRACE_POINTS
#include "trace-events-emulation.h" #include "trace-events-emulation.h"
...@@ -85,6 +86,57 @@ static void remove_emulation_hooks(struct insn_emulation_ops *ops) ...@@ -85,6 +86,57 @@ static void remove_emulation_hooks(struct insn_emulation_ops *ops)
pr_notice("Removed %s emulation handler\n", ops->name); pr_notice("Removed %s emulation handler\n", ops->name);
} }
static void enable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
if (insn->ops->set_hw_mode)
insn->ops->set_hw_mode(true);
}
static void disable_insn_hw_mode(void *data)
{
struct insn_emulation *insn = (struct insn_emulation *)data;
if (insn->ops->set_hw_mode)
insn->ops->set_hw_mode(false);
}
/* Run set_hw_mode(mode) on all active CPUs */
static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
{
if (!insn->ops->set_hw_mode)
return -EINVAL;
if (enable)
on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
else
on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
return 0;
}
/*
* Run set_hw_mode for all insns on a starting CPU.
* Returns:
* 0 - If all the hooks ran successfully.
* -EINVAL - At least one hook is not supported by the CPU.
*/
static int run_all_insn_set_hw_mode(unsigned long cpu)
{
int rc = 0;
unsigned long flags;
struct insn_emulation *insn;
raw_spin_lock_irqsave(&insn_emulation_lock, flags);
list_for_each_entry(insn, &insn_emulation, node) {
bool enable = (insn->current_mode == INSN_HW);
if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
pr_warn("CPU[%ld] cannot support the emulation of %s",
cpu, insn->ops->name);
rc = -EINVAL;
}
}
raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
return rc;
}
static int update_insn_emulation_mode(struct insn_emulation *insn, static int update_insn_emulation_mode(struct insn_emulation *insn,
enum insn_emulation_mode prev) enum insn_emulation_mode prev)
{ {
...@@ -97,10 +149,8 @@ static int update_insn_emulation_mode(struct insn_emulation *insn, ...@@ -97,10 +149,8 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
remove_emulation_hooks(insn->ops); remove_emulation_hooks(insn->ops);
break; break;
case INSN_HW: case INSN_HW:
if (insn->ops->set_hw_mode) { if (!run_all_cpu_set_hw_mode(insn, false))
insn->ops->set_hw_mode(false);
pr_notice("Disabled %s support\n", insn->ops->name); pr_notice("Disabled %s support\n", insn->ops->name);
}
break; break;
} }
...@@ -111,10 +161,9 @@ static int update_insn_emulation_mode(struct insn_emulation *insn, ...@@ -111,10 +161,9 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
register_emulation_hooks(insn->ops); register_emulation_hooks(insn->ops);
break; break;
case INSN_HW: case INSN_HW:
if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true)) ret = run_all_cpu_set_hw_mode(insn, true);
if (!ret)
pr_notice("Enabled %s support\n", insn->ops->name); pr_notice("Enabled %s support\n", insn->ops->name);
else
ret = -EINVAL;
break; break;
} }
...@@ -133,6 +182,8 @@ static void register_insn_emulation(struct insn_emulation_ops *ops) ...@@ -133,6 +182,8 @@ static void register_insn_emulation(struct insn_emulation_ops *ops)
switch (ops->status) { switch (ops->status) {
case INSN_DEPRECATED: case INSN_DEPRECATED:
insn->current_mode = INSN_EMULATE; insn->current_mode = INSN_EMULATE;
/* Disable the HW mode if it was turned on at early boot time */
run_all_cpu_set_hw_mode(insn, false);
insn->max = INSN_HW; insn->max = INSN_HW;
break; break;
case INSN_OBSOLETE: case INSN_OBSOLETE:
...@@ -453,8 +504,6 @@ static int cp15barrier_handler(struct pt_regs *regs, u32 instr) ...@@ -453,8 +504,6 @@ static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
return 0; return 0;
} }
#define SCTLR_EL1_CP15BEN (1 << 5)
static inline void config_sctlr_el1(u32 clear, u32 set) static inline void config_sctlr_el1(u32 clear, u32 set)
{ {
u32 val; u32 val;
...@@ -465,48 +514,13 @@ static inline void config_sctlr_el1(u32 clear, u32 set) ...@@ -465,48 +514,13 @@ static inline void config_sctlr_el1(u32 clear, u32 set)
asm volatile("msr sctlr_el1, %0" : : "r" (val)); asm volatile("msr sctlr_el1, %0" : : "r" (val));
} }
static void enable_cp15_ben(void *info)
{
config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
}
static void disable_cp15_ben(void *info)
{
config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
}
static int cpu_hotplug_notify(struct notifier_block *b,
unsigned long action, void *hcpu)
{
switch (action) {
case CPU_STARTING:
case CPU_STARTING_FROZEN:
enable_cp15_ben(NULL);
return NOTIFY_DONE;
case CPU_DYING:
case CPU_DYING_FROZEN:
disable_cp15_ben(NULL);
return NOTIFY_DONE;
}
return NOTIFY_OK;
}
static struct notifier_block cpu_hotplug_notifier = {
.notifier_call = cpu_hotplug_notify,
};
static int cp15_barrier_set_hw_mode(bool enable) static int cp15_barrier_set_hw_mode(bool enable)
{ {
if (enable) { if (enable)
register_cpu_notifier(&cpu_hotplug_notifier); config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
on_each_cpu(enable_cp15_ben, NULL, true); else
} else { config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
unregister_cpu_notifier(&cpu_hotplug_notifier); return 0;
on_each_cpu(disable_cp15_ben, NULL, true);
}
return true;
} }
static struct undef_hook cp15_barrier_hooks[] = { static struct undef_hook cp15_barrier_hooks[] = {
...@@ -534,6 +548,93 @@ static struct insn_emulation_ops cp15_barrier_ops = { ...@@ -534,6 +548,93 @@ static struct insn_emulation_ops cp15_barrier_ops = {
.set_hw_mode = cp15_barrier_set_hw_mode, .set_hw_mode = cp15_barrier_set_hw_mode,
}; };
static int setend_set_hw_mode(bool enable)
{
if (!cpu_supports_mixed_endian_el0())
return -EINVAL;
if (enable)
config_sctlr_el1(SCTLR_EL1_SED, 0);
else
config_sctlr_el1(0, SCTLR_EL1_SED);
return 0;
}
static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
{
char *insn;
perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
if (big_endian) {
insn = "setend be";
regs->pstate |= COMPAT_PSR_E_BIT;
} else {
insn = "setend le";
regs->pstate &= ~COMPAT_PSR_E_BIT;
}
trace_instruction_emulation(insn, regs->pc);
pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
current->comm, (unsigned long)current->pid, regs->pc);
return 0;
}
static int a32_setend_handler(struct pt_regs *regs, u32 instr)
{
int rc = compat_setend_handler(regs, (instr >> 9) & 1);
regs->pc += 4;
return rc;
}
static int t16_setend_handler(struct pt_regs *regs, u32 instr)
{
int rc = compat_setend_handler(regs, (instr >> 3) & 1);
regs->pc += 2;
return rc;
}
static struct undef_hook setend_hooks[] = {
{
.instr_mask = 0xfffffdff,
.instr_val = 0xf1010000,
.pstate_mask = COMPAT_PSR_MODE_MASK,
.pstate_val = COMPAT_PSR_MODE_USR,
.fn = a32_setend_handler,
},
{
/* Thumb mode */
.instr_mask = 0x0000fff7,
.instr_val = 0x0000b650,
.pstate_mask = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_MASK),
.pstate_val = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_USR),
.fn = t16_setend_handler,
},
{}
};
static struct insn_emulation_ops setend_ops = {
.name = "setend",
.status = INSN_DEPRECATED,
.hooks = setend_hooks,
.set_hw_mode = setend_set_hw_mode,
};
static int insn_cpu_hotplug_notify(struct notifier_block *b,
unsigned long action, void *hcpu)
{
int rc = 0;
if ((action & ~CPU_TASKS_FROZEN) == CPU_STARTING)
rc = run_all_insn_set_hw_mode((unsigned long)hcpu);
return notifier_from_errno(rc);
}
static struct notifier_block insn_cpu_hotplug_notifier = {
.notifier_call = insn_cpu_hotplug_notify,
};
/* /*
* Invoked as late_initcall, since not needed before init spawned. * Invoked as late_initcall, since not needed before init spawned.
*/ */
...@@ -545,6 +646,14 @@ static int __init armv8_deprecated_init(void) ...@@ -545,6 +646,14 @@ static int __init armv8_deprecated_init(void)
if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
register_insn_emulation(&cp15_barrier_ops); register_insn_emulation(&cp15_barrier_ops);
if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
if(system_supports_mixed_endian_el0())
register_insn_emulation(&setend_ops);
else
pr_info("setend instruction emulation is not supported on the system");
}
register_cpu_notifier(&insn_cpu_hotplug_notifier);
register_insn_emulation_sysctl(ctl_abi); register_insn_emulation_sysctl(ctl_abi);
return 0; return 0;
......
...@@ -152,7 +152,7 @@ int main(void) ...@@ -152,7 +152,7 @@ int main(void)
DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
#endif #endif
#ifdef CONFIG_ARM64_CPU_SUSPEND #ifdef CONFIG_CPU_PM
DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx)); DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx));
DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp));
DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask)); DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask));
......
/*
* ARM64 cacheinfo support
*
* Copyright (C) 2015 ARM Ltd.
* All Rights Reserved
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed "as is" WITHOUT ANY WARRANTY of any
* kind, whether express or implied; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/bitops.h>
#include <linux/cacheinfo.h>
#include <linux/cpu.h>
#include <linux/compiler.h>
#include <linux/of.h>
#include <asm/cachetype.h>
#include <asm/processor.h>
#define MAX_CACHE_LEVEL 7 /* Max 7 level supported */
/* Ctypen, bits[3(n - 1) + 2 : 3(n - 1)], for n = 1 to 7 */
#define CLIDR_CTYPE_SHIFT(level) (3 * (level - 1))
#define CLIDR_CTYPE_MASK(level) (7 << CLIDR_CTYPE_SHIFT(level))
#define CLIDR_CTYPE(clidr, level) \
(((clidr) & CLIDR_CTYPE_MASK(level)) >> CLIDR_CTYPE_SHIFT(level))
static inline enum cache_type get_cache_type(int level)
{
u64 clidr;
if (level > MAX_CACHE_LEVEL)
return CACHE_TYPE_NOCACHE;
asm volatile ("mrs %x0, clidr_el1" : "=r" (clidr));
return CLIDR_CTYPE(clidr, level);
}
/*
* Cache Size Selection Register(CSSELR) selects which Cache Size ID
* Register(CCSIDR) is accessible by specifying the required cache
* level and the cache type. We need to ensure that no one else changes
* CSSELR by calling this in non-preemtible context
*/
u64 __attribute_const__ cache_get_ccsidr(u64 csselr)
{
u64 ccsidr;
WARN_ON(preemptible());
/* Put value into CSSELR */
asm volatile("msr csselr_el1, %x0" : : "r" (csselr));
isb();
/* Read result out of CCSIDR */
asm volatile("mrs %x0, ccsidr_el1" : "=r" (ccsidr));
return ccsidr;
}
static void ci_leaf_init(struct cacheinfo *this_leaf,
enum cache_type type, unsigned int level)
{
bool is_icache = type & CACHE_TYPE_INST;
u64 tmp = cache_get_ccsidr((level - 1) << 1 | is_icache);
this_leaf->level = level;
this_leaf->type = type;
this_leaf->coherency_line_size = CACHE_LINESIZE(tmp);
this_leaf->number_of_sets = CACHE_NUMSETS(tmp);
this_leaf->ways_of_associativity = CACHE_ASSOCIATIVITY(tmp);
this_leaf->size = this_leaf->number_of_sets *
this_leaf->coherency_line_size * this_leaf->ways_of_associativity;
this_leaf->attributes =
((tmp & CCSIDR_EL1_WRITE_THROUGH) ? CACHE_WRITE_THROUGH : 0) |
((tmp & CCSIDR_EL1_WRITE_BACK) ? CACHE_WRITE_BACK : 0) |
((tmp & CCSIDR_EL1_READ_ALLOCATE) ? CACHE_READ_ALLOCATE : 0) |
((tmp & CCSIDR_EL1_WRITE_ALLOCATE) ? CACHE_WRITE_ALLOCATE : 0);
}
static int __init_cache_level(unsigned int cpu)
{
unsigned int ctype, level, leaves;
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
for (level = 1, leaves = 0; level <= MAX_CACHE_LEVEL; level++) {
ctype = get_cache_type(level);
if (ctype == CACHE_TYPE_NOCACHE) {
level--;
break;
}
/* Separate instruction and data caches */
leaves += (ctype == CACHE_TYPE_SEPARATE) ? 2 : 1;
}
this_cpu_ci->num_levels = level;
this_cpu_ci->num_leaves = leaves;
return 0;
}
static int __populate_cache_leaves(unsigned int cpu)
{
unsigned int level, idx;
enum cache_type type;
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
struct cacheinfo *this_leaf = this_cpu_ci->info_list;
for (idx = 0, level = 1; level <= this_cpu_ci->num_levels &&
idx < this_cpu_ci->num_leaves; idx++, level++) {
type = get_cache_type(level);
if (type == CACHE_TYPE_SEPARATE) {
ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level);
ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level);
} else {
ci_leaf_init(this_leaf++, type, level);
}
}
return 0;
}
DEFINE_SMP_CALL_CACHE_FUNCTION(init_cache_level)
DEFINE_SMP_CALL_CACHE_FUNCTION(populate_cache_leaves)
...@@ -29,3 +29,23 @@ int cpu_init_idle(unsigned int cpu) ...@@ -29,3 +29,23 @@ int cpu_init_idle(unsigned int cpu)
of_node_put(cpu_node); of_node_put(cpu_node);
return ret; return ret;
} }
/**
* cpu_suspend() - function to enter a low-power idle state
* @arg: argument to pass to CPU suspend operations
*
* Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU
* operations back-end error code otherwise.
*/
int cpu_suspend(unsigned long arg)
{
int cpu = smp_processor_id();
/*
* If cpu_ops have not been registered or suspend
* has not been initialized, cpu_suspend call fails early.
*/
if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
return -EOPNOTSUPP;
return cpu_ops[cpu]->cpu_suspend(arg);
}
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
*/ */
DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data); DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data);
static struct cpuinfo_arm64 boot_cpu_data; static struct cpuinfo_arm64 boot_cpu_data;
static bool mixed_endian_el0 = true;
static char *icache_policy_str[] = { static char *icache_policy_str[] = {
[ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN", [ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN",
...@@ -68,6 +69,26 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) ...@@ -68,6 +69,26 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info)
pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu);
} }
bool cpu_supports_mixed_endian_el0(void)
{
return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
}
bool system_supports_mixed_endian_el0(void)
{
return mixed_endian_el0;
}
static void update_mixed_endian_el0_support(struct cpuinfo_arm64 *info)
{
mixed_endian_el0 &= id_aa64mmfr0_mixed_endian_el0(info->reg_id_aa64mmfr0);
}
static void update_cpu_features(struct cpuinfo_arm64 *info)
{
update_mixed_endian_el0_support(info);
}
static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu) static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu)
{ {
if ((boot & mask) == (cur & mask)) if ((boot & mask) == (cur & mask))
...@@ -215,6 +236,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) ...@@ -215,6 +236,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
cpuinfo_detect_icache_policy(info); cpuinfo_detect_icache_policy(info);
check_local_cpu_errata(); check_local_cpu_errata();
update_cpu_features(info);
} }
void cpuinfo_store_cpu(void) void cpuinfo_store_cpu(void)
...@@ -231,15 +253,3 @@ void __init cpuinfo_store_boot_cpu(void) ...@@ -231,15 +253,3 @@ void __init cpuinfo_store_boot_cpu(void)
boot_cpu_data = *info; boot_cpu_data = *info;
} }
u64 __attribute_const__ icache_get_ccsidr(void)
{
u64 ccsidr;
WARN_ON(preemptible());
/* Select L1 I-cache and read its size ID register */
asm("msr csselr_el1, %1; isb; mrs %0, ccsidr_el1"
: "=r"(ccsidr) : "r"(1L));
return ccsidr;
}
...@@ -11,27 +11,46 @@ ...@@ -11,27 +11,46 @@
* *
*/ */
#include <linux/atomic.h>
#include <linux/dmi.h> #include <linux/dmi.h>
#include <linux/efi.h> #include <linux/efi.h>
#include <linux/export.h> #include <linux/export.h>
#include <linux/memblock.h> #include <linux/memblock.h>
#include <linux/mm_types.h>
#include <linux/bootmem.h> #include <linux/bootmem.h>
#include <linux/of.h> #include <linux/of.h>
#include <linux/of_fdt.h> #include <linux/of_fdt.h>
#include <linux/preempt.h>
#include <linux/rbtree.h>
#include <linux/rwsem.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/spinlock.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/efi.h> #include <asm/efi.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include <asm/mmu_context.h> #include <asm/mmu_context.h>
#include <asm/mmu.h>
#include <asm/pgtable.h>
struct efi_memory_map memmap; struct efi_memory_map memmap;
static efi_runtime_services_t *runtime;
static u64 efi_system_table; static u64 efi_system_table;
static pgd_t efi_pgd[PTRS_PER_PGD] __page_aligned_bss;
static struct mm_struct efi_mm = {
.mm_rb = RB_ROOT,
.pgd = efi_pgd,
.mm_users = ATOMIC_INIT(2),
.mm_count = ATOMIC_INIT(1),
.mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem),
.page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock),
.mmlist = LIST_HEAD_INIT(efi_mm.mmlist),
INIT_MM_CONTEXT(efi_mm)
};
static int uefi_debug __initdata; static int uefi_debug __initdata;
static int __init uefi_debug_setup(char *str) static int __init uefi_debug_setup(char *str)
{ {
...@@ -48,30 +67,33 @@ static int __init is_normal_ram(efi_memory_desc_t *md) ...@@ -48,30 +67,33 @@ static int __init is_normal_ram(efi_memory_desc_t *md)
return 0; return 0;
} }
static void __init efi_setup_idmap(void) /*
* Translate a EFI virtual address into a physical address: this is necessary,
* as some data members of the EFI system table are virtually remapped after
* SetVirtualAddressMap() has been called.
*/
static phys_addr_t efi_to_phys(unsigned long addr)
{ {
struct memblock_region *r;
efi_memory_desc_t *md; efi_memory_desc_t *md;
u64 paddr, npages, size;
for_each_memblock(memory, r)
create_id_mapping(r->base, r->size, 0);
/* map runtime io spaces */
for_each_efi_memory_desc(&memmap, md) { for_each_efi_memory_desc(&memmap, md) {
if (!(md->attribute & EFI_MEMORY_RUNTIME) || is_normal_ram(md)) if (!(md->attribute & EFI_MEMORY_RUNTIME))
continue; continue;
paddr = md->phys_addr; if (md->virt_addr == 0)
npages = md->num_pages; /* no virtual mapping has been installed by the stub */
memrange_efi_to_native(&paddr, &npages); break;
size = npages << PAGE_SHIFT; if (md->virt_addr <= addr &&
create_id_mapping(paddr, size, 1); (addr - md->virt_addr) < (md->num_pages << EFI_PAGE_SHIFT))
return md->phys_addr + addr - md->virt_addr;
} }
return addr;
} }
static int __init uefi_init(void) static int __init uefi_init(void)
{ {
efi_char16_t *c16; efi_char16_t *c16;
void *config_tables;
u64 table_size;
char vendor[100] = "unknown"; char vendor[100] = "unknown";
int i, retval; int i, retval;
...@@ -99,7 +121,7 @@ static int __init uefi_init(void) ...@@ -99,7 +121,7 @@ static int __init uefi_init(void)
efi.systab->hdr.revision & 0xffff); efi.systab->hdr.revision & 0xffff);
/* Show what we know for posterity */ /* Show what we know for posterity */
c16 = early_memremap(efi.systab->fw_vendor, c16 = early_memremap(efi_to_phys(efi.systab->fw_vendor),
sizeof(vendor)); sizeof(vendor));
if (c16) { if (c16) {
for (i = 0; i < (int) sizeof(vendor) - 1 && *c16; ++i) for (i = 0; i < (int) sizeof(vendor) - 1 && *c16; ++i)
...@@ -112,8 +134,14 @@ static int __init uefi_init(void) ...@@ -112,8 +134,14 @@ static int __init uefi_init(void)
efi.systab->hdr.revision >> 16, efi.systab->hdr.revision >> 16,
efi.systab->hdr.revision & 0xffff, vendor); efi.systab->hdr.revision & 0xffff, vendor);
retval = efi_config_init(NULL); table_size = sizeof(efi_config_table_64_t) * efi.systab->nr_tables;
config_tables = early_memremap(efi_to_phys(efi.systab->tables),
table_size);
retval = efi_config_parse_tables(config_tables, efi.systab->nr_tables,
sizeof(efi_config_table_64_t), NULL);
early_memunmap(config_tables, table_size);
out: out:
early_memunmap(efi.systab, sizeof(efi_system_table_t)); early_memunmap(efi.systab, sizeof(efi_system_table_t));
return retval; return retval;
...@@ -163,9 +191,7 @@ static __init void reserve_regions(void) ...@@ -163,9 +191,7 @@ static __init void reserve_regions(void)
if (is_normal_ram(md)) if (is_normal_ram(md))
early_init_dt_add_memory_arch(paddr, size); early_init_dt_add_memory_arch(paddr, size);
if (is_reserve_region(md) || if (is_reserve_region(md)) {
md->type == EFI_BOOT_SERVICES_CODE ||
md->type == EFI_BOOT_SERVICES_DATA) {
memblock_reserve(paddr, size); memblock_reserve(paddr, size);
if (uefi_debug) if (uefi_debug)
pr_cont("*"); pr_cont("*");
...@@ -178,123 +204,6 @@ static __init void reserve_regions(void) ...@@ -178,123 +204,6 @@ static __init void reserve_regions(void)
set_bit(EFI_MEMMAP, &efi.flags); set_bit(EFI_MEMMAP, &efi.flags);
} }
static u64 __init free_one_region(u64 start, u64 end)
{
u64 size = end - start;
if (uefi_debug)
pr_info(" EFI freeing: 0x%012llx-0x%012llx\n", start, end - 1);
free_bootmem_late(start, size);
return size;
}
static u64 __init free_region(u64 start, u64 end)
{
u64 map_start, map_end, total = 0;
if (end <= start)
return total;
map_start = (u64)memmap.phys_map;
map_end = PAGE_ALIGN(map_start + (memmap.map_end - memmap.map));
map_start &= PAGE_MASK;
if (start < map_end && end > map_start) {
/* region overlaps UEFI memmap */
if (start < map_start)
total += free_one_region(start, map_start);
if (map_end < end)
total += free_one_region(map_end, end);
} else
total += free_one_region(start, end);
return total;
}
static void __init free_boot_services(void)
{
u64 total_freed = 0;
u64 keep_end, free_start, free_end;
efi_memory_desc_t *md;
/*
* If kernel uses larger pages than UEFI, we have to be careful
* not to inadvertantly free memory we want to keep if there is
* overlap at the kernel page size alignment. We do not want to
* free is_reserve_region() memory nor the UEFI memmap itself.
*
* The memory map is sorted, so we keep track of the end of
* any previous region we want to keep, remember any region
* we want to free and defer freeing it until we encounter
* the next region we want to keep. This way, before freeing
* it, we can clip it as needed to avoid freeing memory we
* want to keep for UEFI.
*/
keep_end = 0;
free_start = 0;
for_each_efi_memory_desc(&memmap, md) {
u64 paddr, npages, size;
if (is_reserve_region(md)) {
/*
* We don't want to free any memory from this region.
*/
if (free_start) {
/* adjust free_end then free region */
if (free_end > md->phys_addr)
free_end -= PAGE_SIZE;
total_freed += free_region(free_start, free_end);
free_start = 0;
}
keep_end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT);
continue;
}
if (md->type != EFI_BOOT_SERVICES_CODE &&
md->type != EFI_BOOT_SERVICES_DATA) {
/* no need to free this region */
continue;
}
/*
* We want to free memory from this region.
*/
paddr = md->phys_addr;
npages = md->num_pages;
memrange_efi_to_native(&paddr, &npages);
size = npages << PAGE_SHIFT;
if (free_start) {
if (paddr <= free_end)
free_end = paddr + size;
else {
total_freed += free_region(free_start, free_end);
free_start = paddr;
free_end = paddr + size;
}
} else {
free_start = paddr;
free_end = paddr + size;
}
if (free_start < keep_end) {
free_start += PAGE_SIZE;
if (free_start >= free_end)
free_start = 0;
}
}
if (free_start)
total_freed += free_region(free_start, free_end);
if (total_freed)
pr_info("Freed 0x%llx bytes of EFI boot services memory",
total_freed);
}
void __init efi_init(void) void __init efi_init(void)
{ {
struct efi_fdt_params params; struct efi_fdt_params params;
...@@ -317,159 +226,100 @@ void __init efi_init(void) ...@@ -317,159 +226,100 @@ void __init efi_init(void)
return; return;
reserve_regions(); reserve_regions();
early_memunmap(memmap.map, params.mmap_size);
} }
void __init efi_idmap_init(void) static bool __init efi_virtmap_init(void)
{ {
if (!efi_enabled(EFI_BOOT)) efi_memory_desc_t *md;
return;
/* boot time idmap_pg_dir is incomplete, so fill in missing parts */
efi_setup_idmap();
early_memunmap(memmap.map, memmap.map_end - memmap.map);
}
static int __init remap_region(efi_memory_desc_t *md, void **new)
{
u64 paddr, vaddr, npages, size;
paddr = md->phys_addr;
npages = md->num_pages;
memrange_efi_to_native(&paddr, &npages);
size = npages << PAGE_SHIFT;
if (is_normal_ram(md)) for_each_efi_memory_desc(&memmap, md) {
vaddr = (__force u64)ioremap_cache(paddr, size); u64 paddr, npages, size;
else pgprot_t prot;
vaddr = (__force u64)ioremap(paddr, size);
if (!vaddr) { if (!(md->attribute & EFI_MEMORY_RUNTIME))
pr_err("Unable to remap 0x%llx pages @ %p\n", continue;
npages, (void *)paddr); if (md->virt_addr == 0)
return 0; return false;
}
/* adjust for any rounding when EFI and system pagesize differs */ paddr = md->phys_addr;
md->virt_addr = vaddr + (md->phys_addr - paddr); npages = md->num_pages;
memrange_efi_to_native(&paddr, &npages);
size = npages << PAGE_SHIFT;
if (uefi_debug) pr_info(" EFI remap 0x%016llx => %p\n",
pr_info(" EFI remap 0x%012llx => %p\n",
md->phys_addr, (void *)md->virt_addr); md->phys_addr, (void *)md->virt_addr);
memcpy(*new, md, memmap.desc_size); /*
*new += memmap.desc_size; * Only regions of type EFI_RUNTIME_SERVICES_CODE need to be
* executable, everything else can be mapped with the XN bits
return 1; * set.
*/
if (!is_normal_ram(md))
prot = __pgprot(PROT_DEVICE_nGnRE);
else if (md->type == EFI_RUNTIME_SERVICES_CODE)
prot = PAGE_KERNEL_EXEC;
else
prot = PAGE_KERNEL;
create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size, prot);
}
return true;
} }
/* /*
* Switch UEFI from an identity map to a kernel virtual map * Enable the UEFI Runtime Services if all prerequisites are in place, i.e.,
* non-early mapping of the UEFI system table and virtual mappings for all
* EFI_MEMORY_RUNTIME regions.
*/ */
static int __init arm64_enter_virtual_mode(void) static int __init arm64_enable_runtime_services(void)
{ {
efi_memory_desc_t *md;
phys_addr_t virtmap_phys;
void *virtmap, *virt_md;
efi_status_t status;
u64 mapsize; u64 mapsize;
int count = 0;
unsigned long flags;
if (!efi_enabled(EFI_BOOT)) { if (!efi_enabled(EFI_BOOT)) {
pr_info("EFI services will not be available.\n"); pr_info("EFI services will not be available.\n");
return -1; return -1;
} }
mapsize = memmap.map_end - memmap.map;
if (efi_runtime_disabled()) { if (efi_runtime_disabled()) {
pr_info("EFI runtime services will be disabled.\n"); pr_info("EFI runtime services will be disabled.\n");
return -1; return -1;
} }
pr_info("Remapping and enabling EFI services.\n"); pr_info("Remapping and enabling EFI services.\n");
/* replace early memmap mapping with permanent mapping */
mapsize = memmap.map_end - memmap.map;
memmap.map = (__force void *)ioremap_cache((phys_addr_t)memmap.phys_map, memmap.map = (__force void *)ioremap_cache((phys_addr_t)memmap.phys_map,
mapsize); mapsize);
memmap.map_end = memmap.map + mapsize; if (!memmap.map) {
pr_err("Failed to remap EFI memory map\n");
efi.memmap = &memmap;
/* Map the runtime regions */
virtmap = kmalloc(mapsize, GFP_KERNEL);
if (!virtmap) {
pr_err("Failed to allocate EFI virtual memmap\n");
return -1; return -1;
} }
virtmap_phys = virt_to_phys(virtmap); memmap.map_end = memmap.map + mapsize;
virt_md = virtmap; efi.memmap = &memmap;
for_each_efi_memory_desc(&memmap, md) {
if (!(md->attribute & EFI_MEMORY_RUNTIME))
continue;
if (!remap_region(md, &virt_md))
goto err_unmap;
++count;
}
efi.systab = (__force void *)efi_lookup_mapped_addr(efi_system_table); efi.systab = (__force void *)ioremap_cache(efi_system_table,
sizeof(efi_system_table_t));
if (!efi.systab) { if (!efi.systab) {
/* pr_err("Failed to remap EFI System Table\n");
* If we have no virtual mapping for the System Table at this return -1;
* point, the memory map doesn't cover the physical offset where
* it resides. This means the System Table will be inaccessible
* to Runtime Services themselves once the virtual mapping is
* installed.
*/
pr_err("Failed to remap EFI System Table -- buggy firmware?\n");
goto err_unmap;
} }
set_bit(EFI_SYSTEM_TABLES, &efi.flags); set_bit(EFI_SYSTEM_TABLES, &efi.flags);
local_irq_save(flags); if (!efi_virtmap_init()) {
cpu_switch_mm(idmap_pg_dir, &init_mm); pr_err("No UEFI virtual mapping was installed -- runtime services will not be available\n");
/* Call SetVirtualAddressMap with the physical address of the map */
runtime = efi.systab->runtime;
efi.set_virtual_address_map = runtime->set_virtual_address_map;
status = efi.set_virtual_address_map(count * memmap.desc_size,
memmap.desc_size,
memmap.desc_version,
(efi_memory_desc_t *)virtmap_phys);
cpu_set_reserved_ttbr0();
flush_tlb_all();
local_irq_restore(flags);
kfree(virtmap);
free_boot_services();
if (status != EFI_SUCCESS) {
pr_err("Failed to set EFI virtual address map! [%lx]\n",
status);
return -1; return -1;
} }
/* Set up runtime services function pointers */ /* Set up runtime services function pointers */
runtime = efi.systab->runtime;
efi_native_runtime_setup(); efi_native_runtime_setup();
set_bit(EFI_RUNTIME_SERVICES, &efi.flags); set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
efi.runtime_version = efi.systab->hdr.revision; efi.runtime_version = efi.systab->hdr.revision;
return 0; return 0;
err_unmap:
/* unmap all mappings that succeeded: there are 'count' of those */
for (virt_md = virtmap; count--; virt_md += memmap.desc_size) {
md = virt_md;
iounmap((__force void __iomem *)md->virt_addr);
}
kfree(virtmap);
return -1;
} }
early_initcall(arm64_enter_virtual_mode); early_initcall(arm64_enable_runtime_services);
static int __init arm64_dmi_init(void) static int __init arm64_dmi_init(void)
{ {
...@@ -484,3 +334,23 @@ static int __init arm64_dmi_init(void) ...@@ -484,3 +334,23 @@ static int __init arm64_dmi_init(void)
return 0; return 0;
} }
core_initcall(arm64_dmi_init); core_initcall(arm64_dmi_init);
static void efi_set_pgd(struct mm_struct *mm)
{
cpu_switch_mm(mm->pgd, mm);
flush_tlb_all();
if (icache_is_aivivt())
__flush_icache_all();
}
void efi_virtmap_load(void)
{
preempt_disable();
efi_set_pgd(&efi_mm);
}
void efi_virtmap_unload(void)
{
efi_set_pgd(current->active_mm);
preempt_enable();
}
...@@ -269,18 +269,18 @@ ENDPROC(el1_error_invalid) ...@@ -269,18 +269,18 @@ ENDPROC(el1_error_invalid)
el1_sync: el1_sync:
kernel_entry 1 kernel_entry 1
mrs x1, esr_el1 // read the syndrome register mrs x1, esr_el1 // read the syndrome register
lsr x24, x1, #ESR_EL1_EC_SHIFT // exception class lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class
cmp x24, #ESR_EL1_EC_DABT_EL1 // data abort in EL1 cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1
b.eq el1_da b.eq el1_da
cmp x24, #ESR_EL1_EC_SYS64 // configurable trap cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
b.eq el1_undef b.eq el1_undef
cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
b.eq el1_sp_pc b.eq el1_sp_pc
cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
b.eq el1_sp_pc b.eq el1_sp_pc
cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL1 cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1
b.eq el1_undef b.eq el1_undef
cmp x24, #ESR_EL1_EC_BREAKPT_EL1 // debug exception in EL1 cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1
b.ge el1_dbg b.ge el1_dbg
b el1_inv b el1_inv
el1_da: el1_da:
...@@ -318,7 +318,7 @@ el1_dbg: ...@@ -318,7 +318,7 @@ el1_dbg:
/* /*
* Debug exception handling * Debug exception handling
*/ */
cmp x24, #ESR_EL1_EC_BRK64 // if BRK64 cmp x24, #ESR_ELx_EC_BRK64 // if BRK64
cinc x24, x24, eq // set bit '0' cinc x24, x24, eq // set bit '0'
tbz x24, #0, el1_inv // EL1 only tbz x24, #0, el1_inv // EL1 only
mrs x0, far_el1 mrs x0, far_el1
...@@ -375,26 +375,26 @@ el1_preempt: ...@@ -375,26 +375,26 @@ el1_preempt:
el0_sync: el0_sync:
kernel_entry 0 kernel_entry 0
mrs x25, esr_el1 // read the syndrome register mrs x25, esr_el1 // read the syndrome register
lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state
b.eq el0_svc b.eq el0_svc
cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
b.eq el0_da b.eq el0_da
cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
b.eq el0_ia b.eq el0_ia
cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
b.eq el0_fpsimd_acc b.eq el0_fpsimd_acc
cmp x24, #ESR_EL1_EC_FP_EXC64 // FP/ASIMD exception cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception
b.eq el0_fpsimd_exc b.eq el0_fpsimd_exc
cmp x24, #ESR_EL1_EC_SYS64 // configurable trap cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
b.eq el0_undef b.eq el0_undef
cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
b.eq el0_sp_pc b.eq el0_sp_pc
cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
b.eq el0_sp_pc b.eq el0_sp_pc
cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0 cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
b.eq el0_undef b.eq el0_undef
cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0 cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
b.ge el0_dbg b.ge el0_dbg
b el0_inv b el0_inv
...@@ -403,37 +403,37 @@ el0_sync: ...@@ -403,37 +403,37 @@ el0_sync:
el0_sync_compat: el0_sync_compat:
kernel_entry 0, 32 kernel_entry 0, 32
mrs x25, esr_el1 // read the syndrome register mrs x25, esr_el1 // read the syndrome register
lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state cmp x24, #ESR_ELx_EC_SVC32 // SVC in 32-bit state
b.eq el0_svc_compat b.eq el0_svc_compat
cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
b.eq el0_da b.eq el0_da
cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
b.eq el0_ia b.eq el0_ia
cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
b.eq el0_fpsimd_acc b.eq el0_fpsimd_acc
cmp x24, #ESR_EL1_EC_FP_EXC32 // FP/ASIMD exception cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception
b.eq el0_fpsimd_exc b.eq el0_fpsimd_exc
cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0 cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
b.eq el0_undef b.eq el0_undef
cmp x24, #ESR_EL1_EC_CP15_32 // CP15 MRC/MCR trap cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap
b.eq el0_undef b.eq el0_undef
cmp x24, #ESR_EL1_EC_CP15_64 // CP15 MRRC/MCRR trap cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap
b.eq el0_undef b.eq el0_undef
cmp x24, #ESR_EL1_EC_CP14_MR // CP14 MRC/MCR trap cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap
b.eq el0_undef b.eq el0_undef
cmp x24, #ESR_EL1_EC_CP14_LS // CP14 LDC/STC trap cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap
b.eq el0_undef b.eq el0_undef
cmp x24, #ESR_EL1_EC_CP14_64 // CP14 MRRC/MCRR trap cmp x24, #ESR_ELx_EC_CP14_64 // CP14 MRRC/MCRR trap
b.eq el0_undef b.eq el0_undef
cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0 cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
b.ge el0_dbg b.ge el0_dbg
b el0_inv b el0_inv
el0_svc_compat: el0_svc_compat:
/* /*
* AArch32 syscall handling * AArch32 syscall handling
*/ */
adr stbl, compat_sys_call_table // load compat syscall table pointer adrp stbl, compat_sys_call_table // load compat syscall table pointer
uxtw scno, w7 // syscall number in w7 (r7) uxtw scno, w7 // syscall number in w7 (r7)
mov sc_nr, #__NR_compat_syscalls mov sc_nr, #__NR_compat_syscalls
b el0_svc_naked b el0_svc_naked
......
...@@ -27,26 +27,26 @@ ...@@ -27,26 +27,26 @@
* System call wrappers for the AArch32 compatibility layer. * System call wrappers for the AArch32 compatibility layer.
*/ */
compat_sys_sigreturn_wrapper: ENTRY(compat_sys_sigreturn_wrapper)
mov x0, sp mov x0, sp
mov x27, #0 // prevent syscall restart handling (why) mov x27, #0 // prevent syscall restart handling (why)
b compat_sys_sigreturn b compat_sys_sigreturn
ENDPROC(compat_sys_sigreturn_wrapper) ENDPROC(compat_sys_sigreturn_wrapper)
compat_sys_rt_sigreturn_wrapper: ENTRY(compat_sys_rt_sigreturn_wrapper)
mov x0, sp mov x0, sp
mov x27, #0 // prevent syscall restart handling (why) mov x27, #0 // prevent syscall restart handling (why)
b compat_sys_rt_sigreturn b compat_sys_rt_sigreturn
ENDPROC(compat_sys_rt_sigreturn_wrapper) ENDPROC(compat_sys_rt_sigreturn_wrapper)
compat_sys_statfs64_wrapper: ENTRY(compat_sys_statfs64_wrapper)
mov w3, #84 mov w3, #84
cmp w1, #88 cmp w1, #88
csel w1, w3, w1, eq csel w1, w3, w1, eq
b compat_sys_statfs64 b compat_sys_statfs64
ENDPROC(compat_sys_statfs64_wrapper) ENDPROC(compat_sys_statfs64_wrapper)
compat_sys_fstatfs64_wrapper: ENTRY(compat_sys_fstatfs64_wrapper)
mov w3, #84 mov w3, #84
cmp w1, #88 cmp w1, #88
csel w1, w3, w1, eq csel w1, w3, w1, eq
...@@ -58,33 +58,33 @@ ENDPROC(compat_sys_fstatfs64_wrapper) ...@@ -58,33 +58,33 @@ ENDPROC(compat_sys_fstatfs64_wrapper)
* in registers or that take 32-bit parameters which require sign * in registers or that take 32-bit parameters which require sign
* extension. * extension.
*/ */
compat_sys_pread64_wrapper: ENTRY(compat_sys_pread64_wrapper)
regs_to_64 x3, x4, x5 regs_to_64 x3, x4, x5
b sys_pread64 b sys_pread64
ENDPROC(compat_sys_pread64_wrapper) ENDPROC(compat_sys_pread64_wrapper)
compat_sys_pwrite64_wrapper: ENTRY(compat_sys_pwrite64_wrapper)
regs_to_64 x3, x4, x5 regs_to_64 x3, x4, x5
b sys_pwrite64 b sys_pwrite64
ENDPROC(compat_sys_pwrite64_wrapper) ENDPROC(compat_sys_pwrite64_wrapper)
compat_sys_truncate64_wrapper: ENTRY(compat_sys_truncate64_wrapper)
regs_to_64 x1, x2, x3 regs_to_64 x1, x2, x3
b sys_truncate b sys_truncate
ENDPROC(compat_sys_truncate64_wrapper) ENDPROC(compat_sys_truncate64_wrapper)
compat_sys_ftruncate64_wrapper: ENTRY(compat_sys_ftruncate64_wrapper)
regs_to_64 x1, x2, x3 regs_to_64 x1, x2, x3
b sys_ftruncate b sys_ftruncate
ENDPROC(compat_sys_ftruncate64_wrapper) ENDPROC(compat_sys_ftruncate64_wrapper)
compat_sys_readahead_wrapper: ENTRY(compat_sys_readahead_wrapper)
regs_to_64 x1, x2, x3 regs_to_64 x1, x2, x3
mov w2, w4 mov w2, w4
b sys_readahead b sys_readahead
ENDPROC(compat_sys_readahead_wrapper) ENDPROC(compat_sys_readahead_wrapper)
compat_sys_fadvise64_64_wrapper: ENTRY(compat_sys_fadvise64_64_wrapper)
mov w6, w1 mov w6, w1
regs_to_64 x1, x2, x3 regs_to_64 x1, x2, x3
regs_to_64 x2, x4, x5 regs_to_64 x2, x4, x5
...@@ -92,24 +92,14 @@ compat_sys_fadvise64_64_wrapper: ...@@ -92,24 +92,14 @@ compat_sys_fadvise64_64_wrapper:
b sys_fadvise64_64 b sys_fadvise64_64
ENDPROC(compat_sys_fadvise64_64_wrapper) ENDPROC(compat_sys_fadvise64_64_wrapper)
compat_sys_sync_file_range2_wrapper: ENTRY(compat_sys_sync_file_range2_wrapper)
regs_to_64 x2, x2, x3 regs_to_64 x2, x2, x3
regs_to_64 x3, x4, x5 regs_to_64 x3, x4, x5
b sys_sync_file_range2 b sys_sync_file_range2
ENDPROC(compat_sys_sync_file_range2_wrapper) ENDPROC(compat_sys_sync_file_range2_wrapper)
compat_sys_fallocate_wrapper: ENTRY(compat_sys_fallocate_wrapper)
regs_to_64 x2, x2, x3 regs_to_64 x2, x2, x3
regs_to_64 x3, x4, x5 regs_to_64 x3, x4, x5
b sys_fallocate b sys_fallocate
ENDPROC(compat_sys_fallocate_wrapper) ENDPROC(compat_sys_fallocate_wrapper)
#undef __SYSCALL
#define __SYSCALL(x, y) .quad y // x
/*
* The system calls table must be 4KB aligned.
*/
.align 12
ENTRY(compat_sys_call_table)
#include <asm/unistd32.h>
...@@ -894,7 +894,7 @@ static struct notifier_block hw_breakpoint_reset_nb = { ...@@ -894,7 +894,7 @@ static struct notifier_block hw_breakpoint_reset_nb = {
.notifier_call = hw_breakpoint_reset_notify, .notifier_call = hw_breakpoint_reset_notify,
}; };
#ifdef CONFIG_ARM64_CPU_SUSPEND #ifdef CONFIG_CPU_PM
extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)); extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *));
#else #else
static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
......
...@@ -17,14 +17,19 @@ ...@@ -17,14 +17,19 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <linux/bitops.h> #include <linux/bitops.h>
#include <linux/bug.h>
#include <linux/compiler.h> #include <linux/compiler.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h> #include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/stop_machine.h> #include <linux/stop_machine.h>
#include <linux/types.h>
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/debug-monitors.h> #include <asm/debug-monitors.h>
#include <asm/fixmap.h>
#include <asm/insn.h> #include <asm/insn.h>
#define AARCH64_INSN_SF_BIT BIT(31) #define AARCH64_INSN_SF_BIT BIT(31)
...@@ -72,6 +77,29 @@ bool __kprobes aarch64_insn_is_nop(u32 insn) ...@@ -72,6 +77,29 @@ bool __kprobes aarch64_insn_is_nop(u32 insn)
} }
} }
static DEFINE_SPINLOCK(patch_lock);
static void __kprobes *patch_map(void *addr, int fixmap)
{
unsigned long uintaddr = (uintptr_t) addr;
bool module = !core_kernel_text(uintaddr);
struct page *page;
if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX))
page = vmalloc_to_page(addr);
else
page = virt_to_page(addr);
BUG_ON(!page);
set_fixmap(fixmap, page_to_phys(page));
return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK));
}
static void __kprobes patch_unmap(int fixmap)
{
clear_fixmap(fixmap);
}
/* /*
* In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
* little-endian. * little-endian.
...@@ -88,10 +116,27 @@ int __kprobes aarch64_insn_read(void *addr, u32 *insnp) ...@@ -88,10 +116,27 @@ int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
return ret; return ret;
} }
static int __kprobes __aarch64_insn_write(void *addr, u32 insn)
{
void *waddr = addr;
unsigned long flags = 0;
int ret;
spin_lock_irqsave(&patch_lock, flags);
waddr = patch_map(addr, FIX_TEXT_POKE0);
ret = probe_kernel_write(waddr, &insn, AARCH64_INSN_SIZE);
patch_unmap(FIX_TEXT_POKE0);
spin_unlock_irqrestore(&patch_lock, flags);
return ret;
}
int __kprobes aarch64_insn_write(void *addr, u32 insn) int __kprobes aarch64_insn_write(void *addr, u32 insn)
{ {
insn = cpu_to_le32(insn); insn = cpu_to_le32(insn);
return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE); return __aarch64_insn_write(addr, insn);
} }
static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn) static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)
......
...@@ -540,8 +540,6 @@ const struct cpu_operations cpu_psci_ops = { ...@@ -540,8 +540,6 @@ const struct cpu_operations cpu_psci_ops = {
.name = "psci", .name = "psci",
#ifdef CONFIG_CPU_IDLE #ifdef CONFIG_CPU_IDLE
.cpu_init_idle = cpu_psci_cpu_init_idle, .cpu_init_idle = cpu_psci_cpu_init_idle,
#endif
#ifdef CONFIG_ARM64_CPU_SUSPEND
.cpu_suspend = cpu_psci_cpu_suspend, .cpu_suspend = cpu_psci_cpu_suspend,
#endif #endif
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
......
...@@ -40,6 +40,7 @@ ...@@ -40,6 +40,7 @@
#include <linux/fs.h> #include <linux/fs.h>
#include <linux/proc_fs.h> #include <linux/proc_fs.h>
#include <linux/memblock.h> #include <linux/memblock.h>
#include <linux/of_iommu.h>
#include <linux/of_fdt.h> #include <linux/of_fdt.h>
#include <linux/of_platform.h> #include <linux/of_platform.h>
#include <linux/efi.h> #include <linux/efi.h>
...@@ -322,25 +323,6 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys) ...@@ -322,25 +323,6 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys)
dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name()); dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name());
} }
/*
* Limit the memory size that was specified via FDT.
*/
static int __init early_mem(char *p)
{
phys_addr_t limit;
if (!p)
return 1;
limit = memparse(p, &p) & PAGE_MASK;
pr_notice("Memory limited to %lldMB\n", limit >> 20);
memblock_enforce_memory_limit(limit);
return 0;
}
early_param("mem", early_mem);
static void __init request_standard_resources(void) static void __init request_standard_resources(void)
{ {
struct memblock_region *region; struct memblock_region *region;
...@@ -401,7 +383,6 @@ void __init setup_arch(char **cmdline_p) ...@@ -401,7 +383,6 @@ void __init setup_arch(char **cmdline_p)
paging_init(); paging_init();
request_standard_resources(); request_standard_resources();
efi_idmap_init();
early_ioremap_reset(); early_ioremap_reset();
unflatten_device_tree(); unflatten_device_tree();
...@@ -425,6 +406,7 @@ void __init setup_arch(char **cmdline_p) ...@@ -425,6 +406,7 @@ void __init setup_arch(char **cmdline_p)
static int __init arm64_device_init(void) static int __init arm64_device_init(void)
{ {
of_iommu_init();
of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL);
return 0; return 0;
} }
......
...@@ -440,7 +440,7 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka, ...@@ -440,7 +440,7 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
{ {
compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler); compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler);
compat_ulong_t retcode; compat_ulong_t retcode;
compat_ulong_t spsr = regs->pstate & ~PSR_f; compat_ulong_t spsr = regs->pstate & ~(PSR_f | COMPAT_PSR_E_BIT);
int thumb; int thumb;
/* Check if the handler is written for ARM or Thumb */ /* Check if the handler is written for ARM or Thumb */
...@@ -454,6 +454,9 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka, ...@@ -454,6 +454,9 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
/* The IT state must be cleared for both ARM and Thumb-2 */ /* The IT state must be cleared for both ARM and Thumb-2 */
spsr &= ~COMPAT_PSR_IT_MASK; spsr &= ~COMPAT_PSR_IT_MASK;
/* Restore the original endianness */
spsr |= COMPAT_PSR_ENDSTATE;
if (ka->sa.sa_flags & SA_RESTORER) { if (ka->sa.sa_flags & SA_RESTORER) {
retcode = ptr_to_compat(ka->sa.sa_restorer); retcode = ptr_to_compat(ka->sa.sa_restorer);
} else { } else {
...@@ -501,7 +504,7 @@ static int compat_setup_sigframe(struct compat_sigframe __user *sf, ...@@ -501,7 +504,7 @@ static int compat_setup_sigframe(struct compat_sigframe __user *sf,
__put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err); __put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err);
/* set the compat FSR WnR */ /* set the compat FSR WnR */
__put_user_error(!!(current->thread.fault_code & ESR_EL1_WRITE) << __put_user_error(!!(current->thread.fault_code & ESR_ELx_WNR) <<
FSR_WRITE_SHIFT, &sf->uc.uc_mcontext.error_code, err); FSR_WRITE_SHIFT, &sf->uc.uc_mcontext.error_code, err);
__put_user_error(current->thread.fault_address, &sf->uc.uc_mcontext.fault_address, err); __put_user_error(current->thread.fault_address, &sf->uc.uc_mcontext.fault_address, err);
__put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err); __put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err);
......
...@@ -65,7 +65,6 @@ struct secondary_data secondary_data; ...@@ -65,7 +65,6 @@ struct secondary_data secondary_data;
enum ipi_msg_type { enum ipi_msg_type {
IPI_RESCHEDULE, IPI_RESCHEDULE,
IPI_CALL_FUNC, IPI_CALL_FUNC,
IPI_CALL_FUNC_SINGLE,
IPI_CPU_STOP, IPI_CPU_STOP,
IPI_TIMER, IPI_TIMER,
IPI_IRQ_WORK, IPI_IRQ_WORK,
...@@ -483,7 +482,6 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { ...@@ -483,7 +482,6 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = {
#define S(x,s) [x] = s #define S(x,s) [x] = s
S(IPI_RESCHEDULE, "Rescheduling interrupts"), S(IPI_RESCHEDULE, "Rescheduling interrupts"),
S(IPI_CALL_FUNC, "Function call interrupts"), S(IPI_CALL_FUNC, "Function call interrupts"),
S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
S(IPI_CPU_STOP, "CPU stop interrupts"), S(IPI_CPU_STOP, "CPU stop interrupts"),
S(IPI_TIMER, "Timer broadcast interrupts"), S(IPI_TIMER, "Timer broadcast interrupts"),
S(IPI_IRQ_WORK, "IRQ work interrupts"), S(IPI_IRQ_WORK, "IRQ work interrupts"),
...@@ -527,7 +525,7 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask) ...@@ -527,7 +525,7 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask)
void arch_send_call_function_single_ipi(int cpu) void arch_send_call_function_single_ipi(int cpu)
{ {
smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE); smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
} }
#ifdef CONFIG_IRQ_WORK #ifdef CONFIG_IRQ_WORK
...@@ -585,12 +583,6 @@ void handle_IPI(int ipinr, struct pt_regs *regs) ...@@ -585,12 +583,6 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
irq_exit(); irq_exit();
break; break;
case IPI_CALL_FUNC_SINGLE:
irq_enter();
generic_smp_call_function_single_interrupt();
irq_exit();
break;
case IPI_CPU_STOP: case IPI_CPU_STOP:
irq_enter(); irq_enter();
ipi_cpu_stop(cpu); ipi_cpu_stop(cpu);
......
#include <linux/percpu.h> #include <linux/percpu.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/cpu_ops.h>
#include <asm/debug-monitors.h> #include <asm/debug-monitors.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/memory.h> #include <asm/memory.h>
...@@ -51,26 +50,6 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) ...@@ -51,26 +50,6 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
hw_breakpoint_restore = hw_bp_restore; hw_breakpoint_restore = hw_bp_restore;
} }
/**
* cpu_suspend() - function to enter a low-power state
* @arg: argument to pass to CPU suspend operations
*
* Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU
* operations back-end error code otherwise.
*/
int cpu_suspend(unsigned long arg)
{
int cpu = smp_processor_id();
/*
* If cpu_ops have not been registered or suspend
* has not been initialized, cpu_suspend call fails early.
*/
if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
return -EOPNOTSUPP;
return cpu_ops[cpu]->cpu_suspend(arg);
}
/* /*
* __cpu_suspend * __cpu_suspend
* *
......
...@@ -39,10 +39,9 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len, ...@@ -39,10 +39,9 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
/* /*
* Wrappers to pass the pt_regs argument. * Wrappers to pass the pt_regs argument.
*/ */
asmlinkage long sys_rt_sigreturn_wrapper(void);
#define sys_rt_sigreturn sys_rt_sigreturn_wrapper #define sys_rt_sigreturn sys_rt_sigreturn_wrapper
#include <asm/syscalls.h>
#undef __SYSCALL #undef __SYSCALL
#define __SYSCALL(nr, sym) [nr] = sym, #define __SYSCALL(nr, sym) [nr] = sym,
...@@ -50,7 +49,7 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len, ...@@ -50,7 +49,7 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
* The sys_call_table array must be 4K aligned to be accessible from * The sys_call_table array must be 4K aligned to be accessible from
* kernel/entry.S. * kernel/entry.S.
*/ */
void *sys_call_table[__NR_syscalls] __aligned(4096) = { void * const sys_call_table[__NR_syscalls] __aligned(4096) = {
[0 ... __NR_syscalls - 1] = sys_ni_syscall, [0 ... __NR_syscalls - 1] = sys_ni_syscall,
#include <asm/unistd.h> #include <asm/unistd.h>
}; };
/*
* arch/arm64/kernel/sys32.c
*
* Copyright (C) 2015 ARM Ltd.
*
* This program is free software(void); you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http(void);//www.gnu.org/licenses/>.
*/
/*
* Needed to avoid conflicting __NR_* macros between uapi/asm/unistd.h and
* asm/unistd32.h.
*/
#define __COMPAT_SYSCALL_NR
#include <linux/compiler.h>
#include <linux/syscalls.h>
asmlinkage long compat_sys_sigreturn_wrapper(void);
asmlinkage long compat_sys_rt_sigreturn_wrapper(void);
asmlinkage long compat_sys_statfs64_wrapper(void);
asmlinkage long compat_sys_fstatfs64_wrapper(void);
asmlinkage long compat_sys_pread64_wrapper(void);
asmlinkage long compat_sys_pwrite64_wrapper(void);
asmlinkage long compat_sys_truncate64_wrapper(void);
asmlinkage long compat_sys_ftruncate64_wrapper(void);
asmlinkage long compat_sys_readahead_wrapper(void);
asmlinkage long compat_sys_fadvise64_64_wrapper(void);
asmlinkage long compat_sys_sync_file_range2_wrapper(void);
asmlinkage long compat_sys_fallocate_wrapper(void);
#undef __SYSCALL
#define __SYSCALL(nr, sym) [nr] = sym,
/*
* The sys_call_table array must be 4K aligned to be accessible from
* kernel/entry.S.
*/
void * const compat_sys_call_table[__NR_compat_syscalls] __aligned(4096) = {
[0 ... __NR_compat_syscalls - 1] = sys_ni_syscall,
#include <asm/unistd32.h>
};
...@@ -33,6 +33,7 @@ ...@@ -33,6 +33,7 @@
#include <asm/atomic.h> #include <asm/atomic.h>
#include <asm/debug-monitors.h> #include <asm/debug-monitors.h>
#include <asm/esr.h>
#include <asm/traps.h> #include <asm/traps.h>
#include <asm/stacktrace.h> #include <asm/stacktrace.h>
#include <asm/exception.h> #include <asm/exception.h>
...@@ -373,6 +374,51 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs) ...@@ -373,6 +374,51 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs)
return sys_ni_syscall(); return sys_ni_syscall();
} }
static const char *esr_class_str[] = {
[0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
[ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized",
[ESR_ELx_EC_WFx] = "WFI/WFE",
[ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC",
[ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC",
[ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC",
[ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC",
[ESR_ELx_EC_FP_ASIMD] = "ASIMD",
[ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS",
[ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC",
[ESR_ELx_EC_ILL] = "PSTATE.IL",
[ESR_ELx_EC_SVC32] = "SVC (AArch32)",
[ESR_ELx_EC_HVC32] = "HVC (AArch32)",
[ESR_ELx_EC_SMC32] = "SMC (AArch32)",
[ESR_ELx_EC_SVC64] = "SVC (AArch64)",
[ESR_ELx_EC_HVC64] = "HVC (AArch64)",
[ESR_ELx_EC_SMC64] = "SMC (AArch64)",
[ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
[ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
[ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
[ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
[ESR_ELx_EC_PC_ALIGN] = "PC Alignment",
[ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)",
[ESR_ELx_EC_DABT_CUR] = "DABT (current EL)",
[ESR_ELx_EC_SP_ALIGN] = "SP Alignment",
[ESR_ELx_EC_FP_EXC32] = "FP (AArch32)",
[ESR_ELx_EC_FP_EXC64] = "FP (AArch64)",
[ESR_ELx_EC_SERROR] = "SError",
[ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)",
[ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)",
[ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)",
[ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)",
[ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)",
[ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)",
[ESR_ELx_EC_BKPT32] = "BKPT (AArch32)",
[ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)",
[ESR_ELx_EC_BRK64] = "BRK (AArch64)",
};
const char *esr_get_class_string(u32 esr)
{
return esr_class_str[esr >> ESR_ELx_EC_SHIFT];
}
/* /*
* bad_mode handles the impossible case in the exception vector. * bad_mode handles the impossible case in the exception vector.
*/ */
...@@ -382,8 +428,8 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) ...@@ -382,8 +428,8 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
void __user *pc = (void __user *)instruction_pointer(regs); void __user *pc = (void __user *)instruction_pointer(regs);
console_verbose(); console_verbose();
pr_crit("Bad mode in %s handler detected, code 0x%08x\n", pr_crit("Bad mode in %s handler detected, code 0x%08x -- %s\n",
handler[reason], esr); handler[reason], esr, esr_get_class_string(esr));
__show_regs(regs); __show_regs(regs);
info.si_signo = SIGILL; info.si_signo = SIGILL;
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include <asm/thread_info.h> #include <asm/thread_info.h>
#include <asm/memory.h> #include <asm/memory.h>
#include <asm/page.h> #include <asm/page.h>
#include <asm/pgtable.h>
#include "image.h" #include "image.h"
...@@ -49,6 +50,14 @@ PECOFF_FILE_ALIGNMENT = 0x200; ...@@ -49,6 +50,14 @@ PECOFF_FILE_ALIGNMENT = 0x200;
#define PECOFF_EDATA_PADDING #define PECOFF_EDATA_PADDING
#endif #endif
#ifdef CONFIG_DEBUG_ALIGN_RODATA
#define ALIGN_DEBUG_RO . = ALIGN(1<<SECTION_SHIFT);
#define ALIGN_DEBUG_RO_MIN(min) ALIGN_DEBUG_RO
#else
#define ALIGN_DEBUG_RO
#define ALIGN_DEBUG_RO_MIN(min) . = ALIGN(min);
#endif
SECTIONS SECTIONS
{ {
/* /*
...@@ -71,6 +80,7 @@ SECTIONS ...@@ -71,6 +80,7 @@ SECTIONS
_text = .; _text = .;
HEAD_TEXT HEAD_TEXT
} }
ALIGN_DEBUG_RO
.text : { /* Real text segment */ .text : { /* Real text segment */
_stext = .; /* Text and read-only data */ _stext = .; /* Text and read-only data */
__exception_text_start = .; __exception_text_start = .;
...@@ -87,19 +97,22 @@ SECTIONS ...@@ -87,19 +97,22 @@ SECTIONS
*(.got) /* Global offset table */ *(.got) /* Global offset table */
} }
ALIGN_DEBUG_RO
RO_DATA(PAGE_SIZE) RO_DATA(PAGE_SIZE)
EXCEPTION_TABLE(8) EXCEPTION_TABLE(8)
NOTES NOTES
ALIGN_DEBUG_RO
_etext = .; /* End of text and rodata section */ _etext = .; /* End of text and rodata section */
. = ALIGN(PAGE_SIZE); ALIGN_DEBUG_RO_MIN(PAGE_SIZE)
__init_begin = .; __init_begin = .;
INIT_TEXT_SECTION(8) INIT_TEXT_SECTION(8)
.exit.text : { .exit.text : {
ARM_EXIT_KEEP(EXIT_TEXT) ARM_EXIT_KEEP(EXIT_TEXT)
} }
. = ALIGN(16);
ALIGN_DEBUG_RO_MIN(16)
.init.data : { .init.data : {
INIT_DATA INIT_DATA
INIT_SETUP(16) INIT_SETUP(16)
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
*/ */
#include <linux/kvm_host.h> #include <linux/kvm_host.h>
#include <asm/esr.h>
#include <asm/kvm_emulate.h> #include <asm/kvm_emulate.h>
/* /*
...@@ -55,8 +56,8 @@ static int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu) ...@@ -55,8 +56,8 @@ static int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu)
{ {
u32 esr = kvm_vcpu_get_hsr(vcpu); u32 esr = kvm_vcpu_get_hsr(vcpu);
if (esr & ESR_EL2_CV) if (esr & ESR_ELx_CV)
return (esr & ESR_EL2_COND) >> ESR_EL2_COND_SHIFT; return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
return -1; return -1;
} }
......
...@@ -21,8 +21,10 @@ ...@@ -21,8 +21,10 @@
#include <linux/kvm.h> #include <linux/kvm.h>
#include <linux/kvm_host.h> #include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/esr.h>
#include <asm/kvm_coproc.h> #include <asm/kvm_coproc.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_mmu.h> #include <asm/kvm_mmu.h>
#include <asm/kvm_psci.h> #include <asm/kvm_psci.h>
...@@ -61,7 +63,7 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run) ...@@ -61,7 +63,7 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
*/ */
static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run) static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
{ {
if (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EC_WFI_ISS_WFE) if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE)
kvm_vcpu_on_spin(vcpu); kvm_vcpu_on_spin(vcpu);
else else
kvm_vcpu_block(vcpu); kvm_vcpu_block(vcpu);
...@@ -72,29 +74,30 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run) ...@@ -72,29 +74,30 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
} }
static exit_handle_fn arm_exit_handlers[] = { static exit_handle_fn arm_exit_handlers[] = {
[ESR_EL2_EC_WFI] = kvm_handle_wfx, [ESR_ELx_EC_WFx] = kvm_handle_wfx,
[ESR_EL2_EC_CP15_32] = kvm_handle_cp15_32, [ESR_ELx_EC_CP15_32] = kvm_handle_cp15_32,
[ESR_EL2_EC_CP15_64] = kvm_handle_cp15_64, [ESR_ELx_EC_CP15_64] = kvm_handle_cp15_64,
[ESR_EL2_EC_CP14_MR] = kvm_handle_cp14_32, [ESR_ELx_EC_CP14_MR] = kvm_handle_cp14_32,
[ESR_EL2_EC_CP14_LS] = kvm_handle_cp14_load_store, [ESR_ELx_EC_CP14_LS] = kvm_handle_cp14_load_store,
[ESR_EL2_EC_CP14_64] = kvm_handle_cp14_64, [ESR_ELx_EC_CP14_64] = kvm_handle_cp14_64,
[ESR_EL2_EC_HVC32] = handle_hvc, [ESR_ELx_EC_HVC32] = handle_hvc,
[ESR_EL2_EC_SMC32] = handle_smc, [ESR_ELx_EC_SMC32] = handle_smc,
[ESR_EL2_EC_HVC64] = handle_hvc, [ESR_ELx_EC_HVC64] = handle_hvc,
[ESR_EL2_EC_SMC64] = handle_smc, [ESR_ELx_EC_SMC64] = handle_smc,
[ESR_EL2_EC_SYS64] = kvm_handle_sys_reg, [ESR_ELx_EC_SYS64] = kvm_handle_sys_reg,
[ESR_EL2_EC_IABT] = kvm_handle_guest_abort, [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,
[ESR_EL2_EC_DABT] = kvm_handle_guest_abort, [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,
}; };
static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)
{ {
u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu); u32 hsr = kvm_vcpu_get_hsr(vcpu);
u8 hsr_ec = hsr >> ESR_ELx_EC_SHIFT;
if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) || if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) ||
!arm_exit_handlers[hsr_ec]) { !arm_exit_handlers[hsr_ec]) {
kvm_err("Unknown exception class: hsr: %#08x\n", kvm_err("Unknown exception class: hsr: %#08x -- %s\n",
(unsigned int)kvm_vcpu_get_hsr(vcpu)); hsr, esr_get_class_string(hsr));
BUG(); BUG();
} }
......
...@@ -17,15 +17,16 @@ ...@@ -17,15 +17,16 @@
#include <linux/linkage.h> #include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/memory.h>
#include <asm/asm-offsets.h> #include <asm/asm-offsets.h>
#include <asm/assembler.h>
#include <asm/debug-monitors.h> #include <asm/debug-monitors.h>
#include <asm/esr.h>
#include <asm/fpsimdmacros.h> #include <asm/fpsimdmacros.h>
#include <asm/kvm.h> #include <asm/kvm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_arm.h> #include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_mmu.h> #include <asm/kvm_mmu.h>
#include <asm/memory.h>
#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x) #define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x) #define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
...@@ -1141,9 +1142,9 @@ el1_sync: // Guest trapped into EL2 ...@@ -1141,9 +1142,9 @@ el1_sync: // Guest trapped into EL2
push x2, x3 push x2, x3
mrs x1, esr_el2 mrs x1, esr_el2
lsr x2, x1, #ESR_EL2_EC_SHIFT lsr x2, x1, #ESR_ELx_EC_SHIFT
cmp x2, #ESR_EL2_EC_HVC64 cmp x2, #ESR_ELx_EC_HVC64
b.ne el1_trap b.ne el1_trap
mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
...@@ -1178,13 +1179,13 @@ el1_trap: ...@@ -1178,13 +1179,13 @@ el1_trap:
* x1: ESR * x1: ESR
* x2: ESR_EC * x2: ESR_EC
*/ */
cmp x2, #ESR_EL2_EC_DABT cmp x2, #ESR_ELx_EC_DABT_LOW
mov x0, #ESR_EL2_EC_IABT mov x0, #ESR_ELx_EC_IABT_LOW
ccmp x2, x0, #4, ne ccmp x2, x0, #4, ne
b.ne 1f // Not an abort we care about b.ne 1f // Not an abort we care about
/* This is an abort. Check for permission fault */ /* This is an abort. Check for permission fault */
and x2, x1, #ESR_EL2_FSC_TYPE and x2, x1, #ESR_ELx_FSC_TYPE
cmp x2, #FSC_PERM cmp x2, #FSC_PERM
b.ne 1f // Not a permission fault b.ne 1f // Not a permission fault
......
...@@ -118,27 +118,27 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr ...@@ -118,27 +118,27 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
* instruction set. Report an external synchronous abort. * instruction set. Report an external synchronous abort.
*/ */
if (kvm_vcpu_trap_il_is32bit(vcpu)) if (kvm_vcpu_trap_il_is32bit(vcpu))
esr |= ESR_EL1_IL; esr |= ESR_ELx_IL;
/* /*
* Here, the guest runs in AArch64 mode when in EL1. If we get * Here, the guest runs in AArch64 mode when in EL1. If we get
* an AArch32 fault, it means we managed to trap an EL0 fault. * an AArch32 fault, it means we managed to trap an EL0 fault.
*/ */
if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t)
esr |= (ESR_EL1_EC_IABT_EL0 << ESR_EL1_EC_SHIFT); esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT);
else else
esr |= (ESR_EL1_EC_IABT_EL1 << ESR_EL1_EC_SHIFT); esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT);
if (!is_iabt) if (!is_iabt)
esr |= ESR_EL1_EC_DABT_EL0; esr |= ESR_ELx_EC_DABT_LOW;
vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_EL2_EC_xABT_xFSR_EXTABT; vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_ELx_FSC_EXTABT;
} }
static void inject_undef64(struct kvm_vcpu *vcpu) static void inject_undef64(struct kvm_vcpu *vcpu)
{ {
unsigned long cpsr = *vcpu_cpsr(vcpu); unsigned long cpsr = *vcpu_cpsr(vcpu);
u32 esr = (ESR_EL1_EC_UNKNOWN << ESR_EL1_EC_SHIFT); u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT);
*vcpu_spsr(vcpu) = cpsr; *vcpu_spsr(vcpu) = cpsr;
*vcpu_elr_el1(vcpu) = *vcpu_pc(vcpu); *vcpu_elr_el1(vcpu) = *vcpu_pc(vcpu);
...@@ -151,7 +151,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu) ...@@ -151,7 +151,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
* set. * set.
*/ */
if (kvm_vcpu_trap_il_is32bit(vcpu)) if (kvm_vcpu_trap_il_is32bit(vcpu))
esr |= ESR_EL1_IL; esr |= ESR_ELx_IL;
vcpu_sys_reg(vcpu, ESR_EL1) = esr; vcpu_sys_reg(vcpu, ESR_EL1) = esr;
} }
......
...@@ -20,17 +20,20 @@ ...@@ -20,17 +20,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <linux/mm.h>
#include <linux/kvm_host.h> #include <linux/kvm_host.h>
#include <linux/mm.h>
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_coproc.h>
#include <asm/kvm_mmu.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/cputype.h> #include <asm/cputype.h>
#include <asm/debug-monitors.h> #include <asm/debug-monitors.h>
#include <asm/esr.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_coproc.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_host.h>
#include <asm/kvm_mmu.h>
#include <trace/events/kvm.h> #include <trace/events/kvm.h>
#include "sys_regs.h" #include "sys_regs.h"
...@@ -760,12 +763,12 @@ static void unhandled_cp_access(struct kvm_vcpu *vcpu, ...@@ -760,12 +763,12 @@ static void unhandled_cp_access(struct kvm_vcpu *vcpu,
int cp; int cp;
switch(hsr_ec) { switch(hsr_ec) {
case ESR_EL2_EC_CP15_32: case ESR_ELx_EC_CP15_32:
case ESR_EL2_EC_CP15_64: case ESR_ELx_EC_CP15_64:
cp = 15; cp = 15;
break; break;
case ESR_EL2_EC_CP14_MR: case ESR_ELx_EC_CP14_MR:
case ESR_EL2_EC_CP14_64: case ESR_ELx_EC_CP14_64:
cp = 14; cp = 14;
break; break;
default: default:
......
...@@ -134,16 +134,17 @@ static void __dma_free_coherent(struct device *dev, size_t size, ...@@ -134,16 +134,17 @@ static void __dma_free_coherent(struct device *dev, size_t size,
swiotlb_free_coherent(dev, size, vaddr, dma_handle); swiotlb_free_coherent(dev, size, vaddr, dma_handle);
} }
static void *__dma_alloc_noncoherent(struct device *dev, size_t size, static void *__dma_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags, dma_addr_t *dma_handle, gfp_t flags,
struct dma_attrs *attrs) struct dma_attrs *attrs)
{ {
struct page *page; struct page *page;
void *ptr, *coherent_ptr; void *ptr, *coherent_ptr;
bool coherent = is_device_dma_coherent(dev);
size = PAGE_ALIGN(size); size = PAGE_ALIGN(size);
if (!(flags & __GFP_WAIT)) { if (!coherent && !(flags & __GFP_WAIT)) {
struct page *page = NULL; struct page *page = NULL;
void *addr = __alloc_from_pool(size, &page); void *addr = __alloc_from_pool(size, &page);
...@@ -151,13 +152,16 @@ static void *__dma_alloc_noncoherent(struct device *dev, size_t size, ...@@ -151,13 +152,16 @@ static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
*dma_handle = phys_to_dma(dev, page_to_phys(page)); *dma_handle = phys_to_dma(dev, page_to_phys(page));
return addr; return addr;
} }
ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs); ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
if (!ptr) if (!ptr)
goto no_mem; goto no_mem;
/* no need for non-cacheable mapping if coherent */
if (coherent)
return ptr;
/* remove any dirty cache lines on the kernel alias */ /* remove any dirty cache lines on the kernel alias */
__dma_flush_range(ptr, ptr + size); __dma_flush_range(ptr, ptr + size);
...@@ -179,15 +183,17 @@ static void *__dma_alloc_noncoherent(struct device *dev, size_t size, ...@@ -179,15 +183,17 @@ static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
return NULL; return NULL;
} }
static void __dma_free_noncoherent(struct device *dev, size_t size, static void __dma_free(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle, void *vaddr, dma_addr_t dma_handle,
struct dma_attrs *attrs) struct dma_attrs *attrs)
{ {
void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle)); void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle));
if (__free_from_pool(vaddr, size)) if (!is_device_dma_coherent(dev)) {
return; if (__free_from_pool(vaddr, size))
vunmap(vaddr); return;
vunmap(vaddr);
}
__dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs); __dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
} }
...@@ -199,7 +205,8 @@ static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page, ...@@ -199,7 +205,8 @@ static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
dma_addr_t dev_addr; dma_addr_t dev_addr;
dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs); dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); if (!is_device_dma_coherent(dev))
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
return dev_addr; return dev_addr;
} }
...@@ -209,7 +216,8 @@ static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr, ...@@ -209,7 +216,8 @@ static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir, size_t size, enum dma_data_direction dir,
struct dma_attrs *attrs) struct dma_attrs *attrs)
{ {
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); if (!is_device_dma_coherent(dev))
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
swiotlb_unmap_page(dev, dev_addr, size, dir, attrs); swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
} }
...@@ -221,9 +229,10 @@ static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl, ...@@ -221,9 +229,10 @@ static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
int i, ret; int i, ret;
ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs); ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
for_each_sg(sgl, sg, ret, i) if (!is_device_dma_coherent(dev))
__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), for_each_sg(sgl, sg, ret, i)
sg->length, dir); __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
return ret; return ret;
} }
...@@ -236,9 +245,10 @@ static void __swiotlb_unmap_sg_attrs(struct device *dev, ...@@ -236,9 +245,10 @@ static void __swiotlb_unmap_sg_attrs(struct device *dev,
struct scatterlist *sg; struct scatterlist *sg;
int i; int i;
for_each_sg(sgl, sg, nelems, i) if (!is_device_dma_coherent(dev))
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), for_each_sg(sgl, sg, nelems, i)
sg->length, dir); __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs); swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
} }
...@@ -246,7 +256,8 @@ static void __swiotlb_sync_single_for_cpu(struct device *dev, ...@@ -246,7 +256,8 @@ static void __swiotlb_sync_single_for_cpu(struct device *dev,
dma_addr_t dev_addr, size_t size, dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir) enum dma_data_direction dir)
{ {
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); if (!is_device_dma_coherent(dev))
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir); swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir);
} }
...@@ -255,7 +266,8 @@ static void __swiotlb_sync_single_for_device(struct device *dev, ...@@ -255,7 +266,8 @@ static void __swiotlb_sync_single_for_device(struct device *dev,
enum dma_data_direction dir) enum dma_data_direction dir)
{ {
swiotlb_sync_single_for_device(dev, dev_addr, size, dir); swiotlb_sync_single_for_device(dev, dev_addr, size, dir);
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); if (!is_device_dma_coherent(dev))
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
} }
static void __swiotlb_sync_sg_for_cpu(struct device *dev, static void __swiotlb_sync_sg_for_cpu(struct device *dev,
...@@ -265,9 +277,10 @@ static void __swiotlb_sync_sg_for_cpu(struct device *dev, ...@@ -265,9 +277,10 @@ static void __swiotlb_sync_sg_for_cpu(struct device *dev,
struct scatterlist *sg; struct scatterlist *sg;
int i; int i;
for_each_sg(sgl, sg, nelems, i) if (!is_device_dma_coherent(dev))
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), for_each_sg(sgl, sg, nelems, i)
sg->length, dir); __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir); swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
} }
...@@ -279,9 +292,10 @@ static void __swiotlb_sync_sg_for_device(struct device *dev, ...@@ -279,9 +292,10 @@ static void __swiotlb_sync_sg_for_device(struct device *dev,
int i; int i;
swiotlb_sync_sg_for_device(dev, sgl, nelems, dir); swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
for_each_sg(sgl, sg, nelems, i) if (!is_device_dma_coherent(dev))
__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), for_each_sg(sgl, sg, nelems, i)
sg->length, dir); __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
sg->length, dir);
} }
/* vma->vm_page_prot must be set appropriately before calling this function */ /* vma->vm_page_prot must be set appropriately before calling this function */
...@@ -308,28 +322,20 @@ static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma, ...@@ -308,28 +322,20 @@ static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
return ret; return ret;
} }
static int __swiotlb_mmap_noncoherent(struct device *dev, static int __swiotlb_mmap(struct device *dev,
struct vm_area_struct *vma, struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size, void *cpu_addr, dma_addr_t dma_addr, size_t size,
struct dma_attrs *attrs) struct dma_attrs *attrs)
{
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);
return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
}
static int __swiotlb_mmap_coherent(struct device *dev,
struct vm_area_struct *vma,
void *cpu_addr, dma_addr_t dma_addr, size_t size,
struct dma_attrs *attrs)
{ {
/* Just use whatever page_prot attributes were specified */ vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
is_device_dma_coherent(dev));
return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
} }
struct dma_map_ops noncoherent_swiotlb_dma_ops = { static struct dma_map_ops swiotlb_dma_ops = {
.alloc = __dma_alloc_noncoherent, .alloc = __dma_alloc,
.free = __dma_free_noncoherent, .free = __dma_free,
.mmap = __swiotlb_mmap_noncoherent, .mmap = __swiotlb_mmap,
.map_page = __swiotlb_map_page, .map_page = __swiotlb_map_page,
.unmap_page = __swiotlb_unmap_page, .unmap_page = __swiotlb_unmap_page,
.map_sg = __swiotlb_map_sg_attrs, .map_sg = __swiotlb_map_sg_attrs,
...@@ -341,24 +347,6 @@ struct dma_map_ops noncoherent_swiotlb_dma_ops = { ...@@ -341,24 +347,6 @@ struct dma_map_ops noncoherent_swiotlb_dma_ops = {
.dma_supported = swiotlb_dma_supported, .dma_supported = swiotlb_dma_supported,
.mapping_error = swiotlb_dma_mapping_error, .mapping_error = swiotlb_dma_mapping_error,
}; };
EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops);
struct dma_map_ops coherent_swiotlb_dma_ops = {
.alloc = __dma_alloc_coherent,
.free = __dma_free_coherent,
.mmap = __swiotlb_mmap_coherent,
.map_page = swiotlb_map_page,
.unmap_page = swiotlb_unmap_page,
.map_sg = swiotlb_map_sg_attrs,
.unmap_sg = swiotlb_unmap_sg_attrs,
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
.sync_single_for_device = swiotlb_sync_single_for_device,
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
.sync_sg_for_device = swiotlb_sync_sg_for_device,
.dma_supported = swiotlb_dma_supported,
.mapping_error = swiotlb_dma_mapping_error,
};
EXPORT_SYMBOL(coherent_swiotlb_dma_ops);
extern int swiotlb_late_init_with_default_size(size_t default_size); extern int swiotlb_late_init_with_default_size(size_t default_size);
...@@ -427,7 +415,7 @@ static int __init swiotlb_late_init(void) ...@@ -427,7 +415,7 @@ static int __init swiotlb_late_init(void)
{ {
size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT); size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT);
dma_ops = &noncoherent_swiotlb_dma_ops; dma_ops = &swiotlb_dma_ops;
return swiotlb_late_init_with_default_size(swiotlb_size); return swiotlb_late_init_with_default_size(swiotlb_size);
} }
......
...@@ -14,14 +14,18 @@ ...@@ -14,14 +14,18 @@
* of the License. * of the License.
*/ */
#include <linux/debugfs.h> #include <linux/debugfs.h>
#include <linux/errno.h>
#include <linux/fs.h> #include <linux/fs.h>
#include <linux/io.h> #include <linux/io.h>
#include <linux/init.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/seq_file.h> #include <linux/seq_file.h>
#include <asm/fixmap.h> #include <asm/fixmap.h>
#include <asm/memory.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/pgtable-hwdef.h>
#define LOWEST_ADDR (UL(0xffffffffffffffff) << VA_BITS) #define LOWEST_ADDR (UL(0xffffffffffffffff) << VA_BITS)
...@@ -37,10 +41,10 @@ enum address_markers_idx { ...@@ -37,10 +41,10 @@ enum address_markers_idx {
VMEMMAP_START_NR, VMEMMAP_START_NR,
VMEMMAP_END_NR, VMEMMAP_END_NR,
#endif #endif
PCI_START_NR,
PCI_END_NR,
FIXADDR_START_NR, FIXADDR_START_NR,
FIXADDR_END_NR, FIXADDR_END_NR,
PCI_START_NR,
PCI_END_NR,
MODULES_START_NR, MODULES_START_NR,
MODUELS_END_NR, MODUELS_END_NR,
KERNEL_SPACE_NR, KERNEL_SPACE_NR,
...@@ -53,10 +57,10 @@ static struct addr_marker address_markers[] = { ...@@ -53,10 +57,10 @@ static struct addr_marker address_markers[] = {
{ 0, "vmemmap start" }, { 0, "vmemmap start" },
{ 0, "vmemmap end" }, { 0, "vmemmap end" },
#endif #endif
{ (unsigned long) PCI_IOBASE, "PCI I/O start" },
{ (unsigned long) PCI_IOBASE + SZ_16M, "PCI I/O end" },
{ FIXADDR_START, "Fixmap start" }, { FIXADDR_START, "Fixmap start" },
{ FIXADDR_TOP, "Fixmap end" }, { FIXADDR_TOP, "Fixmap end" },
{ PCI_IO_START, "PCI I/O start" },
{ PCI_IO_END, "PCI I/O end" },
{ MODULES_VADDR, "Modules start" }, { MODULES_VADDR, "Modules start" },
{ MODULES_END, "Modules end" }, { MODULES_END, "Modules end" },
{ PAGE_OFFSET, "Kernel Mapping" }, { PAGE_OFFSET, "Kernel Mapping" },
...@@ -246,10 +250,12 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) ...@@ -246,10 +250,12 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
for (i = 0; i < PTRS_PER_PMD; i++, pmd++) { for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
addr = start + i * PMD_SIZE; addr = start + i * PMD_SIZE;
if (pmd_none(*pmd) || pmd_sect(*pmd) || pmd_bad(*pmd)) if (pmd_none(*pmd) || pmd_sect(*pmd)) {
note_page(st, addr, 3, pmd_val(*pmd)); note_page(st, addr, 3, pmd_val(*pmd));
else } else {
BUG_ON(pmd_bad(*pmd));
walk_pte(st, pmd, addr); walk_pte(st, pmd, addr);
}
} }
} }
...@@ -261,10 +267,12 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) ...@@ -261,10 +267,12 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
for (i = 0; i < PTRS_PER_PUD; i++, pud++) { for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
addr = start + i * PUD_SIZE; addr = start + i * PUD_SIZE;
if (pud_none(*pud) || pud_sect(*pud) || pud_bad(*pud)) if (pud_none(*pud) || pud_sect(*pud)) {
note_page(st, addr, 2, pud_val(*pud)); note_page(st, addr, 2, pud_val(*pud));
else } else {
BUG_ON(pud_bad(*pud));
walk_pmd(st, pud, addr); walk_pmd(st, pud, addr);
}
} }
} }
...@@ -276,10 +284,12 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm, unsigned long st ...@@ -276,10 +284,12 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm, unsigned long st
for (i = 0; i < PTRS_PER_PGD; i++, pgd++) { for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
addr = start + i * PGDIR_SIZE; addr = start + i * PGDIR_SIZE;
if (pgd_none(*pgd) || pgd_bad(*pgd)) if (pgd_none(*pgd)) {
note_page(st, addr, 1, pgd_val(*pgd)); note_page(st, addr, 1, pgd_val(*pgd));
else } else {
BUG_ON(pgd_bad(*pgd));
walk_pud(st, pgd, addr); walk_pud(st, pgd, addr);
}
} }
} }
......
...@@ -219,7 +219,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, ...@@ -219,7 +219,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
if (esr & ESR_LNX_EXEC) { if (esr & ESR_LNX_EXEC) {
vm_flags = VM_EXEC; vm_flags = VM_EXEC;
} else if ((esr & ESR_EL1_WRITE) && !(esr & ESR_EL1_CM)) { } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
vm_flags = VM_WRITE; vm_flags = VM_WRITE;
mm_flags |= FAULT_FLAG_WRITE; mm_flags |= FAULT_FLAG_WRITE;
} }
......
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
#include <linux/efi.h> #include <linux/efi.h>
#include <asm/fixmap.h> #include <asm/fixmap.h>
#include <asm/memory.h>
#include <asm/sections.h> #include <asm/sections.h>
#include <asm/setup.h> #include <asm/setup.h>
#include <asm/sizes.h> #include <asm/sizes.h>
...@@ -136,10 +137,29 @@ static void arm64_memory_present(void) ...@@ -136,10 +137,29 @@ static void arm64_memory_present(void)
} }
#endif #endif
static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX;
/*
* Limit the memory size that was specified via FDT.
*/
static int __init early_mem(char *p)
{
if (!p)
return 1;
memory_limit = memparse(p, &p) & PAGE_MASK;
pr_notice("Memory limited to %lldMB\n", memory_limit >> 20);
return 0;
}
early_param("mem", early_mem);
void __init arm64_memblock_init(void) void __init arm64_memblock_init(void)
{ {
phys_addr_t dma_phys_limit = 0; phys_addr_t dma_phys_limit = 0;
memblock_enforce_memory_limit(memory_limit);
/* /*
* Register the kernel text, kernel data, initrd, and initial * Register the kernel text, kernel data, initrd, and initial
* pagetables with memblock. * pagetables with memblock.
...@@ -277,8 +297,8 @@ void __init mem_init(void) ...@@ -277,8 +297,8 @@ void __init mem_init(void)
" vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n" " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n"
" 0x%16lx - 0x%16lx (%6ld MB actual)\n" " 0x%16lx - 0x%16lx (%6ld MB actual)\n"
#endif #endif
" PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n"
" fixed : 0x%16lx - 0x%16lx (%6ld KB)\n" " fixed : 0x%16lx - 0x%16lx (%6ld KB)\n"
" PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n"
" modules : 0x%16lx - 0x%16lx (%6ld MB)\n" " modules : 0x%16lx - 0x%16lx (%6ld MB)\n"
" memory : 0x%16lx - 0x%16lx (%6ld MB)\n" " memory : 0x%16lx - 0x%16lx (%6ld MB)\n"
" .init : 0x%p" " - 0x%p" " (%6ld KB)\n" " .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
...@@ -291,8 +311,8 @@ void __init mem_init(void) ...@@ -291,8 +311,8 @@ void __init mem_init(void)
MLM((unsigned long)virt_to_page(PAGE_OFFSET), MLM((unsigned long)virt_to_page(PAGE_OFFSET),
(unsigned long)virt_to_page(high_memory)), (unsigned long)virt_to_page(high_memory)),
#endif #endif
MLM((unsigned long)PCI_IOBASE, (unsigned long)PCI_IOBASE + SZ_16M),
MLK(FIXADDR_START, FIXADDR_TOP), MLK(FIXADDR_START, FIXADDR_TOP),
MLM(PCI_IO_START, PCI_IO_END),
MLM(MODULES_VADDR, MODULES_END), MLM(MODULES_VADDR, MODULES_END),
MLM(PAGE_OFFSET, (unsigned long)high_memory), MLM(PAGE_OFFSET, (unsigned long)high_memory),
MLK_ROUNDUP(__init_begin, __init_end), MLK_ROUNDUP(__init_begin, __init_end),
...@@ -325,6 +345,7 @@ void __init mem_init(void) ...@@ -325,6 +345,7 @@ void __init mem_init(void)
void free_initmem(void) void free_initmem(void)
{ {
fixup_init();
free_initmem_default(0); free_initmem_default(0);
free_alternatives_memory(); free_alternatives_memory();
} }
......
...@@ -62,6 +62,7 @@ static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size, ...@@ -62,6 +62,7 @@ static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size,
if (!area) if (!area)
return NULL; return NULL;
addr = (unsigned long)area->addr; addr = (unsigned long)area->addr;
area->phys_addr = phys_addr;
err = ioremap_page_range(addr, addr + size, phys_addr, prot); err = ioremap_page_range(addr, addr + size, phys_addr, prot);
if (err) { if (err) {
......
extern void __init bootmem_init(void); extern void __init bootmem_init(void);
void fixup_init(void);
...@@ -26,6 +26,8 @@ ...@@ -26,6 +26,8 @@
#include <linux/memblock.h> #include <linux/memblock.h>
#include <linux/fs.h> #include <linux/fs.h>
#include <linux/io.h> #include <linux/io.h>
#include <linux/slab.h>
#include <linux/stop_machine.h>
#include <asm/cputype.h> #include <asm/cputype.h>
#include <asm/fixmap.h> #include <asm/fixmap.h>
...@@ -45,80 +47,6 @@ ...@@ -45,80 +47,6 @@
struct page *empty_zero_page; struct page *empty_zero_page;
EXPORT_SYMBOL(empty_zero_page); EXPORT_SYMBOL(empty_zero_page);
struct cachepolicy {
const char policy[16];
u64 mair;
u64 tcr;
};
static struct cachepolicy cache_policies[] __initdata = {
{
.policy = "uncached",
.mair = 0x44, /* inner, outer non-cacheable */
.tcr = TCR_IRGN_NC | TCR_ORGN_NC,
}, {
.policy = "writethrough",
.mair = 0xaa, /* inner, outer write-through, read-allocate */
.tcr = TCR_IRGN_WT | TCR_ORGN_WT,
}, {
.policy = "writeback",
.mair = 0xee, /* inner, outer write-back, read-allocate */
.tcr = TCR_IRGN_WBnWA | TCR_ORGN_WBnWA,
}
};
/*
* These are useful for identifying cache coherency problems by allowing the
* cache or the cache and writebuffer to be turned off. It changes the Normal
* memory caching attributes in the MAIR_EL1 register.
*/
static int __init early_cachepolicy(char *p)
{
int i;
u64 tmp;
for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
int len = strlen(cache_policies[i].policy);
if (memcmp(p, cache_policies[i].policy, len) == 0)
break;
}
if (i == ARRAY_SIZE(cache_policies)) {
pr_err("ERROR: unknown or unsupported cache policy: %s\n", p);
return 0;
}
flush_cache_all();
/*
* Modify MT_NORMAL attributes in MAIR_EL1.
*/
asm volatile(
" mrs %0, mair_el1\n"
" bfi %0, %1, %2, #8\n"
" msr mair_el1, %0\n"
" isb\n"
: "=&r" (tmp)
: "r" (cache_policies[i].mair), "i" (MT_NORMAL * 8));
/*
* Modify TCR PTW cacheability attributes.
*/
asm volatile(
" mrs %0, tcr_el1\n"
" bic %0, %0, %2\n"
" orr %0, %0, %1\n"
" msr tcr_el1, %0\n"
" isb\n"
: "=&r" (tmp)
: "r" (cache_policies[i].tcr), "r" (TCR_IRGN_MASK | TCR_ORGN_MASK));
flush_cache_all();
return 0;
}
early_param("cachepolicy", early_cachepolicy);
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
unsigned long size, pgprot_t vma_prot) unsigned long size, pgprot_t vma_prot)
{ {
...@@ -133,19 +61,42 @@ EXPORT_SYMBOL(phys_mem_access_prot); ...@@ -133,19 +61,42 @@ EXPORT_SYMBOL(phys_mem_access_prot);
static void __init *early_alloc(unsigned long sz) static void __init *early_alloc(unsigned long sz)
{ {
void *ptr = __va(memblock_alloc(sz, sz)); void *ptr = __va(memblock_alloc(sz, sz));
BUG_ON(!ptr);
memset(ptr, 0, sz); memset(ptr, 0, sz);
return ptr; return ptr;
} }
static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, /*
* remap a PMD into pages
*/
static void split_pmd(pmd_t *pmd, pte_t *pte)
{
unsigned long pfn = pmd_pfn(*pmd);
int i = 0;
do {
/*
* Need to have the least restrictive permissions available
* permissions will be fixed up later
*/
set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC));
pfn++;
} while (pte++, i++, i < PTRS_PER_PTE);
}
static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
unsigned long end, unsigned long pfn, unsigned long end, unsigned long pfn,
pgprot_t prot) pgprot_t prot,
void *(*alloc)(unsigned long size))
{ {
pte_t *pte; pte_t *pte;
if (pmd_none(*pmd)) { if (pmd_none(*pmd) || pmd_sect(*pmd)) {
pte = early_alloc(PTRS_PER_PTE * sizeof(pte_t)); pte = alloc(PTRS_PER_PTE * sizeof(pte_t));
if (pmd_sect(*pmd))
split_pmd(pmd, pte);
__pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE); __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE);
flush_tlb_all();
} }
BUG_ON(pmd_bad(*pmd)); BUG_ON(pmd_bad(*pmd));
...@@ -156,30 +107,42 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, ...@@ -156,30 +107,42 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
} while (pte++, addr += PAGE_SIZE, addr != end); } while (pte++, addr += PAGE_SIZE, addr != end);
} }
static void __init alloc_init_pmd(pud_t *pud, unsigned long addr, void split_pud(pud_t *old_pud, pmd_t *pmd)
unsigned long end, phys_addr_t phys, {
int map_io) unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT;
pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr);
int i = 0;
do {
set_pmd(pmd, __pmd(addr | prot));
addr += PMD_SIZE;
} while (pmd++, i++, i < PTRS_PER_PMD);
}
static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud,
unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
void *(*alloc)(unsigned long size))
{ {
pmd_t *pmd; pmd_t *pmd;
unsigned long next; unsigned long next;
pmdval_t prot_sect;
pgprot_t prot_pte;
if (map_io) {
prot_sect = PROT_SECT_DEVICE_nGnRE;
prot_pte = __pgprot(PROT_DEVICE_nGnRE);
} else {
prot_sect = PROT_SECT_NORMAL_EXEC;
prot_pte = PAGE_KERNEL_EXEC;
}
/* /*
* Check for initial section mappings in the pgd/pud and remove them. * Check for initial section mappings in the pgd/pud and remove them.
*/ */
if (pud_none(*pud) || pud_bad(*pud)) { if (pud_none(*pud) || pud_sect(*pud)) {
pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t)); pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t));
pud_populate(&init_mm, pud, pmd); if (pud_sect(*pud)) {
/*
* need to have the 1G of mappings continue to be
* present
*/
split_pud(pud, pmd);
}
pud_populate(mm, pud, pmd);
flush_tlb_all();
} }
BUG_ON(pud_bad(*pud));
pmd = pmd_offset(pud, addr); pmd = pmd_offset(pud, addr);
do { do {
...@@ -187,31 +150,51 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr, ...@@ -187,31 +150,51 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
/* try section mapping first */ /* try section mapping first */
if (((addr | next | phys) & ~SECTION_MASK) == 0) { if (((addr | next | phys) & ~SECTION_MASK) == 0) {
pmd_t old_pmd =*pmd; pmd_t old_pmd =*pmd;
set_pmd(pmd, __pmd(phys | prot_sect)); set_pmd(pmd, __pmd(phys |
pgprot_val(mk_sect_prot(prot))));
/* /*
* Check for previous table entries created during * Check for previous table entries created during
* boot (__create_page_tables) and flush them. * boot (__create_page_tables) and flush them.
*/ */
if (!pmd_none(old_pmd)) if (!pmd_none(old_pmd)) {
flush_tlb_all(); flush_tlb_all();
if (pmd_table(old_pmd)) {
phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0));
if (!WARN_ON_ONCE(slab_is_available()))
memblock_free(table, PAGE_SIZE);
}
}
} else { } else {
alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
prot_pte); prot, alloc);
} }
phys += next - addr; phys += next - addr;
} while (pmd++, addr = next, addr != end); } while (pmd++, addr = next, addr != end);
} }
static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, static inline bool use_1G_block(unsigned long addr, unsigned long next,
unsigned long end, phys_addr_t phys, unsigned long phys)
int map_io) {
if (PAGE_SHIFT != 12)
return false;
if (((addr | next | phys) & ~PUD_MASK) != 0)
return false;
return true;
}
static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd,
unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot,
void *(*alloc)(unsigned long size))
{ {
pud_t *pud; pud_t *pud;
unsigned long next; unsigned long next;
if (pgd_none(*pgd)) { if (pgd_none(*pgd)) {
pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t)); pud = alloc(PTRS_PER_PUD * sizeof(pud_t));
pgd_populate(&init_mm, pgd, pud); pgd_populate(mm, pgd, pud);
} }
BUG_ON(pgd_bad(*pgd)); BUG_ON(pgd_bad(*pgd));
...@@ -222,10 +205,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, ...@@ -222,10 +205,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
/* /*
* For 4K granule only, attempt to put down a 1GB block * For 4K granule only, attempt to put down a 1GB block
*/ */
if (!map_io && (PAGE_SHIFT == 12) && if (use_1G_block(addr, next, phys)) {
((addr | next | phys) & ~PUD_MASK) == 0) {
pud_t old_pud = *pud; pud_t old_pud = *pud;
set_pud(pud, __pud(phys | PROT_SECT_NORMAL_EXEC)); set_pud(pud, __pud(phys |
pgprot_val(mk_sect_prot(prot))));
/* /*
* If we have an old value for a pud, it will * If we have an old value for a pud, it will
...@@ -235,12 +218,15 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, ...@@ -235,12 +218,15 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
* Look up the old pmd table and free it. * Look up the old pmd table and free it.
*/ */
if (!pud_none(old_pud)) { if (!pud_none(old_pud)) {
phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
memblock_free(table, PAGE_SIZE);
flush_tlb_all(); flush_tlb_all();
if (pud_table(old_pud)) {
phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
if (!WARN_ON_ONCE(slab_is_available()))
memblock_free(table, PAGE_SIZE);
}
} }
} else { } else {
alloc_init_pmd(pud, addr, next, phys, map_io); alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc);
} }
phys += next - addr; phys += next - addr;
} while (pud++, addr = next, addr != end); } while (pud++, addr = next, addr != end);
...@@ -250,9 +236,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, ...@@ -250,9 +236,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
* Create the page directory entries and any necessary page tables for the * Create the page directory entries and any necessary page tables for the
* mapping specified by 'md'. * mapping specified by 'md'.
*/ */
static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys, static void __create_mapping(struct mm_struct *mm, pgd_t *pgd,
unsigned long virt, phys_addr_t size, phys_addr_t phys, unsigned long virt,
int map_io) phys_addr_t size, pgprot_t prot,
void *(*alloc)(unsigned long size))
{ {
unsigned long addr, length, end, next; unsigned long addr, length, end, next;
...@@ -262,31 +249,95 @@ static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys, ...@@ -262,31 +249,95 @@ static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
end = addr + length; end = addr + length;
do { do {
next = pgd_addr_end(addr, end); next = pgd_addr_end(addr, end);
alloc_init_pud(pgd, addr, next, phys, map_io); alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc);
phys += next - addr; phys += next - addr;
} while (pgd++, addr = next, addr != end); } while (pgd++, addr = next, addr != end);
} }
static void __init create_mapping(phys_addr_t phys, unsigned long virt, static void *late_alloc(unsigned long size)
phys_addr_t size) {
void *ptr;
BUG_ON(size > PAGE_SIZE);
ptr = (void *)__get_free_page(PGALLOC_GFP);
BUG_ON(!ptr);
return ptr;
}
static void __ref create_mapping(phys_addr_t phys, unsigned long virt,
phys_addr_t size, pgprot_t prot)
{ {
if (virt < VMALLOC_START) { if (virt < VMALLOC_START) {
pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
&phys, virt); &phys, virt);
return; return;
} }
__create_mapping(pgd_offset_k(virt & PAGE_MASK), phys, virt, size, 0); __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt,
size, prot, early_alloc);
}
void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot)
{
__create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot,
late_alloc);
} }
void __init create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io) static void create_mapping_late(phys_addr_t phys, unsigned long virt,
phys_addr_t size, pgprot_t prot)
{ {
if ((addr >> PGDIR_SHIFT) >= ARRAY_SIZE(idmap_pg_dir)) { if (virt < VMALLOC_START) {
pr_warn("BUG: not creating id mapping for %pa\n", &addr); pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
&phys, virt);
return; return;
} }
__create_mapping(&idmap_pg_dir[pgd_index(addr)],
addr, addr, size, map_io); return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK),
phys, virt, size, prot, late_alloc);
}
#ifdef CONFIG_DEBUG_RODATA
static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
{
/*
* Set up the executable regions using the existing section mappings
* for now. This will get more fine grained later once all memory
* is mapped
*/
unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE);
unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
if (end < kernel_x_start) {
create_mapping(start, __phys_to_virt(start),
end - start, PAGE_KERNEL);
} else if (start >= kernel_x_end) {
create_mapping(start, __phys_to_virt(start),
end - start, PAGE_KERNEL);
} else {
if (start < kernel_x_start)
create_mapping(start, __phys_to_virt(start),
kernel_x_start - start,
PAGE_KERNEL);
create_mapping(kernel_x_start,
__phys_to_virt(kernel_x_start),
kernel_x_end - kernel_x_start,
PAGE_KERNEL_EXEC);
if (kernel_x_end < end)
create_mapping(kernel_x_end,
__phys_to_virt(kernel_x_end),
end - kernel_x_end,
PAGE_KERNEL);
}
} }
#else
static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
{
create_mapping(start, __phys_to_virt(start), end - start,
PAGE_KERNEL_EXEC);
}
#endif
static void __init map_mem(void) static void __init map_mem(void)
{ {
...@@ -332,14 +383,53 @@ static void __init map_mem(void) ...@@ -332,14 +383,53 @@ static void __init map_mem(void)
memblock_set_current_limit(limit); memblock_set_current_limit(limit);
} }
#endif #endif
__map_memblock(start, end);
create_mapping(start, __phys_to_virt(start), end - start);
} }
/* Limit no longer required. */ /* Limit no longer required. */
memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
} }
void __init fixup_executable(void)
{
#ifdef CONFIG_DEBUG_RODATA
/* now that we are actually fully mapped, make the start/end more fine grained */
if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) {
unsigned long aligned_start = round_down(__pa(_stext),
SECTION_SIZE);
create_mapping(aligned_start, __phys_to_virt(aligned_start),
__pa(_stext) - aligned_start,
PAGE_KERNEL);
}
if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) {
unsigned long aligned_end = round_up(__pa(__init_end),
SECTION_SIZE);
create_mapping(__pa(__init_end), (unsigned long)__init_end,
aligned_end - __pa(__init_end),
PAGE_KERNEL);
}
#endif
}
#ifdef CONFIG_DEBUG_RODATA
void mark_rodata_ro(void)
{
create_mapping_late(__pa(_stext), (unsigned long)_stext,
(unsigned long)_etext - (unsigned long)_stext,
PAGE_KERNEL_EXEC | PTE_RDONLY);
}
#endif
void fixup_init(void)
{
create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin,
(unsigned long)__init_end - (unsigned long)__init_begin,
PAGE_KERNEL);
}
/* /*
* paging_init() sets up the page tables, initialises the zone memory * paging_init() sets up the page tables, initialises the zone memory
* maps and sets up the zero page. * maps and sets up the zero page.
...@@ -349,13 +439,7 @@ void __init paging_init(void) ...@@ -349,13 +439,7 @@ void __init paging_init(void)
void *zero_page; void *zero_page;
map_mem(); map_mem();
fixup_executable();
/*
* Finally flush the caches and tlb to ensure that we're in a
* consistent state.
*/
flush_cache_all();
flush_tlb_all();
/* allocate the zero page. */ /* allocate the zero page. */
zero_page = early_alloc(PAGE_SIZE); zero_page = early_alloc(PAGE_SIZE);
......
...@@ -102,7 +102,7 @@ ENTRY(cpu_do_idle) ...@@ -102,7 +102,7 @@ ENTRY(cpu_do_idle)
ret ret
ENDPROC(cpu_do_idle) ENDPROC(cpu_do_idle)
#ifdef CONFIG_ARM64_CPU_SUSPEND #ifdef CONFIG_CPU_PM
/** /**
* cpu_do_suspend - save CPU registers context * cpu_do_suspend - save CPU registers context
* *
...@@ -244,14 +244,18 @@ ENTRY(__cpu_setup) ...@@ -244,14 +244,18 @@ ENTRY(__cpu_setup)
ENDPROC(__cpu_setup) ENDPROC(__cpu_setup)
/* /*
* We set the desired value explicitly, including those of the
* reserved bits. The values of bits EE & E0E were set early in
* el2_setup, which are left untouched below.
*
* n n T * n n T
* U E WT T UD US IHBS * U E WT T UD US IHBS
* CE0 XWHW CZ ME TEEA S * CE0 XWHW CZ ME TEEA S
* .... .IEE .... NEAI TE.I ..AD DEN0 ACAM * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
* 0011 0... 1101 ..0. ..0. 10.. .... .... < hardware reserved * 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved
* .... .1.. .... 01.1 11.1 ..01 0001 1101 < software settings * .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings
*/ */
.type crval, #object .type crval, #object
crval: crval:
.word 0x000802e2 // clear .word 0xfcffffff // clear
.word 0x0405d11d // set .word 0x34d5d91d // set
...@@ -4,7 +4,6 @@ ...@@ -4,7 +4,6 @@
config ARM64_CPUIDLE config ARM64_CPUIDLE
bool "Generic ARM64 CPU idle Driver" bool "Generic ARM64 CPU idle Driver"
select ARM64_CPU_SUSPEND
select DT_IDLE_STATES select DT_IDLE_STATES
help help
Select this to enable generic cpuidle driver for ARM64. Select this to enable generic cpuidle driver for ARM64.
......
...@@ -19,7 +19,6 @@ ...@@ -19,7 +19,6 @@
#include <linux/of.h> #include <linux/of.h>
#include <asm/cpuidle.h> #include <asm/cpuidle.h>
#include <asm/suspend.h>
#include "dt_idle_states.h" #include "dt_idle_states.h"
......
...@@ -297,29 +297,15 @@ static __init int match_config_table(efi_guid_t *guid, ...@@ -297,29 +297,15 @@ static __init int match_config_table(efi_guid_t *guid,
return 0; return 0;
} }
int __init efi_config_init(efi_config_table_type_t *arch_tables) int __init efi_config_parse_tables(void *config_tables, int count, int sz,
efi_config_table_type_t *arch_tables)
{ {
void *config_tables, *tablep; void *tablep;
int i, sz; int i;
if (efi_enabled(EFI_64BIT))
sz = sizeof(efi_config_table_64_t);
else
sz = sizeof(efi_config_table_32_t);
/*
* Let's see what config tables the firmware passed to us.
*/
config_tables = early_memremap(efi.systab->tables,
efi.systab->nr_tables * sz);
if (config_tables == NULL) {
pr_err("Could not map Configuration table!\n");
return -ENOMEM;
}
tablep = config_tables; tablep = config_tables;
pr_info(""); pr_info("");
for (i = 0; i < efi.systab->nr_tables; i++) { for (i = 0; i < count; i++) {
efi_guid_t guid; efi_guid_t guid;
unsigned long table; unsigned long table;
...@@ -332,8 +318,6 @@ int __init efi_config_init(efi_config_table_type_t *arch_tables) ...@@ -332,8 +318,6 @@ int __init efi_config_init(efi_config_table_type_t *arch_tables)
if (table64 >> 32) { if (table64 >> 32) {
pr_cont("\n"); pr_cont("\n");
pr_err("Table located above 4GB, disabling EFI.\n"); pr_err("Table located above 4GB, disabling EFI.\n");
early_memunmap(config_tables,
efi.systab->nr_tables * sz);
return -EINVAL; return -EINVAL;
} }
#endif #endif
...@@ -348,13 +332,37 @@ int __init efi_config_init(efi_config_table_type_t *arch_tables) ...@@ -348,13 +332,37 @@ int __init efi_config_init(efi_config_table_type_t *arch_tables)
tablep += sz; tablep += sz;
} }
pr_cont("\n"); pr_cont("\n");
early_memunmap(config_tables, efi.systab->nr_tables * sz);
set_bit(EFI_CONFIG_TABLES, &efi.flags); set_bit(EFI_CONFIG_TABLES, &efi.flags);
return 0; return 0;
} }
int __init efi_config_init(efi_config_table_type_t *arch_tables)
{
void *config_tables;
int sz, ret;
if (efi_enabled(EFI_64BIT))
sz = sizeof(efi_config_table_64_t);
else
sz = sizeof(efi_config_table_32_t);
/*
* Let's see what config tables the firmware passed to us.
*/
config_tables = early_memremap(efi.systab->tables,
efi.systab->nr_tables * sz);
if (config_tables == NULL) {
pr_err("Could not map Configuration table!\n");
return -ENOMEM;
}
ret = efi_config_parse_tables(config_tables, efi.systab->nr_tables, sz,
arch_tables);
early_memunmap(config_tables, efi.systab->nr_tables * sz);
return ret;
}
#ifdef CONFIG_EFI_VARS_MODULE #ifdef CONFIG_EFI_VARS_MODULE
static int __init efi_load_efivars(void) static int __init efi_load_efivars(void)
{ {
......
...@@ -295,3 +295,62 @@ unsigned long efi_entry(void *handle, efi_system_table_t *sys_table, ...@@ -295,3 +295,62 @@ unsigned long efi_entry(void *handle, efi_system_table_t *sys_table,
fail: fail:
return EFI_ERROR; return EFI_ERROR;
} }
/*
* This is the base address at which to start allocating virtual memory ranges
* for UEFI Runtime Services. This is in the low TTBR0 range so that we can use
* any allocation we choose, and eliminate the risk of a conflict after kexec.
* The value chosen is the largest non-zero power of 2 suitable for this purpose
* both on 32-bit and 64-bit ARM CPUs, to maximize the likelihood that it can
* be mapped efficiently.
*/
#define EFI_RT_VIRTUAL_BASE 0x40000000
/*
* efi_get_virtmap() - create a virtual mapping for the EFI memory map
*
* This function populates the virt_addr fields of all memory region descriptors
* in @memory_map whose EFI_MEMORY_RUNTIME attribute is set. Those descriptors
* are also copied to @runtime_map, and their total count is returned in @count.
*/
void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size,
unsigned long desc_size, efi_memory_desc_t *runtime_map,
int *count)
{
u64 efi_virt_base = EFI_RT_VIRTUAL_BASE;
efi_memory_desc_t *out = runtime_map;
int l;
for (l = 0; l < map_size; l += desc_size) {
efi_memory_desc_t *in = (void *)memory_map + l;
u64 paddr, size;
if (!(in->attribute & EFI_MEMORY_RUNTIME))
continue;
/*
* Make the mapping compatible with 64k pages: this allows
* a 4k page size kernel to kexec a 64k page size kernel and
* vice versa.
*/
paddr = round_down(in->phys_addr, SZ_64K);
size = round_up(in->num_pages * EFI_PAGE_SIZE +
in->phys_addr - paddr, SZ_64K);
/*
* Avoid wasting memory on PTEs by choosing a virtual base that
* is compatible with section mappings if this region has the
* appropriate size and physical alignment. (Sections are 2 MB
* on 4k granule kernels)
*/
if (IS_ALIGNED(in->phys_addr, SZ_2M) && size >= SZ_2M)
efi_virt_base = round_up(efi_virt_base, SZ_2M);
in->virt_addr = efi_virt_base + in->phys_addr - paddr;
efi_virt_base += size;
memcpy(out, in, desc_size);
out = (void *)out + desc_size;
++*count;
}
}
...@@ -32,6 +32,15 @@ ...@@ -32,6 +32,15 @@
static unsigned long __chunk_size = EFI_READ_CHUNK_SIZE; static unsigned long __chunk_size = EFI_READ_CHUNK_SIZE;
/*
* Allow the platform to override the allocation granularity: this allows
* systems that have the capability to run with a larger page size to deal
* with the allocations for initrd and fdt more efficiently.
*/
#ifndef EFI_ALLOC_ALIGN
#define EFI_ALLOC_ALIGN EFI_PAGE_SIZE
#endif
struct file_info { struct file_info {
efi_file_handle_t *handle; efi_file_handle_t *handle;
u64 size; u64 size;
...@@ -154,10 +163,10 @@ efi_status_t efi_high_alloc(efi_system_table_t *sys_table_arg, ...@@ -154,10 +163,10 @@ efi_status_t efi_high_alloc(efi_system_table_t *sys_table_arg,
* a specific address. We are doing page-based allocations, * a specific address. We are doing page-based allocations,
* so we must be aligned to a page. * so we must be aligned to a page.
*/ */
if (align < EFI_PAGE_SIZE) if (align < EFI_ALLOC_ALIGN)
align = EFI_PAGE_SIZE; align = EFI_ALLOC_ALIGN;
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE; nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
again: again:
for (i = 0; i < map_size / desc_size; i++) { for (i = 0; i < map_size / desc_size; i++) {
efi_memory_desc_t *desc; efi_memory_desc_t *desc;
...@@ -239,10 +248,10 @@ efi_status_t efi_low_alloc(efi_system_table_t *sys_table_arg, ...@@ -239,10 +248,10 @@ efi_status_t efi_low_alloc(efi_system_table_t *sys_table_arg,
* a specific address. We are doing page-based allocations, * a specific address. We are doing page-based allocations,
* so we must be aligned to a page. * so we must be aligned to a page.
*/ */
if (align < EFI_PAGE_SIZE) if (align < EFI_ALLOC_ALIGN)
align = EFI_PAGE_SIZE; align = EFI_ALLOC_ALIGN;
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE; nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
for (i = 0; i < map_size / desc_size; i++) { for (i = 0; i < map_size / desc_size; i++) {
efi_memory_desc_t *desc; efi_memory_desc_t *desc;
unsigned long m = (unsigned long)map; unsigned long m = (unsigned long)map;
...@@ -296,7 +305,7 @@ void efi_free(efi_system_table_t *sys_table_arg, unsigned long size, ...@@ -296,7 +305,7 @@ void efi_free(efi_system_table_t *sys_table_arg, unsigned long size,
if (!size) if (!size)
return; return;
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE; nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
efi_call_early(free_pages, addr, nr_pages); efi_call_early(free_pages, addr, nr_pages);
} }
...@@ -565,7 +574,7 @@ efi_status_t efi_relocate_kernel(efi_system_table_t *sys_table_arg, ...@@ -565,7 +574,7 @@ efi_status_t efi_relocate_kernel(efi_system_table_t *sys_table_arg,
* to the preferred address. If that fails, allocate as low * to the preferred address. If that fails, allocate as low
* as possible while respecting the required alignment. * as possible while respecting the required alignment.
*/ */
nr_pages = round_up(alloc_size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE; nr_pages = round_up(alloc_size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
status = efi_call_early(allocate_pages, status = efi_call_early(allocate_pages,
EFI_ALLOCATE_ADDRESS, EFI_LOADER_DATA, EFI_ALLOCATE_ADDRESS, EFI_LOADER_DATA,
nr_pages, &efi_addr); nr_pages, &efi_addr);
......
...@@ -39,4 +39,8 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, ...@@ -39,4 +39,8 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
void *get_fdt(efi_system_table_t *sys_table); void *get_fdt(efi_system_table_t *sys_table);
void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size,
unsigned long desc_size, efi_memory_desc_t *runtime_map,
int *count);
#endif #endif
...@@ -14,6 +14,8 @@ ...@@ -14,6 +14,8 @@
#include <linux/libfdt.h> #include <linux/libfdt.h>
#include <asm/efi.h> #include <asm/efi.h>
#include "efistub.h"
efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt, efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt,
unsigned long orig_fdt_size, unsigned long orig_fdt_size,
void *fdt, int new_fdt_size, char *cmdline_ptr, void *fdt, int new_fdt_size, char *cmdline_ptr,
...@@ -193,9 +195,26 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, ...@@ -193,9 +195,26 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
unsigned long map_size, desc_size; unsigned long map_size, desc_size;
u32 desc_ver; u32 desc_ver;
unsigned long mmap_key; unsigned long mmap_key;
efi_memory_desc_t *memory_map; efi_memory_desc_t *memory_map, *runtime_map;
unsigned long new_fdt_size; unsigned long new_fdt_size;
efi_status_t status; efi_status_t status;
int runtime_entry_count = 0;
/*
* Get a copy of the current memory map that we will use to prepare
* the input for SetVirtualAddressMap(). We don't have to worry about
* subsequent allocations adding entries, since they could not affect
* the number of EFI_MEMORY_RUNTIME regions.
*/
status = efi_get_memory_map(sys_table, &runtime_map, &map_size,
&desc_size, &desc_ver, &mmap_key);
if (status != EFI_SUCCESS) {
pr_efi_err(sys_table, "Unable to retrieve UEFI memory map.\n");
return status;
}
pr_efi(sys_table,
"Exiting boot services and installing virtual address map...\n");
/* /*
* Estimate size of new FDT, and allocate memory for it. We * Estimate size of new FDT, and allocate memory for it. We
...@@ -248,12 +267,48 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, ...@@ -248,12 +267,48 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
} }
} }
/*
* Update the memory map with virtual addresses. The function will also
* populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME
* entries so that we can pass it straight into SetVirtualAddressMap()
*/
efi_get_virtmap(memory_map, map_size, desc_size, runtime_map,
&runtime_entry_count);
/* Now we are ready to exit_boot_services.*/ /* Now we are ready to exit_boot_services.*/
status = sys_table->boottime->exit_boot_services(handle, mmap_key); status = sys_table->boottime->exit_boot_services(handle, mmap_key);
if (status == EFI_SUCCESS) {
efi_set_virtual_address_map_t *svam;
if (status == EFI_SUCCESS) /* Install the new virtual address map */
return status; svam = sys_table->runtime->set_virtual_address_map;
status = svam(runtime_entry_count * desc_size, desc_size,
desc_ver, runtime_map);
/*
* We are beyond the point of no return here, so if the call to
* SetVirtualAddressMap() failed, we need to signal that to the
* incoming kernel but proceed normally otherwise.
*/
if (status != EFI_SUCCESS) {
int l;
/*
* Set the virtual address field of all
* EFI_MEMORY_RUNTIME entries to 0. This will signal
* the incoming kernel that no virtual translation has
* been installed.
*/
for (l = 0; l < map_size; l += desc_size) {
efi_memory_desc_t *p = (void *)memory_map + l;
if (p->attribute & EFI_MEMORY_RUNTIME)
p->virt_addr = 0;
}
}
return EFI_SUCCESS;
}
pr_efi_err(sys_table, "Exit boot services failed.\n"); pr_efi_err(sys_table, "Exit boot services failed.\n");
...@@ -264,6 +319,7 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, ...@@ -264,6 +319,7 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
efi_free(sys_table, new_fdt_size, *new_fdt_addr); efi_free(sys_table, new_fdt_size, *new_fdt_addr);
fail: fail:
sys_table->boottime->free_pool(runtime_map);
return EFI_LOAD_ERROR; return EFI_LOAD_ERROR;
} }
......
...@@ -689,6 +689,15 @@ asmlinkage long compat_sys_sendfile64(int out_fd, int in_fd, ...@@ -689,6 +689,15 @@ asmlinkage long compat_sys_sendfile64(int out_fd, int in_fd,
asmlinkage long compat_sys_sigaltstack(const compat_stack_t __user *uss_ptr, asmlinkage long compat_sys_sigaltstack(const compat_stack_t __user *uss_ptr,
compat_stack_t __user *uoss_ptr); compat_stack_t __user *uoss_ptr);
#ifdef __ARCH_WANT_SYS_SIGPENDING
asmlinkage long compat_sys_sigpending(compat_old_sigset_t __user *set);
#endif
#ifdef __ARCH_WANT_SYS_SIGPROCMASK
asmlinkage long compat_sys_sigprocmask(int how, compat_old_sigset_t __user *nset,
compat_old_sigset_t __user *oset);
#endif
int compat_restore_altstack(const compat_stack_t __user *uss); int compat_restore_altstack(const compat_stack_t __user *uss);
int __compat_save_altstack(compat_stack_t __user *, unsigned long); int __compat_save_altstack(compat_stack_t __user *, unsigned long);
#define compat_save_altstack_ex(uss, sp) do { \ #define compat_save_altstack_ex(uss, sp) do { \
......
...@@ -875,6 +875,8 @@ static inline efi_status_t efi_query_variable_store(u32 attributes, unsigned lon ...@@ -875,6 +875,8 @@ static inline efi_status_t efi_query_variable_store(u32 attributes, unsigned lon
#endif #endif
extern void __iomem *efi_lookup_mapped_addr(u64 phys_addr); extern void __iomem *efi_lookup_mapped_addr(u64 phys_addr);
extern int efi_config_init(efi_config_table_type_t *arch_tables); extern int efi_config_init(efi_config_table_type_t *arch_tables);
extern int efi_config_parse_tables(void *config_tables, int count, int sz,
efi_config_table_type_t *arch_tables);
extern u64 efi_get_iobase (void); extern u64 efi_get_iobase (void);
extern u32 efi_mem_type (unsigned long phys_addr); extern u32 efi_mem_type (unsigned long phys_addr);
extern u64 efi_mem_attributes (unsigned long phys_addr); extern u64 efi_mem_attributes (unsigned long phys_addr);
......
...@@ -410,12 +410,16 @@ asmlinkage long sys_newlstat(const char __user *filename, ...@@ -410,12 +410,16 @@ asmlinkage long sys_newlstat(const char __user *filename,
struct stat __user *statbuf); struct stat __user *statbuf);
asmlinkage long sys_newfstat(unsigned int fd, struct stat __user *statbuf); asmlinkage long sys_newfstat(unsigned int fd, struct stat __user *statbuf);
asmlinkage long sys_ustat(unsigned dev, struct ustat __user *ubuf); asmlinkage long sys_ustat(unsigned dev, struct ustat __user *ubuf);
#if BITS_PER_LONG == 32 #if defined(__ARCH_WANT_STAT64) || defined(__ARCH_WANT_COMPAT_STAT64)
asmlinkage long sys_stat64(const char __user *filename, asmlinkage long sys_stat64(const char __user *filename,
struct stat64 __user *statbuf); struct stat64 __user *statbuf);
asmlinkage long sys_fstat64(unsigned long fd, struct stat64 __user *statbuf); asmlinkage long sys_fstat64(unsigned long fd, struct stat64 __user *statbuf);
asmlinkage long sys_lstat64(const char __user *filename, asmlinkage long sys_lstat64(const char __user *filename,
struct stat64 __user *statbuf); struct stat64 __user *statbuf);
asmlinkage long sys_fstatat64(int dfd, const char __user *filename,
struct stat64 __user *statbuf, int flag);
#endif
#if BITS_PER_LONG == 32
asmlinkage long sys_truncate64(const char __user *path, loff_t length); asmlinkage long sys_truncate64(const char __user *path, loff_t length);
asmlinkage long sys_ftruncate64(unsigned int fd, loff_t length); asmlinkage long sys_ftruncate64(unsigned int fd, loff_t length);
#endif #endif
...@@ -771,8 +775,6 @@ asmlinkage long sys_openat(int dfd, const char __user *filename, int flags, ...@@ -771,8 +775,6 @@ asmlinkage long sys_openat(int dfd, const char __user *filename, int flags,
umode_t mode); umode_t mode);
asmlinkage long sys_newfstatat(int dfd, const char __user *filename, asmlinkage long sys_newfstatat(int dfd, const char __user *filename,
struct stat __user *statbuf, int flag); struct stat __user *statbuf, int flag);
asmlinkage long sys_fstatat64(int dfd, const char __user *filename,
struct stat64 __user *statbuf, int flag);
asmlinkage long sys_readlinkat(int dfd, const char __user *path, char __user *buf, asmlinkage long sys_readlinkat(int dfd, const char __user *path, char __user *buf,
int bufsiz); int bufsiz);
asmlinkage long sys_utimensat(int dfd, const char __user *filename, asmlinkage long sys_utimensat(int dfd, const char __user *filename,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment