Commit 7e3a68be authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman

powerpc/64: vmlinux support building with PCREL addresing

PC-Relative or PCREL addressing is an extension to the ELF ABI which
uses Power ISA v3.1 PC-relative instructions to calculate addresses,
rather than the traditional TOC scheme.

Add an option to build vmlinux using pcrel addressing. Modules continue
to use TOC addressing.

- TOC address helpers and r2 are poisoned with -1 when running vmlinux.
  r2 could be used for something useful once things are ironed out.

- Assembly must call C functions with @notoc annotation, or the linker
  complains aobut a missing nop after the call. This is done with the
  CFUNC macro introduced earlier.

- Boot: with the exception of prom_init, the execution branches to the
  kernel virtual address early in boot, before any addresses are
  generated, which ensures 34-bit pcrel addressing does not miss the
  high PAGE_OFFSET bits. TOC relative addressing has a similar
  requirement. prom_init does not go to the virtual address and its
  addresses should not carry over to the post-prom kernel.

- Ftrace trampolines are converted from TOC addressing to pcrel
  addressing, including module ftrace trampolines that currently use the
  kernel TOC to find ftrace target functions.

- BPF function prologue and function calling generation are converted
  from TOC to pcrel.

- copypage_64.S has an interesting problem, prefixed instructions have
  alignment restrictions so the linker can add padding, which makes the
  assembler treat the difference between two local labels as
  non-constant even if alignment is arranged so padding is not required.
  This may need toolchain help to solve nicely, for now move the prefix
  instruction out of the alternate patch section to work around it.

This reduces kernel text size by about 6%.
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://msgid.link/20230408021752.862660-6-npiggin@gmail.com
parent 4e991e3c
...@@ -7,6 +7,9 @@ config CC_HAS_ELFV2 ...@@ -7,6 +7,9 @@ config CC_HAS_ELFV2
config CC_HAS_PREFIXED config CC_HAS_PREFIXED
def_bool PPC64 && $(cc-option, -mcpu=power10 -mprefixed) def_bool PPC64 && $(cc-option, -mcpu=power10 -mprefixed)
config CC_HAS_PCREL
def_bool PPC64 && $(cc-option, -mcpu=power10 -mpcrel)
config 32BIT config 32BIT
bool bool
default y if PPC32 default y if PPC32
......
...@@ -107,6 +107,9 @@ LDFLAGS_vmlinux-$(CONFIG_RELOCATABLE) += -z notext ...@@ -107,6 +107,9 @@ LDFLAGS_vmlinux-$(CONFIG_RELOCATABLE) += -z notext
LDFLAGS_vmlinux := $(LDFLAGS_vmlinux-y) LDFLAGS_vmlinux := $(LDFLAGS_vmlinux-y)
ifdef CONFIG_PPC64 ifdef CONFIG_PPC64
ifdef CONFIG_PPC_KERNEL_PCREL
KBUILD_CFLAGS_MODULE += $(call cc-option,-mno-pcrel)
endif
ifeq ($(call cc-option-yn,-mcmodel=medium),y) ifeq ($(call cc-option-yn,-mcmodel=medium),y)
# -mcmodel=medium breaks modules because it uses 32bit offsets from # -mcmodel=medium breaks modules because it uses 32bit offsets from
# the TOC pointer to create pointers where possible. Pointers into the # the TOC pointer to create pointers where possible. Pointers into the
...@@ -185,7 +188,11 @@ KBUILD_CFLAGS += $(call cc-option,-mprefixed) ...@@ -185,7 +188,11 @@ KBUILD_CFLAGS += $(call cc-option,-mprefixed)
else else
KBUILD_CFLAGS += $(call cc-option,-mno-prefixed) KBUILD_CFLAGS += $(call cc-option,-mno-prefixed)
endif endif
ifdef CONFIG_PPC_KERNEL_PCREL
KBUILD_CFLAGS += $(call cc-option,-mpcrel)
else
KBUILD_CFLAGS += $(call cc-option,-mno-pcrel) KBUILD_CFLAGS += $(call cc-option,-mno-pcrel)
endif
# No AltiVec or VSX or MMA instructions when building kernel # No AltiVec or VSX or MMA instructions when building kernel
KBUILD_CFLAGS += $(call cc-option,-mno-altivec) KBUILD_CFLAGS += $(call cc-option,-mno-altivec)
......
...@@ -88,7 +88,9 @@ struct paca_struct { ...@@ -88,7 +88,9 @@ struct paca_struct {
u16 lock_token; /* Constant 0x8000, used in locks */ u16 lock_token; /* Constant 0x8000, used in locks */
#endif #endif
#ifndef CONFIG_PPC_KERNEL_PCREL
u64 kernel_toc; /* Kernel TOC address */ u64 kernel_toc; /* Kernel TOC address */
#endif
u64 kernelbase; /* Base address of kernel */ u64 kernelbase; /* Base address of kernel */
u64 kernel_msr; /* MSR while running in kernel */ u64 kernel_msr; /* MSR while running in kernel */
void *emergency_sp; /* pointer to emergency stack */ void *emergency_sp; /* pointer to emergency stack */
......
...@@ -120,11 +120,18 @@ ...@@ -120,11 +120,18 @@
* 16-bit immediate helper macros: HA() is for use with sign-extending instrs * 16-bit immediate helper macros: HA() is for use with sign-extending instrs
* (e.g. LD, ADDI). If the bottom 16 bits is "-ve", add another bit into the * (e.g. LD, ADDI). If the bottom 16 bits is "-ve", add another bit into the
* top half to negate the effect (i.e. 0xffff + 1 = 0x(1)0000). * top half to negate the effect (i.e. 0xffff + 1 = 0x(1)0000).
*
* XXX: should these mask out possible sign bits?
*/ */
#define IMM_H(i) ((uintptr_t)(i)>>16) #define IMM_H(i) ((uintptr_t)(i)>>16)
#define IMM_HA(i) (((uintptr_t)(i)>>16) + \ #define IMM_HA(i) (((uintptr_t)(i)>>16) + \
(((uintptr_t)(i) & 0x8000) >> 15)) (((uintptr_t)(i) & 0x8000) >> 15))
/*
* 18-bit immediate helper for prefix 18-bit upper immediate si0 field.
*/
#define IMM_H18(i) (((uintptr_t)(i)>>16) & 0x3ffff)
/* opcode and xopcode for instructions */ /* opcode and xopcode for instructions */
#define OP_PREFIX 1 #define OP_PREFIX 1
...@@ -306,6 +313,7 @@ ...@@ -306,6 +313,7 @@
#define PPC_PREFIX_8LS 0x04000000 #define PPC_PREFIX_8LS 0x04000000
/* Prefixed instructions */ /* Prefixed instructions */
#define PPC_INST_PADDI 0x38000000
#define PPC_INST_PLD 0xe4000000 #define PPC_INST_PLD 0xe4000000
#define PPC_INST_PSTD 0xf4000000 #define PPC_INST_PSTD 0xf4000000
......
...@@ -183,7 +183,11 @@ ...@@ -183,7 +183,11 @@
/* /*
* Used to name C functions called from asm * Used to name C functions called from asm
*/ */
#if defined(CONFIG_PPC_KERNEL_PCREL) && !defined(MODULE)
#define CFUNC(name) name@notoc
#else
#define CFUNC(name) name #define CFUNC(name) name
#endif
/* /*
* We use __powerpc64__ here because we want the compat VDSO to use the 32-bit * We use __powerpc64__ here because we want the compat VDSO to use the 32-bit
...@@ -212,6 +216,9 @@ ...@@ -212,6 +216,9 @@
.globl name; \ .globl name; \
name: name:
#if defined(CONFIG_PPC_KERNEL_PCREL) && !defined(MODULE)
#define _GLOBAL_TOC _GLOBAL
#else
#define _GLOBAL_TOC(name) \ #define _GLOBAL_TOC(name) \
.align 2 ; \ .align 2 ; \
.type name,@function; \ .type name,@function; \
...@@ -220,6 +227,7 @@ name: \ ...@@ -220,6 +227,7 @@ name: \
0: addis r2,r12,(.TOC.-0b)@ha; \ 0: addis r2,r12,(.TOC.-0b)@ha; \
addi r2,r2,(.TOC.-0b)@l; \ addi r2,r2,(.TOC.-0b)@l; \
.localentry name,.-name .localentry name,.-name
#endif
#define DOTSYM(a) a #define DOTSYM(a) a
...@@ -351,8 +359,13 @@ GLUE(.,name): ...@@ -351,8 +359,13 @@ GLUE(.,name):
#ifdef __powerpc64__ #ifdef __powerpc64__
#ifdef CONFIG_PPC_KERNEL_PCREL
#define __LOAD_PACA_TOC(reg) \
li reg,-1
#else
#define __LOAD_PACA_TOC(reg) \ #define __LOAD_PACA_TOC(reg) \
ld reg,PACATOC(r13) ld reg,PACATOC(r13)
#endif
#define LOAD_PACA_TOC() \ #define LOAD_PACA_TOC() \
__LOAD_PACA_TOC(r2) __LOAD_PACA_TOC(r2)
...@@ -366,9 +379,15 @@ GLUE(.,name): ...@@ -366,9 +379,15 @@ GLUE(.,name):
ori reg, reg, (expr)@l; \ ori reg, reg, (expr)@l; \
rldimi reg, tmp, 32, 0 rldimi reg, tmp, 32, 0
#if defined(CONFIG_PPC_KERNEL_PCREL) && !defined(MODULE)
#define LOAD_REG_ADDR(reg,name) \
pla reg,name@pcrel
#else
#define LOAD_REG_ADDR(reg,name) \ #define LOAD_REG_ADDR(reg,name) \
addis reg,r2,name@toc@ha; \ addis reg,r2,name@toc@ha; \
addi reg,reg,name@toc@l addi reg,reg,name@toc@l
#endif
#ifdef CONFIG_PPC_BOOK3E_64 #ifdef CONFIG_PPC_BOOK3E_64
/* /*
......
...@@ -46,10 +46,15 @@ extern char end_virt_trampolines[]; ...@@ -46,10 +46,15 @@ extern char end_virt_trampolines[];
*/ */
static inline unsigned long kernel_toc_addr(void) static inline unsigned long kernel_toc_addr(void)
{ {
#ifdef CONFIG_PPC_KERNEL_PCREL
BUILD_BUG();
return -1UL;
#else
unsigned long toc_ptr; unsigned long toc_ptr;
asm volatile("mr %0, 2" : "=r" (toc_ptr)); asm volatile("mr %0, 2" : "=r" (toc_ptr));
return toc_ptr; return toc_ptr;
#endif
} }
static inline int overlaps_interrupt_vector_text(unsigned long start, static inline int overlaps_interrupt_vector_text(unsigned long start,
......
...@@ -185,7 +185,9 @@ int main(void) ...@@ -185,7 +185,9 @@ int main(void)
offsetof(struct task_struct, thread_info)); offsetof(struct task_struct, thread_info));
OFFSET(PACASAVEDMSR, paca_struct, saved_msr); OFFSET(PACASAVEDMSR, paca_struct, saved_msr);
OFFSET(PACAR1, paca_struct, saved_r1); OFFSET(PACAR1, paca_struct, saved_r1);
#ifndef CONFIG_PPC_KERNEL_PCREL
OFFSET(PACATOC, paca_struct, kernel_toc); OFFSET(PACATOC, paca_struct, kernel_toc);
#endif
OFFSET(PACAKBASE, paca_struct, kernelbase); OFFSET(PACAKBASE, paca_struct, kernelbase);
OFFSET(PACAKMSR, paca_struct, kernel_msr); OFFSET(PACAKMSR, paca_struct, kernel_msr);
#ifdef CONFIG_PPC_BOOK3S_64 #ifdef CONFIG_PPC_BOOK3S_64
......
...@@ -330,6 +330,12 @@ _GLOBAL(fsl_secondary_thread_init) ...@@ -330,6 +330,12 @@ _GLOBAL(fsl_secondary_thread_init)
*/ */
_GLOBAL(generic_secondary_smp_init) _GLOBAL(generic_secondary_smp_init)
FIXUP_ENDIAN FIXUP_ENDIAN
li r13,0
/* Poison TOC */
li r2,-1
mr r24,r3 mr r24,r3
mr r25,r4 mr r25,r4
...@@ -528,6 +534,9 @@ __start_initialization_multiplatform: ...@@ -528,6 +534,9 @@ __start_initialization_multiplatform:
/* Zero r13 (paca) so early program check / mce don't use it */ /* Zero r13 (paca) so early program check / mce don't use it */
li r13,0 li r13,0
/* Poison TOC */
li r2,-1
/* /*
* Are we booted from a PROM Of-type client-interface ? * Are we booted from a PROM Of-type client-interface ?
*/ */
...@@ -921,6 +930,10 @@ SYM_FUNC_END(enable_64b_mode) ...@@ -921,6 +930,10 @@ SYM_FUNC_END(enable_64b_mode)
* this. * this.
*/ */
_GLOBAL(relative_toc) _GLOBAL(relative_toc)
#ifdef CONFIG_PPC_KERNEL_PCREL
tdnei r2,-1
blr
#else
mflr r0 mflr r0
bcl 20,31,$+4 bcl 20,31,$+4
0: mflr r11 0: mflr r11
...@@ -931,6 +944,7 @@ _GLOBAL(relative_toc) ...@@ -931,6 +944,7 @@ _GLOBAL(relative_toc)
.balign 8 .balign 8
p_toc: .8byte .TOC. - 0b p_toc: .8byte .TOC. - 0b
#endif
/* /*
* This is where the main kernel code starts. * This is where the main kernel code starts.
......
...@@ -206,7 +206,11 @@ static __always_inline void call_do_softirq(const void *sp) ...@@ -206,7 +206,11 @@ static __always_inline void call_do_softirq(const void *sp)
asm volatile ( asm volatile (
PPC_STLU " %%r1, %[offset](%[sp]) ;" PPC_STLU " %%r1, %[offset](%[sp]) ;"
"mr %%r1, %[sp] ;" "mr %%r1, %[sp] ;"
#ifdef CONFIG_PPC_KERNEL_PCREL
"bl %[callee]@notoc ;"
#else
"bl %[callee] ;" "bl %[callee] ;"
#endif
PPC_LL " %%r1, 0(%%r1) ;" PPC_LL " %%r1, 0(%%r1) ;"
: // Outputs : // Outputs
: // Inputs : // Inputs
...@@ -259,7 +263,11 @@ static __always_inline void call_do_irq(struct pt_regs *regs, void *sp) ...@@ -259,7 +263,11 @@ static __always_inline void call_do_irq(struct pt_regs *regs, void *sp)
PPC_STLU " %%r1, %[offset](%[sp]) ;" PPC_STLU " %%r1, %[offset](%[sp]) ;"
"mr %%r4, %%r1 ;" "mr %%r4, %%r1 ;"
"mr %%r1, %[sp] ;" "mr %%r1, %[sp] ;"
#ifdef CONFIG_PPC_KERNEL_PCREL
"bl %[callee]@notoc ;"
#else
"bl %[callee] ;" "bl %[callee] ;"
#endif
PPC_LL " %%r1, 0(%%r1) ;" PPC_LL " %%r1, 0(%%r1) ;"
: // Outputs : // Outputs
"+r" (r3) "+r" (r3)
......
...@@ -101,17 +101,18 @@ static unsigned long stub_func_addr(func_desc_t func) ...@@ -101,17 +101,18 @@ static unsigned long stub_func_addr(func_desc_t func)
/* Like PPC32, we need little trampolines to do > 24-bit jumps (into /* Like PPC32, we need little trampolines to do > 24-bit jumps (into
the kernel itself). But on PPC64, these need to be used for every the kernel itself). But on PPC64, these need to be used for every
jump, actually, to reset r2 (TOC+0x8000). */ jump, actually, to reset r2 (TOC+0x8000). */
struct ppc64_stub_entry struct ppc64_stub_entry {
{ /*
/* 28 byte jump instruction sequence (7 instructions). We only * 28 byte jump instruction sequence (7 instructions) that can
* need 6 instructions on ABIv2 but we always allocate 7 so * hold ppc64_stub_insns or stub_insns. Must be 8-byte aligned
* so we don't have to modify the trampoline load instruction. */ * with PCREL kernels that use prefix instructions in the stub.
*/
u32 jump[7]; u32 jump[7];
/* Used by ftrace to identify stubs */ /* Used by ftrace to identify stubs */
u32 magic; u32 magic;
/* Data for the above code */ /* Data for the above code */
func_desc_t funcdata; func_desc_t funcdata;
}; } __aligned(8);
/* /*
* PPC64 uses 24 bit jumps, but we need to jump into other modules or * PPC64 uses 24 bit jumps, but we need to jump into other modules or
...@@ -333,11 +334,21 @@ int module_frob_arch_sections(Elf64_Ehdr *hdr, ...@@ -333,11 +334,21 @@ int module_frob_arch_sections(Elf64_Ehdr *hdr,
#ifdef CONFIG_MPROFILE_KERNEL #ifdef CONFIG_MPROFILE_KERNEL
static u32 stub_insns[] = { static u32 stub_insns[] = {
#ifdef CONFIG_PPC_KERNEL_PCREL
PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernelbase)),
PPC_RAW_NOP(), /* align the prefix insn */
/* paddi r12,r12,addr */
PPC_PREFIX_MLS | __PPC_PRFX_R(0),
PPC_INST_PADDI | ___PPC_RT(_R12) | ___PPC_RA(_R12),
PPC_RAW_MTCTR(_R12),
PPC_RAW_BCTR(),
#else
PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)), PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)),
PPC_RAW_ADDIS(_R12, _R12, 0), PPC_RAW_ADDIS(_R12, _R12, 0),
PPC_RAW_ADDI(_R12, _R12, 0), PPC_RAW_ADDI(_R12, _R12, 0),
PPC_RAW_MTCTR(_R12), PPC_RAW_MTCTR(_R12),
PPC_RAW_BCTR(), PPC_RAW_BCTR(),
#endif
}; };
/* /*
...@@ -358,8 +369,26 @@ static inline int create_ftrace_stub(struct ppc64_stub_entry *entry, ...@@ -358,8 +369,26 @@ static inline int create_ftrace_stub(struct ppc64_stub_entry *entry,
{ {
long reladdr; long reladdr;
if ((unsigned long)entry->jump % 8 != 0) {
pr_err("%s: Address of stub entry is not 8-byte aligned\n", me->name);
return 0;
}
BUILD_BUG_ON(sizeof(stub_insns) > sizeof(entry->jump));
memcpy(entry->jump, stub_insns, sizeof(stub_insns)); memcpy(entry->jump, stub_insns, sizeof(stub_insns));
if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
/* Stub uses address relative to kernel base (from the paca) */
reladdr = addr - local_paca->kernelbase;
if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) {
pr_err("%s: Address of %ps out of range of 34-bit relative address.\n",
me->name, (void *)addr);
return 0;
}
entry->jump[2] |= IMM_H18(reladdr);
entry->jump[3] |= IMM_L(reladdr);
} else {
/* Stub uses address relative to kernel toc (from the paca) */ /* Stub uses address relative to kernel toc (from the paca) */
reladdr = addr - kernel_toc_addr(); reladdr = addr - kernel_toc_addr();
if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
...@@ -370,6 +399,7 @@ static inline int create_ftrace_stub(struct ppc64_stub_entry *entry, ...@@ -370,6 +399,7 @@ static inline int create_ftrace_stub(struct ppc64_stub_entry *entry,
entry->jump[1] |= PPC_HA(reladdr); entry->jump[1] |= PPC_HA(reladdr);
entry->jump[2] |= PPC_LO(reladdr); entry->jump[2] |= PPC_LO(reladdr);
}
/* Even though we don't use funcdata in the stub, it's needed elsewhere. */ /* Even though we don't use funcdata in the stub, it's needed elsewhere. */
entry->funcdata = func_desc(addr); entry->funcdata = func_desc(addr);
......
...@@ -191,7 +191,9 @@ void __init initialise_paca(struct paca_struct *new_paca, int cpu) ...@@ -191,7 +191,9 @@ void __init initialise_paca(struct paca_struct *new_paca, int cpu)
#endif #endif
new_paca->lock_token = 0x8000; new_paca->lock_token = 0x8000;
new_paca->paca_index = cpu; new_paca->paca_index = cpu;
#ifndef CONFIG_PPC_KERNEL_PCREL
new_paca->kernel_toc = kernel_toc_addr(); new_paca->kernel_toc = kernel_toc_addr();
#endif
new_paca->kernelbase = (unsigned long) _stext; new_paca->kernelbase = (unsigned long) _stext;
/* Only set MSR:IR/DR when MMU is initialized */ /* Only set MSR:IR/DR when MMU is initialized */
new_paca->kernel_msr = MSR_KERNEL & ~(MSR_IR | MSR_DR); new_paca->kernel_msr = MSR_KERNEL & ~(MSR_IR | MSR_DR);
......
...@@ -727,6 +727,15 @@ int __init ftrace_dyn_arch_init(void) ...@@ -727,6 +727,15 @@ int __init ftrace_dyn_arch_init(void)
{ {
int i; int i;
unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init }; unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init };
#ifdef CONFIG_PPC_KERNEL_PCREL
u32 stub_insns[] = {
/* pla r12,addr */
PPC_PREFIX_MLS | __PPC_PRFX_R(1),
PPC_INST_PADDI | ___PPC_RT(_R12),
PPC_RAW_MTCTR(_R12),
PPC_RAW_BCTR()
};
#else
u32 stub_insns[] = { u32 stub_insns[] = {
PPC_RAW_LD(_R12, _R13, PACATOC), PPC_RAW_LD(_R12, _R13, PACATOC),
PPC_RAW_ADDIS(_R12, _R12, 0), PPC_RAW_ADDIS(_R12, _R12, 0),
...@@ -734,6 +743,8 @@ int __init ftrace_dyn_arch_init(void) ...@@ -734,6 +743,8 @@ int __init ftrace_dyn_arch_init(void)
PPC_RAW_MTCTR(_R12), PPC_RAW_MTCTR(_R12),
PPC_RAW_BCTR() PPC_RAW_BCTR()
}; };
#endif
unsigned long addr; unsigned long addr;
long reladdr; long reladdr;
...@@ -742,9 +753,25 @@ int __init ftrace_dyn_arch_init(void) ...@@ -742,9 +753,25 @@ int __init ftrace_dyn_arch_init(void)
else else
addr = ppc_global_function_entry((void *)ftrace_caller); addr = ppc_global_function_entry((void *)ftrace_caller);
if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
for (i = 0; i < 2; i++) {
reladdr = addr - (unsigned long)tramp[i];
if (reladdr >= (long)SZ_8G || reladdr < -(long)SZ_8G) {
pr_err("Address of %ps out of range of pcrel address.\n",
(void *)addr);
return -1;
}
memcpy(tramp[i], stub_insns, sizeof(stub_insns));
tramp[i][0] |= IMM_H18(reladdr);
tramp[i][1] |= IMM_L(reladdr);
add_ftrace_tramp((unsigned long)tramp[i]);
}
} else {
reladdr = addr - kernel_toc_addr(); reladdr = addr - kernel_toc_addr();
if (reladdr >= SZ_2G || reladdr < -(long)SZ_2G) { if (reladdr >= (long)SZ_2G || reladdr < -(long)SZ_2G) {
pr_err("Address of %ps out of range of kernel_toc.\n", pr_err("Address of %ps out of range of kernel_toc.\n",
(void *)addr); (void *)addr);
return -1; return -1;
...@@ -756,6 +783,7 @@ int __init ftrace_dyn_arch_init(void) ...@@ -756,6 +783,7 @@ int __init ftrace_dyn_arch_init(void)
tramp[i][2] |= PPC_LO(reladdr); tramp[i][2] |= PPC_LO(reladdr);
add_ftrace_tramp((unsigned long)tramp[i]); add_ftrace_tramp((unsigned long)tramp[i]);
} }
}
return 0; return 0;
} }
......
...@@ -177,9 +177,15 @@ fpone: ...@@ -177,9 +177,15 @@ fpone:
fphalf: fphalf:
.quad 0x3fe0000000000000 /* 0.5 */ .quad 0x3fe0000000000000 /* 0.5 */
#ifdef CONFIG_PPC_KERNEL_PCREL
#define LDCONST(fr, name) \
pla r11,name@pcrel; \
lfd fr,0(r11)
#else
#define LDCONST(fr, name) \ #define LDCONST(fr, name) \
addis r11,r2,name@toc@ha; \ addis r11,r2,name@toc@ha; \
lfd fr,name@toc@l(r11) lfd fr,name@toc@l(r11)
#endif
#endif #endif
.text .text
/* /*
......
...@@ -169,12 +169,18 @@ SECTIONS ...@@ -169,12 +169,18 @@ SECTIONS
} }
#else /* CONFIG_PPC32 */ #else /* CONFIG_PPC32 */
#ifndef CONFIG_PPC_KERNEL_PCREL
.toc1 : AT(ADDR(.toc1) - LOAD_OFFSET) { .toc1 : AT(ADDR(.toc1) - LOAD_OFFSET) {
*(.toc1) *(.toc1)
} }
#endif
.got : AT(ADDR(.got) - LOAD_OFFSET) ALIGN(256) { .got : AT(ADDR(.got) - LOAD_OFFSET) ALIGN(256) {
#ifdef CONFIG_PPC_KERNEL_PCREL
*(.got)
#else
*(.got .toc) *(.got .toc)
#endif
} }
SOFT_MASK_TABLE(8) SOFT_MASK_TABLE(8)
......
...@@ -18,8 +18,18 @@ FTR_SECTION_ELSE ...@@ -18,8 +18,18 @@ FTR_SECTION_ELSE
#endif #endif
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY) ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
ori r5,r5,PAGE_SIZE@l ori r5,r5,PAGE_SIZE@l
#ifdef CONFIG_PPC_KERNEL_PCREL
/*
* Hack for toolchain - prefixed instructions cause label difference to
* be non-constant even if 8 byte alignment is known, so they can not
* be put in FTR sections.
*/
LOAD_REG_ADDR(r10, ppc64_caches)
BEGIN_FTR_SECTION
#else
BEGIN_FTR_SECTION BEGIN_FTR_SECTION
LOAD_REG_ADDR(r10, ppc64_caches) LOAD_REG_ADDR(r10, ppc64_caches)
#endif
lwz r11,DCACHEL1LOGBLOCKSIZE(r10) /* log2 of cache block size */ lwz r11,DCACHEL1LOGBLOCKSIZE(r10) /* log2 of cache block size */
lwz r12,DCACHEL1BLOCKSIZE(r10) /* get cache block size */ lwz r12,DCACHEL1BLOCKSIZE(r10) /* get cache block size */
li r9,0 li r9,0
......
...@@ -19,6 +19,8 @@ ...@@ -19,6 +19,8 @@
#define FUNCTION_DESCR_SIZE 0 #define FUNCTION_DESCR_SIZE 0
#endif #endif
#define CTX_NIA(ctx) ((unsigned long)ctx->idx * 4)
#define PLANT_INSTR(d, idx, instr) \ #define PLANT_INSTR(d, idx, instr) \
do { if (d) { (d)[idx] = instr; } idx++; } while (0) do { if (d) { (d)[idx] = instr; } idx++; } while (0)
#define EMIT(instr) PLANT_INSTR(image, ctx->idx, instr) #define EMIT(instr) PLANT_INSTR(image, ctx->idx, instr)
...@@ -26,7 +28,7 @@ ...@@ -26,7 +28,7 @@
/* Long jump; (unconditional 'branch') */ /* Long jump; (unconditional 'branch') */
#define PPC_JMP(dest) \ #define PPC_JMP(dest) \
do { \ do { \
long offset = (long)(dest) - (ctx->idx * 4); \ long offset = (long)(dest) - CTX_NIA(ctx); \
if ((dest) != 0 && !is_offset_in_branch_range(offset)) { \ if ((dest) != 0 && !is_offset_in_branch_range(offset)) { \
pr_err_ratelimited("Branch offset 0x%lx (@%u) out of range\n", offset, ctx->idx); \ pr_err_ratelimited("Branch offset 0x%lx (@%u) out of range\n", offset, ctx->idx); \
return -ERANGE; \ return -ERANGE; \
...@@ -40,7 +42,7 @@ ...@@ -40,7 +42,7 @@
/* "cond" here covers BO:BI fields. */ /* "cond" here covers BO:BI fields. */
#define PPC_BCC_SHORT(cond, dest) \ #define PPC_BCC_SHORT(cond, dest) \
do { \ do { \
long offset = (long)(dest) - (ctx->idx * 4); \ long offset = (long)(dest) - CTX_NIA(ctx); \
if ((dest) != 0 && !is_offset_in_cond_branch_range(offset)) { \ if ((dest) != 0 && !is_offset_in_cond_branch_range(offset)) { \
pr_err_ratelimited("Conditional branch offset 0x%lx (@%u) out of range\n", offset, ctx->idx); \ pr_err_ratelimited("Conditional branch offset 0x%lx (@%u) out of range\n", offset, ctx->idx); \
return -ERANGE; \ return -ERANGE; \
...@@ -92,12 +94,12 @@ ...@@ -92,12 +94,12 @@
* state. * state.
*/ */
#define PPC_BCC(cond, dest) do { \ #define PPC_BCC(cond, dest) do { \
if (is_offset_in_cond_branch_range((long)(dest) - (ctx->idx * 4))) { \ if (is_offset_in_cond_branch_range((long)(dest) - CTX_NIA(ctx))) { \
PPC_BCC_SHORT(cond, dest); \ PPC_BCC_SHORT(cond, dest); \
EMIT(PPC_RAW_NOP()); \ EMIT(PPC_RAW_NOP()); \
} else { \ } else { \
/* Flip the 'T or F' bit to invert comparison */ \ /* Flip the 'T or F' bit to invert comparison */ \
PPC_BCC_SHORT(cond ^ COND_CMP_TRUE, (ctx->idx+2)*4); \ PPC_BCC_SHORT(cond ^ COND_CMP_TRUE, CTX_NIA(ctx) + 2*4); \
PPC_JMP(dest); \ PPC_JMP(dest); \
} } while(0) } } while(0)
......
...@@ -126,8 +126,10 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) ...@@ -126,8 +126,10 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
{ {
int i; int i;
#ifndef CONFIG_PPC_KERNEL_PCREL
if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2)) if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2))
EMIT(PPC_RAW_LD(_R2, _R13, offsetof(struct paca_struct, kernel_toc))); EMIT(PPC_RAW_LD(_R2, _R13, offsetof(struct paca_struct, kernel_toc)));
#endif
/* /*
* Initialize tail_call_cnt if we do tail calls. * Initialize tail_call_cnt if we do tail calls.
...@@ -208,6 +210,21 @@ static int bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx, u ...@@ -208,6 +210,21 @@ static int bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx, u
if (WARN_ON_ONCE(!core_kernel_text(func_addr))) if (WARN_ON_ONCE(!core_kernel_text(func_addr)))
return -EINVAL; return -EINVAL;
if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
reladdr = func_addr - CTX_NIA(ctx);
if (reladdr >= (long)SZ_8G || reladdr < -(long)SZ_8G) {
pr_err("eBPF: address of %ps out of range of pcrel address.\n",
(void *)func);
return -ERANGE;
}
/* pla r12,addr */
EMIT(PPC_PREFIX_MLS | __PPC_PRFX_R(1) | IMM_H18(reladdr));
EMIT(PPC_INST_PADDI | ___PPC_RT(_R12) | IMM_L(reladdr));
EMIT(PPC_RAW_MTCTR(_R12));
EMIT(PPC_RAW_BCTR());
} else {
reladdr = func_addr - kernel_toc_addr(); reladdr = func_addr - kernel_toc_addr();
if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
pr_err("eBPF: address of %ps out of range of kernel_toc.\n", (void *)func); pr_err("eBPF: address of %ps out of range of kernel_toc.\n", (void *)func);
...@@ -218,6 +235,7 @@ static int bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx, u ...@@ -218,6 +235,7 @@ static int bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx, u
EMIT(PPC_RAW_ADDI(_R12, _R12, PPC_LO(reladdr))); EMIT(PPC_RAW_ADDI(_R12, _R12, PPC_LO(reladdr)));
EMIT(PPC_RAW_MTCTR(_R12)); EMIT(PPC_RAW_MTCTR(_R12));
EMIT(PPC_RAW_BCTRL()); EMIT(PPC_RAW_BCTRL());
}
return 0; return 0;
} }
......
...@@ -181,6 +181,7 @@ config POWER10_CPU ...@@ -181,6 +181,7 @@ config POWER10_CPU
depends on PPC_BOOK3S_64 depends on PPC_BOOK3S_64
select ARCH_HAS_FAST_MULTIPLIER select ARCH_HAS_FAST_MULTIPLIER
select PPC_HAVE_PREFIXED_SUPPORT select PPC_HAVE_PREFIXED_SUPPORT
select PPC_HAVE_PCREL_SUPPORT
config E5500_CPU config E5500_CPU
bool "Freescale e5500" bool "Freescale e5500"
...@@ -471,6 +472,20 @@ config PPC_KERNEL_PREFIXED ...@@ -471,6 +472,20 @@ config PPC_KERNEL_PREFIXED
Kernel support for prefixed instructions in applications and guests Kernel support for prefixed instructions in applications and guests
is not affected by this option. is not affected by this option.
config PPC_KERNEL_PCREL
depends on PPC_HAVE_PCREL_SUPPORT
depends on PPC_HAVE_PREFIXED_SUPPORT
depends on CC_HAS_PCREL
default n
select PPC_KERNEL_PREFIXED
bool "Build Kernel with PC-Relative addressing model"
help
POWER10 and later CPUs support pc relative addressing. Recent
compilers have support for an ELF ABI extension for a pc relative
ABI.
This option builds the kernel with the pc relative ABI model.
config PPC_KUEP config PPC_KUEP
bool "Kernel Userspace Execution Prevention" if !40x bool "Kernel Userspace Execution Prevention" if !40x
default y if !40x default y if !40x
...@@ -510,6 +525,9 @@ config PPC_HAVE_PMU_SUPPORT ...@@ -510,6 +525,9 @@ config PPC_HAVE_PMU_SUPPORT
config PPC_HAVE_PREFIXED_SUPPORT config PPC_HAVE_PREFIXED_SUPPORT
bool bool
config PPC_HAVE_PCREL_SUPPORT
bool
config PMU_SYSFS config PMU_SYSFS
bool "Create PMU SPRs sysfs file" bool "Create PMU SPRs sysfs file"
default n default n
......
...@@ -2634,7 +2634,9 @@ static void dump_one_paca(int cpu) ...@@ -2634,7 +2634,9 @@ static void dump_one_paca(int cpu)
DUMP(p, lock_token, "%#-*x"); DUMP(p, lock_token, "%#-*x");
DUMP(p, paca_index, "%#-*x"); DUMP(p, paca_index, "%#-*x");
#ifndef CONFIG_PPC_KERNEL_PCREL
DUMP(p, kernel_toc, "%#-*llx"); DUMP(p, kernel_toc, "%#-*llx");
#endif
DUMP(p, kernelbase, "%#-*llx"); DUMP(p, kernelbase, "%#-*llx");
DUMP(p, kernel_msr, "%#-*llx"); DUMP(p, kernel_msr, "%#-*llx");
DUMP(p, emergency_sp, "%-*px"); DUMP(p, emergency_sp, "%-*px");
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment