Commit dea5e2a4 authored by Marc Zyngier's avatar Marc Zyngier

arm64: alternatives: Add dynamic patching feature

We've so far relied on a patching infrastructure that only gave us
a single alternative, without any way to provide a range of potential
replacement instructions. For a single feature, this is an all or
nothing thing.

It would be interesting to have a more flexible grained way of patching
the kernel though, where we could dynamically tune the code that gets
injected.

In order to achive this, let's introduce a new form of dynamic patching,
assiciating a callback to a patching site. This callback gets source and
target locations of the patching request, as well as the number of
instructions to be patched.

Dynamic patching is declared with the new ALTERNATIVE_CB and alternative_cb
directives:

	asm volatile(ALTERNATIVE_CB("mov %0, #0\n", callback)
		     : "r" (v));
or
	alternative_cb callback
		mov	x0, #0
	alternative_cb_end

where callback is the C function computing the alternative.
Reviewed-by: default avatarChristoffer Dall <christoffer.dall@linaro.org>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Signed-off-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
parent 2d0e63e0
...@@ -5,6 +5,8 @@ ...@@ -5,6 +5,8 @@
#include <asm/cpucaps.h> #include <asm/cpucaps.h>
#include <asm/insn.h> #include <asm/insn.h>
#define ARM64_CB_PATCH ARM64_NCAPS
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/init.h> #include <linux/init.h>
...@@ -22,12 +24,19 @@ struct alt_instr { ...@@ -22,12 +24,19 @@ struct alt_instr {
u8 alt_len; /* size of new instruction(s), <= orig_len */ u8 alt_len; /* size of new instruction(s), <= orig_len */
}; };
typedef void (*alternative_cb_t)(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr, int nr_inst);
void __init apply_alternatives_all(void); void __init apply_alternatives_all(void);
void apply_alternatives(void *start, size_t length); void apply_alternatives(void *start, size_t length);
#define ALTINSTR_ENTRY(feature) \ #define ALTINSTR_ENTRY(feature,cb) \
" .word 661b - .\n" /* label */ \ " .word 661b - .\n" /* label */ \
" .if " __stringify(cb) " == 0\n" \
" .word 663f - .\n" /* new instruction */ \ " .word 663f - .\n" /* new instruction */ \
" .else\n" \
" .word " __stringify(cb) "- .\n" /* callback */ \
" .endif\n" \
" .hword " __stringify(feature) "\n" /* feature bit */ \ " .hword " __stringify(feature) "\n" /* feature bit */ \
" .byte 662b-661b\n" /* source len */ \ " .byte 662b-661b\n" /* source len */ \
" .byte 664f-663f\n" /* replacement len */ " .byte 664f-663f\n" /* replacement len */
...@@ -45,15 +54,18 @@ void apply_alternatives(void *start, size_t length); ...@@ -45,15 +54,18 @@ void apply_alternatives(void *start, size_t length);
* but most assemblers die if insn1 or insn2 have a .inst. This should * but most assemblers die if insn1 or insn2 have a .inst. This should
* be fixed in a binutils release posterior to 2.25.51.0.2 (anything * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
* containing commit 4e4d08cf7399b606 or c1baaddf8861). * containing commit 4e4d08cf7399b606 or c1baaddf8861).
*
* Alternatives with callbacks do not generate replacement instructions.
*/ */
#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \ #define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled, cb) \
".if "__stringify(cfg_enabled)" == 1\n" \ ".if "__stringify(cfg_enabled)" == 1\n" \
"661:\n\t" \ "661:\n\t" \
oldinstr "\n" \ oldinstr "\n" \
"662:\n" \ "662:\n" \
".pushsection .altinstructions,\"a\"\n" \ ".pushsection .altinstructions,\"a\"\n" \
ALTINSTR_ENTRY(feature) \ ALTINSTR_ENTRY(feature,cb) \
".popsection\n" \ ".popsection\n" \
" .if " __stringify(cb) " == 0\n" \
".pushsection .altinstr_replacement, \"a\"\n" \ ".pushsection .altinstr_replacement, \"a\"\n" \
"663:\n\t" \ "663:\n\t" \
newinstr "\n" \ newinstr "\n" \
...@@ -61,11 +73,17 @@ void apply_alternatives(void *start, size_t length); ...@@ -61,11 +73,17 @@ void apply_alternatives(void *start, size_t length);
".popsection\n\t" \ ".popsection\n\t" \
".org . - (664b-663b) + (662b-661b)\n\t" \ ".org . - (664b-663b) + (662b-661b)\n\t" \
".org . - (662b-661b) + (664b-663b)\n" \ ".org . - (662b-661b) + (664b-663b)\n" \
".else\n\t" \
"663:\n\t" \
"664:\n\t" \
".endif\n" \
".endif\n" ".endif\n"
#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \ #define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
__ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg)) __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg), 0)
#define ALTERNATIVE_CB(oldinstr, cb) \
__ALTERNATIVE_CFG(oldinstr, "NOT_AN_INSTRUCTION", ARM64_CB_PATCH, 1, cb)
#else #else
#include <asm/assembler.h> #include <asm/assembler.h>
...@@ -132,6 +150,14 @@ void apply_alternatives(void *start, size_t length); ...@@ -132,6 +150,14 @@ void apply_alternatives(void *start, size_t length);
661: 661:
.endm .endm
.macro alternative_cb cb
.set .Lasm_alt_mode, 0
.pushsection .altinstructions, "a"
altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
.popsection
661:
.endm
/* /*
* Provide the other half of the alternative code sequence. * Provide the other half of the alternative code sequence.
*/ */
...@@ -157,6 +183,13 @@ void apply_alternatives(void *start, size_t length); ...@@ -157,6 +183,13 @@ void apply_alternatives(void *start, size_t length);
.org . - (662b-661b) + (664b-663b) .org . - (662b-661b) + (664b-663b)
.endm .endm
/*
* Callback-based alternative epilogue
*/
.macro alternative_cb_end
662:
.endm
/* /*
* Provides a trivial alternative or default sequence consisting solely * Provides a trivial alternative or default sequence consisting solely
* of NOPs. The number of NOPs is chosen automatically to match the * of NOPs. The number of NOPs is chosen automatically to match the
......
...@@ -107,32 +107,53 @@ static u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnp ...@@ -107,32 +107,53 @@ static u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnp
return insn; return insn;
} }
static void patch_alternative(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr, int nr_inst)
{
__le32 *replptr;
int i;
replptr = ALT_REPL_PTR(alt);
for (i = 0; i < nr_inst; i++) {
u32 insn;
insn = get_alt_insn(alt, origptr + i, replptr + i);
updptr[i] = cpu_to_le32(insn);
}
}
static void __apply_alternatives(void *alt_region, bool use_linear_alias) static void __apply_alternatives(void *alt_region, bool use_linear_alias)
{ {
struct alt_instr *alt; struct alt_instr *alt;
struct alt_region *region = alt_region; struct alt_region *region = alt_region;
__le32 *origptr, *replptr, *updptr; __le32 *origptr, *updptr;
alternative_cb_t alt_cb;
for (alt = region->begin; alt < region->end; alt++) { for (alt = region->begin; alt < region->end; alt++) {
u32 insn; int nr_inst;
int i, nr_inst;
if (!cpus_have_cap(alt->cpufeature)) /* Use ARM64_CB_PATCH as an unconditional patch */
if (alt->cpufeature < ARM64_CB_PATCH &&
!cpus_have_cap(alt->cpufeature))
continue; continue;
if (alt->cpufeature == ARM64_CB_PATCH)
BUG_ON(alt->alt_len != 0);
else
BUG_ON(alt->alt_len != alt->orig_len); BUG_ON(alt->alt_len != alt->orig_len);
pr_info_once("patching kernel code\n"); pr_info_once("patching kernel code\n");
origptr = ALT_ORIG_PTR(alt); origptr = ALT_ORIG_PTR(alt);
replptr = ALT_REPL_PTR(alt);
updptr = use_linear_alias ? lm_alias(origptr) : origptr; updptr = use_linear_alias ? lm_alias(origptr) : origptr;
nr_inst = alt->alt_len / sizeof(insn); nr_inst = alt->orig_len / AARCH64_INSN_SIZE;
for (i = 0; i < nr_inst; i++) { if (alt->cpufeature < ARM64_CB_PATCH)
insn = get_alt_insn(alt, origptr + i, replptr + i); alt_cb = patch_alternative;
updptr[i] = cpu_to_le32(insn); else
} alt_cb = ALT_REPL_PTR(alt);
alt_cb(alt, origptr, updptr, nr_inst);
flush_icache_range((uintptr_t)origptr, flush_icache_range((uintptr_t)origptr,
(uintptr_t)(origptr + nr_inst)); (uintptr_t)(origptr + nr_inst));
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment