Commit 9732cafd authored by Jiang Liu's avatar Jiang Liu Committed by Catalin Marinas

arm64, jump label: optimize jump label implementation

Optimize jump label implementation for ARM64 by dynamically patching
kernel text.
Reviewed-by: default avatarWill Deacon <will.deacon@arm.com>
Signed-off-by: default avatarJiang Liu <liuj97@gmail.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent f3c003f7
...@@ -25,6 +25,7 @@ config ARM64 ...@@ -25,6 +25,7 @@ config ARM64
select GENERIC_STRNLEN_USER select GENERIC_STRNLEN_USER
select GENERIC_TIME_VSYSCALL select GENERIC_TIME_VSYSCALL
select HARDIRQS_SW_RESEND select HARDIRQS_SW_RESEND
select HAVE_ARCH_JUMP_LABEL
select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRACEHOOK
select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_BUGVERBOSE
select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_KMEMLEAK
......
/*
* Copyright (C) 2013 Huawei Ltd.
* Author: Jiang Liu <liuj97@gmail.com>
*
* Based on arch/arm/include/asm/jump_label.h
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_JUMP_LABEL_H
#define __ASM_JUMP_LABEL_H
#include <linux/types.h>
#include <asm/insn.h>
#ifdef __KERNEL__
#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1: nop\n\t"
".pushsection __jump_table, \"aw\"\n\t"
".align 3\n\t"
".quad 1b, %l[l_yes], %c0\n\t"
".popsection\n\t"
: : "i"(key) : : l_yes);
return false;
l_yes:
return true;
}
#endif /* __KERNEL__ */
typedef u64 jump_label_t;
struct jump_entry {
jump_label_t code;
jump_label_t target;
jump_label_t key;
};
#endif /* __ASM_JUMP_LABEL_H */
...@@ -19,6 +19,7 @@ arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o ...@@ -19,6 +19,7 @@ arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o
arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o
obj-y += $(arm64-obj-y) vdso/ obj-y += $(arm64-obj-y) vdso/
obj-m += $(arm64-obj-m) obj-m += $(arm64-obj-m)
......
/*
* Copyright (C) 2013 Huawei Ltd.
* Author: Jiang Liu <liuj97@gmail.com>
*
* Based on arch/arm/kernel/jump_label.c
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/kernel.h>
#include <linux/jump_label.h>
#include <asm/insn.h>
#ifdef HAVE_JUMP_LABEL
static void __arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type,
bool is_static)
{
void *addr = (void *)entry->code;
u32 insn;
if (type == JUMP_LABEL_ENABLE) {
insn = aarch64_insn_gen_branch_imm(entry->code,
entry->target,
AARCH64_INSN_BRANCH_NOLINK);
} else {
insn = aarch64_insn_gen_nop();
}
if (is_static)
aarch64_insn_patch_text_nosync(addr, insn);
else
aarch64_insn_patch_text(&addr, &insn, 1);
}
void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
{
__arch_jump_label_transform(entry, type, false);
}
void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type)
{
__arch_jump_label_transform(entry, type, true);
}
#endif /* HAVE_JUMP_LABEL */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment