Commit 33352244 authored by Josh Poimboeuf's avatar Josh Poimboeuf Committed by Ingo Molnar

jump_label: Explicitly disable jump labels in __init code

After initmem has been freed, any jump labels in __init code are
prevented from being written to by the kernel_text_address() check in
__jump_label_update().  However, this check is quite broad.  If
kernel_text_address() were to return false for any other reason, the
jump label write would fail silently with no warning.

For jump labels in module init code, entry->code is set to zero to
indicate that the entry is disabled.  Do the same thing for core kernel
init code.  This makes the behavior more consistent, and will also make
it more straightforward to detect non-init jump label write failures in
the next patch.
Signed-off-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Acked-by: default avatarPeter Zijlstra <peterz@infradead.org>
Cc: Borislav Petkov <bp@suse.de>
Cc: Jason Baron <jbaron@akamai.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/c52825c73f3a174e8398b6898284ec20d4deb126.1519051220.git.jpoimboe@redhat.comSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent f3d415ea
...@@ -151,6 +151,7 @@ extern struct jump_entry __start___jump_table[]; ...@@ -151,6 +151,7 @@ extern struct jump_entry __start___jump_table[];
extern struct jump_entry __stop___jump_table[]; extern struct jump_entry __stop___jump_table[];
extern void jump_label_init(void); extern void jump_label_init(void);
extern void jump_label_invalidate_init(void);
extern void jump_label_lock(void); extern void jump_label_lock(void);
extern void jump_label_unlock(void); extern void jump_label_unlock(void);
extern void arch_jump_label_transform(struct jump_entry *entry, extern void arch_jump_label_transform(struct jump_entry *entry,
...@@ -198,6 +199,8 @@ static __always_inline void jump_label_init(void) ...@@ -198,6 +199,8 @@ static __always_inline void jump_label_init(void)
static_key_initialized = true; static_key_initialized = true;
} }
static inline void jump_label_invalidate_init(void) {}
static __always_inline bool static_key_false(struct static_key *key) static __always_inline bool static_key_false(struct static_key *key)
{ {
if (unlikely(static_key_count(key) > 0)) if (unlikely(static_key_count(key) > 0))
......
...@@ -89,6 +89,7 @@ ...@@ -89,6 +89,7 @@
#include <linux/io.h> #include <linux/io.h>
#include <linux/cache.h> #include <linux/cache.h>
#include <linux/rodata_test.h> #include <linux/rodata_test.h>
#include <linux/jump_label.h>
#include <asm/io.h> #include <asm/io.h>
#include <asm/bugs.h> #include <asm/bugs.h>
...@@ -1000,6 +1001,7 @@ static int __ref kernel_init(void *unused) ...@@ -1000,6 +1001,7 @@ static int __ref kernel_init(void *unused)
/* need to finish all async __init code before freeing the memory */ /* need to finish all async __init code before freeing the memory */
async_synchronize_full(); async_synchronize_full();
ftrace_free_init_mem(); ftrace_free_init_mem();
jump_label_invalidate_init();
free_initmem(); free_initmem();
mark_readonly(); mark_readonly();
system_state = SYSTEM_RUNNING; system_state = SYSTEM_RUNNING;
......
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
#include <linux/jump_label_ratelimit.h> #include <linux/jump_label_ratelimit.h>
#include <linux/bug.h> #include <linux/bug.h>
#include <linux/cpu.h> #include <linux/cpu.h>
#include <asm/sections.h>
#ifdef HAVE_JUMP_LABEL #ifdef HAVE_JUMP_LABEL
...@@ -417,6 +418,20 @@ void __init jump_label_init(void) ...@@ -417,6 +418,20 @@ void __init jump_label_init(void)
cpus_read_unlock(); cpus_read_unlock();
} }
/* Disable any jump label entries in __init code */
void __init jump_label_invalidate_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __stop___jump_table;
struct jump_entry *iter;
for (iter = iter_start; iter < iter_stop; iter++) {
if (iter->code >= (unsigned long)_sinittext &&
iter->code < (unsigned long)_einittext)
iter->code = 0;
}
}
#ifdef CONFIG_MODULES #ifdef CONFIG_MODULES
static enum jump_label_type jump_label_init_type(struct jump_entry *entry) static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
...@@ -633,6 +648,7 @@ static void jump_label_del_module(struct module *mod) ...@@ -633,6 +648,7 @@ static void jump_label_del_module(struct module *mod)
} }
} }
/* Disable any jump label entries in module init code */
static void jump_label_invalidate_module_init(struct module *mod) static void jump_label_invalidate_module_init(struct module *mod)
{ {
struct jump_entry *iter_start = mod->jump_entries; struct jump_entry *iter_start = mod->jump_entries;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment