Commit 375074cc authored by Andy Lutomirski's avatar Andy Lutomirski Committed by Ingo Molnar

x86: Clean up cr4 manipulation

CR4 manipulation was split, seemingly at random, between direct
(write_cr4) and using a helper (set/clear_in_cr4).  Unfortunately,
the set_in_cr4 and clear_in_cr4 helpers also poke at the boot code,
which only a small subset of users actually wanted.

This patch replaces all cr4 access in functions that don't leave cr4
exactly the way they found it with new helpers cr4_set_bits,
cr4_clear_bits, and cr4_set_bits_and_update_boot.
Signed-off-by: default avatarAndy Lutomirski <luto@amacapital.net>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Vince Weaver <vince@deater.net>
Cc: "hillf.zj" <hillf.zj@alibaba-inc.com>
Cc: Valdis Kletnieks <Valdis.Kletnieks@vt.edu>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/495a10bdc9e67016b8fd3945700d46cfd5c12c2f.1414190806.git.luto@amacapital.netSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 0967160a
...@@ -579,39 +579,6 @@ static inline void load_sp0(struct tss_struct *tss, ...@@ -579,39 +579,6 @@ static inline void load_sp0(struct tss_struct *tss,
#define set_iopl_mask native_set_iopl_mask #define set_iopl_mask native_set_iopl_mask
#endif /* CONFIG_PARAVIRT */ #endif /* CONFIG_PARAVIRT */
/*
* Save the cr4 feature set we're using (ie
* Pentium 4MB enable and PPro Global page
* enable), so that any CPU's that boot up
* after us can get the correct flags.
*/
extern unsigned long mmu_cr4_features;
extern u32 *trampoline_cr4_features;
static inline void set_in_cr4(unsigned long mask)
{
unsigned long cr4;
mmu_cr4_features |= mask;
if (trampoline_cr4_features)
*trampoline_cr4_features = mmu_cr4_features;
cr4 = read_cr4();
cr4 |= mask;
write_cr4(cr4);
}
static inline void clear_in_cr4(unsigned long mask)
{
unsigned long cr4;
mmu_cr4_features &= ~mask;
if (trampoline_cr4_features)
*trampoline_cr4_features = mmu_cr4_features;
cr4 = read_cr4();
cr4 &= ~mask;
write_cr4(cr4);
}
typedef struct { typedef struct {
unsigned long seg; unsigned long seg;
} mm_segment_t; } mm_segment_t;
......
...@@ -15,6 +15,43 @@ ...@@ -15,6 +15,43 @@
#define __flush_tlb_single(addr) __native_flush_tlb_single(addr) #define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
#endif #endif
/* Set in this cpu's CR4. */
static inline void cr4_set_bits(unsigned long mask)
{
unsigned long cr4;
cr4 = read_cr4();
cr4 |= mask;
write_cr4(cr4);
}
/* Clear in this cpu's CR4. */
static inline void cr4_clear_bits(unsigned long mask)
{
unsigned long cr4;
cr4 = read_cr4();
cr4 &= ~mask;
write_cr4(cr4);
}
/*
* Save some of cr4 feature set we're using (e.g. Pentium 4MB
* enable and PPro Global page enable), so that any CPU's that boot
* up after us can get the correct flags. This should only be used
* during boot on the boot cpu.
*/
extern unsigned long mmu_cr4_features;
extern u32 *trampoline_cr4_features;
static inline void cr4_set_bits_and_update_boot(unsigned long mask)
{
mmu_cr4_features |= mask;
if (trampoline_cr4_features)
*trampoline_cr4_features = mmu_cr4_features;
cr4_set_bits(mask);
}
static inline void __native_flush_tlb(void) static inline void __native_flush_tlb(void)
{ {
native_write_cr3(native_read_cr3()); native_write_cr3(native_read_cr3());
......
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
#include <asm/vmx.h> #include <asm/vmx.h>
#include <asm/svm.h> #include <asm/svm.h>
#include <asm/tlbflush.h>
/* /*
* VMX functions: * VMX functions:
...@@ -40,7 +41,7 @@ static inline int cpu_has_vmx(void) ...@@ -40,7 +41,7 @@ static inline int cpu_has_vmx(void)
static inline void cpu_vmxoff(void) static inline void cpu_vmxoff(void)
{ {
asm volatile (ASM_VMX_VMXOFF : : : "cc"); asm volatile (ASM_VMX_VMXOFF : : : "cc");
write_cr4(read_cr4() & ~X86_CR4_VMXE); cr4_clear_bits(X86_CR4_VMXE);
} }
static inline int cpu_vmx_enabled(void) static inline int cpu_vmx_enabled(void)
......
...@@ -278,7 +278,7 @@ __setup("nosmep", setup_disable_smep); ...@@ -278,7 +278,7 @@ __setup("nosmep", setup_disable_smep);
static __always_inline void setup_smep(struct cpuinfo_x86 *c) static __always_inline void setup_smep(struct cpuinfo_x86 *c)
{ {
if (cpu_has(c, X86_FEATURE_SMEP)) if (cpu_has(c, X86_FEATURE_SMEP))
set_in_cr4(X86_CR4_SMEP); cr4_set_bits(X86_CR4_SMEP);
} }
static __init int setup_disable_smap(char *arg) static __init int setup_disable_smap(char *arg)
...@@ -298,9 +298,9 @@ static __always_inline void setup_smap(struct cpuinfo_x86 *c) ...@@ -298,9 +298,9 @@ static __always_inline void setup_smap(struct cpuinfo_x86 *c)
if (cpu_has(c, X86_FEATURE_SMAP)) { if (cpu_has(c, X86_FEATURE_SMAP)) {
#ifdef CONFIG_X86_SMAP #ifdef CONFIG_X86_SMAP
set_in_cr4(X86_CR4_SMAP); cr4_set_bits(X86_CR4_SMAP);
#else #else
clear_in_cr4(X86_CR4_SMAP); cr4_clear_bits(X86_CR4_SMAP);
#endif #endif
} }
} }
...@@ -1312,7 +1312,7 @@ void cpu_init(void) ...@@ -1312,7 +1312,7 @@ void cpu_init(void)
pr_debug("Initializing CPU#%d\n", cpu); pr_debug("Initializing CPU#%d\n", cpu);
clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE); cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
/* /*
* Initialize the per-CPU GDT with the boot GDT, * Initialize the per-CPU GDT with the boot GDT,
...@@ -1393,7 +1393,7 @@ void cpu_init(void) ...@@ -1393,7 +1393,7 @@ void cpu_init(void)
printk(KERN_INFO "Initializing CPU#%d\n", cpu); printk(KERN_INFO "Initializing CPU#%d\n", cpu);
if (cpu_feature_enabled(X86_FEATURE_VME) || cpu_has_tsc || cpu_has_de) if (cpu_feature_enabled(X86_FEATURE_VME) || cpu_has_tsc || cpu_has_de)
clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE); cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
load_current_idt(); load_current_idt();
switch_to_new_gdt(cpu); switch_to_new_gdt(cpu);
......
...@@ -44,6 +44,7 @@ ...@@ -44,6 +44,7 @@
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/traps.h> #include <asm/traps.h>
#include <asm/tlbflush.h>
#include <asm/mce.h> #include <asm/mce.h>
#include <asm/msr.h> #include <asm/msr.h>
...@@ -1449,7 +1450,7 @@ static void __mcheck_cpu_init_generic(void) ...@@ -1449,7 +1450,7 @@ static void __mcheck_cpu_init_generic(void)
bitmap_fill(all_banks, MAX_NR_BANKS); bitmap_fill(all_banks, MAX_NR_BANKS);
machine_check_poll(MCP_UC | m_fl, &all_banks); machine_check_poll(MCP_UC | m_fl, &all_banks);
set_in_cr4(X86_CR4_MCE); cr4_set_bits(X86_CR4_MCE);
rdmsrl(MSR_IA32_MCG_CAP, cap); rdmsrl(MSR_IA32_MCG_CAP, cap);
if (cap & MCG_CTL_P) if (cap & MCG_CTL_P)
......
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/traps.h> #include <asm/traps.h>
#include <asm/tlbflush.h>
#include <asm/mce.h> #include <asm/mce.h>
#include <asm/msr.h> #include <asm/msr.h>
...@@ -65,7 +66,7 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c) ...@@ -65,7 +66,7 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c)
"Intel old style machine check architecture supported.\n"); "Intel old style machine check architecture supported.\n");
/* Enable MCE: */ /* Enable MCE: */
set_in_cr4(X86_CR4_MCE); cr4_set_bits(X86_CR4_MCE);
printk(KERN_INFO printk(KERN_INFO
"Intel old style machine check reporting enabled on CPU#%d.\n", "Intel old style machine check reporting enabled on CPU#%d.\n",
smp_processor_id()); smp_processor_id());
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/traps.h> #include <asm/traps.h>
#include <asm/tlbflush.h>
#include <asm/mce.h> #include <asm/mce.h>
#include <asm/msr.h> #include <asm/msr.h>
...@@ -36,7 +37,7 @@ void winchip_mcheck_init(struct cpuinfo_x86 *c) ...@@ -36,7 +37,7 @@ void winchip_mcheck_init(struct cpuinfo_x86 *c)
lo &= ~(1<<4); /* Enable MCE */ lo &= ~(1<<4); /* Enable MCE */
wrmsr(MSR_IDT_FCR1, lo, hi); wrmsr(MSR_IDT_FCR1, lo, hi);
set_in_cr4(X86_CR4_MCE); cr4_set_bits(X86_CR4_MCE);
printk(KERN_INFO printk(KERN_INFO
"Winchip machine check reporting enabled on CPU#0.\n"); "Winchip machine check reporting enabled on CPU#0.\n");
......
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include <asm/nmi.h> #include <asm/nmi.h>
#include <asm/smp.h> #include <asm/smp.h>
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/tlbflush.h>
#include <asm/timer.h> #include <asm/timer.h>
#include <asm/desc.h> #include <asm/desc.h>
#include <asm/ldt.h> #include <asm/ldt.h>
...@@ -1328,7 +1329,7 @@ x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) ...@@ -1328,7 +1329,7 @@ x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
case CPU_STARTING: case CPU_STARTING:
if (x86_pmu.attr_rdpmc) if (x86_pmu.attr_rdpmc)
set_in_cr4(X86_CR4_PCE); cr4_set_bits(X86_CR4_PCE);
if (x86_pmu.cpu_starting) if (x86_pmu.cpu_starting)
x86_pmu.cpu_starting(cpu); x86_pmu.cpu_starting(cpu);
break; break;
...@@ -1834,9 +1835,9 @@ static void change_rdpmc(void *info) ...@@ -1834,9 +1835,9 @@ static void change_rdpmc(void *info)
bool enable = !!(unsigned long)info; bool enable = !!(unsigned long)info;
if (enable) if (enable)
set_in_cr4(X86_CR4_PCE); cr4_set_bits(X86_CR4_PCE);
else else
clear_in_cr4(X86_CR4_PCE); cr4_clear_bits(X86_CR4_PCE);
} }
static ssize_t set_attr_rdpmc(struct device *cdev, static ssize_t set_attr_rdpmc(struct device *cdev,
......
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
#include <asm/sigcontext.h> #include <asm/sigcontext.h>
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/math_emu.h> #include <asm/math_emu.h>
#include <asm/tlbflush.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include <asm/ptrace.h> #include <asm/ptrace.h>
#include <asm/i387.h> #include <asm/i387.h>
...@@ -180,7 +181,7 @@ void fpu_init(void) ...@@ -180,7 +181,7 @@ void fpu_init(void)
if (cpu_has_xmm) if (cpu_has_xmm)
cr4_mask |= X86_CR4_OSXMMEXCPT; cr4_mask |= X86_CR4_OSXMMEXCPT;
if (cr4_mask) if (cr4_mask)
set_in_cr4(cr4_mask); cr4_set_bits(cr4_mask);
cr0 = read_cr0(); cr0 = read_cr0();
cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */ cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
......
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
#include <asm/fpu-internal.h> #include <asm/fpu-internal.h>
#include <asm/debugreg.h> #include <asm/debugreg.h>
#include <asm/nmi.h> #include <asm/nmi.h>
#include <asm/tlbflush.h>
/* /*
* per-CPU TSS segments. Threads are completely 'soft' on Linux, * per-CPU TSS segments. Threads are completely 'soft' on Linux,
...@@ -141,7 +142,7 @@ void flush_thread(void) ...@@ -141,7 +142,7 @@ void flush_thread(void)
static void hard_disable_TSC(void) static void hard_disable_TSC(void)
{ {
write_cr4(read_cr4() | X86_CR4_TSD); cr4_set_bits(X86_CR4_TSD);
} }
void disable_TSC(void) void disable_TSC(void)
...@@ -158,7 +159,7 @@ void disable_TSC(void) ...@@ -158,7 +159,7 @@ void disable_TSC(void)
static void hard_enable_TSC(void) static void hard_enable_TSC(void)
{ {
write_cr4(read_cr4() & ~X86_CR4_TSD); cr4_clear_bits(X86_CR4_TSD);
} }
static void enable_TSC(void) static void enable_TSC(void)
......
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
#include <asm/i387.h> #include <asm/i387.h>
#include <asm/fpu-internal.h> #include <asm/fpu-internal.h>
#include <asm/sigframe.h> #include <asm/sigframe.h>
#include <asm/tlbflush.h>
#include <asm/xcr.h> #include <asm/xcr.h>
/* /*
...@@ -453,7 +454,7 @@ static void prepare_fx_sw_frame(void) ...@@ -453,7 +454,7 @@ static void prepare_fx_sw_frame(void)
*/ */
static inline void xstate_enable(void) static inline void xstate_enable(void)
{ {
set_in_cr4(X86_CR4_OSXSAVE); cr4_set_bits(X86_CR4_OSXSAVE);
xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask); xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask);
} }
......
...@@ -2812,7 +2812,7 @@ static int hardware_enable(void) ...@@ -2812,7 +2812,7 @@ static int hardware_enable(void)
/* enable and lock */ /* enable and lock */
wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits); wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
} }
write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */ cr4_set_bits(X86_CR4_VMXE);
if (vmm_exclusive) { if (vmm_exclusive) {
kvm_cpu_vmxon(phys_addr); kvm_cpu_vmxon(phys_addr);
...@@ -2849,7 +2849,7 @@ static void hardware_disable(void) ...@@ -2849,7 +2849,7 @@ static void hardware_disable(void)
vmclear_local_loaded_vmcss(); vmclear_local_loaded_vmcss();
kvm_cpu_vmxoff(); kvm_cpu_vmxoff();
} }
write_cr4(read_cr4() & ~X86_CR4_VMXE); cr4_clear_bits(X86_CR4_VMXE);
} }
static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
......
...@@ -173,11 +173,11 @@ static void __init probe_page_size_mask(void) ...@@ -173,11 +173,11 @@ static void __init probe_page_size_mask(void)
/* Enable PSE if available */ /* Enable PSE if available */
if (cpu_has_pse) if (cpu_has_pse)
set_in_cr4(X86_CR4_PSE); cr4_set_bits_and_update_boot(X86_CR4_PSE);
/* Enable PGE if available */ /* Enable PGE if available */
if (cpu_has_pge) { if (cpu_has_pge) {
set_in_cr4(X86_CR4_PGE); cr4_set_bits_and_update_boot(X86_CR4_PGE);
__supported_pte_mask |= _PAGE_GLOBAL; __supported_pte_mask |= _PAGE_GLOBAL;
} }
} }
......
...@@ -1494,10 +1494,10 @@ static void xen_pvh_set_cr_flags(int cpu) ...@@ -1494,10 +1494,10 @@ static void xen_pvh_set_cr_flags(int cpu)
* set them here. For all, OSFXSR OSXMMEXCPT are set in fpu_init. * set them here. For all, OSFXSR OSXMMEXCPT are set in fpu_init.
*/ */
if (cpu_has_pse) if (cpu_has_pse)
set_in_cr4(X86_CR4_PSE); cr4_set_bits_and_update_boot(X86_CR4_PSE);
if (cpu_has_pge) if (cpu_has_pge)
set_in_cr4(X86_CR4_PGE); cr4_set_bits_and_update_boot(X86_CR4_PGE);
} }
/* /*
......
...@@ -47,6 +47,7 @@ ...@@ -47,6 +47,7 @@
#include <asm/lguest.h> #include <asm/lguest.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include <asm/i387.h> #include <asm/i387.h>
#include <asm/tlbflush.h>
#include "../lg.h" #include "../lg.h"
static int cpu_had_pge; static int cpu_had_pge;
...@@ -452,9 +453,9 @@ void lguest_arch_handle_trap(struct lg_cpu *cpu) ...@@ -452,9 +453,9 @@ void lguest_arch_handle_trap(struct lg_cpu *cpu)
static void adjust_pge(void *on) static void adjust_pge(void *on)
{ {
if (on) if (on)
write_cr4(read_cr4() | X86_CR4_PGE); cr4_set_bits(X86_CR4_PGE);
else else
write_cr4(read_cr4() & ~X86_CR4_PGE); cr4_clear_bits(X86_CR4_PGE);
} }
/*H:020 /*H:020
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment