Commit 32b49b3c authored by Ingo Molnar's avatar Ingo Molnar

x86/fpu: Factor out FPU hw activation/deactivation

We have repeat patterns of:

	if (!use_eager_fpu())
		clts();

... to activate FPU registers, and:

	if (!use_eager_fpu())
		stts();

... to deactivate them.

Encapsulate these in:

	__fpregs_activate_hw();
	__fpregs_activate_hw();

and use them accordingly.

Doing this synchronizes the idiom with the fpu->fpregs_active
software-flag's handling functions, creating clear patterns of:

	__fpregs_activate_hw();
	__fpregs_activate(fpu);

etc., which improves readability.
Reviewed-by: default avatarBorislav Petkov <bp@alien8.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 67ee658e
...@@ -324,14 +324,31 @@ static inline int restore_fpu_checking(struct fpu *fpu) ...@@ -324,14 +324,31 @@ static inline int restore_fpu_checking(struct fpu *fpu)
return fpu_restore_checking(fpu); return fpu_restore_checking(fpu);
} }
/* Must be paired with an 'stts' after! */ /*
* Wrap lazy FPU TS handling in a 'hw fpregs activation/deactivation'
* idiom, which is then paired with the sw-flag (fpregs_active) later on:
*/
static inline void __fpregs_activate_hw(void)
{
if (!use_eager_fpu())
clts();
}
static inline void __fpregs_deactivate_hw(void)
{
if (!use_eager_fpu())
stts();
}
/* Must be paired with an 'stts' (fpregs_deactivate_hw()) after! */
static inline void __fpregs_deactivate(struct fpu *fpu) static inline void __fpregs_deactivate(struct fpu *fpu)
{ {
fpu->fpregs_active = 0; fpu->fpregs_active = 0;
this_cpu_write(fpu_fpregs_owner_ctx, NULL); this_cpu_write(fpu_fpregs_owner_ctx, NULL);
} }
/* Must be paired with a 'clts' before! */ /* Must be paired with a 'clts' (fpregs_activate_hw()) before! */
static inline void __fpregs_activate(struct fpu *fpu) static inline void __fpregs_activate(struct fpu *fpu)
{ {
fpu->fpregs_active = 1; fpu->fpregs_active = 1;
...@@ -362,16 +379,14 @@ static inline int user_has_fpu(void) ...@@ -362,16 +379,14 @@ static inline int user_has_fpu(void)
*/ */
static inline void fpregs_activate(struct fpu *fpu) static inline void fpregs_activate(struct fpu *fpu)
{ {
if (!use_eager_fpu()) __fpregs_activate_hw();
clts();
__fpregs_activate(fpu); __fpregs_activate(fpu);
} }
static inline void fpregs_deactivate(struct fpu *fpu) static inline void fpregs_deactivate(struct fpu *fpu)
{ {
__fpregs_deactivate(fpu); __fpregs_deactivate(fpu);
if (!use_eager_fpu()) __fpregs_deactivate_hw();
stts();
} }
static inline void drop_fpu(struct fpu *fpu) static inline void drop_fpu(struct fpu *fpu)
...@@ -455,8 +470,9 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu) ...@@ -455,8 +470,9 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
new_fpu->counter++; new_fpu->counter++;
__fpregs_activate(new_fpu); __fpregs_activate(new_fpu);
prefetch(&new_fpu->state); prefetch(&new_fpu->state);
} else if (!use_eager_fpu()) } else {
stts(); __fpregs_deactivate_hw();
}
} else { } else {
old_fpu->counter = 0; old_fpu->counter = 0;
old_fpu->last_cpu = -1; old_fpu->last_cpu = -1;
......
...@@ -105,8 +105,7 @@ void __kernel_fpu_begin(void) ...@@ -105,8 +105,7 @@ void __kernel_fpu_begin(void)
copy_fpregs_to_fpstate(fpu); copy_fpregs_to_fpstate(fpu);
} else { } else {
this_cpu_write(fpu_fpregs_owner_ctx, NULL); this_cpu_write(fpu_fpregs_owner_ctx, NULL);
if (!use_eager_fpu()) __fpregs_activate_hw();
clts();
} }
} }
EXPORT_SYMBOL(__kernel_fpu_begin); EXPORT_SYMBOL(__kernel_fpu_begin);
...@@ -118,8 +117,8 @@ void __kernel_fpu_end(void) ...@@ -118,8 +117,8 @@ void __kernel_fpu_end(void)
if (fpu->fpregs_active) { if (fpu->fpregs_active) {
if (WARN_ON(restore_fpu_checking(fpu))) if (WARN_ON(restore_fpu_checking(fpu)))
fpu_reset_state(fpu); fpu_reset_state(fpu);
} else if (!use_eager_fpu()) { } else {
stts(); __fpregs_deactivate_hw();
} }
kernel_fpu_enable(); kernel_fpu_enable();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment