xstate.c 32.9 KB
Newer Older
1 2 3 4 5 6
/*
 * xsave/xrstor support.
 *
 * Author: Suresh Siddha <suresh.b.siddha@intel.com>
 */
#include <linux/compat.h>
7
#include <linux/cpu.h>
8
#include <linux/mman.h>
9
#include <linux/pkeys.h>
10

11
#include <asm/fpu/api.h>
12
#include <asm/fpu/internal.h>
13
#include <asm/fpu/signal.h>
14
#include <asm/fpu/regset.h>
15
#include <asm/fpu/xstate.h>
16

17
#include <asm/tlbflush.h>
18
#include <asm/cpufeature.h>
19

20 21 22 23 24
/*
 * Although we spell it out in here, the Processor Trace
 * xfeature is completely unused.  We use other mechanisms
 * to save/restore PT state in Linux.
 */
25 26 27 28 29 30 31 32 33 34
static const char *xfeature_names[] =
{
	"x87 floating point registers"	,
	"SSE registers"			,
	"AVX registers"			,
	"MPX bounds registers"		,
	"MPX CSR"			,
	"AVX-512 opmask"		,
	"AVX-512 Hi256"			,
	"AVX-512 ZMM_Hi256"		,
35
	"Processor Trace (unused)"	,
36
	"Protection Keys User registers",
37 38 39
	"unknown xstate feature"	,
};

40 41 42 43 44 45 46 47 48 49 50 51 52
static short xsave_cpuid_features[] __initdata = {
	X86_FEATURE_FPU,
	X86_FEATURE_XMM,
	X86_FEATURE_AVX,
	X86_FEATURE_MPX,
	X86_FEATURE_MPX,
	X86_FEATURE_AVX512F,
	X86_FEATURE_AVX512F,
	X86_FEATURE_AVX512F,
	X86_FEATURE_INTEL_PT,
	X86_FEATURE_PKU,
};

53
/*
54
 * Mask of xstate features supported by the CPU and the kernel:
55
 */
56
u64 xfeatures_mask __read_mostly;
57

58 59
static unsigned int xstate_offsets[XFEATURE_MAX] = { [ 0 ... XFEATURE_MAX - 1] = -1};
static unsigned int xstate_sizes[XFEATURE_MAX]   = { [ 0 ... XFEATURE_MAX - 1] = -1};
60
static unsigned int xstate_comp_offsets[sizeof(xfeatures_mask)*8];
61

62 63 64 65 66 67 68
/*
 * The XSAVE area of kernel can be in standard or compacted format;
 * it is always in standard format for user mode. This is the user
 * mode standard format size used for signal and ptrace frames.
 */
unsigned int fpu_user_xstate_size;

69 70 71 72 73 74 75 76 77
/*
 * Clear all of the X86_FEATURE_* bits that are unavailable
 * when the CPU has no XSAVE support.
 */
void fpu__xstate_clear_all_cpu_caps(void)
{
	setup_clear_cpu_cap(X86_FEATURE_XSAVE);
}

78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
/*
 * Return whether the system supports a given xfeature.
 *
 * Also return the name of the (most advanced) feature that the caller requested:
 */
int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
{
	u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask;

	if (unlikely(feature_name)) {
		long xfeature_idx, max_idx;
		u64 xfeatures_print;
		/*
		 * So we use FLS here to be able to print the most advanced
		 * feature that was requested but is missing. So if a driver
Dave Hansen's avatar
Dave Hansen committed
93
		 * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
		 * missing AVX feature - this is the most informative message
		 * to users:
		 */
		if (xfeatures_missing)
			xfeatures_print = xfeatures_missing;
		else
			xfeatures_print = xfeatures_needed;

		xfeature_idx = fls64(xfeatures_print)-1;
		max_idx = ARRAY_SIZE(xfeature_names)-1;
		xfeature_idx = min(xfeature_idx, max_idx);

		*feature_name = xfeature_names[xfeature_idx];
	}

	if (xfeatures_missing)
		return 0;

	return 1;
}
EXPORT_SYMBOL_GPL(cpu_has_xfeatures);

116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
static int xfeature_is_supervisor(int xfeature_nr)
{
	/*
	 * We currently do not support supervisor states, but if
	 * we did, we could find out like this.
	 *
	 * SDM says: If state component 'i' is a user state component,
	 * ECX[0] return 0; if state component i is a supervisor
	 * state component, ECX[0] returns 1.
	 */
	u32 eax, ebx, ecx, edx;

	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
	return !!(ecx & 1);
}

static int xfeature_is_user(int xfeature_nr)
{
	return !xfeature_is_supervisor(xfeature_nr);
}

137
/*
138 139 140 141 142
 * When executing XSAVEOPT (or other optimized XSAVE instructions), if
 * a processor implementation detects that an FPU state component is still
 * (or is again) in its initialized state, it may clear the corresponding
 * bit in the header.xfeatures field, and can skip the writeout of registers
 * to the corresponding memory layout.
143 144 145 146 147 148 149 150
 *
 * This means that when the bit is zero, the state component might still contain
 * some previous - non-initialized register state.
 *
 * Before writing xstate information to user-space we sanitize those components,
 * to always ensure that the memory layout of a feature will be in the init state
 * if the corresponding header bit is zero. This is to ensure that user-space doesn't
 * see some stale state in the memory layout during signal handling, debugging etc.
151
 */
152
void fpstate_sanitize_xstate(struct fpu *fpu)
153
{
154
	struct fxregs_state *fx = &fpu->state.fxsave;
155
	int feature_bit;
156
	u64 xfeatures;
157

158
	if (!use_xsaveopt())
159 160
		return;

161
	xfeatures = fpu->state.xsave.header.xfeatures;
162 163 164

	/*
	 * None of the feature bits are in init state. So nothing else
165
	 * to do for us, as the memory layout is up to date.
166
	 */
167
	if ((xfeatures & xfeatures_mask) == xfeatures_mask)
168 169 170 171 172
		return;

	/*
	 * FP is in init state
	 */
Dave Hansen's avatar
Dave Hansen committed
173
	if (!(xfeatures & XFEATURE_MASK_FP)) {
174 175 176 177 178 179 180 181 182 183 184 185
		fx->cwd = 0x37f;
		fx->swd = 0;
		fx->twd = 0;
		fx->fop = 0;
		fx->rip = 0;
		fx->rdp = 0;
		memset(&fx->st_space[0], 0, 128);
	}

	/*
	 * SSE is in init state
	 */
Dave Hansen's avatar
Dave Hansen committed
186
	if (!(xfeatures & XFEATURE_MASK_SSE))
187 188
		memset(&fx->xmm_space[0], 0, 256);

189 190 191 192 193
	/*
	 * First two features are FPU and SSE, which above we handled
	 * in a special way already:
	 */
	feature_bit = 0x2;
194
	xfeatures = (xfeatures_mask & ~xfeatures) >> 2;
195 196

	/*
197 198 199
	 * Update all the remaining memory layouts according to their
	 * standard xstate layout, if their header bit is in the init
	 * state:
200
	 */
201 202
	while (xfeatures) {
		if (xfeatures & 0x1) {
203
			int offset = xstate_comp_offsets[feature_bit];
204 205
			int size = xstate_sizes[feature_bit];

206
			memcpy((void *)fx + offset,
207
			       (void *)&init_fpstate.xsave + offset,
208 209 210
			       size);
		}

211
		xfeatures >>= 1;
212 213 214 215
		feature_bit++;
	}
}

216
/*
217 218
 * Enable the extended processor state save/restore feature.
 * Called once per CPU onlining.
219
 */
220
void fpu__init_cpu_xstate(void)
221
{
222
	if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask)
223
		return;
224 225 226 227 228 229 230 231 232
	/*
	 * Make it clear that XSAVES supervisor states are not yet
	 * implemented should anyone expect it to work by changing
	 * bits in XFEATURE_MASK_* macros and XCR0.
	 */
	WARN_ONCE((xfeatures_mask & XFEATURE_MASK_SUPERVISOR),
		"x86/fpu: XSAVES supervisor states are not yet implemented.\n");

	xfeatures_mask &= ~XFEATURE_MASK_SUPERVISOR;
233

234
	cr4_set_bits(X86_CR4_OSXSAVE);
235
	xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
236 237
}

238 239 240 241 242 243 244 245 246 247
/*
 * Note that in the future we will likely need a pair of
 * functions here: one for user xstates and the other for
 * system xstates.  For now, they are the same.
 */
static int xfeature_enabled(enum xfeature xfeature)
{
	return !!(xfeatures_mask & (1UL << xfeature));
}

248
/*
249 250
 * Record the offsets and sizes of various xstates contained
 * in the XSAVE state memory layout.
251
 */
252
static void __init setup_xstate_features(void)
253
{
254
	u32 eax, ebx, ecx, edx, i;
255 256 257
	/* start at the beginnning of the "extended state" */
	unsigned int last_good_offset = offsetof(struct xregs_state,
						 extended_state_area);
258 259 260 261 262 263 264 265 266
	/*
	 * The FP xstates and SSE xstates are legacy states. They are always
	 * in the fixed offsets in the xsave area in either compacted form
	 * or standard form.
	 */
	xstate_offsets[0] = 0;
	xstate_sizes[0] = offsetof(struct fxregs_state, xmm_space);
	xstate_offsets[1] = xstate_sizes[0];
	xstate_sizes[1] = FIELD_SIZEOF(struct fxregs_state, xmm_space);
267

268
	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
269 270
		if (!xfeature_enabled(i))
			continue;
271

272
		cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
273 274 275 276 277 278 279 280

		/*
		 * If an xfeature is supervisor state, the offset
		 * in EBX is invalid. We leave it to -1.
		 */
		if (xfeature_is_user(i))
			xstate_offsets[i] = ebx;

281
		xstate_sizes[i] = eax;
282 283 284 285 286 287 288 289
		/*
		 * In our xstate size checks, we assume that the
		 * highest-numbered xstate feature has the
		 * highest offset in the buffer.  Ensure it does.
		 */
		WARN_ONCE(last_good_offset > xstate_offsets[i],
			"x86/fpu: misordered xstate at %d\n", last_good_offset);
		last_good_offset = xstate_offsets[i];
290
	}
291 292
}

293
static void __init print_xstate_feature(u64 xstate_mask)
294
{
295
	const char *feature_name;
296

297
	if (cpu_has_xfeatures(xstate_mask, &feature_name))
298
		pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name);
299 300 301 302 303
}

/*
 * Print out all the supported xstate features:
 */
304
static void __init print_xstate_features(void)
305
{
Dave Hansen's avatar
Dave Hansen committed
306 307 308 309 310 311 312 313
	print_xstate_feature(XFEATURE_MASK_FP);
	print_xstate_feature(XFEATURE_MASK_SSE);
	print_xstate_feature(XFEATURE_MASK_YMM);
	print_xstate_feature(XFEATURE_MASK_BNDREGS);
	print_xstate_feature(XFEATURE_MASK_BNDCSR);
	print_xstate_feature(XFEATURE_MASK_OPMASK);
	print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
	print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
314
	print_xstate_feature(XFEATURE_MASK_PKRU);
315 316
}

317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
/*
 * This check is important because it is easy to get XSTATE_*
 * confused with XSTATE_BIT_*.
 */
#define CHECK_XFEATURE(nr) do {		\
	WARN_ON(nr < FIRST_EXTENDED_XFEATURE);	\
	WARN_ON(nr >= XFEATURE_MAX);	\
} while (0)

/*
 * We could cache this like xstate_size[], but we only use
 * it here, so it would be a waste of space.
 */
static int xfeature_is_aligned(int xfeature_nr)
{
	u32 eax, ebx, ecx, edx;

	CHECK_XFEATURE(xfeature_nr);
	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
	/*
	 * The value returned by ECX[1] indicates the alignment
	 * of state component 'i' when the compacted format
	 * of the extended region of an XSAVE area is used:
	 */
	return !!(ecx & 2);
}

344 345 346 347 348
/*
 * This function sets up offsets and sizes of all extended states in
 * xsave area. This supports both standard format and compacted format
 * of the xsave aread.
 */
349
static void __init setup_xstate_comp(void)
350
{
351
	unsigned int xstate_comp_sizes[sizeof(xfeatures_mask)*8];
352 353
	int i;

354 355 356 357 358 359
	/*
	 * The FP xstates and SSE xstates are legacy states. They are always
	 * in the fixed offsets in the xsave area in either compacted form
	 * or standard form.
	 */
	xstate_comp_offsets[0] = 0;
360
	xstate_comp_offsets[1] = offsetof(struct fxregs_state, xmm_space);
361

362
	if (!boot_cpu_has(X86_FEATURE_XSAVES)) {
363
		for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
364
			if (xfeature_enabled(i)) {
365 366 367 368 369 370 371
				xstate_comp_offsets[i] = xstate_offsets[i];
				xstate_comp_sizes[i] = xstate_sizes[i];
			}
		}
		return;
	}

372 373
	xstate_comp_offsets[FIRST_EXTENDED_XFEATURE] =
		FXSAVE_SIZE + XSAVE_HDR_SIZE;
374

375
	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
376
		if (xfeature_enabled(i))
377 378 379 380
			xstate_comp_sizes[i] = xstate_sizes[i];
		else
			xstate_comp_sizes[i] = 0;

381
		if (i > FIRST_EXTENDED_XFEATURE) {
382 383 384
			xstate_comp_offsets[i] = xstate_comp_offsets[i-1]
					+ xstate_comp_sizes[i-1];

385 386 387 388
			if (xfeature_is_aligned(i))
				xstate_comp_offsets[i] =
					ALIGN(xstate_comp_offsets[i], 64);
		}
389 390 391
	}
}

392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
/*
 * Print out xstate component offsets and sizes
 */
static void __init print_xstate_offset_size(void)
{
	int i;

	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
		if (!xfeature_enabled(i))
			continue;
		pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n",
			 i, xstate_comp_offsets[i], i, xstate_sizes[i]);
	}
}

407 408 409
/*
 * setup the xstate image representing the init state
 */
410
static void __init setup_init_fpu_buf(void)
411
{
412
	static int on_boot_cpu __initdata = 1;
413 414 415 416

	WARN_ON_FPU(!on_boot_cpu);
	on_boot_cpu = 0;

417
	if (!boot_cpu_has(X86_FEATURE_XSAVE))
418 419 420
		return;

	setup_xstate_features();
421
	print_xstate_features();
422

423
	if (boot_cpu_has(X86_FEATURE_XSAVES))
424
		init_fpstate.xsave.header.xcomp_bv = (u64)1 << 63 | xfeatures_mask;
425

426
	/*
427
	 * Init all the features state with header.xfeatures being 0x0
428
	 */
429
	copy_kernel_to_xregs_booting(&init_fpstate.xsave);
430

431 432 433 434
	/*
	 * Dump the init state again. This is to identify the init state
	 * of any feature which is not represented by all zero's.
	 */
435
	copy_xregs_to_kernel_booting(&init_fpstate.xsave);
436 437
}

438 439 440 441
static int xfeature_uncompacted_offset(int xfeature_nr)
{
	u32 eax, ebx, ecx, edx;

442 443 444 445 446 447 448 449 450 451
	/*
	 * Only XSAVES supports supervisor states and it uses compacted
	 * format. Checking a supervisor state's uncompacted offset is
	 * an error.
	 */
	if (XFEATURE_MASK_SUPERVISOR & (1 << xfeature_nr)) {
		WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr);
		return -1;
	}

452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474
	CHECK_XFEATURE(xfeature_nr);
	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
	return ebx;
}

static int xfeature_size(int xfeature_nr)
{
	u32 eax, ebx, ecx, edx;

	CHECK_XFEATURE(xfeature_nr);
	cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
	return eax;
}

/*
 * 'XSAVES' implies two different things:
 * 1. saving of supervisor/system state
 * 2. using the compacted format
 *
 * Use this function when dealing with the compacted format so
 * that it is obvious which aspect of 'XSAVES' is being handled
 * by the calling code.
 */
475
int using_compacted_format(void)
476
{
477
	return boot_cpu_has(X86_FEATURE_XSAVES);
478 479
}

480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503
/* Validate an xstate header supplied by userspace (ptrace or sigreturn) */
int validate_xstate_header(const struct xstate_header *hdr)
{
	/* No unknown or supervisor features may be set */
	if (hdr->xfeatures & (~xfeatures_mask | XFEATURE_MASK_SUPERVISOR))
		return -EINVAL;

	/* Userspace must use the uncompacted format */
	if (hdr->xcomp_bv)
		return -EINVAL;

	/*
	 * If 'reserved' is shrunken to add a new field, make sure to validate
	 * that new field here!
	 */
	BUILD_BUG_ON(sizeof(hdr->reserved) != 48);

	/* No reserved bits may be set */
	if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved)))
		return -EINVAL;

	return 0;
}

504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529
static void __xstate_dump_leaves(void)
{
	int i;
	u32 eax, ebx, ecx, edx;
	static int should_dump = 1;

	if (!should_dump)
		return;
	should_dump = 0;
	/*
	 * Dump out a few leaves past the ones that we support
	 * just in case there are some goodies up there
	 */
	for (i = 0; i < XFEATURE_MAX + 10; i++) {
		cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
		pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n",
			XSTATE_CPUID, i, eax, ebx, ecx, edx);
	}
}

#define XSTATE_WARN_ON(x) do {							\
	if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) {	\
		__xstate_dump_leaves();						\
	}									\
} while (0)

530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559
#define XCHECK_SZ(sz, nr, nr_macro, __struct) do {			\
	if ((nr == nr_macro) &&						\
	    WARN_ONCE(sz != sizeof(__struct),				\
		"%s: struct is %zu bytes, cpu state %d bytes\n",	\
		__stringify(nr_macro), sizeof(__struct), sz)) {		\
		__xstate_dump_leaves();					\
	}								\
} while (0)

/*
 * We have a C struct for each 'xstate'.  We need to ensure
 * that our software representation matches what the CPU
 * tells us about the state's size.
 */
static void check_xstate_against_struct(int nr)
{
	/*
	 * Ask the CPU for the size of the state.
	 */
	int sz = xfeature_size(nr);
	/*
	 * Match each CPU state with the corresponding software
	 * structure.
	 */
	XCHECK_SZ(sz, nr, XFEATURE_YMM,       struct ymmh_struct);
	XCHECK_SZ(sz, nr, XFEATURE_BNDREGS,   struct mpx_bndreg_state);
	XCHECK_SZ(sz, nr, XFEATURE_BNDCSR,    struct mpx_bndcsr_state);
	XCHECK_SZ(sz, nr, XFEATURE_OPMASK,    struct avx_512_opmask_state);
	XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);
	XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM,  struct avx_512_hi16_state);
560
	XCHECK_SZ(sz, nr, XFEATURE_PKRU,      struct pkru_state);
561 562 563 564 565 566 567

	/*
	 * Make *SURE* to add any feature numbers in below if
	 * there are "holes" in the xsave state component
	 * numbers.
	 */
	if ((nr < XFEATURE_YMM) ||
568 569
	    (nr >= XFEATURE_MAX) ||
	    (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR)) {
570 571 572 573 574
		WARN_ONCE(1, "no structure for xstate: %d\n", nr);
		XSTATE_WARN_ON(1);
	}
}

575 576 577 578 579 580 581 582 583 584 585 586 587
/*
 * This essentially double-checks what the cpu told us about
 * how large the XSAVE buffer needs to be.  We are recalculating
 * it to be safe.
 */
static void do_extra_xstate_size_checks(void)
{
	int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
	int i;

	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
		if (!xfeature_enabled(i))
			continue;
588 589

		check_xstate_against_struct(i);
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613
		/*
		 * Supervisor state components can be managed only by
		 * XSAVES, which is compacted-format only.
		 */
		if (!using_compacted_format())
			XSTATE_WARN_ON(xfeature_is_supervisor(i));

		/* Align from the end of the previous feature */
		if (xfeature_is_aligned(i))
			paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64);
		/*
		 * The offset of a given state in the non-compacted
		 * format is given to us in a CPUID leaf.  We check
		 * them for being ordered (increasing offsets) in
		 * setup_xstate_features().
		 */
		if (!using_compacted_format())
			paranoid_xstate_size = xfeature_uncompacted_offset(i);
		/*
		 * The compacted-format offset always depends on where
		 * the previous state ended.
		 */
		paranoid_xstate_size += xfeature_size(i);
	}
614
	XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size);
615 616
}

617

618
/*
619
 * Get total size of enabled xstates in XCR0/xfeatures_mask.
620 621 622 623 624 625 626 627
 *
 * Note the SDM's wording here.  "sub-function 0" only enumerates
 * the size of the *user* states.  If we use it to size a buffer
 * that we use 'XSAVES' on, we could potentially overflow the
 * buffer because 'XSAVES' saves system states too.
 *
 * Note that we do not currently set any bits on IA32_XSS so
 * 'XCR0 | IA32_XSS == XCR0' for now.
628
 */
629
static unsigned int __init get_xsaves_size(void)
630 631
{
	unsigned int eax, ebx, ecx, edx;
632 633 634 635 636 637 638 639 640 641 642
	/*
	 * - CPUID function 0DH, sub-function 1:
	 *    EBX enumerates the size (in bytes) required by
	 *    the XSAVES instruction for an XSAVE area
	 *    containing all the state components
	 *    corresponding to bits currently set in
	 *    XCR0 | IA32_XSS.
	 */
	cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
	return ebx;
}
643

644 645 646 647 648 649 650 651 652 653 654 655
static unsigned int __init get_xsave_size(void)
{
	unsigned int eax, ebx, ecx, edx;
	/*
	 * - CPUID function 0DH, sub-function 0:
	 *    EBX enumerates the size (in bytes) required by
	 *    the XSAVE instruction for an XSAVE area
	 *    containing all the *user* state components
	 *    corresponding to bits currently set in XCR0.
	 */
	cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
	return ebx;
656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
}

/*
 * Will the runtime-enumerated 'xstate_size' fit in the init
 * task's statically-allocated buffer?
 */
static bool is_supported_xstate_size(unsigned int test_xstate_size)
{
	if (test_xstate_size <= sizeof(union fpregs_state))
		return true;

	pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n",
			sizeof(union fpregs_state), test_xstate_size);
	return false;
}

static int init_xstate_size(void)
{
	/* Recompute the context size for enabled features: */
675 676 677 678 679 680 681 682 683
	unsigned int possible_xstate_size;
	unsigned int xsave_size;

	xsave_size = get_xsave_size();

	if (boot_cpu_has(X86_FEATURE_XSAVES))
		possible_xstate_size = get_xsaves_size();
	else
		possible_xstate_size = xsave_size;
684 685 686 687 688 689 690 691 692

	/* Ensure we have the space to store all enabled: */
	if (!is_supported_xstate_size(possible_xstate_size))
		return -EINVAL;

	/*
	 * The size is OK, we are definitely going to use xsave,
	 * make it known to the world that we need more space.
	 */
693
	fpu_kernel_xstate_size = possible_xstate_size;
694
	do_extra_xstate_size_checks();
695 696 697 698 699

	/*
	 * User space is always in standard format.
	 */
	fpu_user_xstate_size = xsave_size;
700 701 702
	return 0;
}

Dave Hansen's avatar
Dave Hansen committed
703 704 705 706 707
/*
 * We enabled the XSAVE hardware, but something went wrong and
 * we can not use it.  Disable it.
 */
static void fpu__init_disable_system_xstate(void)
708 709 710 711
{
	xfeatures_mask = 0;
	cr4_clear_bits(X86_CR4_OSXSAVE);
	fpu__xstate_clear_all_cpu_caps();
712 713
}

714 715
/*
 * Enable and initialize the xsave feature.
716
 * Called once per system bootup.
717
 */
718
void __init fpu__init_system_xstate(void)
719 720
{
	unsigned int eax, ebx, ecx, edx;
721
	static int on_boot_cpu __initdata = 1;
722
	int err;
723
	int i;
724 725 726

	WARN_ON_FPU(!on_boot_cpu);
	on_boot_cpu = 0;
727

728 729 730 731 732
	if (!boot_cpu_has(X86_FEATURE_FPU)) {
		pr_info("x86/fpu: No FPU detected\n");
		return;
	}

733
	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
734 735
		pr_info("x86/fpu: x87 FPU will use %s\n",
			boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE");
736 737 738
		return;
	}

739
	if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
740
		WARN_ON_FPU(1);
741 742 743 744
		return;
	}

	cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
745
	xfeatures_mask = eax + ((u64)edx << 32);
746

Dave Hansen's avatar
Dave Hansen committed
747
	if ((xfeatures_mask & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
748 749 750 751 752
		/*
		 * This indicates that something really unexpected happened
		 * with the enumeration.  Disable XSAVE and try to continue
		 * booting without it.  This is too early to BUG().
		 */
753
		pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", xfeatures_mask);
754
		goto out_disable;
755 756
	}

757 758 759 760 761 762 763 764
	/*
	 * Clear XSAVE features that are disabled in the normal CPUID.
	 */
	for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) {
		if (!boot_cpu_has(xsave_cpuid_features[i]))
			xfeatures_mask &= ~BIT(i);
	}

765
	xfeatures_mask &= fpu__get_supported_xfeatures_mask();
766

767 768
	/* Enable xstate instructions to be able to continue with initialization: */
	fpu__init_cpu_xstate();
769
	err = init_xstate_size();
770 771
	if (err)
		goto out_disable;
772

773 774 775 776 777 778
	/*
	 * Update info used for ptrace frames; use standard-format size and no
	 * supervisor xstates:
	 */
	update_regset_xstate_info(fpu_user_xstate_size,	xfeatures_mask & ~XFEATURE_MASK_SUPERVISOR);

779
	fpu__init_prepare_fx_sw_frame();
780
	setup_init_fpu_buf();
781
	setup_xstate_comp();
782
	print_xstate_offset_size();
783

784
	pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
785
		xfeatures_mask,
786
		fpu_kernel_xstate_size,
787
		boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard");
788 789 790 791 792
	return;

out_disable:
	/* something went wrong, try to boot without any XSAVE support */
	fpu__init_disable_system_xstate();
793
}
794

795 796 797 798 799 800 801 802
/*
 * Restore minimal FPU state after suspend:
 */
void fpu__resume_cpu(void)
{
	/*
	 * Restore XCR0 on xsave capable CPUs:
	 */
803
	if (boot_cpu_has(X86_FEATURE_XSAVE))
804 805 806
		xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask);
}

807 808 809 810 811 812 813 814 815 816 817
/*
 * Given an xstate feature mask, calculate where in the xsave
 * buffer the state is.  Callers should ensure that the buffer
 * is valid.
 *
 * Note: does not work for compacted buffers.
 */
void *__raw_xsave_addr(struct xregs_state *xsave, int xstate_feature_mask)
{
	int feature_nr = fls64(xstate_feature_mask) - 1;

818 819 820 821 822
	if (!xfeature_enabled(feature_nr)) {
		WARN_ON_FPU(1);
		return NULL;
	}

823 824
	return (void *)xsave + xstate_comp_offsets[feature_nr];
}
825 826 827 828 829 830 831
/*
 * Given the xsave area and a state inside, this function returns the
 * address of the state.
 *
 * This is the API that is called to get xstate address in either
 * standard format or compacted format of xsave area.
 *
832 833 834
 * Note that if there is no data for the field in the xsave buffer
 * this will return NULL.
 *
835
 * Inputs:
836 837
 *	xstate: the thread's storage area for all FPU data
 *	xstate_feature: state which is defined in xsave.h (e.g.
Dave Hansen's avatar
Dave Hansen committed
838
 *	XFEATURE_MASK_FP, XFEATURE_MASK_SSE, etc...)
839
 * Output:
840 841
 *	address of the state in the xsave area, or NULL if the
 *	field is not present in the xsave buffer.
842
 */
843
void *get_xsave_addr(struct xregs_state *xsave, int xstate_feature)
844
{
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869
	/*
	 * Do we even *have* xsave state?
	 */
	if (!boot_cpu_has(X86_FEATURE_XSAVE))
		return NULL;

	/*
	 * We should not ever be requesting features that we
	 * have not enabled.  Remember that pcntxt_mask is
	 * what we write to the XCR0 register.
	 */
	WARN_ONCE(!(xfeatures_mask & xstate_feature),
		  "get of unsupported state");
	/*
	 * This assumes the last 'xsave*' instruction to
	 * have requested that 'xstate_feature' be saved.
	 * If it did not, we might be seeing and old value
	 * of the field in the buffer.
	 *
	 * This can happen because the last 'xsave' did not
	 * request that this feature be saved (unlikely)
	 * or because the "init optimization" caused it
	 * to not be saved.
	 */
	if (!(xsave->header.xfeatures & xstate_feature))
870 871
		return NULL;

872
	return __raw_xsave_addr(xsave, xstate_feature);
873
}
Paolo Bonzini's avatar
Paolo Bonzini committed
874
EXPORT_SYMBOL_GPL(get_xsave_addr);
875 876 877 878 879 880 881 882 883 884 885 886

/*
 * This wraps up the common operations that need to occur when retrieving
 * data from xsave state.  It first ensures that the current task was
 * using the FPU and retrieves the data in to a buffer.  It then calculates
 * the offset of the requested field in the buffer.
 *
 * This function is safe to call whether the FPU is in use or not.
 *
 * Note that this only works on the current task.
 *
 * Inputs:
Dave Hansen's avatar
Dave Hansen committed
887 888
 *	@xsave_state: state which is defined in xsave.h (e.g. XFEATURE_MASK_FP,
 *	XFEATURE_MASK_SSE, etc...)
889 890 891 892 893 894 895 896
 * Output:
 *	address of the state in the xsave area or NULL if the state
 *	is not present or is in its 'init state'.
 */
const void *get_xsave_field_ptr(int xsave_state)
{
	struct fpu *fpu = &current->thread.fpu;

897
	if (!fpu->initialized)
898 899 900 901 902 903 904 905 906
		return NULL;
	/*
	 * fpu__save() takes the CPU's xstate registers
	 * and saves them off to the 'fpu memory buffer.
	 */
	fpu__save(fpu);

	return get_xsave_addr(&fpu->state.xsave, xsave_state);
}
907

908 909
#ifdef CONFIG_ARCH_HAS_PKEYS

910 911 912
#define NR_VALID_PKRU_BITS (CONFIG_NR_PROTECTION_KEYS * 2)
#define PKRU_VALID_MASK (NR_VALID_PKRU_BITS - 1)
/*
913 914
 * This will go out and modify PKRU register to set the access
 * rights for @pkey to @init_val.
915 916 917 918
 */
int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
		unsigned long init_val)
{
919
	u32 old_pkru;
920 921 922 923 924 925 926 927 928 929
	int pkey_shift = (pkey * PKRU_BITS_PER_PKEY);
	u32 new_pkru_bits = 0;

	/*
	 * This check implies XSAVE support.  OSPKE only gets
	 * set if we enable XSAVE and we enable PKU in XCR0.
	 */
	if (!boot_cpu_has(X86_FEATURE_OSPKE))
		return -EINVAL;

930
	/* Set the bits we need in PKRU:  */
931 932 933 934 935
	if (init_val & PKEY_DISABLE_ACCESS)
		new_pkru_bits |= PKRU_AD_BIT;
	if (init_val & PKEY_DISABLE_WRITE)
		new_pkru_bits |= PKRU_WD_BIT;

936
	/* Shift the bits in to the correct place in PKRU for pkey: */
937 938
	new_pkru_bits <<= pkey_shift;

939 940 941
	/* Get old PKRU and mask off any old bits in place: */
	old_pkru = read_pkru();
	old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift);
942

943 944
	/* Write old part along with new part: */
	write_pkru(old_pkru | new_pkru_bits);
945 946 947

	return 0;
}
948
#endif /* ! CONFIG_ARCH_HAS_PKEYS */
949

950 951 952 953 954 955 956 957 958
/*
 * Weird legacy quirk: SSE and YMM states store information in the
 * MXCSR and MXCSR_FLAGS fields of the FP area. That means if the FP
 * area is marked as unused in the xfeatures header, we need to copy
 * MXCSR and MXCSR_FLAGS if either SSE or YMM are in use.
 */
static inline bool xfeatures_mxcsr_quirk(u64 xfeatures)
{
	if (!(xfeatures & (XFEATURE_MASK_SSE|XFEATURE_MASK_YMM)))
959
		return false;
960 961

	if (xfeatures & XFEATURE_MASK_FP)
962
		return false;
963

964
	return true;
965 966
}

967
static void fill_gap(unsigned to, void **kbuf, unsigned *pos, unsigned *count)
968
{
969 970 971 972 973 974 975 976 977 978 979
	if (*pos < to) {
		unsigned size = to - *pos;

		if (size > *count)
			size = *count;
		memcpy(*kbuf, (void *)&init_fpstate.xsave + *pos, size);
		*kbuf += size;
		*pos += size;
		*count -= size;
	}
}
980

981 982 983 984 985 986 987 988 989 990 991
static void copy_part(unsigned offset, unsigned size, void *from,
			void **kbuf, unsigned *pos, unsigned *count)
{
	fill_gap(offset, kbuf, pos, count);
	if (size > *count)
		size = *count;
	if (size) {
		memcpy(*kbuf, from, size);
		*kbuf += size;
		*pos += size;
		*count -= size;
992 993 994 995 996 997 998 999 1000 1001
	}
}

/*
 * Convert from kernel XSAVES compacted format to standard format and copy
 * to a kernel-space ptrace buffer.
 *
 * It supports partial copy but pos always starts from zero. This is called
 * from xstateregs_get() and there we check the CPU has XSAVES.
 */
1002
int copy_xstate_to_kernel(void *kbuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total)
1003 1004
{
	struct xstate_header header;
1005 1006
	const unsigned off_mxcsr = offsetof(struct fxregs_state, mxcsr);
	unsigned count = size_total;
1007
	int i;
1008 1009 1010 1011

	/*
	 * Currently copy_regset_to_user() starts from pos 0:
	 */
1012
	if (unlikely(offset_start != 0))
1013 1014 1015 1016 1017 1018 1019 1020 1021
		return -EFAULT;

	/*
	 * The destination is a ptrace buffer; we put in only user xstates:
	 */
	memset(&header, 0, sizeof(header));
	header.xfeatures = xsave->header.xfeatures;
	header.xfeatures &= ~XFEATURE_MASK_SUPERVISOR;

1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
	if (header.xfeatures & XFEATURE_MASK_FP)
		copy_part(0, off_mxcsr,
			  &xsave->i387, &kbuf, &offset_start, &count);
	if (header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM))
		copy_part(off_mxcsr, MXCSR_AND_FLAGS_SIZE,
			  &xsave->i387.mxcsr, &kbuf, &offset_start, &count);
	if (header.xfeatures & XFEATURE_MASK_FP)
		copy_part(offsetof(struct fxregs_state, st_space), 128,
			  &xsave->i387.st_space, &kbuf, &offset_start, &count);
	if (header.xfeatures & XFEATURE_MASK_SSE)
1032
		copy_part(xstate_offsets[XFEATURE_SSE], 256,
1033 1034 1035 1036 1037 1038
			  &xsave->i387.xmm_space, &kbuf, &offset_start, &count);
	/*
	 * Fill xsave->i387.sw_reserved value for ptrace frame:
	 */
	copy_part(offsetof(struct fxregs_state, sw_reserved), 48,
		  xstate_fx_sw_bytes, &kbuf, &offset_start, &count);
1039 1040 1041
	/*
	 * Copy xregs_state->header:
	 */
1042 1043
	copy_part(offsetof(struct xregs_state, header), sizeof(header),
		  &header, &kbuf, &offset_start, &count);
1044

1045
	for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
1046 1047 1048 1049 1050 1051
		/*
		 * Copy only in-use xstates:
		 */
		if ((header.xfeatures >> i) & 1) {
			void *src = __raw_xsave_addr(xsave, 1 << i);

1052 1053
			copy_part(xstate_offsets[i], xstate_sizes[i],
				  src, &kbuf, &offset_start, &count);
1054 1055 1056
		}

	}
1057
	fill_gap(size_total, &kbuf, &offset_start, &count);
1058 1059 1060 1061 1062

	return 0;
}

static inline int
1063
__copy_xstate_to_user(void __user *ubuf, const void *data, unsigned int offset, unsigned int size, unsigned int size_total)
1064
{
1065
	if (!size)
1066 1067
		return 0;

1068 1069
	if (offset < size_total) {
		unsigned int copy = min(size, size_total - offset);
1070

1071
		if (__copy_to_user(ubuf + offset, data, copy))
1072
			return -EFAULT;
1073 1074 1075 1076 1077 1078
	}
	return 0;
}

/*
 * Convert from kernel XSAVES compacted format to standard format and copy
1079
 * to a user-space buffer. It supports partial copy but pos always starts from
1080 1081 1082
 * zero. This is called from xstateregs_get() and there we check the CPU
 * has XSAVES.
 */
1083
int copy_xstate_to_user(void __user *ubuf, struct xregs_state *xsave, unsigned int offset_start, unsigned int size_total)
1084 1085 1086 1087 1088 1089 1090 1091
{
	unsigned int offset, size;
	int ret, i;
	struct xstate_header header;

	/*
	 * Currently copy_regset_to_user() starts from pos 0:
	 */
1092
	if (unlikely(offset_start != 0))
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107
		return -EFAULT;

	/*
	 * The destination is a ptrace buffer; we put in only user xstates:
	 */
	memset(&header, 0, sizeof(header));
	header.xfeatures = xsave->header.xfeatures;
	header.xfeatures &= ~XFEATURE_MASK_SUPERVISOR;

	/*
	 * Copy xregs_state->header:
	 */
	offset = offsetof(struct xregs_state, header);
	size = sizeof(header);

1108
	ret = __copy_xstate_to_user(ubuf, &header, offset, size, size_total);
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
	if (ret)
		return ret;

	for (i = 0; i < XFEATURE_MAX; i++) {
		/*
		 * Copy only in-use xstates:
		 */
		if ((header.xfeatures >> i) & 1) {
			void *src = __raw_xsave_addr(xsave, 1 << i);

			offset = xstate_offsets[i];
			size = xstate_sizes[i];

1122 1123 1124 1125
			/* The next component has to fit fully into the output buffer: */
			if (offset + size > size_total)
				break;

1126
			ret = __copy_xstate_to_user(ubuf, src, offset, size, size_total);
1127 1128 1129 1130 1131 1132
			if (ret)
				return ret;
		}

	}

1133 1134 1135 1136 1137 1138
	if (xfeatures_mxcsr_quirk(header.xfeatures)) {
		offset = offsetof(struct fxregs_state, mxcsr);
		size = MXCSR_AND_FLAGS_SIZE;
		__copy_xstate_to_user(ubuf, &xsave->i387.mxcsr, offset, size, size_total);
	}

1139 1140 1141 1142 1143 1144
	/*
	 * Fill xsave->i387.sw_reserved value for ptrace frame:
	 */
	offset = offsetof(struct fxregs_state, sw_reserved);
	size = sizeof(xstate_fx_sw_bytes);

1145
	ret = __copy_xstate_to_user(ubuf, xstate_fx_sw_bytes, offset, size, size_total);
1146 1147 1148 1149 1150 1151 1152
	if (ret)
		return ret;

	return 0;
}

/*
1153
 * Convert from a ptrace standard-format kernel buffer to kernel XSAVES format
1154
 * and copy to the target thread. This is called from xstateregs_set().
1155
 */
1156
int copy_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf)
1157 1158 1159
{
	unsigned int offset, size;
	int i;
1160
	struct xstate_header hdr;
1161 1162

	offset = offsetof(struct xregs_state, header);
1163
	size = sizeof(hdr);
1164

1165
	memcpy(&hdr, kbuf + offset, size);
1166

1167
	if (validate_xstate_header(&hdr))
1168 1169 1170 1171 1172
		return -EINVAL;

	for (i = 0; i < XFEATURE_MAX; i++) {
		u64 mask = ((u64)1 << i);

1173
		if (hdr.xfeatures & mask) {
1174 1175 1176 1177 1178
			void *dst = __raw_xsave_addr(xsave, 1 << i);

			offset = xstate_offsets[i];
			size = xstate_sizes[i];

1179
			memcpy(dst, kbuf + offset, size);
1180 1181 1182
		}
	}

1183
	if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1184 1185 1186 1187 1188
		offset = offsetof(struct fxregs_state, mxcsr);
		size = MXCSR_AND_FLAGS_SIZE;
		memcpy(&xsave->i387.mxcsr, kbuf + offset, size);
	}

1189 1190 1191 1192 1193 1194 1195 1196 1197
	/*
	 * The state that came in from userspace was user-state only.
	 * Mask all the user states out of 'xfeatures':
	 */
	xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR;

	/*
	 * Add back in the features that came in from userspace:
	 */
1198
	xsave->header.xfeatures |= hdr.xfeatures;
1199 1200 1201 1202 1203

	return 0;
}

/*
1204 1205 1206 1207
 * Convert from a ptrace or sigreturn standard-format user-space buffer to
 * kernel XSAVES format and copy to the target thread. This is called from
 * xstateregs_set(), as well as potentially from the sigreturn() and
 * rt_sigreturn() system calls.
1208
 */
1209
int copy_user_to_xstate(struct xregs_state *xsave, const void __user *ubuf)
1210 1211 1212
{
	unsigned int offset, size;
	int i;
1213
	struct xstate_header hdr;
1214 1215

	offset = offsetof(struct xregs_state, header);
1216
	size = sizeof(hdr);
1217

1218
	if (__copy_from_user(&hdr, ubuf + offset, size))
1219
		return -EFAULT;
1220

1221
	if (validate_xstate_header(&hdr))
1222 1223 1224 1225 1226
		return -EINVAL;

	for (i = 0; i < XFEATURE_MAX; i++) {
		u64 mask = ((u64)1 << i);

1227
		if (hdr.xfeatures & mask) {
1228 1229 1230 1231 1232
			void *dst = __raw_xsave_addr(xsave, 1 << i);

			offset = xstate_offsets[i];
			size = xstate_sizes[i];

1233 1234
			if (__copy_from_user(dst, ubuf + offset, size))
				return -EFAULT;
1235 1236 1237
		}
	}

1238
	if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1239 1240 1241 1242 1243 1244
		offset = offsetof(struct fxregs_state, mxcsr);
		size = MXCSR_AND_FLAGS_SIZE;
		if (__copy_from_user(&xsave->i387.mxcsr, ubuf + offset, size))
			return -EFAULT;
	}

1245 1246 1247 1248 1249 1250 1251 1252 1253
	/*
	 * The state that came in from userspace was user-state only.
	 * Mask all the user states out of 'xfeatures':
	 */
	xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR;

	/*
	 * Add back in the features that came in from userspace:
	 */
1254
	xsave->header.xfeatures |= hdr.xfeatures;
1255 1256 1257

	return 0;
}