x86.c 279 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
Ben-Ami Yassour's avatar
Ben-Ami Yassour committed
8 9
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11 12 13 14
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
Ben-Ami Yassour's avatar
Ben-Ami Yassour committed
15 16
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
17 18
 */

19
#include <linux/kvm_host.h>
20
#include "irq.h"
21
#include "ioapic.h"
22
#include "mmu.h"
Sheng Yang's avatar
Sheng Yang committed
23
#include "i8254.h"
24
#include "tss.h"
25
#include "kvm_cache_regs.h"
26
#include "kvm_emulate.h"
27
#include "x86.h"
28
#include "cpuid.h"
29
#include "pmu.h"
30
#include "hyperv.h"
31
#include "lapic.h"
32

33
#include <linux/clocksource.h>
Ben-Ami Yassour's avatar
Ben-Ami Yassour committed
34
#include <linux/interrupt.h>
35 36 37
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
38 39
#include <linux/export.h>
#include <linux/moduleparam.h>
40
#include <linux/mman.h>
41
#include <linux/highmem.h>
42
#include <linux/iommu.h>
43
#include <linux/intel-iommu.h>
44
#include <linux/cpufreq.h>
45
#include <linux/user-return-notifier.h>
46
#include <linux/srcu.h>
47
#include <linux/slab.h>
48
#include <linux/perf_event.h>
49
#include <linux/uaccess.h>
50
#include <linux/hash.h>
51
#include <linux/pci.h>
52 53
#include <linux/timekeeper_internal.h>
#include <linux/pvclock_gtod.h>
54 55
#include <linux/kvm_irqfd.h>
#include <linux/irqbypass.h>
56
#include <linux/sched/stat.h>
57
#include <linux/sched/isolation.h>
58
#include <linux/mem_encrypt.h>
59

Avi Kivity's avatar
Avi Kivity committed
60
#include <trace/events/kvm.h>
Xiao Guangrong's avatar
Xiao Guangrong committed
61

62
#include <asm/debugreg.h>
63
#include <asm/msr.h>
64
#include <asm/desc.h>
Huang Ying's avatar
Huang Ying committed
65
#include <asm/mce.h>
66
#include <linux/kernel_stat.h>
67
#include <asm/fpu/internal.h> /* Ugh! */
68
#include <asm/pvclock.h>
69
#include <asm/div64.h>
70
#include <asm/irq_remapping.h>
71
#include <asm/mshyperv.h>
72
#include <asm/hypervisor.h>
73
#include <asm/intel_pt.h>
74
#include <asm/emulate_prefix.h>
75
#include <clocksource/hyperv_timer.h>
76

77 78 79
#define CREATE_TRACE_POINTS
#include "trace.h"

80
#define MAX_IO_MSRS 256
Huang Ying's avatar
Huang Ying committed
81
#define KVM_MAX_MCE_BANKS 32
82 83
u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P;
EXPORT_SYMBOL_GPL(kvm_mce_cap_supported);
Huang Ying's avatar
Huang Ying committed
84

85
#define emul_to_vcpu(ctxt) \
86
	((struct kvm_vcpu *)(ctxt)->vcpu)
87

88 89 90 91 92
/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
93 94
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
95
#else
96
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
97
#endif
98

99 100
static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;

101 102
#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
                                    KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
103

104
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
Avi Kivity's avatar
Avi Kivity committed
105
static void process_nmi(struct kvm_vcpu *vcpu);
106
static void enter_smm(struct kvm_vcpu *vcpu);
107
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
Ken Hofsass's avatar
Ken Hofsass committed
108 109
static void store_regs(struct kvm_vcpu *vcpu);
static int sync_regs(struct kvm_vcpu *vcpu);
110

111
struct kvm_x86_ops kvm_x86_ops __read_mostly;
112
EXPORT_SYMBOL_GPL(kvm_x86_ops);
113

114
static bool __read_mostly ignore_msrs = 0;
115
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
116

117 118 119
static bool __read_mostly report_ignored_msrs = true;
module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);

120
unsigned int min_timer_period_us = 200;
121 122
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

123 124 125
static bool __read_mostly kvmclock_periodic_sync = true;
module_param(kvmclock_periodic_sync, bool, S_IRUGO);

126
bool __read_mostly kvm_has_tsc_control;
127
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
128
u32  __read_mostly kvm_max_guest_tsc_khz;
129
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
130 131 132 133
u8   __read_mostly kvm_tsc_scaling_ratio_frac_bits;
EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits);
u64  __read_mostly kvm_max_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
134 135
u64 __read_mostly kvm_default_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
136

137
/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
138
static u32 __read_mostly tsc_tolerance_ppm = 250;
139 140
module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);

141 142 143 144 145 146 147
/*
 * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
 * adaptive tuning starting from default advancment of 1000ns.  '0' disables
 * advancement entirely.  Any other value is used as-is and disables adaptive
 * tuning, i.e. allows priveleged userspace to set an exact advancement time.
 */
static int __read_mostly lapic_timer_advance_ns = -1;
148
module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
149

150 151 152
static bool __read_mostly vector_hashing = true;
module_param(vector_hashing, bool, S_IRUGO);

153 154 155 156
bool __read_mostly enable_vmware_backdoor = false;
module_param(enable_vmware_backdoor, bool, S_IRUGO);
EXPORT_SYMBOL_GPL(enable_vmware_backdoor);

157 158 159
static bool __read_mostly force_emulation_prefix = false;
module_param(force_emulation_prefix, bool, S_IRUGO);

160 161 162
int __read_mostly pi_inject_timer = -1;
module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR);

163 164 165 166
#define KVM_NR_SHARED_MSRS 16

struct kvm_shared_msrs_global {
	int nr;
167
	u32 msrs[KVM_NR_SHARED_MSRS];
168 169 170 171 172
};

struct kvm_shared_msrs {
	struct user_return_notifier urn;
	bool registered;
173 174 175 176
	struct kvm_shared_msr_values {
		u64 host;
		u64 curr;
	} values[KVM_NR_SHARED_MSRS];
177 178 179
};

static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
180
static struct kvm_shared_msrs __percpu *shared_msrs;
181

182 183 184 185 186
#define KVM_SUPPORTED_XCR0     (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \
				| XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \
				| XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \
				| XFEATURE_MASK_PKRU)

187 188 189
u64 __read_mostly host_efer;
EXPORT_SYMBOL_GPL(host_efer);

190
static u64 __read_mostly host_xss;
191 192
u64 __read_mostly supported_xss;
EXPORT_SYMBOL_GPL(supported_xss);
193

194
struct kvm_stats_debugfs_item debugfs_entries[] = {
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
	VCPU_STAT("pf_fixed", pf_fixed),
	VCPU_STAT("pf_guest", pf_guest),
	VCPU_STAT("tlb_flush", tlb_flush),
	VCPU_STAT("invlpg", invlpg),
	VCPU_STAT("exits", exits),
	VCPU_STAT("io_exits", io_exits),
	VCPU_STAT("mmio_exits", mmio_exits),
	VCPU_STAT("signal_exits", signal_exits),
	VCPU_STAT("irq_window", irq_window_exits),
	VCPU_STAT("nmi_window", nmi_window_exits),
	VCPU_STAT("halt_exits", halt_exits),
	VCPU_STAT("halt_successful_poll", halt_successful_poll),
	VCPU_STAT("halt_attempted_poll", halt_attempted_poll),
	VCPU_STAT("halt_poll_invalid", halt_poll_invalid),
	VCPU_STAT("halt_wakeup", halt_wakeup),
	VCPU_STAT("hypercalls", hypercalls),
	VCPU_STAT("request_irq", request_irq_exits),
	VCPU_STAT("irq_exits", irq_exits),
	VCPU_STAT("host_state_reload", host_state_reload),
	VCPU_STAT("fpu_reload", fpu_reload),
	VCPU_STAT("insn_emulation", insn_emulation),
	VCPU_STAT("insn_emulation_fail", insn_emulation_fail),
	VCPU_STAT("irq_injections", irq_injections),
	VCPU_STAT("nmi_injections", nmi_injections),
	VCPU_STAT("req_event", req_event),
	VCPU_STAT("l1d_flush", l1d_flush),
221 222
	VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns),
	VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns),
223 224 225 226 227 228 229 230 231 232 233 234
	VM_STAT("mmu_shadow_zapped", mmu_shadow_zapped),
	VM_STAT("mmu_pte_write", mmu_pte_write),
	VM_STAT("mmu_pte_updated", mmu_pte_updated),
	VM_STAT("mmu_pde_zapped", mmu_pde_zapped),
	VM_STAT("mmu_flooded", mmu_flooded),
	VM_STAT("mmu_recycled", mmu_recycled),
	VM_STAT("mmu_cache_miss", mmu_cache_miss),
	VM_STAT("mmu_unsync", mmu_unsync),
	VM_STAT("remote_tlb_flush", remote_tlb_flush),
	VM_STAT("largepages", lpages, .mode = 0444),
	VM_STAT("nx_largepages_splitted", nx_lpage_splits, .mode = 0444),
	VM_STAT("max_mmu_page_hash_collisions", max_mmu_page_hash_collisions),
235 236 237
	{ NULL }
};

238
u64 __read_mostly host_xcr0;
239 240
u64 __read_mostly supported_xcr0;
EXPORT_SYMBOL_GPL(supported_xcr0);
241

242 243 244
struct kmem_cache *x86_fpu_cache;
EXPORT_SYMBOL_GPL(x86_fpu_cache);

245 246 247 248
static struct kmem_cache *x86_emulator_cache;

static struct kmem_cache *kvm_alloc_emulator_cache(void)
{
249 250 251 252
	unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src);
	unsigned int size = sizeof(struct x86_emulate_ctxt);

	return kmem_cache_create_usercopy("x86_emulator", size,
253
					  __alignof__(struct x86_emulate_ctxt),
254 255
					  SLAB_ACCOUNT, useroffset,
					  size - useroffset, NULL);
256 257
}

258
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
259

260 261 262
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
	int i;
263
	for (i = 0; i < ASYNC_PF_PER_VCPU; i++)
264 265 266
		vcpu->arch.apf.gfns[i] = ~0;
}

267 268 269 270 271
static void kvm_on_user_return(struct user_return_notifier *urn)
{
	unsigned slot;
	struct kvm_shared_msrs *locals
		= container_of(urn, struct kvm_shared_msrs, urn);
272
	struct kvm_shared_msr_values *values;
273 274 275 276 277 278 279 280 281 282 283 284
	unsigned long flags;

	/*
	 * Disabling irqs at this point since the following code could be
	 * interrupted and executed through kvm_arch_hardware_disable()
	 */
	local_irq_save(flags);
	if (locals->registered) {
		locals->registered = false;
		user_return_notifier_unregister(urn);
	}
	local_irq_restore(flags);
285
	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
286 287 288 289
		values = &locals->values[slot];
		if (values->host != values->curr) {
			wrmsrl(shared_msrs_global.msrs[slot], values->host);
			values->curr = values->host;
290 291 292 293
		}
	}
}

294 295
void kvm_define_shared_msr(unsigned slot, u32 msr)
{
296
	BUG_ON(slot >= KVM_NR_SHARED_MSRS);
297
	shared_msrs_global.msrs[slot] = msr;
298 299 300 301 302 303 304
	if (slot >= shared_msrs_global.nr)
		shared_msrs_global.nr = slot + 1;
}
EXPORT_SYMBOL_GPL(kvm_define_shared_msr);

static void kvm_shared_msr_cpu_online(void)
{
305 306 307 308
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
	u64 value;
	int i;
309

310 311 312 313 314
	for (i = 0; i < shared_msrs_global.nr; ++i) {
		rdmsrl_safe(shared_msrs_global.msrs[i], &value);
		smsr->values[i].host = value;
		smsr->values[i].curr = value;
	}
315 316
}

317
int kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
318
{
319 320
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
321
	int err;
322

323 324
	value = (value & mask) | (smsr->values[slot].host & ~mask);
	if (value == smsr->values[slot].curr)
325 326 327 328 329
		return 0;
	err = wrmsrl_safe(shared_msrs_global.msrs[slot], value);
	if (err)
		return 1;

330
	smsr->values[slot].curr = value;
331 332 333 334 335
	if (!smsr->registered) {
		smsr->urn.on_user_return = kvm_on_user_return;
		user_return_notifier_register(&smsr->urn);
		smsr->registered = true;
	}
336
	return 0;
337 338 339
}
EXPORT_SYMBOL_GPL(kvm_set_shared_msr);

340
static void drop_user_return_notifiers(void)
341
{
342 343
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
344 345 346 347 348

	if (smsr->registered)
		kvm_on_user_return(&smsr->urn);
}

349 350
u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
351
	return vcpu->arch.apic_base;
352 353 354
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

355 356 357 358 359 360
enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
{
	return kvm_apic_mode(kvm_get_apic_base(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_apic_mode);

361 362
int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
363 364
	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
365 366
	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |
		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
367

368
	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
369
		return 1;
370 371 372 373 374 375
	if (!msr_info->host_initiated) {
		if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
			return 1;
		if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
			return 1;
	}
376 377

	kvm_lapic_set_base(vcpu, msr_info->data);
378
	kvm_recalculate_apic_map(vcpu->kvm);
379
	return 0;
380 381 382
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

383
asmlinkage __visible void kvm_spurious_fault(void)
384 385
{
	/* Fault while not rebooting.  We want the trace. */
386
	BUG_ON(!kvm_rebooting);
387 388 389
}
EXPORT_SYMBOL_GPL(kvm_spurious_fault);

390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
#define EXCPT_BENIGN		0
#define EXCPT_CONTRIBUTORY	1
#define EXCPT_PF		2

static int exception_class(int vector)
{
	switch (vector) {
	case PF_VECTOR:
		return EXCPT_PF;
	case DE_VECTOR:
	case TS_VECTOR:
	case NP_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
		return EXCPT_CONTRIBUTORY;
	default:
		break;
	}
	return EXCPT_BENIGN;
}

411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
#define EXCPT_FAULT		0
#define EXCPT_TRAP		1
#define EXCPT_ABORT		2
#define EXCPT_INTERRUPT		3

static int exception_type(int vector)
{
	unsigned int mask;

	if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
		return EXCPT_INTERRUPT;

	mask = 1 << vector;

	/* #DB is trap, as instruction watchpoints are handled elsewhere */
	if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR)))
		return EXCPT_TRAP;

	if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
		return EXCPT_ABORT;

	/* Reserved exceptions will result in fault */
	return EXCPT_FAULT;
}

436 437 438 439 440 441 442 443 444 445
void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
{
	unsigned nr = vcpu->arch.exception.nr;
	bool has_payload = vcpu->arch.exception.has_payload;
	unsigned long payload = vcpu->arch.exception.payload;

	if (!has_payload)
		return;

	switch (nr) {
446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
	case DB_VECTOR:
		/*
		 * "Certain debug exceptions may clear bit 0-3.  The
		 * remaining contents of the DR6 register are never
		 * cleared by the processor".
		 */
		vcpu->arch.dr6 &= ~DR_TRAP_BITS;
		/*
		 * DR6.RTM is set by all #DB exceptions that don't clear it.
		 */
		vcpu->arch.dr6 |= DR6_RTM;
		vcpu->arch.dr6 |= payload;
		/*
		 * Bit 16 should be set in the payload whenever the #DB
		 * exception should clear DR6.RTM. This makes the payload
		 * compatible with the pending debug exceptions under VMX.
		 * Though not currently documented in the SDM, this also
		 * makes the payload compatible with the exit qualification
		 * for #DB exceptions under VMX.
		 */
		vcpu->arch.dr6 ^= payload & DR6_RTM;
467 468 469 470 471 472 473 474

		/*
		 * The #DB payload is defined as compatible with the 'pending
		 * debug exceptions' field under VMX, not DR6. While bit 12 is
		 * defined in the 'pending debug exceptions' field (enabled
		 * breakpoint), it is reserved and must be zero in DR6.
		 */
		vcpu->arch.dr6 &= ~BIT(12);
475
		break;
476 477 478 479 480 481 482 483 484 485
	case PF_VECTOR:
		vcpu->arch.cr2 = payload;
		break;
	}

	vcpu->arch.exception.has_payload = false;
	vcpu->arch.exception.payload = 0;
}
EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);

486
static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
487
		unsigned nr, bool has_error, u32 error_code,
488
	        bool has_payload, unsigned long payload, bool reinject)
489 490 491 492
{
	u32 prev_nr;
	int class1, class2;

493 494
	kvm_make_request(KVM_REQ_EVENT, vcpu);

495
	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
496
	queue:
497 498
		if (has_error && !is_protmode(vcpu))
			has_error = false;
499 500 501 502 503 504 505 506 507 508 509
		if (reinject) {
			/*
			 * On vmentry, vcpu->arch.exception.pending is only
			 * true if an event injection was blocked by
			 * nested_run_pending.  In that case, however,
			 * vcpu_enter_guest requests an immediate exit,
			 * and the guest shouldn't proceed far enough to
			 * need reinjection.
			 */
			WARN_ON_ONCE(vcpu->arch.exception.pending);
			vcpu->arch.exception.injected = true;
510 511 512 513 514 515 516 517
			if (WARN_ON_ONCE(has_payload)) {
				/*
				 * A reinjected event has already
				 * delivered its payload.
				 */
				has_payload = false;
				payload = 0;
			}
518 519 520 521
		} else {
			vcpu->arch.exception.pending = true;
			vcpu->arch.exception.injected = false;
		}
522 523 524
		vcpu->arch.exception.has_error_code = has_error;
		vcpu->arch.exception.nr = nr;
		vcpu->arch.exception.error_code = error_code;
525 526
		vcpu->arch.exception.has_payload = has_payload;
		vcpu->arch.exception.payload = payload;
527
		if (!is_guest_mode(vcpu))
528
			kvm_deliver_exception_payload(vcpu);
529 530 531 532 533 534 535
		return;
	}

	/* to check exception */
	prev_nr = vcpu->arch.exception.nr;
	if (prev_nr == DF_VECTOR) {
		/* triple fault -> shutdown */
536
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
537 538 539 540 541 542
		return;
	}
	class1 = exception_class(prev_nr);
	class2 = exception_class(nr);
	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
543 544 545 546 547
		/*
		 * Generate double fault per SDM Table 5-5.  Set
		 * exception.pending = true so that the double fault
		 * can trigger a nested vmexit.
		 */
548
		vcpu->arch.exception.pending = true;
549
		vcpu->arch.exception.injected = false;
550 551 552
		vcpu->arch.exception.has_error_code = true;
		vcpu->arch.exception.nr = DF_VECTOR;
		vcpu->arch.exception.error_code = 0;
553 554
		vcpu->arch.exception.has_payload = false;
		vcpu->arch.exception.payload = 0;
555 556 557 558 559 560 561
	} else
		/* replace previous exception with a new one in a hope
		   that instruction re-execution will regenerate lost
		   exception */
		goto queue;
}

562 563
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
564
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
565 566 567
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

568 569
void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
570
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
571 572 573
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

574 575
void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
			   unsigned long payload)
576 577 578
{
	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
}
579
EXPORT_SYMBOL_GPL(kvm_queue_exception_p);
580

581 582 583 584 585 586 587
static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
				    u32 error_code, unsigned long payload)
{
	kvm_multiple_exception(vcpu, nr, true, error_code,
			       true, payload, false);
}

588
int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
589
{
590 591 592
	if (err)
		kvm_inject_gp(vcpu, 0);
	else
593 594 595
		return kvm_skip_emulated_instruction(vcpu);

	return 1;
596 597
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
598

599
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
600 601
{
	++vcpu->stat.pf_guest;
602 603
	vcpu->arch.exception.nested_apf =
		is_guest_mode(vcpu) && fault->async_page_fault;
604
	if (vcpu->arch.exception.nested_apf) {
605
		vcpu->arch.apf.nested_apf_token = fault->address;
606 607 608 609 610
		kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
	} else {
		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
					fault->address);
	}
611
}
Nadav Har'El's avatar
Nadav Har'El committed
612
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
613

614 615
bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
				    struct x86_exception *fault)
616
{
617
	struct kvm_mmu *fault_mmu;
618 619
	WARN_ON_ONCE(fault->vector != PF_VECTOR);

620 621
	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
					       vcpu->arch.walk_mmu;
622

623 624 625 626 627 628 629 630 631 632
	/*
	 * Invalidate the TLB entry for the faulting address, if it exists,
	 * else the access will fault indefinitely (and to emulate hardware).
	 */
	if ((fault->error_code & PFERR_PRESENT_MASK) &&
	    !(fault->error_code & PFERR_RSVD_MASK))
		kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,
				       fault_mmu->root_hpa);

	fault_mmu->inject_page_fault(vcpu, fault);
633
	return fault->nested_page_fault;
634
}
635
EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault);
636

637 638
void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
Avi Kivity's avatar
Avi Kivity committed
639 640
	atomic_inc(&vcpu->arch.nmi_queued);
	kvm_make_request(KVM_REQ_NMI, vcpu);
641 642 643
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

644 645
void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
646
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
647 648 649
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

650 651
void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
652
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
653 654 655
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

656 657 658 659 660
/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
661
{
662
	if (kvm_x86_ops.get_cpl(vcpu) <= required_cpl)
663 664 665
		return true;
	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
	return false;
666
}
667
EXPORT_SYMBOL_GPL(kvm_require_cpl);
668

669 670 671 672 673 674 675 676 677 678
bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
{
	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
		return true;

	kvm_queue_exception(vcpu, UD_VECTOR);
	return false;
}
EXPORT_SYMBOL_GPL(kvm_require_dr);

679 680
/*
 * This function will be used to read from the physical memory of the currently
681
 * running guest. The difference to kvm_vcpu_read_guest_page is that this function
682 683 684 685 686 687
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gfn_t ngfn, void *data, int offset, int len,
			    u32 access)
{
688
	struct x86_exception exception;
689 690 691 692
	gfn_t real_gfn;
	gpa_t ngpa;

	ngpa     = gfn_to_gpa(ngfn);
693
	real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);
694 695 696 697 698
	if (real_gfn == UNMAPPED_GVA)
		return -EFAULT;

	real_gfn = gpa_to_gfn(real_gfn);

699
	return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len);
700 701 702
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

703
static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
704 705 706 707 708 709
			       void *data, int offset, int len, u32 access)
{
	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
				       data, offset, len, access);
}

710 711 712 713 714 715
static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
{
	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) | rsvd_bits(5, 8) |
	       rsvd_bits(1, 2);
}

716
/*
717
 * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
718
 */
719
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
720 721 722 723 724
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
	int i;
	int ret;
725
	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
726

727 728 729
	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
				      offset * sizeof(u64), sizeof(pdpte),
				      PFERR_USER_MASK|PFERR_WRITE_MASK);
730 731 732 733 734
	if (ret < 0) {
		ret = 0;
		goto out;
	}
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
735
		if ((pdpte[i] & PT_PRESENT_MASK) &&
736
		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
737 738 739 740 741 742
			ret = 0;
			goto out;
		}
	}
	ret = 1;

743
	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
744 745
	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);

746 747 748 749
out:

	return ret;
}
750
EXPORT_SYMBOL_GPL(load_pdptrs);
751

752
bool pdptrs_changed(struct kvm_vcpu *vcpu)
753
{
754
	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
755 756
	int offset;
	gfn_t gfn;
757 758
	int r;

759
	if (!is_pae_paging(vcpu))
760 761
		return false;

762
	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
Avi Kivity's avatar
Avi Kivity committed
763 764
		return true;

765 766
	gfn = (kvm_read_cr3(vcpu) & 0xffffffe0ul) >> PAGE_SHIFT;
	offset = (kvm_read_cr3(vcpu) & 0xffffffe0ul) & (PAGE_SIZE - 1);
767 768
	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
				       PFERR_USER_MASK | PFERR_WRITE_MASK);
769
	if (r < 0)
770
		return true;
771

772
	return memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
773
}
774
EXPORT_SYMBOL_GPL(pdptrs_changed);
775

776
int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
777
{
778
	unsigned long old_cr0 = kvm_read_cr0(vcpu);
779
	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP;
780

781 782
	cr0 |= X86_CR0_ET;

783
#ifdef CONFIG_X86_64
784 785
	if (cr0 & 0xffffffff00000000UL)
		return 1;
786 787 788
#endif

	cr0 &= ~CR0_RESERVED_BITS;
789

790 791
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
		return 1;
792

793 794
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
		return 1;
795 796 797

	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
#ifdef CONFIG_X86_64
798
		if ((vcpu->arch.efer & EFER_LME)) {
799 800
			int cs_db, cs_l;

801 802
			if (!is_pae(vcpu))
				return 1;
803
			kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
804 805
			if (cs_l)
				return 1;
806 807
		} else
#endif
808
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
809
						 kvm_read_cr3(vcpu)))
810
			return 1;
811 812
	}

813 814 815
	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
		return 1;

816
	kvm_x86_ops.set_cr0(vcpu, cr0);
817

818
	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
819
		kvm_clear_async_pf_completion_queue(vcpu);
820 821
		kvm_async_pf_hash_reset(vcpu);
	}
822

823 824
	if ((cr0 ^ old_cr0) & update_bits)
		kvm_mmu_reset_context(vcpu);
825

826 827 828
	if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
829 830
		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);

831 832
	return 0;
}
833
EXPORT_SYMBOL_GPL(kvm_set_cr0);
834

835
void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
836
{
837
	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
838
}
839
EXPORT_SYMBOL_GPL(kvm_lmsw);
840

841
void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
842
{
843 844 845 846 847 848 849 850 851
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {

		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);

		if (vcpu->arch.xsaves_enabled &&
		    vcpu->arch.ia32_xss != host_xss)
			wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
	}
852 853 854 855 856 857

	if (static_cpu_has(X86_FEATURE_PKU) &&
	    (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||
	     (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU)) &&
	    vcpu->arch.pkru != vcpu->arch.host_pkru)
		__write_pkru(vcpu->arch.pkru);
858
}
859
EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state);
860

861
void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
862
{
863 864 865 866 867 868 869 870
	if (static_cpu_has(X86_FEATURE_PKU) &&
	    (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) ||
	     (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU))) {
		vcpu->arch.pkru = rdpkru();
		if (vcpu->arch.pkru != vcpu->arch.host_pkru)
			__write_pkru(vcpu->arch.host_pkru);
	}

871 872 873 874 875 876 877 878 879 880
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {

		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);

		if (vcpu->arch.xsaves_enabled &&
		    vcpu->arch.ia32_xss != host_xss)
			wrmsrl(MSR_IA32_XSS, host_xss);
	}

881
}
882
EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state);
883

884
static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
885
{
886 887
	u64 xcr0 = xcr;
	u64 old_xcr0 = vcpu->arch.xcr0;
888
	u64 valid_bits;
889 890 891 892

	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
	if (index != XCR_XFEATURE_ENABLED_MASK)
		return 1;
Dave Hansen's avatar
Dave Hansen committed
893
	if (!(xcr0 & XFEATURE_MASK_FP))
894
		return 1;
Dave Hansen's avatar
Dave Hansen committed
895
	if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
896
		return 1;
897 898 899 900 901 902

	/*
	 * Do not allow the guest to set bits that we do not support
	 * saving.  However, xcr0 bit 0 is always set, even if the
	 * emulated CPU does not support XSAVE (see fx_init).
	 */
Dave Hansen's avatar
Dave Hansen committed
903
	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
904
	if (xcr0 & ~valid_bits)
905
		return 1;
906

Dave Hansen's avatar
Dave Hansen committed
907 908
	if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
	    (!(xcr0 & XFEATURE_MASK_BNDCSR)))
909 910
		return 1;

Dave Hansen's avatar
Dave Hansen committed
911 912
	if (xcr0 & XFEATURE_MASK_AVX512) {
		if (!(xcr0 & XFEATURE_MASK_YMM))
913
			return 1;
Dave Hansen's avatar
Dave Hansen committed
914
		if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
915 916
			return 1;
	}
917
	vcpu->arch.xcr0 = xcr0;
918

Dave Hansen's avatar
Dave Hansen committed
919
	if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
920
		kvm_update_cpuid(vcpu);
921 922 923 924 925
	return 0;
}

int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
926
	if (kvm_x86_ops.get_cpl(vcpu) != 0 ||
927
	    __kvm_set_xcr(vcpu, index, xcr)) {
928 929 930 931 932 933 934
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_xcr);

935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950
#define __cr4_reserved_bits(__cpu_has, __c)		\
({							\
	u64 __reserved_bits = CR4_RESERVED_BITS;	\
							\
	if (!__cpu_has(__c, X86_FEATURE_XSAVE))		\
		__reserved_bits |= X86_CR4_OSXSAVE;	\
	if (!__cpu_has(__c, X86_FEATURE_SMEP))		\
		__reserved_bits |= X86_CR4_SMEP;	\
	if (!__cpu_has(__c, X86_FEATURE_SMAP))		\
		__reserved_bits |= X86_CR4_SMAP;	\
	if (!__cpu_has(__c, X86_FEATURE_FSGSBASE))	\
		__reserved_bits |= X86_CR4_FSGSBASE;	\
	if (!__cpu_has(__c, X86_FEATURE_PKU))		\
		__reserved_bits |= X86_CR4_PKE;		\
	if (!__cpu_has(__c, X86_FEATURE_LA57))		\
		__reserved_bits |= X86_CR4_LA57;	\
951 952
	if (!__cpu_has(__c, X86_FEATURE_UMIP))		\
		__reserved_bits |= X86_CR4_UMIP;	\
953 954
	__reserved_bits;				\
})
955

956
static int kvm_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
957
{
958
	if (cr4 & cr4_reserved_bits)
959
		return -EINVAL;
960

961
	if (cr4 & __cr4_reserved_bits(guest_cpuid_has, vcpu))
962 963 964 965 966 967 968 969 970 971 972 973
		return -EINVAL;

	return 0;
}

int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
	unsigned long old_cr4 = kvm_read_cr4(vcpu);
	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE |
				   X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE;

	if (kvm_valid_cr4(vcpu, cr4))
974 975
		return 1;

976
	if (is_long_mode(vcpu)) {
977 978
		if (!(cr4 & X86_CR4_PAE))
			return 1;
979 980
	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
		   && ((cr4 ^ old_cr4) & pdptr_bits)
981 982
		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
				   kvm_read_cr3(vcpu)))
983 984
		return 1;

985
	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
986
		if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
987 988 989 990 991 992 993
			return 1;

		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
			return 1;
	}

994
	if (kvm_x86_ops.set_cr4(vcpu, cr4))
995
		return 1;
996

997 998
	if (((cr4 ^ old_cr4) & pdptr_bits) ||
	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
999
		kvm_mmu_reset_context(vcpu);
1000

1001
	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
1002
		kvm_update_cpuid(vcpu);
1003

1004 1005
	return 0;
}
1006
EXPORT_SYMBOL_GPL(kvm_set_cr4);
1007

1008
int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1009
{
1010
	bool skip_tlb_flush = false;
1011
#ifdef CONFIG_X86_64
1012 1013
	bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);

1014
	if (pcid_enabled) {
1015 1016
		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
		cr3 &= ~X86_CR3_PCID_NOFLUSH;
1017
	}
1018
#endif
1019

1020
	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
1021 1022
		if (!skip_tlb_flush) {
			kvm_mmu_sync_roots(vcpu);
1023
			kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1024
		}
1025
		return 0;
1026 1027
	}

1028
	if (is_long_mode(vcpu) &&
1029
	    (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63)))
1030
		return 1;
1031 1032
	else if (is_pae_paging(vcpu) &&
		 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
Nadav Amit's avatar
Nadav Amit committed
1033
		return 1;
1034

1035
	kvm_mmu_new_pgd(vcpu, cr3, skip_tlb_flush, skip_tlb_flush);
1036
	vcpu->arch.cr3 = cr3;
1037
	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
1038

1039 1040
	return 0;
}
1041
EXPORT_SYMBOL_GPL(kvm_set_cr3);
1042

Andre Przywara's avatar
Andre Przywara committed
1043
int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
1044
{
1045 1046
	if (cr8 & CR8_RESERVED_BITS)
		return 1;
1047
	if (lapic_in_kernel(vcpu))
1048 1049
		kvm_lapic_set_tpr(vcpu, cr8);
	else
1050
		vcpu->arch.cr8 = cr8;
1051 1052
	return 0;
}
1053
EXPORT_SYMBOL_GPL(kvm_set_cr8);
1054

1055
unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1056
{
1057
	if (lapic_in_kernel(vcpu))
1058 1059
		return kvm_lapic_get_cr8(vcpu);
	else
1060
		return vcpu->arch.cr8;
1061
}
1062
EXPORT_SYMBOL_GPL(kvm_get_cr8);
1063

1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
{
	int i;

	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
	}
}

1075
void kvm_update_dr7(struct kvm_vcpu *vcpu)
1076 1077 1078 1079 1080 1081 1082
{
	unsigned long dr7;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		dr7 = vcpu->arch.guest_debug_dr7;
	else
		dr7 = vcpu->arch.dr7;
1083
	kvm_x86_ops.set_dr7(vcpu, dr7);
1084 1085 1086
	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
	if (dr7 & DR7_BP_EN_MASK)
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1087
}
1088
EXPORT_SYMBOL_GPL(kvm_update_dr7);
1089

1090 1091 1092 1093
static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
{
	u64 fixed = DR6_FIXED_1;

1094
	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1095 1096 1097 1098
		fixed |= DR6_RTM;
	return fixed;
}

1099
static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1100
{
1101 1102
	size_t size = ARRAY_SIZE(vcpu->arch.db);

1103 1104
	switch (dr) {
	case 0 ... 3:
1105
		vcpu->arch.db[array_index_nospec(dr, size)] = val;
1106 1107 1108 1109 1110 1111
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = val;
		break;
	case 4:
		/* fall through */
	case 6:
1112 1113
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
1114
		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
1115 1116 1117 1118
		break;
	case 5:
		/* fall through */
	default: /* 7 */
1119
		if (!kvm_dr7_valid(val))
1120
			return -1; /* #GP */
1121
		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1122
		kvm_update_dr7(vcpu);
1123 1124 1125 1126 1127
		break;
	}

	return 0;
}
1128 1129 1130

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
1131
	if (__kvm_set_dr(vcpu, dr, val)) {
1132
		kvm_inject_gp(vcpu, 0);
1133 1134 1135
		return 1;
	}
	return 0;
1136
}
1137 1138
EXPORT_SYMBOL_GPL(kvm_set_dr);

1139
int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1140
{
1141 1142
	size_t size = ARRAY_SIZE(vcpu->arch.db);

1143 1144
	switch (dr) {
	case 0 ... 3:
1145
		*val = vcpu->arch.db[array_index_nospec(dr, size)];
1146 1147 1148 1149
		break;
	case 4:
		/* fall through */
	case 6:
1150
		*val = vcpu->arch.dr6;
1151 1152 1153 1154 1155 1156 1157
		break;
	case 5:
		/* fall through */
	default: /* 7 */
		*val = vcpu->arch.dr7;
		break;
	}
1158 1159
	return 0;
}
1160 1161
EXPORT_SYMBOL_GPL(kvm_get_dr);

Avi Kivity's avatar
Avi Kivity committed
1162 1163
bool kvm_rdpmc(struct kvm_vcpu *vcpu)
{
1164
	u32 ecx = kvm_rcx_read(vcpu);
Avi Kivity's avatar
Avi Kivity committed
1165 1166 1167
	u64 data;
	int err;

1168
	err = kvm_pmu_rdpmc(vcpu, ecx, &data);
Avi Kivity's avatar
Avi Kivity committed
1169 1170
	if (err)
		return err;
1171 1172
	kvm_rax_write(vcpu, (u32)data);
	kvm_rdx_write(vcpu, data >> 32);
Avi Kivity's avatar
Avi Kivity committed
1173 1174 1175 1176
	return err;
}
EXPORT_SYMBOL_GPL(kvm_rdpmc);

1177 1178 1179 1180
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
1181 1182 1183
 * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features)
 * extract the supported MSRs from the related const lists.
 * msrs_to_save is selected from the msrs_to_save_all to reflect the
1184
 * capabilities of the host cpu. This capabilities test skips MSRs that are
1185
 * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs
1186
 * may depend on host virtualization features rather than host cpu features.
1187
 */
1188

1189
static const u32 msrs_to_save_all[] = {
1190
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
Brian Gerst's avatar
Brian Gerst committed
1191
	MSR_STAR,
1192 1193 1194
#ifdef CONFIG_X86_64
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
1195
	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
1196
	MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
1197
	MSR_IA32_SPEC_CTRL,
1198 1199 1200 1201 1202 1203
	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
1204 1205
	MSR_IA32_UMWAIT_CONTROL,

1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
	MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
	MSR_ARCH_PERFMON_FIXED_CTR0 + 2, MSR_ARCH_PERFMON_FIXED_CTR0 + 3,
	MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
	MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
	MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
	MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
	MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
	MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
	MSR_ARCH_PERFMON_PERFCTR0 + 8, MSR_ARCH_PERFMON_PERFCTR0 + 9,
	MSR_ARCH_PERFMON_PERFCTR0 + 10, MSR_ARCH_PERFMON_PERFCTR0 + 11,
	MSR_ARCH_PERFMON_PERFCTR0 + 12, MSR_ARCH_PERFMON_PERFCTR0 + 13,
	MSR_ARCH_PERFMON_PERFCTR0 + 14, MSR_ARCH_PERFMON_PERFCTR0 + 15,
	MSR_ARCH_PERFMON_PERFCTR0 + 16, MSR_ARCH_PERFMON_PERFCTR0 + 17,
	MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
	MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
	MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
	MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
	MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
	MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
	MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
	MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
	MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
1228 1229
};

1230
static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_all)];
1231 1232
static unsigned num_msrs_to_save;

1233
static const u32 emulated_msrs_all[] = {
1234 1235 1236 1237
	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
1238
	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
1239 1240
	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
1241
	HV_X64_MSR_RESET,
1242
	HV_X64_MSR_VP_INDEX,
1243
	HV_X64_MSR_VP_RUNTIME,
1244
	HV_X64_MSR_SCONTROL,
1245
	HV_X64_MSR_STIMER0_CONFIG,
1246
	HV_X64_MSR_VP_ASSIST_PAGE,
1247 1248 1249 1250
	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
	HV_X64_MSR_TSC_EMULATION_STATUS,

	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
1251
	MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK,
1252

1253
	MSR_IA32_TSC_ADJUST,
1254
	MSR_IA32_TSCDEADLINE,
1255
	MSR_IA32_ARCH_CAPABILITIES,
1256
	MSR_IA32_MISC_ENABLE,
1257 1258
	MSR_IA32_MCG_STATUS,
	MSR_IA32_MCG_CTL,
1259
	MSR_IA32_MCG_EXT_CTL,
1260
	MSR_IA32_SMBASE,
1261
	MSR_SMI_COUNT,
1262 1263
	MSR_PLATFORM_INFO,
	MSR_MISC_FEATURES_ENABLES,
1264
	MSR_AMD64_VIRT_SPEC_CTRL,
1265
	MSR_IA32_POWER_CTL,
1266
	MSR_IA32_UCODE_REV,
1267

1268 1269 1270 1271 1272
	/*
	 * The following list leaves out MSRs whose values are determined
	 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
	 * We always support the "true" VMX control MSRs, even if the host
	 * processor does not, so I am putting these registers here rather
1273
	 * than in msrs_to_save_all.
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287
	 */
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1288
	MSR_K7_HWCR,
1289
	MSR_KVM_POLL_CONTROL,
1290 1291
};

1292
static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
1293 1294
static unsigned num_emulated_msrs;

1295 1296 1297 1298
/*
 * List of msr numbers which are used to expose MSR-based features that
 * can be used by a hypervisor to validate requested CPU features.
 */
1299
static const u32 msr_based_features_all[] = {
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR0_FIXED1,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_CR4_FIXED1,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1319
	MSR_F10H_DECFG,
1320
	MSR_IA32_UCODE_REV,
1321
	MSR_IA32_ARCH_CAPABILITIES,
1322 1323
};

1324
static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)];
1325 1326
static unsigned int num_msr_based_features;

1327
static u64 kvm_get_arch_capabilities(void)
1328
{
1329
	u64 data = 0;
1330

1331 1332
	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);
1333

1334 1335 1336 1337 1338 1339 1340 1341
	/*
	 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
	 * the nested hypervisor runs with NX huge pages.  If it is not,
	 * L1 is anyway vulnerable to ITLB_MULTIHIT explots from other
	 * L1 guests, so it need not worry about its own (L2) guests.
	 */
	data |= ARCH_CAP_PSCHANGE_MC_NO;

1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353
	/*
	 * If we're doing cache flushes (either "always" or "cond")
	 * we will do one whenever the guest does a vmlaunch/vmresume.
	 * If an outer hypervisor is doing the cache flush for us
	 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
	 * capability to the guest too, and if EPT is disabled we're not
	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
	 * require a nested hypervisor to do a flush of its own.
	 */
	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;

1354 1355 1356 1357 1358 1359 1360
	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
		data |= ARCH_CAP_RDCL_NO;
	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
		data |= ARCH_CAP_SSB_NO;
	if (!boot_cpu_has_bug(X86_BUG_MDS))
		data |= ARCH_CAP_MDS_NO;

1361
	/*
1362 1363 1364 1365
	 * On TAA affected systems:
	 *      - nothing to do if TSX is disabled on the host.
	 *      - we emulate TSX_CTRL if present on the host.
	 *	  This lets the guest use VERW to clear CPU buffers.
1366
	 */
1367
	if (!boot_cpu_has(X86_FEATURE_RTM))
1368
		data &= ~(ARCH_CAP_TAA_NO | ARCH_CAP_TSX_CTRL_MSR);
1369 1370
	else if (!boot_cpu_has_bug(X86_BUG_TAA))
		data |= ARCH_CAP_TAA_NO;
1371

1372 1373 1374
	return data;
}

1375 1376 1377
static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
{
	switch (msr->index) {
1378
	case MSR_IA32_ARCH_CAPABILITIES:
1379 1380 1381
		msr->data = kvm_get_arch_capabilities();
		break;
	case MSR_IA32_UCODE_REV:
1382
		rdmsrl_safe(msr->index, &msr->data);
1383
		break;
1384
	default:
1385
		if (kvm_x86_ops.get_msr_feature(msr))
1386 1387 1388 1389 1390
			return 1;
	}
	return 0;
}

1391 1392 1393
static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	struct kvm_msr_entry msr;
1394
	int r;
1395 1396

	msr.index = index;
1397 1398 1399
	r = kvm_get_msr_feature(&msr);
	if (r)
		return r;
1400 1401 1402 1403 1404 1405

	*data = msr.data;

	return 0;
}

1406
static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1407
{
1408
	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1409
		return false;
Alexander Graf's avatar
Alexander Graf committed
1410

1411
	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1412
		return false;
1413

1414 1415 1416 1417 1418 1419
	if (efer & (EFER_LME | EFER_LMA) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))
		return false;

	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
		return false;
1420

1421
	return true;
1422 1423 1424 1425 1426 1427 1428 1429

}
bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	if (efer & efer_reserved_bits)
		return false;

	return __kvm_valid_efer(vcpu, efer);
1430 1431 1432
}
EXPORT_SYMBOL_GPL(kvm_valid_efer);

1433
static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1434 1435
{
	u64 old_efer = vcpu->arch.efer;
1436
	u64 efer = msr_info->data;
1437

1438
	if (efer & efer_reserved_bits)
1439
		return 1;
1440

1441 1442 1443 1444 1445 1446 1447 1448
	if (!msr_info->host_initiated) {
		if (!__kvm_valid_efer(vcpu, efer))
			return 1;

		if (is_paging(vcpu) &&
		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
			return 1;
	}
1449

1450
	efer &= ~EFER_LMA;
1451
	efer |= vcpu->arch.efer & EFER_LMA;
1452

1453
	kvm_x86_ops.set_efer(vcpu, efer);
1454

1455 1456 1457 1458
	/* Update reserved bits */
	if ((efer ^ old_efer) & EFER_NX)
		kvm_mmu_reset_context(vcpu);

1459
	return 0;
1460 1461
}

1462 1463 1464 1465 1466 1467
void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);

1468
/*
1469 1470
 * Write @data into the MSR specified by @index.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
1471 1472 1473
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1474 1475
static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
			 bool host_initiated)
1476
{
1477 1478 1479
	struct msr_data msr;

	switch (index) {
1480 1481 1482 1483 1484
	case MSR_FS_BASE:
	case MSR_GS_BASE:
	case MSR_KERNEL_GS_BASE:
	case MSR_CSTAR:
	case MSR_LSTAR:
1485
		if (is_noncanonical_address(data, vcpu))
1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
			return 1;
		break;
	case MSR_IA32_SYSENTER_EIP:
	case MSR_IA32_SYSENTER_ESP:
		/*
		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
		 * non-canonical address is written on Intel but not on
		 * AMD (which ignores the top 32-bits, because it does
		 * not implement 64-bit SYSENTER).
		 *
		 * 64-bit code should hence be able to write a non-canonical
		 * value on AMD.  Making the address canonical ensures that
		 * vmentry does not fail on Intel after writing a non-canonical
		 * value, and that something deterministic happens if the guest
		 * invokes 64-bit SYSENTER.
		 */
1502
		data = get_canonical(data, vcpu_virt_addr_bits(vcpu));
1503
	}
1504 1505 1506 1507 1508

	msr.data = data;
	msr.index = index;
	msr.host_initiated = host_initiated;

1509
	return kvm_x86_ops.set_msr(vcpu, &msr);
1510 1511
}

1512
/*
1513 1514 1515 1516
 * Read the MSR specified by @index into @data.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
1517
 */
1518 1519
int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
		  bool host_initiated)
1520 1521
{
	struct msr_data msr;
1522
	int ret;
1523 1524

	msr.index = index;
1525
	msr.host_initiated = host_initiated;
1526

1527
	ret = kvm_x86_ops.get_msr(vcpu, &msr);
1528 1529 1530
	if (!ret)
		*data = msr.data;
	return ret;
1531 1532
}

1533
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1534
{
1535 1536 1537
	return __kvm_get_msr(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_get_msr);
1538

1539 1540 1541 1542 1543 1544
int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
{
	return __kvm_set_msr(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_set_msr);

1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579
int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data;

	if (kvm_get_msr(vcpu, ecx, &data)) {
		trace_kvm_msr_read_ex(ecx);
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

	trace_kvm_msr_read(ecx, data);

	kvm_rax_write(vcpu, data & -1u);
	kvm_rdx_write(vcpu, (data >> 32) & -1u);
	return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);

int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data = kvm_read_edx_eax(vcpu);

	if (kvm_set_msr(vcpu, ecx, data)) {
		trace_kvm_msr_write_ex(ecx, data);
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

	trace_kvm_msr_write(ecx, data);
	return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);

1580 1581 1582 1583 1584 1585 1586
bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
{
	return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||
		need_resched() || signal_pending(current);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_exit_request);

1587 1588 1589 1590 1591 1592 1593 1594 1595
/*
 * The fast path for frequent and performance sensitive wrmsr emulation,
 * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces
 * the latency of virtual IPI by avoiding the expensive bits of transitioning
 * from guest to host, e.g. reacquiring KVM's SRCU lock. In contrast to the
 * other cases which must be called after interrupts are enabled on the host.
 */
static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
{
1596 1597 1598 1599
	if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
		return 1;

	if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) &&
1600
		((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) &&
1601 1602
		((data & APIC_MODE_MASK) == APIC_DM_FIXED) &&
		((u32)(data >> 32) != X2APIC_BROADCAST)) {
1603

1604 1605
		data &= ~(1 << 12);
		kvm_apic_send_ipi(vcpu->arch.apic, (u32)data, (u32)(data >> 32));
1606
		kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR2, (u32)(data >> 32));
1607 1608 1609
		kvm_lapic_set_reg(vcpu->arch.apic, APIC_ICR, (u32)data);
		trace_kvm_apic_write(APIC_ICR, (u32)data);
		return 0;
1610 1611 1612 1613 1614
	}

	return 1;
}

1615 1616 1617 1618 1619 1620 1621 1622 1623
static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)
{
	if (!kvm_can_use_hv_timer(vcpu))
		return 1;

	kvm_set_lapic_tscdeadline_msr(vcpu, data);
	return 0;
}

1624
fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
1625 1626
{
	u32 msr = kvm_rcx_read(vcpu);
1627
	u64 data;
1628
	fastpath_t ret = EXIT_FASTPATH_NONE;
1629 1630 1631

	switch (msr) {
	case APIC_BASE_MSR + (APIC_ICR >> 4):
1632
		data = kvm_read_edx_eax(vcpu);
1633 1634 1635
		if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {
			kvm_skip_emulated_instruction(vcpu);
			ret = EXIT_FASTPATH_EXIT_HANDLED;
1636
		}
1637
		break;
1638 1639 1640 1641 1642 1643 1644
	case MSR_IA32_TSCDEADLINE:
		data = kvm_read_edx_eax(vcpu);
		if (!handle_fastpath_set_tscdeadline(vcpu, data)) {
			kvm_skip_emulated_instruction(vcpu);
			ret = EXIT_FASTPATH_REENTER_GUEST;
		}
		break;
1645
	default:
1646
		break;
1647 1648
	}

1649
	if (ret != EXIT_FASTPATH_NONE)
1650 1651
		trace_kvm_msr_write(msr, data);

1652
	return ret;
1653 1654 1655
}
EXPORT_SYMBOL_GPL(handle_fastpath_set_msr_irqoff);

1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return __kvm_get_msr(vcpu, index, data, true);
}

static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return __kvm_set_msr(vcpu, index, *data, true);
1667 1668
}

1669
#ifdef CONFIG_X86_64
1670 1671 1672 1673 1674 1675
struct pvclock_clock {
	int vclock_mode;
	u64 cycle_last;
	u64 mask;
	u32 mult;
	u32 shift;
1676 1677
	u64 base_cycles;
	u64 offset;
1678 1679
};

1680 1681 1682
struct pvclock_gtod_data {
	seqcount_t	seq;

1683 1684
	struct pvclock_clock clock; /* extract of a clocksource struct */
	struct pvclock_clock raw_clock; /* extract of a clocksource struct */
1685

1686
	ktime_t		offs_boot;
1687
	u64		wall_time_sec;
1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698
};

static struct pvclock_gtod_data pvclock_gtod_data;

static void update_pvclock_gtod(struct timekeeper *tk)
{
	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;

	write_seqcount_begin(&vdata->seq);

	/* copy pvclock gtod data */
1699
	vdata->clock.vclock_mode	= tk->tkr_mono.clock->vdso_clock_mode;
1700 1701 1702 1703
	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
	vdata->clock.mask		= tk->tkr_mono.mask;
	vdata->clock.mult		= tk->tkr_mono.mult;
	vdata->clock.shift		= tk->tkr_mono.shift;
1704 1705
	vdata->clock.base_cycles	= tk->tkr_mono.xtime_nsec;
	vdata->clock.offset		= tk->tkr_mono.base;
1706

1707
	vdata->raw_clock.vclock_mode	= tk->tkr_raw.clock->vdso_clock_mode;
1708 1709 1710 1711
	vdata->raw_clock.cycle_last	= tk->tkr_raw.cycle_last;
	vdata->raw_clock.mask		= tk->tkr_raw.mask;
	vdata->raw_clock.mult		= tk->tkr_raw.mult;
	vdata->raw_clock.shift		= tk->tkr_raw.shift;
1712 1713
	vdata->raw_clock.base_cycles	= tk->tkr_raw.xtime_nsec;
	vdata->raw_clock.offset		= tk->tkr_raw.base;
1714

1715 1716
	vdata->wall_time_sec            = tk->xtime_sec;

1717
	vdata->offs_boot		= tk->offs_boot;
1718

1719 1720
	write_seqcount_end(&vdata->seq);
}
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732

static s64 get_kvmclock_base_ns(void)
{
	/* Count up from boot time, but with the frequency of the raw clock.  */
	return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
}
#else
static s64 get_kvmclock_base_ns(void)
{
	/* Master clock not used, so we can just use CLOCK_BOOTTIME.  */
	return ktime_get_boottime_ns();
}
1733 1734
#endif

1735 1736 1737
void kvm_set_pending_timer(struct kvm_vcpu *vcpu)
{
	kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
1738
	kvm_vcpu_kick(vcpu);
1739
}
1740

1741 1742
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
{
1743 1744
	int version;
	int r;
1745
	struct pvclock_wall_clock wc;
1746
	u64 wall_nsec;
1747 1748 1749 1750

	if (!wall_clock)
		return;

1751 1752 1753 1754 1755 1756 1757 1758
	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
	if (r)
		return;

	if (version & 1)
		++version;  /* first time write, random junk */

	++version;
1759

1760 1761
	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
		return;
1762

1763 1764
	/*
	 * The guest calculates current wall clock time by adding
1765
	 * system time (updated by kvm_guest_time_update below) to the
1766
	 * wall clock specified here.  We do the reverse here.
1767
	 */
1768
	wall_nsec = ktime_get_real_ns() - get_kvmclock_ns(kvm);
1769

1770 1771
	wc.nsec = do_div(wall_nsec, 1000000000);
	wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */
1772
	wc.version = version;
1773 1774 1775 1776 1777 1778 1779

	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));

	version++;
	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
}

1780 1781
static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
1782 1783
	do_shl32_div32(dividend, divisor);
	return dividend;
1784 1785
}

1786
static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
1787
			       s8 *pshift, u32 *pmultiplier)
1788
{
1789
	uint64_t scaled64;
1790 1791 1792 1793
	int32_t  shift = 0;
	uint64_t tps64;
	uint32_t tps32;

1794 1795
	tps64 = base_hz;
	scaled64 = scaled_hz;
1796
	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
1797 1798 1799 1800 1801
		tps64 >>= 1;
		shift--;
	}

	tps32 = (uint32_t)tps64;
1802 1803
	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
1804 1805 1806
			scaled64 >>= 1;
		else
			tps32 <<= 1;
1807 1808 1809
		shift++;
	}

1810 1811
	*pshift = shift;
	*pmultiplier = div_frac(scaled64, tps32);
1812 1813
}

1814
#ifdef CONFIG_X86_64
1815
static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
1816
#endif
1817

1818
static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
1819
static unsigned long max_tsc_khz;
1820

1821
static u32 adjust_tsc_khz(u32 khz, s32 ppm)
1822
{
1823 1824 1825
	u64 v = (u64)khz * (1000000 + ppm);
	do_div(v, 1000000);
	return v;
1826 1827
}

1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844
static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
{
	u64 ratio;

	/* Guest TSC same frequency as host TSC? */
	if (!scale) {
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
		return 0;
	}

	/* TSC scaling supported? */
	if (!kvm_has_tsc_control) {
		if (user_tsc_khz > tsc_khz) {
			vcpu->arch.tsc_catchup = 1;
			vcpu->arch.tsc_always_catchup = 1;
			return 0;
		} else {
1845
			pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
1846 1847 1848 1849 1850 1851 1852 1853 1854
			return -1;
		}
	}

	/* TSC scaling required  - calculate ratio */
	ratio = mul_u64_u32_div(1ULL << kvm_tsc_scaling_ratio_frac_bits,
				user_tsc_khz, tsc_khz);

	if (ratio == 0 || ratio >= kvm_max_tsc_scaling_ratio) {
1855 1856
		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
			            user_tsc_khz);
1857 1858 1859 1860 1861 1862 1863
		return -1;
	}

	vcpu->arch.tsc_scaling_ratio = ratio;
	return 0;
}

1864
static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
1865
{
1866 1867
	u32 thresh_lo, thresh_hi;
	int use_scaling = 0;
1868

1869
	/* tsc_khz can be zero if TSC calibration fails */
1870
	if (user_tsc_khz == 0) {
1871 1872
		/* set tsc_scaling_ratio to a safe value */
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
1873
		return -1;
1874
	}
1875

Zachary Amsden's avatar
Zachary Amsden committed
1876
	/* Compute a scale to convert nanoseconds in TSC cycles */
1877
	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
1878 1879
			   &vcpu->arch.virtual_tsc_shift,
			   &vcpu->arch.virtual_tsc_mult);
1880
	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
1881 1882 1883 1884 1885 1886 1887 1888 1889

	/*
	 * Compute the variation in TSC rate which is acceptable
	 * within the range of tolerance and decide if the
	 * rate being applied is within that bounds of the hardware
	 * rate.  If so, no scaling or compensation need be done.
	 */
	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
1890 1891
	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", user_tsc_khz, thresh_lo, thresh_hi);
1892 1893
		use_scaling = 1;
	}
1894
	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
Zachary Amsden's avatar
Zachary Amsden committed
1895 1896 1897 1898
}

static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
1899
	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
1900 1901
				      vcpu->arch.virtual_tsc_mult,
				      vcpu->arch.virtual_tsc_shift);
1902
	tsc += vcpu->arch.this_tsc_write;
Zachary Amsden's avatar
Zachary Amsden committed
1903 1904 1905
	return tsc;
}

1906 1907
static inline int gtod_is_based_on_tsc(int mode)
{
1908
	return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK;
1909 1910
}

1911
static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
1912 1913 1914 1915 1916 1917 1918 1919 1920
{
#ifdef CONFIG_X86_64
	bool vcpus_matched;
	struct kvm_arch *ka = &vcpu->kvm->arch;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			 atomic_read(&vcpu->kvm->online_vcpus));

1921 1922 1923 1924 1925 1926 1927 1928 1929
	/*
	 * Once the masterclock is enabled, always perform request in
	 * order to update it.
	 *
	 * In order to enable masterclock, the host clocksource must be TSC
	 * and the vcpus need to have matched TSCs.  When that happens,
	 * perform request to enable masterclock.
	 */
	if (ka->use_master_clock ||
1930
	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
1931 1932 1933 1934 1935 1936 1937 1938
		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);

	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
			    atomic_read(&vcpu->kvm->online_vcpus),
		            ka->use_master_clock, gtod->clock.vclock_mode);
#endif
}

1939 1940
static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
{
1941
	u64 curr_offset = vcpu->arch.l1_tsc_offset;
1942 1943 1944
	vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
}

1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
/*
 * Multiply tsc by a fixed point number represented by ratio.
 *
 * The most significant 64-N bits (mult) of ratio represent the
 * integral part of the fixed point number; the remaining N bits
 * (frac) represent the fractional part, ie. ratio represents a fixed
 * point number (mult + frac * 2^(-N)).
 *
 * N equals to kvm_tsc_scaling_ratio_frac_bits.
 */
static inline u64 __scale_tsc(u64 ratio, u64 tsc)
{
	return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits);
}

u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc)
{
	u64 _tsc = tsc;
	u64 ratio = vcpu->arch.tsc_scaling_ratio;

	if (ratio != kvm_default_tsc_scaling_ratio)
		_tsc = __scale_tsc(ratio, tsc);

	return _tsc;
}
EXPORT_SYMBOL_GPL(kvm_scale_tsc);

1972 1973 1974 1975 1976 1977 1978 1979 1980
static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
{
	u64 tsc;

	tsc = kvm_scale_tsc(vcpu, rdtsc());

	return target_tsc - tsc;
}

1981 1982
u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
{
1983
	return vcpu->arch.l1_tsc_offset + kvm_scale_tsc(vcpu, host_tsc);
1984 1985 1986
}
EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);

1987 1988
static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
{
1989
	vcpu->arch.l1_tsc_offset = offset;
1990
	vcpu->arch.tsc_offset = kvm_x86_ops.write_l1_tsc_offset(vcpu, offset);
1991 1992
}

1993 1994 1995 1996 1997 1998 1999
static inline bool kvm_check_tsc_unstable(void)
{
#ifdef CONFIG_X86_64
	/*
	 * TSC is marked unstable when we're running on Hyper-V,
	 * 'TSC page' clocksource is good.
	 */
2000
	if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK)
2001 2002 2003 2004 2005
		return false;
#endif
	return check_tsc_unstable();
}

2006
void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
2007 2008
{
	struct kvm *kvm = vcpu->kvm;
2009
	u64 offset, ns, elapsed;
2010
	unsigned long flags;
2011
	bool matched;
Tomasz Grabiec's avatar
Tomasz Grabiec committed
2012
	bool already_matched;
2013
	u64 data = msr->data;
2014
	bool synchronizing = false;
2015

2016
	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
2017
	offset = kvm_compute_tsc_offset(vcpu, data);
2018
	ns = get_kvmclock_base_ns();
2019
	elapsed = ns - kvm->arch.last_tsc_nsec;
2020

2021
	if (vcpu->arch.virtual_tsc_khz) {
2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040
		if (data == 0 && msr->host_initiated) {
			/*
			 * detection of vcpu initialization -- need to sync
			 * with other vCPUs. This particularly helps to keep
			 * kvm_clock stable after CPU hotplug
			 */
			synchronizing = true;
		} else {
			u64 tsc_exp = kvm->arch.last_tsc_write +
						nsec_to_cycles(vcpu, elapsed);
			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
			/*
			 * Special case: TSC write with a small delta (1 second)
			 * of virtual cycle time against real time is
			 * interpreted as an attempt to synchronize the CPU.
			 */
			synchronizing = data < tsc_exp + tsc_hz &&
					data + tsc_hz > tsc_exp;
		}
2041
	}
2042 2043

	/*
2044 2045 2046 2047 2048
	 * For a reliable TSC, we can match TSC offsets, and for an unstable
	 * TSC, we add elapsed time in this computation.  We could let the
	 * compensation code attempt to catch up if we fall behind, but
	 * it's better to try to match offsets from the beginning.
         */
2049
	if (synchronizing &&
2050
	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
2051
		if (!kvm_check_tsc_unstable()) {
2052
			offset = kvm->arch.cur_tsc_offset;
2053
		} else {
2054
			u64 delta = nsec_to_cycles(vcpu, elapsed);
2055
			data += delta;
2056
			offset = kvm_compute_tsc_offset(vcpu, data);
2057
		}
2058
		matched = true;
Tomasz Grabiec's avatar
Tomasz Grabiec committed
2059
		already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
2060 2061 2062 2063 2064 2065
	} else {
		/*
		 * We split periods of matched TSC writes into generations.
		 * For each generation, we track the original measured
		 * nanosecond time, offset, and write, so if TSCs are in
		 * sync, we can match exact offset, and if not, we can match
Guo Chao's avatar
Guo Chao committed
2066
		 * exact software computation in compute_guest_tsc()
2067 2068 2069 2070 2071 2072 2073
		 *
		 * These values are tracked in kvm->arch.cur_xxx variables.
		 */
		kvm->arch.cur_tsc_generation++;
		kvm->arch.cur_tsc_nsec = ns;
		kvm->arch.cur_tsc_write = data;
		kvm->arch.cur_tsc_offset = offset;
2074
		matched = false;
2075
	}
2076 2077 2078 2079 2080

	/*
	 * We also track th most recent recorded KHZ, write and time to
	 * allow the matching interval to be extended at each write.
	 */
2081 2082
	kvm->arch.last_tsc_nsec = ns;
	kvm->arch.last_tsc_write = data;
2083
	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
2084

2085
	vcpu->arch.last_guest_tsc = data;
2086 2087 2088 2089 2090 2091

	/* Keep track of which generation this VCPU has synchronized to */
	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;

2092
	if (!msr->host_initiated && guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST))
2093
		update_ia32_tsc_adjust_msr(vcpu, offset);
2094

2095
	kvm_vcpu_write_tsc_offset(vcpu, offset);
2096
	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
2097 2098

	spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
Tomasz Grabiec's avatar
Tomasz Grabiec committed
2099
	if (!matched) {
2100
		kvm->arch.nr_vcpus_matched_tsc = 0;
Tomasz Grabiec's avatar
Tomasz Grabiec committed
2101 2102 2103
	} else if (!already_matched) {
		kvm->arch.nr_vcpus_matched_tsc++;
	}
2104 2105 2106

	kvm_track_tsc_matching(vcpu);
	spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
2107
}
2108

2109 2110
EXPORT_SYMBOL_GPL(kvm_write_tsc);

2111 2112 2113
static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
					   s64 adjustment)
{
2114
	u64 tsc_offset = vcpu->arch.l1_tsc_offset;
2115
	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2116 2117 2118 2119 2120 2121 2122
}

static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
{
	if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
		WARN_ON(adjustment < 0);
	adjustment = kvm_scale_tsc(vcpu, (u64) adjustment);
2123
	adjust_tsc_offset_guest(vcpu, adjustment);
2124 2125
}

2126 2127
#ifdef CONFIG_X86_64

2128
static u64 read_tsc(void)
2129
{
2130
	u64 ret = (u64)rdtsc_ordered();
2131
	u64 last = pvclock_gtod_data.clock.cycle_last;
2132 2133 2134 2135 2136 2137

	if (likely(ret >= last))
		return ret;

	/*
	 * GCC likes to generate cmov here, but this branch is extremely
2138
	 * predictable (it's just a function of time and the likely is
2139 2140 2141 2142 2143 2144 2145 2146 2147
	 * very likely) and there's a data dependence, so force GCC
	 * to generate a branch instead.  I don't barrier() because
	 * we don't actually need a barrier, and if this function
	 * ever gets inlined it will generate worse code.
	 */
	asm volatile ("");
	return last;
}

2148 2149
static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
			  int *mode)
2150 2151
{
	long v;
2152 2153
	u64 tsc_pg_val;

2154
	switch (clock->vclock_mode) {
2155
	case VDSO_CLOCKMODE_HVCLOCK:
2156 2157 2158 2159
		tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
						  tsc_timestamp);
		if (tsc_pg_val != U64_MAX) {
			/* TSC page valid */
2160
			*mode = VDSO_CLOCKMODE_HVCLOCK;
2161 2162
			v = (tsc_pg_val - clock->cycle_last) &
				clock->mask;
2163 2164
		} else {
			/* TSC page invalid */
2165
			*mode = VDSO_CLOCKMODE_NONE;
2166 2167
		}
		break;
2168 2169
	case VDSO_CLOCKMODE_TSC:
		*mode = VDSO_CLOCKMODE_TSC;
2170
		*tsc_timestamp = read_tsc();
2171 2172
		v = (*tsc_timestamp - clock->cycle_last) &
			clock->mask;
2173 2174
		break;
	default:
2175
		*mode = VDSO_CLOCKMODE_NONE;
2176
	}
2177

2178
	if (*mode == VDSO_CLOCKMODE_NONE)
2179
		*tsc_timestamp = v = 0;
2180

2181
	return v * clock->mult;
2182 2183
}

2184
static int do_monotonic_raw(s64 *t, u64 *tsc_timestamp)
2185
{
2186
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2187 2188
	unsigned long seq;
	int mode;
2189
	u64 ns;
2190 2191 2192

	do {
		seq = read_seqcount_begin(&gtod->seq);
2193
		ns = gtod->raw_clock.base_cycles;
2194
		ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
2195 2196
		ns >>= gtod->raw_clock.shift;
		ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
2197
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2198
	*t = ns;
2199 2200 2201 2202

	return mode;
}

2203
static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
2204 2205 2206 2207 2208 2209 2210 2211 2212
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	unsigned long seq;
	int mode;
	u64 ns;

	do {
		seq = read_seqcount_begin(&gtod->seq);
		ts->tv_sec = gtod->wall_time_sec;
2213
		ns = gtod->clock.base_cycles;
2214
		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
2215 2216 2217 2218 2219 2220 2221 2222 2223
		ns >>= gtod->clock.shift;
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));

	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return mode;
}

2224 2225
/* returns true if host is using TSC based clocksource */
static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
2226 2227
{
	/* checked again under seqlock below */
2228
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2229 2230
		return false;

2231
	return gtod_is_based_on_tsc(do_monotonic_raw(kernel_ns,
2232
						      tsc_timestamp));
2233
}
2234

2235
/* returns true if host is using TSC based clocksource */
2236
static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
2237
					   u64 *tsc_timestamp)
2238 2239
{
	/* checked again under seqlock below */
2240
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2241 2242
		return false;

2243
	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
2244
}
2245 2246 2247 2248
#endif

/*
 *
2249 2250 2251
 * Assuming a stable TSC across physical CPUS, and a stable TSC
 * across virtual CPUs, the following condition is possible.
 * Each numbered line represents an event visible to both
2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283
 * CPUs at the next numbered event.
 *
 * "timespecX" represents host monotonic time. "tscX" represents
 * RDTSC value.
 *
 * 		VCPU0 on CPU0		|	VCPU1 on CPU1
 *
 * 1.  read timespec0,tsc0
 * 2.					| timespec1 = timespec0 + N
 * 					| tsc1 = tsc0 + M
 * 3. transition to guest		| transition to guest
 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
 * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
 * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
 *
 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
 *
 * 	- ret0 < ret1
 *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
 *		...
 *	- 0 < N - M => M < N
 *
 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
 * always the case (the difference between two distinct xtime instances
 * might be smaller then the difference between corresponding TSC reads,
 * when updating guest vcpus pvclock areas).
 *
 * To avoid that problem, do not allow visibility of distinct
 * system_timestamp/tsc_timestamp values simultaneously: use a master
 * copy of host monotonic time values. Update that master copy
 * in lockstep.
 *
2284
 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
2285 2286 2287 2288 2289 2290 2291 2292
 *
 */

static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	struct kvm_arch *ka = &kvm->arch;
	int vclock_mode;
2293 2294 2295 2296
	bool host_tsc_clocksource, vcpus_matched;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			atomic_read(&kvm->online_vcpus));
2297 2298 2299 2300 2301

	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
2302
	host_tsc_clocksource = kvm_get_time_and_clockread(
2303 2304 2305
					&ka->master_kernel_ns,
					&ka->master_cycle_now);

2306
	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
2307
				&& !ka->backwards_tsc_observed
2308
				&& !ka->boot_vcpu_runs_old_kvmclock;
2309

2310 2311 2312 2313
	if (ka->use_master_clock)
		atomic_set(&kvm_guest_has_master_clock, 1);

	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
2314 2315
	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
					vcpus_matched);
2316 2317 2318
#endif
}

2319 2320 2321 2322 2323
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
}

2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336
static void kvm_gen_update_masterclock(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	int i;
	struct kvm_vcpu *vcpu;
	struct kvm_arch *ka = &kvm->arch;

	spin_lock(&ka->pvclock_gtod_sync_lock);
	kvm_make_mclock_inprogress_request(kvm);
	/* no guest entries from this point */
	pvclock_update_vm_gtod_copy(kvm);

	kvm_for_each_vcpu(i, vcpu, kvm)
2337
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2338 2339 2340

	/* guest entries allowed */
	kvm_for_each_vcpu(i, vcpu, kvm)
2341
		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
2342 2343 2344 2345 2346

	spin_unlock(&ka->pvclock_gtod_sync_lock);
#endif
}

2347
u64 get_kvmclock_ns(struct kvm *kvm)
2348 2349
{
	struct kvm_arch *ka = &kvm->arch;
2350
	struct pvclock_vcpu_time_info hv_clock;
2351
	u64 ret;
2352

2353 2354 2355
	spin_lock(&ka->pvclock_gtod_sync_lock);
	if (!ka->use_master_clock) {
		spin_unlock(&ka->pvclock_gtod_sync_lock);
2356
		return get_kvmclock_base_ns() + ka->kvmclock_offset;
2357 2358
	}

2359 2360 2361 2362
	hv_clock.tsc_timestamp = ka->master_cycle_now;
	hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
	spin_unlock(&ka->pvclock_gtod_sync_lock);

2363 2364 2365
	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
	get_cpu();

2366 2367 2368 2369 2370 2371
	if (__this_cpu_read(cpu_tsc_khz)) {
		kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL,
				   &hv_clock.tsc_shift,
				   &hv_clock.tsc_to_system_mul);
		ret = __pvclock_read_cycles(&hv_clock, rdtsc());
	} else
2372
		ret = get_kvmclock_base_ns() + ka->kvmclock_offset;
2373 2374 2375 2376

	put_cpu();

	return ret;
2377 2378
}

2379 2380 2381 2382 2383
static void kvm_setup_pvclock_page(struct kvm_vcpu *v)
{
	struct kvm_vcpu_arch *vcpu = &v->arch;
	struct pvclock_vcpu_time_info guest_hv_clock;

2384
	if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403
		&guest_hv_clock, sizeof(guest_hv_clock))))
		return;

	/* This VCPU is paused, but it's legal for a guest to read another
	 * VCPU's kvmclock, so we really have to follow the specification where
	 * it says that version is odd if data is being modified, and even after
	 * it is consistent.
	 *
	 * Version field updates must be kept separate.  This is because
	 * kvm_write_guest_cached might use a "rep movs" instruction, and
	 * writes within a string instruction are weakly ordered.  So there
	 * are three writes overall.
	 *
	 * As a small optimization, only write the version field in the first
	 * and third write.  The vcpu->pv_time cache is still valid, because the
	 * version field is the first in the struct.
	 */
	BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);

2404 2405 2406
	if (guest_hv_clock.version & 1)
		++guest_hv_clock.version;  /* first time write, random junk */

2407
	vcpu->hv_clock.version = guest_hv_clock.version + 1;
2408 2409 2410
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423

	smp_wmb();

	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
	vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);

	if (vcpu->pvclock_set_guest_stopped_request) {
		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
		vcpu->pvclock_set_guest_stopped_request = false;
	}

	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);

2424 2425 2426
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock));
2427 2428 2429 2430

	smp_wmb();

	vcpu->hv_clock.version++;
2431 2432 2433
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2434 2435
}

2436
static int kvm_guest_time_update(struct kvm_vcpu *v)
2437
{
2438
	unsigned long flags, tgt_tsc_khz;
2439
	struct kvm_vcpu_arch *vcpu = &v->arch;
2440
	struct kvm_arch *ka = &v->kvm->arch;
2441
	s64 kernel_ns;
2442
	u64 tsc_timestamp, host_tsc;
2443
	u8 pvclock_flags;
2444 2445 2446 2447
	bool use_master_clock;

	kernel_ns = 0;
	host_tsc = 0;
2448

2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459
	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
	spin_lock(&ka->pvclock_gtod_sync_lock);
	use_master_clock = ka->use_master_clock;
	if (use_master_clock) {
		host_tsc = ka->master_cycle_now;
		kernel_ns = ka->master_kernel_ns;
	}
	spin_unlock(&ka->pvclock_gtod_sync_lock);
2460 2461 2462

	/* Keep irq disabled to prevent changes to the clock */
	local_irq_save(flags);
2463 2464
	tgt_tsc_khz = __this_cpu_read(cpu_tsc_khz);
	if (unlikely(tgt_tsc_khz == 0)) {
2465 2466 2467 2468
		local_irq_restore(flags);
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
		return 1;
	}
2469
	if (!use_master_clock) {
2470
		host_tsc = rdtsc();
2471
		kernel_ns = get_kvmclock_base_ns();
2472 2473
	}

2474
	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
2475

Zachary Amsden's avatar
Zachary Amsden committed
2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488
	/*
	 * We may have to catch up the TSC to match elapsed wall clock
	 * time for two reasons, even if kvmclock is used.
	 *   1) CPU could have been running below the maximum TSC rate
	 *   2) Broken TSC compensation resets the base at each VCPU
	 *      entry to avoid unknown leaps of TSC even when running
	 *      again on the same CPU.  This may cause apparent elapsed
	 *      time to disappear, and the guest to stand still or run
	 *	very slowly.
	 */
	if (vcpu->tsc_catchup) {
		u64 tsc = compute_guest_tsc(v, kernel_ns);
		if (tsc > tsc_timestamp) {
2489
			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
Zachary Amsden's avatar
Zachary Amsden committed
2490 2491
			tsc_timestamp = tsc;
		}
2492 2493
	}

2494 2495
	local_irq_restore(flags);

2496
	/* With all the info we got, fill in the values */
2497

2498 2499 2500 2501
	if (kvm_has_tsc_control)
		tgt_tsc_khz = kvm_scale_tsc(v, tgt_tsc_khz);

	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
2502
		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
2503 2504
				   &vcpu->hv_clock.tsc_shift,
				   &vcpu->hv_clock.tsc_to_system_mul);
2505
		vcpu->hw_tsc_khz = tgt_tsc_khz;
2506 2507
	}

2508
	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
2509
	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
Zachary Amsden's avatar
Zachary Amsden committed
2510
	vcpu->last_guest_tsc = tsc_timestamp;
2511

2512
	/* If the host uses TSC clocksource, then it is stable */
2513
	pvclock_flags = 0;
2514 2515 2516
	if (use_master_clock)
		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;

2517 2518
	vcpu->hv_clock.flags = pvclock_flags;

2519 2520 2521 2522
	if (vcpu->pv_time_enabled)
		kvm_setup_pvclock_page(v);
	if (v == kvm_get_vcpu(v->kvm, 0))
		kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
2523
	return 0;
2524 2525
}

2526 2527 2528 2529 2530 2531 2532 2533
/*
 * kvmclock updates which are isolated to a given vcpu, such as
 * vcpu->cpu migration, should not allow system_timestamp from
 * the rest of the vcpus to remain static. Otherwise ntp frequency
 * correction applies to one vcpu's system_timestamp but not
 * the others.
 *
 * So in those cases, request a kvmclock update for all vcpus.
2534 2535 2536 2537
 * We need to rate-limit these requests though, as they can
 * considerably slow guests that have a large number of vcpus.
 * The time for a remote vcpu to update its kvmclock is bound
 * by the delay we use to rate-limit the updates.
2538 2539
 */

2540 2541 2542
#define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)

static void kvmclock_update_fn(struct work_struct *work)
2543 2544
{
	int i;
2545 2546 2547 2548
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_update_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);
2549 2550 2551
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm) {
2552
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2553 2554 2555 2556
		kvm_vcpu_kick(vcpu);
	}
}

2557 2558 2559 2560
static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
{
	struct kvm *kvm = v->kvm;

2561
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
2562 2563 2564 2565
	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
					KVMCLOCK_UPDATE_DELAY);
}

2566 2567 2568 2569 2570 2571 2572 2573 2574
#define KVMCLOCK_SYNC_PERIOD (300 * HZ)

static void kvmclock_sync_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_sync_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);

2575 2576 2577
	if (!kvmclock_periodic_sync)
		return;

2578 2579 2580 2581 2582
	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
}

2583 2584 2585 2586 2587 2588
/*
 * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
 */
static bool can_set_mci_status(struct kvm_vcpu *vcpu)
{
	/* McStatusWrEn enabled? */
2589
	if (guest_cpuid_is_amd_or_hygon(vcpu))
2590 2591 2592 2593 2594
		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));

	return false;
}

2595
static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2596
{
Huang Ying's avatar
Huang Ying committed
2597 2598
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2599 2600
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
Huang Ying's avatar
Huang Ying committed
2601

2602 2603
	switch (msr) {
	case MSR_IA32_MCG_STATUS:
Huang Ying's avatar
Huang Ying committed
2604
		vcpu->arch.mcg_status = data;
2605
		break;
2606
	case MSR_IA32_MCG_CTL:
2607 2608
		if (!(mcg_cap & MCG_CTL_P) &&
		    (data || !msr_info->host_initiated))
Huang Ying's avatar
Huang Ying committed
2609 2610
			return 1;
		if (data != 0 && data != ~(u64)0)
2611
			return 1;
Huang Ying's avatar
Huang Ying committed
2612 2613 2614 2615
		vcpu->arch.mcg_ctl = data;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2616
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
2617 2618 2619 2620
			u32 offset = array_index_nospec(
				msr - MSR_IA32_MC0_CTL,
				MSR_IA32_MCx_CTL(bank_num) - MSR_IA32_MC0_CTL);

2621 2622 2623 2624 2625
			/* only 0 or all 1s can be written to IA32_MCi_CTL
			 * some Linux kernels though clear bit 10 in bank 4 to
			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
			 * this to avoid an uncatched #GP in the guest
			 */
Huang Ying's avatar
Huang Ying committed
2626
			if ((offset & 0x3) == 0 &&
2627
			    data != 0 && (data | (1 << 10)) != ~(u64)0)
Huang Ying's avatar
Huang Ying committed
2628
				return -1;
2629 2630

			/* MCi_STATUS */
2631
			if (!msr_info->host_initiated &&
2632 2633 2634 2635 2636
			    (offset & 0x3) == 1 && data != 0) {
				if (!can_set_mci_status(vcpu))
					return -1;
			}

Huang Ying's avatar
Huang Ying committed
2637 2638 2639 2640 2641 2642 2643 2644
			vcpu->arch.mce_banks[offset] = data;
			break;
		}
		return 1;
	}
	return 0;
}

Ed Swierk's avatar
Ed Swierk committed
2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661
static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
	int lm = is_long_mode(vcpu);
	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
		: kvm->arch.xen_hvm_config.blob_size_32;
	u32 page_num = data & ~PAGE_MASK;
	u64 page_addr = data & PAGE_MASK;
	u8 *page;
	int r;

	r = -E2BIG;
	if (page_num >= blob_size)
		goto out;
	r = -ENOMEM;
2662 2663 2664
	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
	if (IS_ERR(page)) {
		r = PTR_ERR(page);
Ed Swierk's avatar
Ed Swierk committed
2665
		goto out;
2666
	}
2667
	if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE))
Ed Swierk's avatar
Ed Swierk committed
2668 2669 2670 2671 2672 2673 2674 2675
		goto out_free;
	r = 0;
out_free:
	kfree(page);
out:
	return r;
}

2676 2677 2678 2679 2680 2681 2682
static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
{
	u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT;

	return (vcpu->arch.apf.msr_en_val & mask) == mask;
}

2683 2684 2685 2686
static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
{
	gpa_t gpa = data & ~0x3f;

2687 2688
	/* Bits 4:5 are reserved, Should be zero */
	if (data & 0x30)
2689 2690
		return 1;

2691
	vcpu->arch.apf.msr_en_val = data;
2692

2693
	if (!kvm_pv_async_pf_enabled(vcpu)) {
2694 2695 2696 2697 2698
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_async_pf_hash_reset(vcpu);
		return 0;
	}

2699
	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
2700
					sizeof(u64)))
2701 2702
		return 1;

2703
	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
2704
	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
2705

2706
	kvm_async_pf_wakeup_all(vcpu);
2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723

	return 0;
}

static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)
{
	/* Bits 8-63 are reserved */
	if (data >> 8)
		return 1;

	if (!lapic_in_kernel(vcpu))
		return 1;

	vcpu->arch.apf.msr_int_val = data;

	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;

2724 2725 2726
	return 0;
}

2727 2728
static void kvmclock_reset(struct kvm_vcpu *vcpu)
{
2729
	vcpu->arch.pv_time_enabled = false;
2730
	vcpu->arch.time = 0;
2731 2732
}

2733
static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
2734 2735
{
	++vcpu->stat.tlb_flush;
2736
	kvm_x86_ops.tlb_flush_all(vcpu);
2737 2738
}

2739 2740 2741 2742 2743 2744
static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
{
	++vcpu->stat.tlb_flush;
	kvm_x86_ops.tlb_flush_guest(vcpu);
}

2745 2746
static void record_steal_time(struct kvm_vcpu *vcpu)
{
2747 2748 2749
	struct kvm_host_map map;
	struct kvm_steal_time *st;

2750 2751 2752
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

2753 2754 2755
	/* -EAGAIN is returned in atomic context so we can just return. */
	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT,
			&map, &vcpu->arch.st.cache, false))
2756 2757
		return;

2758 2759 2760
	st = map.hva +
		offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);

2761 2762 2763 2764
	/*
	 * Doing a TLB flush here, on the guest's behalf, can avoid
	 * expensive IPIs.
	 */
2765
	trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
2766 2767
		st->preempted & KVM_VCPU_FLUSH_TLB);
	if (xchg(&st->preempted, 0) & KVM_VCPU_FLUSH_TLB)
2768
		kvm_vcpu_flush_tlb_guest(vcpu);
2769

2770
	vcpu->arch.st.preempted = 0;
2771

2772 2773
	if (st->version & 1)
		st->version += 1;  /* first time write, random junk */
2774

2775
	st->version += 1;
2776 2777 2778

	smp_wmb();

2779
	st->steal += current->sched_info.run_delay -
2780 2781
		vcpu->arch.st.last_steal;
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
2782 2783 2784

	smp_wmb();

2785
	st->version += 1;
2786

2787
	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, false);
2788 2789
}

2790
int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2791
{
2792
	bool pr = false;
2793 2794
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
2795

2796
	switch (msr) {
2797 2798 2799 2800 2801
	case MSR_AMD64_NB_CFG:
	case MSR_IA32_UCODE_WRITE:
	case MSR_VM_HSAVE_PA:
	case MSR_AMD64_PATCH_LOADER:
	case MSR_AMD64_BU_CFG2:
2802
	case MSR_AMD64_DC_CFG:
2803
	case MSR_F15H_EX_CFG:
2804 2805
		break;

2806 2807 2808 2809
	case MSR_IA32_UCODE_REV:
		if (msr_info->host_initiated)
			vcpu->arch.microcode_version = data;
		break;
2810 2811 2812 2813 2814
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.arch_capabilities = data;
		break;
2815
	case MSR_EFER:
2816
		return set_efer(vcpu, msr_info);
2817 2818
	case MSR_K7_HWCR:
		data &= ~(u64)0x40;	/* ignore flush filter disable */
2819
		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
2820
		data &= ~(u64)0x8;	/* ignore TLB cache disable */
2821 2822 2823 2824 2825

		/* Handle McStatusWrEn */
		if (data == BIT_ULL(18)) {
			vcpu->arch.msr_hwcr = data;
		} else if (data != 0) {
2826 2827
			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
				    data);
2828 2829
			return 1;
		}
2830
		break;
2831 2832
	case MSR_FAM10H_MMIO_CONF_BASE:
		if (data != 0) {
2833 2834
			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
				    "0x%llx\n", data);
2835 2836
			return 1;
		}
2837
		break;
2838 2839 2840 2841 2842 2843 2844 2845 2846
	case MSR_IA32_DEBUGCTLMSR:
		if (!data) {
			/* We support the non-activated case already */
			break;
		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
			/* Values other than LBR and BTF are vendor-specific,
			   thus reserved and should throw a #GP */
			return 1;
		}
2847 2848
		vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
			    __func__, data);
2849
		break;
Avi Kivity's avatar
Avi Kivity committed
2850
	case 0x200 ... 0x2ff:
2851
		return kvm_mtrr_set_msr(vcpu, msr, data);
2852
	case MSR_IA32_APICBASE:
2853
		return kvm_set_apic_base(vcpu, msr_info);
2854 2855
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_write(vcpu, msr, data);
2856 2857 2858
	case MSR_IA32_TSCDEADLINE:
		kvm_set_lapic_tscdeadline_msr(vcpu, data);
		break;
2859
	case MSR_IA32_TSC_ADJUST:
2860
		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
2861
			if (!msr_info->host_initiated) {
2862
				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
2863
				adjust_tsc_offset_guest(vcpu, adj);
2864 2865 2866 2867
			}
			vcpu->arch.ia32_tsc_adjust_msr = data;
		}
		break;
2868
	case MSR_IA32_MISC_ENABLE:
2869 2870 2871 2872 2873 2874 2875 2876 2877
		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
		    ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
				return 1;
			vcpu->arch.ia32_misc_enable_msr = data;
			kvm_update_cpuid(vcpu);
		} else {
			vcpu->arch.ia32_misc_enable_msr = data;
		}
2878
		break;
2879 2880 2881 2882 2883
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smbase = data;
		break;
2884 2885 2886
	case MSR_IA32_POWER_CTL:
		vcpu->arch.msr_ia32_power_ctl = data;
		break;
2887 2888 2889
	case MSR_IA32_TSC:
		kvm_write_tsc(vcpu, msr_info);
		break;
2890 2891 2892 2893 2894
	case MSR_IA32_XSS:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
			return 1;
		/*
2895 2896 2897
		 * KVM supports exposing PT to the guest, but does not support
		 * IA32_XSS[bit 8]. Guests have to use RDMSR/WRMSR rather than
		 * XSAVES/XRSTORS to save/restore PT MSRs.
2898
		 */
2899
		if (data & ~supported_xss)
2900 2901 2902
			return 1;
		vcpu->arch.ia32_xss = data;
		break;
2903 2904 2905 2906 2907
	case MSR_SMI_COUNT:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smi_count = data;
		break;
2908
	case MSR_KVM_WALL_CLOCK_NEW:
2909 2910 2911 2912
	case MSR_KVM_WALL_CLOCK:
		vcpu->kvm->arch.wall_clock = data;
		kvm_write_wall_clock(vcpu->kvm, data);
		break;
2913
	case MSR_KVM_SYSTEM_TIME_NEW:
2914
	case MSR_KVM_SYSTEM_TIME: {
2915 2916 2917 2918 2919 2920
		struct kvm_arch *ka = &vcpu->kvm->arch;

		if (vcpu->vcpu_id == 0 && !msr_info->host_initiated) {
			bool tmp = (msr == MSR_KVM_SYSTEM_TIME);

			if (ka->boot_vcpu_runs_old_kvmclock != tmp)
2921
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2922 2923 2924 2925

			ka->boot_vcpu_runs_old_kvmclock = tmp;
		}

2926
		vcpu->arch.time = data;
2927
		kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2928 2929

		/* we verify if the enable bit is set... */
2930
		vcpu->arch.pv_time_enabled = false;
2931 2932 2933
		if (!(data & 1))
			break;

2934
		if (!kvm_gfn_to_hva_cache_init(vcpu->kvm,
2935 2936
		     &vcpu->arch.pv_time, data & ~1ULL,
		     sizeof(struct pvclock_vcpu_time_info)))
2937
			vcpu->arch.pv_time_enabled = true;
2938

2939 2940
		break;
	}
2941 2942 2943 2944
	case MSR_KVM_ASYNC_PF_EN:
		if (kvm_pv_enable_async_pf(vcpu, data))
			return 1;
		break;
2945 2946 2947 2948
	case MSR_KVM_ASYNC_PF_INT:
		if (kvm_pv_enable_async_pf_int(vcpu, data))
			return 1;
		break;
2949 2950 2951 2952 2953 2954
	case MSR_KVM_ASYNC_PF_ACK:
		if (data & 0x1) {
			vcpu->arch.apf.pageready_pending = false;
			kvm_check_async_pf_completion(vcpu);
		}
		break;
2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970
	case MSR_KVM_STEAL_TIME:

		if (unlikely(!sched_info_on()))
			return 1;

		if (data & KVM_STEAL_RESERVED_MASK)
			return 1;

		vcpu->arch.st.msr_val = data;

		if (!(data & KVM_MSR_ENABLED))
			break;

		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);

		break;
2971
	case MSR_KVM_PV_EOI_EN:
2972
		if (kvm_lapic_enable_pv_eoi(vcpu, data, sizeof(u8)))
2973 2974
			return 1;
		break;
2975

2976 2977 2978 2979 2980 2981 2982 2983
	case MSR_KVM_POLL_CONTROL:
		/* only enable bit supported */
		if (data & (-1ULL << 1))
			return 1;

		vcpu->arch.msr_kvm_poll_control = data;
		break;

Huang Ying's avatar
Huang Ying committed
2984 2985
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
2986
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
2987
		return set_msr_mce(vcpu, msr_info);
2988

2989 2990 2991 2992 2993
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
		pr = true; /* fall through */
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
2994
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2995
			return kvm_pmu_set_msr(vcpu, msr_info);
2996 2997

		if (pr || data != 0)
2998 2999
			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
				    "0x%x data 0x%llx\n", msr, data);
3000
		break;
3001 3002 3003 3004 3005
	case MSR_K7_CLK_CTL:
		/*
		 * Ignore all writes to this no longer documented MSR.
		 * Writes are only relevant for old K7 processors,
		 * all pre-dating SVM, but a recommended workaround from
Guo Chao's avatar
Guo Chao committed
3006
		 * AMD for these chips. It is possible to specify the
3007 3008 3009 3010
		 * affected processor models on the command line, hence
		 * the need to ignore the workaround.
		 */
		break;
3011
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
3012 3013
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
3014
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
3015 3016 3017
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
3018 3019
		return kvm_hv_set_msr_common(vcpu, msr, data,
					     msr_info->host_initiated);
3020 3021 3022 3023
	case MSR_IA32_BBL_CR_CTL3:
		/* Drop writes to this legacy MSR -- see rdmsr
		 * counterpart for further detail.
		 */
3024 3025 3026
		if (report_ignored_msrs)
			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
				msr, data);
3027
		break;
3028
	case MSR_AMD64_OSVW_ID_LENGTH:
3029
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3030 3031 3032 3033
			return 1;
		vcpu->arch.osvw.length = data;
		break;
	case MSR_AMD64_OSVW_STATUS:
3034
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3035 3036 3037
			return 1;
		vcpu->arch.osvw.status = data;
		break;
3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051
	case MSR_PLATFORM_INFO:
		if (!msr_info->host_initiated ||
		    (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
		     cpuid_fault_enabled(vcpu)))
			return 1;
		vcpu->arch.msr_platform_info = data;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
		     !supports_cpuid_fault(vcpu)))
			return 1;
		vcpu->arch.msr_misc_features_enables = data;
		break;
3052
	default:
Ed Swierk's avatar
Ed Swierk committed
3053 3054
		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
			return xen_hvm_config(vcpu, data);
3055
		if (kvm_pmu_is_valid_msr(vcpu, msr))
3056
			return kvm_pmu_set_msr(vcpu, msr_info);
3057
		if (!ignore_msrs) {
3058
			vcpu_debug_ratelimited(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n",
3059
				    msr, data);
3060 3061
			return 1;
		} else {
3062 3063 3064 3065
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu,
					"ignored wrmsr: 0x%x data 0x%llx\n",
					msr, data);
3066 3067
			break;
		}
3068 3069 3070 3071 3072
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);

3073
static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
3074 3075
{
	u64 data;
Huang Ying's avatar
Huang Ying committed
3076 3077
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
3078 3079 3080 3081

	switch (msr) {
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
Huang Ying's avatar
Huang Ying committed
3082 3083
		data = 0;
		break;
3084
	case MSR_IA32_MCG_CAP:
Huang Ying's avatar
Huang Ying committed
3085 3086
		data = vcpu->arch.mcg_cap;
		break;
3087
	case MSR_IA32_MCG_CTL:
3088
		if (!(mcg_cap & MCG_CTL_P) && !host)
Huang Ying's avatar
Huang Ying committed
3089 3090 3091 3092 3093 3094 3095 3096
			return 1;
		data = vcpu->arch.mcg_ctl;
		break;
	case MSR_IA32_MCG_STATUS:
		data = vcpu->arch.mcg_status;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
3097
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
3098 3099 3100 3101
			u32 offset = array_index_nospec(
				msr - MSR_IA32_MC0_CTL,
				MSR_IA32_MCx_CTL(bank_num) - MSR_IA32_MC0_CTL);

Huang Ying's avatar
Huang Ying committed
3102 3103 3104 3105 3106 3107 3108 3109 3110
			data = vcpu->arch.mce_banks[offset];
			break;
		}
		return 1;
	}
	*pdata = data;
	return 0;
}

3111
int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
Huang Ying's avatar
Huang Ying committed
3112
{
3113
	switch (msr_info->index) {
Huang Ying's avatar
Huang Ying committed
3114
	case MSR_IA32_PLATFORM_ID:
3115
	case MSR_IA32_EBL_CR_POWERON:
3116 3117 3118 3119 3120
	case MSR_IA32_DEBUGCTLMSR:
	case MSR_IA32_LASTBRANCHFROMIP:
	case MSR_IA32_LASTBRANCHTOIP:
	case MSR_IA32_LASTINTFROMIP:
	case MSR_IA32_LASTINTTOIP:
3121
	case MSR_K8_SYSCFG:
3122 3123
	case MSR_K8_TSEG_ADDR:
	case MSR_K8_TSEG_MASK:
3124
	case MSR_VM_HSAVE_PA:
3125
	case MSR_K8_INT_PENDING_MSG:
3126
	case MSR_AMD64_NB_CFG:
3127
	case MSR_FAM10H_MMIO_CONF_BASE:
3128
	case MSR_AMD64_BU_CFG2:
3129
	case MSR_IA32_PERF_CTL:
3130
	case MSR_AMD64_DC_CFG:
3131
	case MSR_F15H_EX_CFG:
3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142
	/*
	 * Intel Sandy Bridge CPUs must support the RAPL (running average power
	 * limit) MSRs. Just return 0, as we do not want to expose the host
	 * data here. Do not conditionalize this on CPUID, as KVM does not do
	 * so for existing CPU-specific MSRs.
	 */
	case MSR_RAPL_POWER_UNIT:
	case MSR_PP0_ENERGY_STATUS:	/* Power plane 0 (core) */
	case MSR_PP1_ENERGY_STATUS:	/* Power plane 1 (graphics uncore) */
	case MSR_PKG_ENERGY_STATUS:	/* Total package */
	case MSR_DRAM_ENERGY_STATUS:	/* DRAM controller */
3143
		msr_info->data = 0;
3144
		break;
3145
	case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
3146 3147 3148 3149
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
3150
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3151
			return kvm_pmu_get_msr(vcpu, msr_info);
3152
		msr_info->data = 0;
3153
		break;
3154
	case MSR_IA32_UCODE_REV:
3155
		msr_info->data = vcpu->arch.microcode_version;
3156
		break;
3157 3158 3159 3160 3161 3162
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
			return 1;
		msr_info->data = vcpu->arch.arch_capabilities;
		break;
3163 3164 3165
	case MSR_IA32_POWER_CTL:
		msr_info->data = vcpu->arch.msr_ia32_power_ctl;
		break;
3166 3167 3168
	case MSR_IA32_TSC:
		msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + vcpu->arch.tsc_offset;
		break;
Avi Kivity's avatar
Avi Kivity committed
3169 3170
	case MSR_MTRRcap:
	case 0x200 ... 0x2ff:
3171
		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
3172
	case 0xcd: /* fsb frequency */
3173
		msr_info->data = 3;
3174
		break;
3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186
		/*
		 * MSR_EBC_FREQUENCY_ID
		 * Conservative value valid for even the basic CPU models.
		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
		 * and 266MHz for model 3, or 4. Set Core Clock
		 * Frequency to System Bus Frequency Ratio to 1 (bits
		 * 31:24) even though these are only valid for CPU
		 * models > 2, however guests may end up dividing or
		 * multiplying by zero otherwise.
		 */
	case MSR_EBC_FREQUENCY_ID:
3187
		msr_info->data = 1 << 24;
3188
		break;
3189
	case MSR_IA32_APICBASE:
3190
		msr_info->data = kvm_get_apic_base(vcpu);
3191
		break;
3192
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
3193
		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
3194
	case MSR_IA32_TSCDEADLINE:
3195
		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
3196
		break;
3197
	case MSR_IA32_TSC_ADJUST:
3198
		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
3199
		break;
3200
	case MSR_IA32_MISC_ENABLE:
3201
		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
3202
		break;
3203 3204 3205 3206
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		msr_info->data = vcpu->arch.smbase;
3207
		break;
3208 3209 3210
	case MSR_SMI_COUNT:
		msr_info->data = vcpu->arch.smi_count;
		break;
3211 3212
	case MSR_IA32_PERF_STATUS:
		/* TSC increment by tick */
3213
		msr_info->data = 1000ULL;
3214
		/* CPU multiplier */
3215
		msr_info->data |= (((uint64_t)4ULL) << 40);
3216
		break;
3217
	case MSR_EFER:
3218
		msr_info->data = vcpu->arch.efer;
3219
		break;
3220
	case MSR_KVM_WALL_CLOCK:
3221
	case MSR_KVM_WALL_CLOCK_NEW:
3222
		msr_info->data = vcpu->kvm->arch.wall_clock;
3223 3224
		break;
	case MSR_KVM_SYSTEM_TIME:
3225
	case MSR_KVM_SYSTEM_TIME_NEW:
3226
		msr_info->data = vcpu->arch.time;
3227
		break;
3228
	case MSR_KVM_ASYNC_PF_EN:
3229 3230 3231 3232
		msr_info->data = vcpu->arch.apf.msr_en_val;
		break;
	case MSR_KVM_ASYNC_PF_INT:
		msr_info->data = vcpu->arch.apf.msr_int_val;
3233
		break;
3234 3235 3236
	case MSR_KVM_ASYNC_PF_ACK:
		msr_info->data = 0;
		break;
3237
	case MSR_KVM_STEAL_TIME:
3238
		msr_info->data = vcpu->arch.st.msr_val;
3239
		break;
3240
	case MSR_KVM_PV_EOI_EN:
3241
		msr_info->data = vcpu->arch.pv_eoi.msr_val;
3242
		break;
3243 3244 3245
	case MSR_KVM_POLL_CONTROL:
		msr_info->data = vcpu->arch.msr_kvm_poll_control;
		break;
Huang Ying's avatar
Huang Ying committed
3246 3247 3248 3249 3250
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
3251
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3252 3253
		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
				   msr_info->host_initiated);
3254 3255 3256 3257 3258 3259
	case MSR_IA32_XSS:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
			return 1;
		msr_info->data = vcpu->arch.ia32_xss;
		break;
3260 3261 3262 3263 3264 3265 3266 3267 3268 3269
	case MSR_K7_CLK_CTL:
		/*
		 * Provide expected ramp-up count for K7. All other
		 * are set to zero, indicating minimum divisors for
		 * every field.
		 *
		 * This prevents guest kernels on AMD host with CPU
		 * type 6, model 8 and higher from exploding due to
		 * the rdmsr failing.
		 */
3270
		msr_info->data = 0x20000000;
3271
		break;
3272
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
3273 3274
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
3275
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
3276 3277 3278
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
3279
		return kvm_hv_get_msr_common(vcpu,
3280 3281
					     msr_info->index, &msr_info->data,
					     msr_info->host_initiated);
3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292
	case MSR_IA32_BBL_CR_CTL3:
		/* This legacy MSR exists but isn't fully documented in current
		 * silicon.  It is however accessed by winxp in very narrow
		 * scenarios where it sets bit #19, itself documented as
		 * a "reserved" bit.  Best effort attempt to source coherent
		 * read data here should the balance of the register be
		 * interpreted by the guest:
		 *
		 * L2 cache control register 3: 64GB range, 256KB size,
		 * enabled, latency 0x1, configured
		 */
3293
		msr_info->data = 0xbe702111;
3294
		break;
3295
	case MSR_AMD64_OSVW_ID_LENGTH:
3296
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3297
			return 1;
3298
		msr_info->data = vcpu->arch.osvw.length;
3299 3300
		break;
	case MSR_AMD64_OSVW_STATUS:
3301
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3302
			return 1;
3303
		msr_info->data = vcpu->arch.osvw.status;
3304
		break;
3305
	case MSR_PLATFORM_INFO:
3306 3307 3308
		if (!msr_info->host_initiated &&
		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
			return 1;
3309 3310 3311 3312 3313
		msr_info->data = vcpu->arch.msr_platform_info;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		msr_info->data = vcpu->arch.msr_misc_features_enables;
		break;
3314 3315 3316
	case MSR_K7_HWCR:
		msr_info->data = vcpu->arch.msr_hwcr;
		break;
3317
	default:
3318
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3319
			return kvm_pmu_get_msr(vcpu, msr_info);
3320
		if (!ignore_msrs) {
3321 3322
			vcpu_debug_ratelimited(vcpu, "unhandled rdmsr: 0x%x\n",
					       msr_info->index);
3323 3324
			return 1;
		} else {
3325 3326 3327
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n",
					msr_info->index);
3328
			msr_info->data = 0;
3329 3330
		}
		break;
3331 3332 3333 3334 3335
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

3336 3337 3338 3339 3340 3341 3342 3343 3344 3345
/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
3346
	int i;
3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370

	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
3371
	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
3372 3373 3374 3375 3376 3377 3378
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
3379 3380 3381
	entries = memdup_user(user_msrs->entries, size);
	if (IS_ERR(entries)) {
		r = PTR_ERR(entries);
3382
		goto out;
3383
	}
3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395

	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
3396
	kfree(entries);
3397 3398 3399 3400
out:
	return r;
}

3401 3402 3403
static inline bool kvm_can_mwait_in_guest(void)
{
	return boot_cpu_has(X86_FEATURE_MWAIT) &&
3404 3405
		!boot_cpu_has_bug(X86_BUG_MONITOR) &&
		boot_cpu_has(X86_FEATURE_ARAT);
3406 3407
}

3408
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
3409
{
3410
	int r = 0;
3411 3412 3413 3414 3415 3416

	switch (ext) {
	case KVM_CAP_IRQCHIP:
	case KVM_CAP_HLT:
	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
	case KVM_CAP_SET_TSS_ADDR:
3417
	case KVM_CAP_EXT_CPUID:
3418
	case KVM_CAP_EXT_EMUL_CPUID:
3419
	case KVM_CAP_CLOCKSOURCE:
Sheng Yang's avatar
Sheng Yang committed
3420
	case KVM_CAP_PIT:
3421
	case KVM_CAP_NOP_IO_DELAY:
3422
	case KVM_CAP_MP_STATE:
3423
	case KVM_CAP_SYNC_MMU:
3424
	case KVM_CAP_USER_NMI:
3425
	case KVM_CAP_REINJECT_CONTROL:
3426
	case KVM_CAP_IRQ_INJECT_STATUS:
Gregory Haskins's avatar
Gregory Haskins committed
3427
	case KVM_CAP_IOEVENTFD:
3428
	case KVM_CAP_IOEVENTFD_NO_LENGTH:
3429
	case KVM_CAP_PIT2:
3430
	case KVM_CAP_PIT_STATE2:
3431
	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
Ed Swierk's avatar
Ed Swierk committed
3432
	case KVM_CAP_XEN_HVM:
3433
	case KVM_CAP_VCPU_EVENTS:
3434
	case KVM_CAP_HYPERV:
3435
	case KVM_CAP_HYPERV_VAPIC:
3436
	case KVM_CAP_HYPERV_SPIN:
3437
	case KVM_CAP_HYPERV_SYNIC:
3438
	case KVM_CAP_HYPERV_SYNIC2:
3439
	case KVM_CAP_HYPERV_VP_INDEX:
3440
	case KVM_CAP_HYPERV_EVENTFD:
3441
	case KVM_CAP_HYPERV_TLBFLUSH:
3442
	case KVM_CAP_HYPERV_SEND_IPI:
3443
	case KVM_CAP_HYPERV_CPUID:
3444
	case KVM_CAP_PCI_SEGMENT:
3445
	case KVM_CAP_DEBUGREGS:
3446
	case KVM_CAP_X86_ROBUST_SINGLESTEP:
3447
	case KVM_CAP_XSAVE:
3448
	case KVM_CAP_ASYNC_PF:
3449
	case KVM_CAP_ASYNC_PF_INT:
3450
	case KVM_CAP_GET_TSC_KHZ:
3451
	case KVM_CAP_KVMCLOCK_CTRL:
3452
	case KVM_CAP_READONLY_MEM:
3453
	case KVM_CAP_HYPERV_TIME:
3454
	case KVM_CAP_IOAPIC_POLARITY_IGNORED:
3455
	case KVM_CAP_TSC_DEADLINE_TIMER:
3456
	case KVM_CAP_DISABLE_QUIRKS:
3457
	case KVM_CAP_SET_BOOT_CPU_ID:
3458
 	case KVM_CAP_SPLIT_IRQCHIP:
3459
	case KVM_CAP_IMMEDIATE_EXIT:
Eric Hankland's avatar
Eric Hankland committed
3460
	case KVM_CAP_PMU_EVENT_FILTER:
3461
	case KVM_CAP_GET_MSR_FEATURES:
3462
	case KVM_CAP_MSR_PLATFORM_INFO:
3463
	case KVM_CAP_EXCEPTION_PAYLOAD:
3464
	case KVM_CAP_SET_GUEST_DEBUG:
3465 3466
		r = 1;
		break;
Ken Hofsass's avatar
Ken Hofsass committed
3467 3468 3469
	case KVM_CAP_SYNC_REGS:
		r = KVM_SYNC_X86_VALID_FIELDS;
		break;
3470 3471 3472
	case KVM_CAP_ADJUST_CLOCK:
		r = KVM_CLOCK_TSC_STABLE;
		break;
3473
	case KVM_CAP_X86_DISABLE_EXITS:
3474 3475
		r |=  KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE |
		      KVM_X86_DISABLE_EXITS_CSTATE;
3476 3477
		if(kvm_can_mwait_in_guest())
			r |= KVM_X86_DISABLE_EXITS_MWAIT;
3478
		break;
3479 3480 3481 3482 3483 3484 3485 3486 3487
	case KVM_CAP_X86_SMM:
		/* SMBASE is usually relocated above 1M on modern chipsets,
		 * and SMM handlers might indeed rely on 4G segment limits,
		 * so do not report SMM to be available if real mode is
		 * emulated via vm86 mode.  Still, do not go to great lengths
		 * to avoid userspace's usage of the feature, because it is a
		 * fringe case that is not enabled except via specific settings
		 * of the module parameters.
		 */
3488
		r = kvm_x86_ops.has_emulated_msr(MSR_IA32_SMBASE);
3489
		break;
3490
	case KVM_CAP_VAPIC:
3491
		r = !kvm_x86_ops.cpu_has_accelerated_tpr();
3492
		break;
3493
	case KVM_CAP_NR_VCPUS:
3494 3495 3496
		r = KVM_SOFT_MAX_VCPUS;
		break;
	case KVM_CAP_MAX_VCPUS:
3497 3498
		r = KVM_MAX_VCPUS;
		break;
3499 3500 3501
	case KVM_CAP_MAX_VCPU_ID:
		r = KVM_MAX_VCPU_ID;
		break;
3502 3503
	case KVM_CAP_PV_MMU:	/* obsolete */
		r = 0;
3504
		break;
Huang Ying's avatar
Huang Ying committed
3505 3506 3507
	case KVM_CAP_MCE:
		r = KVM_MAX_MCE_BANKS;
		break;
3508
	case KVM_CAP_XCRS:
3509
		r = boot_cpu_has(X86_FEATURE_XSAVE);
3510
		break;
3511 3512 3513
	case KVM_CAP_TSC_CONTROL:
		r = kvm_has_tsc_control;
		break;
3514 3515 3516
	case KVM_CAP_X2APIC_API:
		r = KVM_X2APIC_API_VALID_FLAGS;
		break;
3517
	case KVM_CAP_NESTED_STATE:
3518 3519
		r = kvm_x86_ops.nested_ops->get_state ?
			kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;
3520
		break;
3521
	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
3522
		r = kvm_x86_ops.enable_direct_tlbflush != NULL;
3523 3524
		break;
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
3525
		r = kvm_x86_ops.nested_ops->enable_evmcs != NULL;
3526
		break;
3527 3528 3529 3530 3531 3532 3533
	default:
		break;
	}
	return r;

}

3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546
long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {
	case KVM_GET_MSR_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
3547
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
3548 3549
			goto out;
		n = msr_list.nmsrs;
3550
		msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
3551
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
3552 3553
			goto out;
		r = -E2BIG;
Jan Kiszka's avatar
Jan Kiszka committed
3554
		if (n < msr_list.nmsrs)
3555 3556 3557 3558 3559
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
				 num_msrs_to_save * sizeof(u32)))
			goto out;
Jan Kiszka's avatar
Jan Kiszka committed
3560
		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
3561
				 &emulated_msrs,
3562
				 num_emulated_msrs * sizeof(u32)))
3563 3564 3565 3566
			goto out;
		r = 0;
		break;
	}
3567 3568
	case KVM_GET_SUPPORTED_CPUID:
	case KVM_GET_EMULATED_CPUID: {
3569 3570 3571 3572
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
3573
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
3574
			goto out;
3575 3576 3577

		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
					    ioctl);
3578 3579 3580 3581
		if (r)
			goto out;

		r = -EFAULT;
3582
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
3583 3584 3585 3586
			goto out;
		r = 0;
		break;
	}
3587
	case KVM_X86_GET_MCE_CAP_SUPPORTED:
Huang Ying's avatar
Huang Ying committed
3588
		r = -EFAULT;
3589 3590
		if (copy_to_user(argp, &kvm_mce_cap_supported,
				 sizeof(kvm_mce_cap_supported)))
Huang Ying's avatar
Huang Ying committed
3591 3592 3593
			goto out;
		r = 0;
		break;
3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618
	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned int n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
			goto out;
		n = msr_list.nmsrs;
		msr_list.nmsrs = num_msr_based_features;
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
			goto out;
		r = -E2BIG;
		if (n < msr_list.nmsrs)
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msr_based_features,
				 num_msr_based_features * sizeof(u32)))
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_MSRS:
		r = msr_io(NULL, argp, do_get_msr_feature, 1);
		break;
3619 3620
	default:
		r = -EINVAL;
3621
		break;
3622 3623 3624 3625 3626
	}
out:
	return r;
}

3627 3628 3629 3630 3631 3632 3633
static void wbinvd_ipi(void *garbage)
{
	wbinvd();
}

static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
3634
	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
3635 3636
}

3637 3638
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
3639 3640
	/* Address WBINVD may be executed by guest */
	if (need_emulate_wbinvd(vcpu)) {
3641
		if (kvm_x86_ops.has_wbinvd_exit())
3642 3643 3644 3645 3646 3647
			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
			smp_call_function_single(vcpu->cpu,
					wbinvd_ipi, NULL, 1);
	}

3648
	kvm_x86_ops.vcpu_load(vcpu, cpu);
3649

3650 3651 3652
	/* Save host pkru register if supported */
	vcpu->arch.host_pkru = read_pkru();

3653 3654 3655 3656
	/* Apply any externally detected TSC adjustments (due to suspend) */
	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
		vcpu->arch.tsc_offset_adjustment = 0;
3657
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3658
	}
3659

3660
	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
3661
		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
3662
				rdtsc() - vcpu->arch.last_host_tsc;
Zachary Amsden's avatar
Zachary Amsden committed
3663 3664
		if (tsc_delta < 0)
			mark_tsc_unstable("KVM discovered backwards TSC");
3665

3666
		if (kvm_check_tsc_unstable()) {
3667
			u64 offset = kvm_compute_tsc_offset(vcpu,
3668
						vcpu->arch.last_guest_tsc);
3669
			kvm_vcpu_write_tsc_offset(vcpu, offset);
Zachary Amsden's avatar
Zachary Amsden committed
3670 3671
			vcpu->arch.tsc_catchup = 1;
		}
3672 3673 3674 3675

		if (kvm_lapic_hv_timer_in_use(vcpu))
			kvm_lapic_restart_hv_timer(vcpu);

3676 3677 3678 3679 3680
		/*
		 * On a host with synchronized TSC, there is no need to update
		 * kvmclock on vcpu->cpu migration
		 */
		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
3681
			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
Zachary Amsden's avatar
Zachary Amsden committed
3682
		if (vcpu->cpu != cpu)
3683
			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
Zachary Amsden's avatar
Zachary Amsden committed
3684
		vcpu->cpu = cpu;
3685
	}
3686 3687

	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3688 3689
}

3690 3691
static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
{
3692 3693 3694
	struct kvm_host_map map;
	struct kvm_steal_time *st;

3695 3696 3697
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

3698
	if (vcpu->arch.st.preempted)
3699 3700
		return;

3701 3702 3703 3704 3705 3706
	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT, &map,
			&vcpu->arch.st.cache, true))
		return;

	st = map.hva +
		offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);
3707

3708
	st->preempted = vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
3709

3710
	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, true);
3711 3712
}

3713 3714
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
3715
	int idx;
3716 3717

	if (vcpu->preempted)
3718
		vcpu->arch.preempted_in_kernel = !kvm_x86_ops.get_cpl(vcpu);
3719

3720 3721 3722 3723 3724 3725 3726 3727 3728
	/*
	 * Disable page faults because we're in atomic context here.
	 * kvm_write_guest_offset_cached() would call might_fault()
	 * that relies on pagefault_disable() to tell if there's a
	 * bug. NOTE: the write to guest memory may not go through if
	 * during postcopy live migration or if there's heavy guest
	 * paging.
	 */
	pagefault_disable();
3729 3730 3731 3732 3733
	/*
	 * kvm_memslots() will be called by
	 * kvm_write_guest_offset_cached() so take the srcu lock.
	 */
	idx = srcu_read_lock(&vcpu->kvm->srcu);
3734
	kvm_steal_time_set_preempted(vcpu);
3735
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
3736
	pagefault_enable();
3737
	kvm_x86_ops.vcpu_put(vcpu);
3738
	vcpu->arch.last_host_tsc = rdtsc();
3739
	/*
3740 3741 3742
	 * If userspace has set any breakpoints or watchpoints, dr6 is restored
	 * on every vmexit, but if not, we might have a stale dr6 from the
	 * guest. do_debug expects dr6 to be cleared after it runs, do the same.
3743
	 */
3744
	set_debugreg(0, 6);
3745 3746 3747 3748 3749
}

static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3750
	if (vcpu->arch.apicv_active)
3751
		kvm_x86_ops.sync_pir_to_irr(vcpu);
3752

3753
	return kvm_apic_get_state(vcpu, s);
3754 3755 3756 3757 3758
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3759 3760 3761 3762 3763
	int r;

	r = kvm_apic_set_state(vcpu, s);
	if (r)
		return r;
3764
	update_cr8_intercept(vcpu);
3765 3766 3767 3768

	return 0;
}

3769 3770 3771 3772 3773 3774
static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
{
	return (!lapic_in_kernel(vcpu) ||
		kvm_apic_accept_pic_intr(vcpu));
}

3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788
/*
 * if userspace requested an interrupt window, check that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
{
	return kvm_arch_interrupt_allowed(vcpu) &&
		!kvm_cpu_has_interrupt(vcpu) &&
		!kvm_event_needs_reinjection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);
}

3789 3790 3791
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
{
3792
	if (irq->irq >= KVM_NR_INTERRUPTS)
3793
		return -EINVAL;
3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805

	if (!irqchip_in_kernel(vcpu->kvm)) {
		kvm_queue_interrupt(vcpu, irq->irq, false);
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		return 0;
	}

	/*
	 * With in-kernel LAPIC, we only use this to inject EXTINT, so
	 * fail for in-kernel 8259.
	 */
	if (pic_in_kernel(vcpu->kvm))
3806 3807
		return -ENXIO;

3808 3809
	if (vcpu->arch.pending_external_vector != -1)
		return -EEXIST;
3810

3811
	vcpu->arch.pending_external_vector = irq->irq;
3812
	kvm_make_request(KVM_REQ_EVENT, vcpu);
3813 3814 3815
	return 0;
}

3816 3817 3818 3819 3820 3821 3822
static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
{
	kvm_inject_nmi(vcpu);

	return 0;
}

3823 3824
static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
{
3825 3826
	kvm_make_request(KVM_REQ_SMI, vcpu);

3827 3828 3829
	return 0;
}

3830 3831 3832 3833 3834 3835 3836 3837 3838
static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
					   struct kvm_tpr_access_ctl *tac)
{
	if (tac->flags)
		return -EINVAL;
	vcpu->arch.tpr_access_reporting = !!tac->enabled;
	return 0;
}

Huang Ying's avatar
Huang Ying committed
3839 3840 3841 3842 3843 3844 3845
static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
					u64 mcg_cap)
{
	int r;
	unsigned bank_num = mcg_cap & 0xff, bank;

	r = -EINVAL;
3846
	if (!bank_num || bank_num > KVM_MAX_MCE_BANKS)
Huang Ying's avatar
Huang Ying committed
3847
		goto out;
3848
	if (mcg_cap & ~(kvm_mce_cap_supported | 0xff | 0xff0000))
Huang Ying's avatar
Huang Ying committed
3849 3850 3851 3852 3853 3854 3855 3856 3857
		goto out;
	r = 0;
	vcpu->arch.mcg_cap = mcg_cap;
	/* Init IA32_MCG_CTL to all 1s */
	if (mcg_cap & MCG_CTL_P)
		vcpu->arch.mcg_ctl = ~(u64)0;
	/* Init IA32_MCi_CTL to all 1s */
	for (bank = 0; bank < bank_num; bank++)
		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
3858

3859
	kvm_x86_ops.setup_mce(vcpu);
Huang Ying's avatar
Huang Ying committed
3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888
out:
	return r;
}

static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
				      struct kvm_x86_mce *mce)
{
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
	u64 *banks = vcpu->arch.mce_banks;

	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
		return -EINVAL;
	/*
	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
	 * reporting is disabled
	 */
	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
	    vcpu->arch.mcg_ctl != ~(u64)0)
		return 0;
	banks += 4 * mce->bank;
	/*
	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
	 * reporting is disabled for the bank
	 */
	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
		return 0;
	if (mce->status & MCI_STATUS_UC) {
		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
3889
		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
3890
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
Huang Ying's avatar
Huang Ying committed
3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911
			return 0;
		}
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		vcpu->arch.mcg_status = mce->mcg_status;
		banks[1] = mce->status;
		kvm_queue_exception(vcpu, MC_VECTOR);
	} else if (!(banks[1] & MCI_STATUS_VAL)
		   || !(banks[1] & MCI_STATUS_UC)) {
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		banks[1] = mce->status;
	} else
		banks[1] |= MCI_STATUS_OVER;
	return 0;
}

3912 3913 3914
static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
					       struct kvm_vcpu_events *events)
{
Avi Kivity's avatar
Avi Kivity committed
3915
	process_nmi(vcpu);
3916

3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931
	/*
	 * In guest mode, payload delivery should be deferred,
	 * so that the L1 hypervisor can intercept #PF before
	 * CR2 is modified (or intercept #DB before DR6 is
	 * modified under nVMX). Unless the per-VM capability,
	 * KVM_CAP_EXCEPTION_PAYLOAD, is set, we may not defer the delivery of
	 * an exception payload and handle after a KVM_GET_VCPU_EVENTS. Since we
	 * opportunistically defer the exception payload, deliver it if the
	 * capability hasn't been requested before processing a
	 * KVM_GET_VCPU_EVENTS.
	 */
	if (!vcpu->kvm->arch.exception_payload_enabled &&
	    vcpu->arch.exception.pending && vcpu->arch.exception.has_payload)
		kvm_deliver_exception_payload(vcpu);

3932
	/*
3933 3934 3935 3936
	 * The API doesn't provide the instruction length for software
	 * exceptions, so don't report them. As long as the guest RIP
	 * isn't advanced, we should expect to encounter the exception
	 * again.
3937
	 */
3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952
	if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {
		events->exception.injected = 0;
		events->exception.pending = 0;
	} else {
		events->exception.injected = vcpu->arch.exception.injected;
		events->exception.pending = vcpu->arch.exception.pending;
		/*
		 * For ABI compatibility, deliberately conflate
		 * pending and injected exceptions when
		 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
		 */
		if (!vcpu->kvm->arch.exception_payload_enabled)
			events->exception.injected |=
				vcpu->arch.exception.pending;
	}
3953 3954 3955
	events->exception.nr = vcpu->arch.exception.nr;
	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
	events->exception.error_code = vcpu->arch.exception.error_code;
3956 3957
	events->exception_has_payload = vcpu->arch.exception.has_payload;
	events->exception_payload = vcpu->arch.exception.payload;
3958

3959
	events->interrupt.injected =
3960
		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
3961
	events->interrupt.nr = vcpu->arch.interrupt.nr;
3962
	events->interrupt.soft = 0;
3963
	events->interrupt.shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);
3964 3965

	events->nmi.injected = vcpu->arch.nmi_injected;
Avi Kivity's avatar
Avi Kivity committed
3966
	events->nmi.pending = vcpu->arch.nmi_pending != 0;
3967
	events->nmi.masked = kvm_x86_ops.get_nmi_mask(vcpu);
3968
	events->nmi.pad = 0;
3969

3970
	events->sipi_vector = 0; /* never valid when reporting to user space */
3971

3972 3973 3974 3975 3976 3977
	events->smi.smm = is_smm(vcpu);
	events->smi.pending = vcpu->arch.smi_pending;
	events->smi.smm_inside_nmi =
		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
	events->smi.latched_init = kvm_lapic_latched_init(vcpu);

3978
	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
3979 3980
			 | KVM_VCPUEVENT_VALID_SHADOW
			 | KVM_VCPUEVENT_VALID_SMM);
3981 3982 3983
	if (vcpu->kvm->arch.exception_payload_enabled)
		events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;

3984
	memset(&events->reserved, 0, sizeof(events->reserved));
3985 3986
}

3987
static void kvm_smm_changed(struct kvm_vcpu *vcpu);
3988

3989 3990 3991
static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
					      struct kvm_vcpu_events *events)
{
3992
	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
3993
			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
3994
			      | KVM_VCPUEVENT_VALID_SHADOW
3995 3996
			      | KVM_VCPUEVENT_VALID_SMM
			      | KVM_VCPUEVENT_VALID_PAYLOAD))
3997 3998
		return -EINVAL;

3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012
	if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
		if (!vcpu->kvm->arch.exception_payload_enabled)
			return -EINVAL;
		if (events->exception.pending)
			events->exception.injected = 0;
		else
			events->exception_has_payload = 0;
	} else {
		events->exception.pending = 0;
		events->exception_has_payload = 0;
	}

	if ((events->exception.injected || events->exception.pending) &&
	    (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
4013 4014
		return -EINVAL;

4015 4016 4017 4018 4019 4020
	/* INITs are latched while in SMM */
	if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
	    (events->smi.smm || events->smi.pending) &&
	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
		return -EINVAL;

Avi Kivity's avatar
Avi Kivity committed
4021
	process_nmi(vcpu);
4022 4023
	vcpu->arch.exception.injected = events->exception.injected;
	vcpu->arch.exception.pending = events->exception.pending;
4024 4025 4026
	vcpu->arch.exception.nr = events->exception.nr;
	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
	vcpu->arch.exception.error_code = events->exception.error_code;
4027 4028
	vcpu->arch.exception.has_payload = events->exception_has_payload;
	vcpu->arch.exception.payload = events->exception_payload;
4029

4030
	vcpu->arch.interrupt.injected = events->interrupt.injected;
4031 4032
	vcpu->arch.interrupt.nr = events->interrupt.nr;
	vcpu->arch.interrupt.soft = events->interrupt.soft;
4033
	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
4034
		kvm_x86_ops.set_interrupt_shadow(vcpu,
4035
						  events->interrupt.shadow);
4036 4037

	vcpu->arch.nmi_injected = events->nmi.injected;
4038 4039
	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
		vcpu->arch.nmi_pending = events->nmi.pending;
4040
	kvm_x86_ops.set_nmi_mask(vcpu, events->nmi.masked);
4041

4042
	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
4043
	    lapic_in_kernel(vcpu))
4044
		vcpu->arch.apic->sipi_vector = events->sipi_vector;
4045

4046
	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
4047 4048 4049 4050 4051 4052 4053
		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
			if (events->smi.smm)
				vcpu->arch.hflags |= HF_SMM_MASK;
			else
				vcpu->arch.hflags &= ~HF_SMM_MASK;
			kvm_smm_changed(vcpu);
		}
4054

4055
		vcpu->arch.smi_pending = events->smi.pending;
4056 4057 4058 4059

		if (events->smi.smm) {
			if (events->smi.smm_inside_nmi)
				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
4060
			else
4061
				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
4062 4063 4064 4065 4066 4067 4068
		}

		if (lapic_in_kernel(vcpu)) {
			if (events->smi.latched_init)
				set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
			else
				clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
4069 4070 4071
		}
	}

4072 4073
	kvm_make_request(KVM_REQ_EVENT, vcpu);

4074 4075 4076
	return 0;
}

4077 4078 4079
static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
					     struct kvm_debugregs *dbgregs)
{
Jan Kiszka's avatar
Jan Kiszka committed
4080 4081
	unsigned long val;

4082
	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
4083
	kvm_get_dr(vcpu, 6, &val);
Jan Kiszka's avatar
Jan Kiszka committed
4084
	dbgregs->dr6 = val;
4085 4086
	dbgregs->dr7 = vcpu->arch.dr7;
	dbgregs->flags = 0;
4087
	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
4088 4089 4090 4091 4092 4093 4094 4095
}

static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
					    struct kvm_debugregs *dbgregs)
{
	if (dbgregs->flags)
		return -EINVAL;

4096 4097 4098 4099 4100
	if (dbgregs->dr6 & ~0xffffffffull)
		return -EINVAL;
	if (dbgregs->dr7 & ~0xffffffffull)
		return -EINVAL;

4101
	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
4102
	kvm_update_dr0123(vcpu);
4103 4104
	vcpu->arch.dr6 = dbgregs->dr6;
	vcpu->arch.dr7 = dbgregs->dr7;
4105
	kvm_update_dr7(vcpu);
4106 4107 4108 4109

	return 0;
}

4110 4111 4112 4113
#define XSTATE_COMPACTION_ENABLED (1ULL << 63)

static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu)
{
4114
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
4115
	u64 xstate_bv = xsave->header.xfeatures;
4116 4117 4118 4119 4120 4121 4122 4123 4124
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(dest, xsave, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV */
4125
	xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;
4126 4127 4128 4129 4130 4131
	*(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv;

	/*
	 * Copy each region from the possibly compacted offset to the
	 * non-compacted offset.
	 */
Dave Hansen's avatar
Dave Hansen committed
4132
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
4133
	while (valid) {
4134 4135 4136
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *src = get_xsave_addr(xsave, xfeature_nr);
4137 4138 4139

		if (src) {
			u32 size, offset, ecx, edx;
4140
			cpuid_count(XSTATE_CPUID, xfeature_nr,
4141
				    &size, &offset, &ecx, &edx);
4142
			if (xfeature_nr == XFEATURE_PKRU)
4143 4144 4145 4146 4147
				memcpy(dest + offset, &vcpu->arch.pkru,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest + offset, src, size);

4148 4149
		}

4150
		valid -= xfeature_mask;
4151 4152 4153 4154 4155
	}
}

static void load_xsave(struct kvm_vcpu *vcpu, u8 *src)
{
4156
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
4157 4158 4159 4160 4161 4162 4163 4164 4165 4166
	u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET);
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(xsave, src, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV and possibly XCOMP_BV.  */
4167
	xsave->header.xfeatures = xstate_bv;
4168
	if (boot_cpu_has(X86_FEATURE_XSAVES))
4169
		xsave->header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED;
4170 4171 4172 4173 4174

	/*
	 * Copy each region from the non-compacted offset to the
	 * possibly compacted offset.
	 */
Dave Hansen's avatar
Dave Hansen committed
4175
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
4176
	while (valid) {
4177 4178 4179
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *dest = get_xsave_addr(xsave, xfeature_nr);
4180 4181 4182

		if (dest) {
			u32 size, offset, ecx, edx;
4183
			cpuid_count(XSTATE_CPUID, xfeature_nr,
4184
				    &size, &offset, &ecx, &edx);
4185
			if (xfeature_nr == XFEATURE_PKRU)
4186 4187 4188 4189
				memcpy(&vcpu->arch.pkru, src + offset,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest, src + offset, size);
4190
		}
4191

4192
		valid -= xfeature_mask;
4193 4194 4195
	}
}

4196 4197 4198
static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
					 struct kvm_xsave *guest_xsave)
{
4199
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
4200 4201
		memset(guest_xsave, 0, sizeof(struct kvm_xsave));
		fill_xsave((u8 *) guest_xsave->region, vcpu);
4202
	} else {
4203
		memcpy(guest_xsave->region,
4204
			&vcpu->arch.guest_fpu->state.fxsave,
4205
			sizeof(struct fxregs_state));
4206
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
Dave Hansen's avatar
Dave Hansen committed
4207
			XFEATURE_MASK_FPSSE;
4208 4209 4210
	}
}

4211 4212
#define XSAVE_MXCSR_OFFSET 24

4213 4214 4215 4216 4217
static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
					struct kvm_xsave *guest_xsave)
{
	u64 xstate_bv =
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
4218
	u32 mxcsr = *(u32 *)&guest_xsave->region[XSAVE_MXCSR_OFFSET / sizeof(u32)];
4219

4220
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
4221 4222 4223 4224 4225
		/*
		 * Here we allow setting states that are not present in
		 * CPUID leaf 0xD, index 0, EDX:EAX.  This is for compatibility
		 * with old userspace.
		 */
4226
		if (xstate_bv & ~supported_xcr0 || mxcsr & ~mxcsr_feature_mask)
4227
			return -EINVAL;
4228
		load_xsave(vcpu, (u8 *)guest_xsave->region);
4229
	} else {
4230 4231
		if (xstate_bv & ~XFEATURE_MASK_FPSSE ||
			mxcsr & ~mxcsr_feature_mask)
4232
			return -EINVAL;
4233
		memcpy(&vcpu->arch.guest_fpu->state.fxsave,
4234
			guest_xsave->region, sizeof(struct fxregs_state));
4235 4236 4237 4238 4239 4240 4241
	}
	return 0;
}

static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
					struct kvm_xcrs *guest_xcrs)
{
4242
	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257
		guest_xcrs->nr_xcrs = 0;
		return;
	}

	guest_xcrs->nr_xcrs = 1;
	guest_xcrs->flags = 0;
	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
}

static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
				       struct kvm_xcrs *guest_xcrs)
{
	int i, r = 0;

4258
	if (!boot_cpu_has(X86_FEATURE_XSAVE))
4259 4260 4261 4262 4263 4264 4265
		return -EINVAL;

	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
		return -EINVAL;

	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
		/* Only support XCR0 currently */
4266
		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
4267
			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
4268
				guest_xcrs->xcrs[i].value);
4269 4270 4271 4272 4273 4274 4275
			break;
		}
	if (r)
		r = -EINVAL;
	return r;
}

4276 4277 4278 4279 4280 4281 4282 4283
/*
 * kvm_set_guest_paused() indicates to the guest kernel that it has been
 * stopped by the hypervisor.  This function will be called from the host only.
 * EINVAL is returned when the host attempts to set the flag for a guest that
 * does not support pv clocks.
 */
static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
{
4284
	if (!vcpu->arch.pv_time_enabled)
4285
		return -EINVAL;
4286
	vcpu->arch.pvclock_set_guest_stopped_request = true;
4287 4288 4289 4290
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	return 0;
}

4291 4292 4293
static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
				     struct kvm_enable_cap *cap)
{
4294 4295 4296 4297
	int r;
	uint16_t vmcs_version;
	void __user *user_ptr;

4298 4299 4300 4301
	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
4302 4303 4304
	case KVM_CAP_HYPERV_SYNIC2:
		if (cap->args[0])
			return -EINVAL;
4305 4306
		/* fall through */

4307
	case KVM_CAP_HYPERV_SYNIC:
4308 4309
		if (!irqchip_in_kernel(vcpu->kvm))
			return -EINVAL;
4310 4311
		return kvm_hv_activate_synic(vcpu, cap->cap ==
					     KVM_CAP_HYPERV_SYNIC2);
4312
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
4313
		if (!kvm_x86_ops.nested_ops->enable_evmcs)
4314
			return -ENOTTY;
4315
		r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);
4316 4317 4318 4319 4320 4321 4322
		if (!r) {
			user_ptr = (void __user *)(uintptr_t)cap->args[0];
			if (copy_to_user(user_ptr, &vmcs_version,
					 sizeof(vmcs_version)))
				r = -EFAULT;
		}
		return r;
4323
	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
4324
		if (!kvm_x86_ops.enable_direct_tlbflush)
4325 4326
			return -ENOTTY;

4327
		return kvm_x86_ops.enable_direct_tlbflush(vcpu);
4328

4329 4330 4331 4332 4333
	default:
		return -EINVAL;
	}
}

4334 4335 4336 4337 4338 4339
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r;
4340 4341 4342 4343 4344 4345 4346
	union {
		struct kvm_lapic_state *lapic;
		struct kvm_xsave *xsave;
		struct kvm_xcrs *xcrs;
		void *buffer;
	} u;

4347 4348
	vcpu_load(vcpu);

4349
	u.buffer = NULL;
4350 4351
	switch (ioctl) {
	case KVM_GET_LAPIC: {
4352
		r = -EINVAL;
4353
		if (!lapic_in_kernel(vcpu))
4354
			goto out;
4355 4356
		u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
				GFP_KERNEL_ACCOUNT);
4357

4358
		r = -ENOMEM;
4359
		if (!u.lapic)
4360
			goto out;
4361
		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
4362 4363 4364
		if (r)
			goto out;
		r = -EFAULT;
4365
		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
4366 4367 4368 4369 4370
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
4371
		r = -EINVAL;
4372
		if (!lapic_in_kernel(vcpu))
4373
			goto out;
4374
		u.lapic = memdup_user(argp, sizeof(*u.lapic));
4375 4376 4377 4378
		if (IS_ERR(u.lapic)) {
			r = PTR_ERR(u.lapic);
			goto out_nofree;
		}
4379

4380
		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
4381 4382
		break;
	}
4383 4384 4385 4386
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
4387
		if (copy_from_user(&irq, argp, sizeof(irq)))
4388 4389 4390 4391
			goto out;
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
		break;
	}
4392 4393 4394 4395
	case KVM_NMI: {
		r = kvm_vcpu_ioctl_nmi(vcpu);
		break;
	}
4396 4397 4398 4399
	case KVM_SMI: {
		r = kvm_vcpu_ioctl_smi(vcpu);
		break;
	}
4400 4401 4402 4403 4404
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
4405
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4406 4407 4408 4409
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		break;
	}
4410 4411 4412 4413 4414
	case KVM_SET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4415
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4416 4417
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
4418
					      cpuid_arg->entries);
4419 4420 4421 4422 4423 4424 4425
		break;
	}
	case KVM_GET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4426
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4427 4428
			goto out;
		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
4429
					      cpuid_arg->entries);
4430 4431 4432
		if (r)
			goto out;
		r = -EFAULT;
4433
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4434 4435 4436 4437
			goto out;
		r = 0;
		break;
	}
4438 4439
	case KVM_GET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4440
		r = msr_io(vcpu, argp, do_get_msr, 1);
4441
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4442
		break;
4443 4444 4445
	}
	case KVM_SET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4446
		r = msr_io(vcpu, argp, do_set_msr, 0);
4447
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4448
		break;
4449
	}
4450 4451 4452 4453
	case KVM_TPR_ACCESS_REPORTING: {
		struct kvm_tpr_access_ctl tac;

		r = -EFAULT;
4454
		if (copy_from_user(&tac, argp, sizeof(tac)))
4455 4456 4457 4458 4459
			goto out;
		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
		if (r)
			goto out;
		r = -EFAULT;
4460
		if (copy_to_user(argp, &tac, sizeof(tac)))
4461 4462 4463 4464
			goto out;
		r = 0;
		break;
	};
Avi Kivity's avatar
Avi Kivity committed
4465 4466
	case KVM_SET_VAPIC_ADDR: {
		struct kvm_vapic_addr va;
4467
		int idx;
Avi Kivity's avatar
Avi Kivity committed
4468 4469

		r = -EINVAL;
4470
		if (!lapic_in_kernel(vcpu))
Avi Kivity's avatar
Avi Kivity committed
4471 4472
			goto out;
		r = -EFAULT;
4473
		if (copy_from_user(&va, argp, sizeof(va)))
Avi Kivity's avatar
Avi Kivity committed
4474
			goto out;
4475
		idx = srcu_read_lock(&vcpu->kvm->srcu);
4476
		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
4477
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
Avi Kivity's avatar
Avi Kivity committed
4478 4479
		break;
	}
Huang Ying's avatar
Huang Ying committed
4480 4481 4482 4483
	case KVM_X86_SETUP_MCE: {
		u64 mcg_cap;

		r = -EFAULT;
4484
		if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
Huang Ying's avatar
Huang Ying committed
4485 4486 4487 4488 4489 4490 4491 4492
			goto out;
		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
		break;
	}
	case KVM_X86_SET_MCE: {
		struct kvm_x86_mce mce;

		r = -EFAULT;
4493
		if (copy_from_user(&mce, argp, sizeof(mce)))
Huang Ying's avatar
Huang Ying committed
4494 4495 4496 4497
			goto out;
		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
		break;
	}
4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);

		r = -EFAULT;
		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		r = -EFAULT;
		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
			break;

		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
		break;
	}
4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541
	case KVM_GET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);

		r = -EFAULT;
		if (copy_to_user(argp, &dbgregs,
				 sizeof(struct kvm_debugregs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		r = -EFAULT;
		if (copy_from_user(&dbgregs, argp,
				   sizeof(struct kvm_debugregs)))
			break;

		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
		break;
	}
4542
	case KVM_GET_XSAVE: {
4543
		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
4544
		r = -ENOMEM;
4545
		if (!u.xsave)
4546 4547
			break;

4548
		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
4549 4550

		r = -EFAULT;
4551
		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
4552 4553 4554 4555 4556
			break;
		r = 0;
		break;
	}
	case KVM_SET_XSAVE: {
4557
		u.xsave = memdup_user(argp, sizeof(*u.xsave));
4558 4559 4560 4561
		if (IS_ERR(u.xsave)) {
			r = PTR_ERR(u.xsave);
			goto out_nofree;
		}
4562

4563
		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
4564 4565 4566
		break;
	}
	case KVM_GET_XCRS: {
4567
		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
4568
		r = -ENOMEM;
4569
		if (!u.xcrs)
4570 4571
			break;

4572
		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
4573 4574

		r = -EFAULT;
4575
		if (copy_to_user(argp, u.xcrs,
4576 4577 4578 4579 4580 4581
				 sizeof(struct kvm_xcrs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XCRS: {
4582
		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
4583 4584 4585 4586
		if (IS_ERR(u.xcrs)) {
			r = PTR_ERR(u.xcrs);
			goto out_nofree;
		}
4587

4588
		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
4589 4590
		break;
	}
4591 4592 4593 4594 4595 4596 4597 4598 4599
	case KVM_SET_TSC_KHZ: {
		u32 user_tsc_khz;

		r = -EINVAL;
		user_tsc_khz = (u32)arg;

		if (user_tsc_khz >= kvm_max_guest_tsc_khz)
			goto out;

4600 4601 4602
		if (user_tsc_khz == 0)
			user_tsc_khz = tsc_khz;

4603 4604
		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
			r = 0;
4605 4606 4607 4608

		goto out;
	}
	case KVM_GET_TSC_KHZ: {
4609
		r = vcpu->arch.virtual_tsc_khz;
4610 4611
		goto out;
	}
4612 4613 4614 4615
	case KVM_KVMCLOCK_CTRL: {
		r = kvm_set_guest_paused(vcpu);
		goto out;
	}
4616 4617 4618 4619 4620 4621 4622 4623 4624
	case KVM_ENABLE_CAP: {
		struct kvm_enable_cap cap;

		r = -EFAULT;
		if (copy_from_user(&cap, argp, sizeof(cap)))
			goto out;
		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
		break;
	}
4625 4626 4627 4628 4629
	case KVM_GET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		u32 user_data_size;

		r = -EINVAL;
4630
		if (!kvm_x86_ops.nested_ops->get_state)
4631 4632 4633
			break;

		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
4634
		r = -EFAULT;
4635
		if (get_user(user_data_size, &user_kvm_nested_state->size))
4636
			break;
4637

4638 4639
		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
						     user_data_size);
4640
		if (r < 0)
4641
			break;
4642 4643 4644

		if (r > user_data_size) {
			if (put_user(r, &user_kvm_nested_state->size))
4645 4646 4647 4648
				r = -EFAULT;
			else
				r = -E2BIG;
			break;
4649
		}
4650

4651 4652 4653 4654 4655 4656
		r = 0;
		break;
	}
	case KVM_SET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		struct kvm_nested_state kvm_state;
4657
		int idx;
4658 4659

		r = -EINVAL;
4660
		if (!kvm_x86_ops.nested_ops->set_state)
4661 4662
			break;

4663
		r = -EFAULT;
4664
		if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
4665
			break;
4666

4667
		r = -EINVAL;
4668
		if (kvm_state.size < sizeof(kvm_state))
4669
			break;
4670 4671

		if (kvm_state.flags &
4672
		    ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
4673 4674
		      | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING
		      | KVM_STATE_NESTED_GIF_SET))
4675
			break;
4676 4677

		/* nested_run_pending implies guest_mode.  */
4678 4679
		if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
		    && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
4680
			break;
4681

4682
		idx = srcu_read_lock(&vcpu->kvm->srcu);
4683
		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
4684
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4685 4686
		break;
	}
4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705
	case KVM_GET_SUPPORTED_HV_CPUID: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
			goto out;

		r = kvm_vcpu_ioctl_get_hv_cpuid(vcpu, &cpuid,
						cpuid_arg->entries);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
			goto out;
		r = 0;
		break;
	}
4706 4707 4708 4709
	default:
		r = -EINVAL;
	}
out:
4710
	kfree(u.buffer);
4711 4712
out_nofree:
	vcpu_put(vcpu);
4713 4714 4715
	return r;
}

4716
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4717 4718 4719 4720
{
	return VM_FAULT_SIGBUS;
}

4721 4722 4723 4724 4725
static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
{
	int ret;

	if (addr > (unsigned int)(-3 * PAGE_SIZE))
4726
		return -EINVAL;
4727
	ret = kvm_x86_ops.set_tss_addr(kvm, addr);
4728 4729 4730
	return ret;
}

4731 4732 4733
static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
					      u64 ident_addr)
{
4734
	return kvm_x86_ops.set_identity_map_addr(kvm, ident_addr);
4735 4736
}

4737
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
4738
					 unsigned long kvm_nr_mmu_pages)
4739 4740 4741 4742
{
	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
		return -EINVAL;

4743
	mutex_lock(&kvm->slots_lock);
4744 4745

	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
4746
	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
4747

4748
	mutex_unlock(&kvm->slots_lock);
4749 4750 4751
	return 0;
}

4752
static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
4753
{
4754
	return kvm->arch.n_max_mmu_pages;
4755 4756 4757 4758
}

static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4759
	struct kvm_pic *pic = kvm->arch.vpic;
4760 4761 4762 4763 4764
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4765
		memcpy(&chip->chip.pic, &pic->pics[0],
4766 4767 4768
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4769
		memcpy(&chip->chip.pic, &pic->pics[1],
4770 4771 4772
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_IOAPIC:
4773
		kvm_get_ioapic(kvm, &chip->chip.ioapic);
4774 4775 4776 4777 4778 4779 4780 4781 4782 4783
		break;
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4784
	struct kvm_pic *pic = kvm->arch.vpic;
4785 4786 4787 4788 4789
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4790 4791
		spin_lock(&pic->lock);
		memcpy(&pic->pics[0], &chip->chip.pic,
4792
			sizeof(struct kvm_pic_state));
4793
		spin_unlock(&pic->lock);
4794 4795
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4796 4797
		spin_lock(&pic->lock);
		memcpy(&pic->pics[1], &chip->chip.pic,
4798
			sizeof(struct kvm_pic_state));
4799
		spin_unlock(&pic->lock);
4800 4801
		break;
	case KVM_IRQCHIP_IOAPIC:
4802
		kvm_set_ioapic(kvm, &chip->chip.ioapic);
4803 4804 4805 4806 4807
		break;
	default:
		r = -EINVAL;
		break;
	}
4808
	kvm_pic_update_irq(pic);
4809 4810 4811
	return r;
}

4812 4813
static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4814 4815 4816 4817 4818 4819 4820
	struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;

	BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));

	mutex_lock(&kps->lock);
	memcpy(ps, &kps->channels, sizeof(*ps));
	mutex_unlock(&kps->lock);
4821
	return 0;
4822 4823 4824 4825
}

static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4826
	int i;
4827 4828 4829
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
4830
	memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
4831
	for (i = 0; i < 3; i++)
4832 4833
		kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
	mutex_unlock(&pit->pit_state.lock);
4834
	return 0;
4835 4836 4837 4838 4839 4840 4841 4842 4843
}

static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
		sizeof(ps->channels));
	ps->flags = kvm->arch.vpit->pit_state.flags;
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
4844
	memset(&ps->reserved, 0, sizeof(ps->reserved));
4845
	return 0;
4846 4847 4848 4849
}

static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
4850
	int start = 0;
4851
	int i;
4852
	u32 prev_legacy, cur_legacy;
4853 4854 4855 4856
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
	prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
4857 4858 4859
	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
	if (!prev_legacy && cur_legacy)
		start = 1;
4860 4861 4862
	memcpy(&pit->pit_state.channels, &ps->channels,
	       sizeof(pit->pit_state.channels));
	pit->pit_state.flags = ps->flags;
4863
	for (i = 0; i < 3; i++)
4864
		kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
4865
				   start && i == 0);
4866
	mutex_unlock(&pit->pit_state.lock);
4867
	return 0;
4868 4869
}

4870 4871 4872
static int kvm_vm_ioctl_reinject(struct kvm *kvm,
				 struct kvm_reinject_control *control)
{
4873 4874 4875 4876 4877 4878 4879 4880 4881
	struct kvm_pit *pit = kvm->arch.vpit;

	/* pit->pit_state.lock was overloaded to prevent userspace from getting
	 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
	 * ioctls in parallel.  Use a separate lock if that ioctl isn't rare.
	 */
	mutex_lock(&pit->pit_state.lock);
	kvm_pit_set_reinject(pit, control->pit_reinject);
	mutex_unlock(&pit->pit_state.lock);
4882

4883 4884 4885
	return 0;
}

4886
void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
4887
{
4888 4889 4890
	/*
	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
	 */
4891 4892
	if (kvm_x86_ops.flush_log_dirty)
		kvm_x86_ops.flush_log_dirty(kvm);
4893 4894
}

4895 4896
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
			bool line_status)
4897 4898 4899 4900 4901
{
	if (!irqchip_in_kernel(kvm))
		return -ENXIO;

	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
4902 4903
					irq_event->irq, irq_event->level,
					line_status);
4904 4905 4906
	return 0;
}

4907 4908
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
			    struct kvm_enable_cap *cap)
4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919
{
	int r;

	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
	case KVM_CAP_DISABLE_QUIRKS:
		kvm->arch.disabled_quirks = cap->args[0];
		r = 0;
		break;
4920 4921
	case KVM_CAP_SPLIT_IRQCHIP: {
		mutex_lock(&kvm->lock);
4922 4923 4924
		r = -EINVAL;
		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
			goto split_irqchip_unlock;
4925 4926 4927
		r = -EEXIST;
		if (irqchip_in_kernel(kvm))
			goto split_irqchip_unlock;
4928
		if (kvm->created_vcpus)
4929 4930
			goto split_irqchip_unlock;
		r = kvm_setup_empty_irq_routing(kvm);
4931
		if (r)
4932 4933 4934
			goto split_irqchip_unlock;
		/* Pairs with irqchip_in_kernel. */
		smp_wmb();
4935
		kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
4936
		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
4937 4938 4939 4940 4941
		r = 0;
split_irqchip_unlock:
		mutex_unlock(&kvm->lock);
		break;
	}
4942 4943 4944 4945 4946 4947 4948
	case KVM_CAP_X2APIC_API:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
			break;

		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
			kvm->arch.x2apic_format = true;
4949 4950
		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
			kvm->arch.x2apic_broadcast_quirk_disabled = true;
4951 4952 4953

		r = 0;
		break;
4954 4955 4956 4957 4958 4959 4960 4961
	case KVM_CAP_X86_DISABLE_EXITS:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
			break;

		if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
			kvm_can_mwait_in_guest())
			kvm->arch.mwait_in_guest = true;
4962
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
4963
			kvm->arch.hlt_in_guest = true;
4964 4965
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
			kvm->arch.pause_in_guest = true;
4966 4967
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
			kvm->arch.cstate_in_guest = true;
4968 4969
		r = 0;
		break;
4970 4971 4972
	case KVM_CAP_MSR_PLATFORM_INFO:
		kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
		r = 0;
4973 4974 4975 4976
		break;
	case KVM_CAP_EXCEPTION_PAYLOAD:
		kvm->arch.exception_payload_enabled = cap->args[0];
		r = 0;
4977
		break;
4978 4979 4980 4981 4982 4983 4984
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

4985 4986 4987 4988 4989
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
4990
	int r = -ENOTTY;
4991 4992 4993 4994 4995 4996 4997
	/*
	 * This union makes it completely explicit to gcc-3.x
	 * that these two variables' stack usage should be
	 * combined, not added together.
	 */
	union {
		struct kvm_pit_state ps;
4998
		struct kvm_pit_state2 ps2;
4999
		struct kvm_pit_config pit_config;
5000
	} u;
5001 5002 5003 5004 5005

	switch (ioctl) {
	case KVM_SET_TSS_ADDR:
		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
		break;
5006 5007 5008
	case KVM_SET_IDENTITY_MAP_ADDR: {
		u64 ident_addr;

5009 5010 5011 5012
		mutex_lock(&kvm->lock);
		r = -EINVAL;
		if (kvm->created_vcpus)
			goto set_identity_unlock;
5013
		r = -EFAULT;
5014
		if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
5015
			goto set_identity_unlock;
5016
		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
5017 5018
set_identity_unlock:
		mutex_unlock(&kvm->lock);
5019 5020
		break;
	}
5021 5022 5023 5024 5025 5026
	case KVM_SET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
		break;
	case KVM_GET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
		break;
5027 5028
	case KVM_CREATE_IRQCHIP: {
		mutex_lock(&kvm->lock);
5029

5030
		r = -EEXIST;
5031
		if (irqchip_in_kernel(kvm))
5032
			goto create_irqchip_unlock;
5033

5034
		r = -EINVAL;
5035
		if (kvm->created_vcpus)
5036
			goto create_irqchip_unlock;
5037 5038 5039

		r = kvm_pic_init(kvm);
		if (r)
5040
			goto create_irqchip_unlock;
5041 5042 5043 5044

		r = kvm_ioapic_init(kvm);
		if (r) {
			kvm_pic_destroy(kvm);
5045
			goto create_irqchip_unlock;
5046 5047
		}

5048 5049
		r = kvm_setup_default_irq_routing(kvm);
		if (r) {
5050
			kvm_ioapic_destroy(kvm);
5051
			kvm_pic_destroy(kvm);
5052
			goto create_irqchip_unlock;
5053
		}
5054
		/* Write kvm->irq_routing before enabling irqchip_in_kernel. */
5055
		smp_wmb();
5056
		kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
5057 5058
	create_irqchip_unlock:
		mutex_unlock(&kvm->lock);
5059
		break;
5060
	}
Sheng Yang's avatar
Sheng Yang committed
5061
	case KVM_CREATE_PIT:
5062 5063 5064 5065 5066 5067 5068 5069
		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
		goto create_pit;
	case KVM_CREATE_PIT2:
		r = -EFAULT;
		if (copy_from_user(&u.pit_config, argp,
				   sizeof(struct kvm_pit_config)))
			goto out;
	create_pit:
5070
		mutex_lock(&kvm->lock);
Avi Kivity's avatar
Avi Kivity committed
5071 5072 5073
		r = -EEXIST;
		if (kvm->arch.vpit)
			goto create_pit_unlock;
Sheng Yang's avatar
Sheng Yang committed
5074
		r = -ENOMEM;
5075
		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
Sheng Yang's avatar
Sheng Yang committed
5076 5077
		if (kvm->arch.vpit)
			r = 0;
Avi Kivity's avatar
Avi Kivity committed
5078
	create_pit_unlock:
5079
		mutex_unlock(&kvm->lock);
Sheng Yang's avatar
Sheng Yang committed
5080
		break;
5081 5082
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
5083
		struct kvm_irqchip *chip;
5084

5085 5086 5087
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
5088
			goto out;
5089 5090
		}

5091
		r = -ENXIO;
5092
		if (!irqchip_kernel(kvm))
5093 5094
			goto get_irqchip_out;
		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
5095
		if (r)
5096
			goto get_irqchip_out;
5097
		r = -EFAULT;
5098
		if (copy_to_user(argp, chip, sizeof(*chip)))
5099
			goto get_irqchip_out;
5100
		r = 0;
5101 5102
	get_irqchip_out:
		kfree(chip);
5103 5104 5105 5106
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
5107
		struct kvm_irqchip *chip;
5108

5109 5110 5111
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
5112
			goto out;
5113 5114
		}

5115
		r = -ENXIO;
5116
		if (!irqchip_kernel(kvm))
5117 5118 5119 5120
			goto set_irqchip_out;
		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
	set_irqchip_out:
		kfree(chip);
5121 5122
		break;
	}
5123 5124
	case KVM_GET_PIT: {
		r = -EFAULT;
5125
		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
5126 5127 5128 5129
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
5130
		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
5131 5132 5133
		if (r)
			goto out;
		r = -EFAULT;
5134
		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
5135 5136 5137 5138 5139 5140
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT: {
		r = -EFAULT;
5141
		if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
5142
			goto out;
5143
		mutex_lock(&kvm->lock);
5144 5145
		r = -ENXIO;
		if (!kvm->arch.vpit)
5146
			goto set_pit_out;
5147
		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
5148 5149
set_pit_out:
		mutex_unlock(&kvm->lock);
5150 5151
		break;
	}
5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168
	case KVM_GET_PIT2: {
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT2: {
		r = -EFAULT;
		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
			goto out;
5169
		mutex_lock(&kvm->lock);
5170 5171
		r = -ENXIO;
		if (!kvm->arch.vpit)
5172
			goto set_pit2_out;
5173
		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
5174 5175
set_pit2_out:
		mutex_unlock(&kvm->lock);
5176 5177
		break;
	}
5178 5179 5180 5181 5182
	case KVM_REINJECT_CONTROL: {
		struct kvm_reinject_control control;
		r =  -EFAULT;
		if (copy_from_user(&control, argp, sizeof(control)))
			goto out;
5183 5184 5185
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
5186 5187 5188
		r = kvm_vm_ioctl_reinject(kvm, &control);
		break;
	}
5189 5190 5191
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
		mutex_lock(&kvm->lock);
5192
		if (kvm->created_vcpus)
5193 5194 5195 5196 5197
			r = -EBUSY;
		else
			kvm->arch.bsp_vcpu_id = arg;
		mutex_unlock(&kvm->lock);
		break;
Ed Swierk's avatar
Ed Swierk committed
5198
	case KVM_XEN_HVM_CONFIG: {
5199
		struct kvm_xen_hvm_config xhc;
Ed Swierk's avatar
Ed Swierk committed
5200
		r = -EFAULT;
5201
		if (copy_from_user(&xhc, argp, sizeof(xhc)))
Ed Swierk's avatar
Ed Swierk committed
5202 5203
			goto out;
		r = -EINVAL;
5204
		if (xhc.flags)
Ed Swierk's avatar
Ed Swierk committed
5205
			goto out;
5206
		memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));
Ed Swierk's avatar
Ed Swierk committed
5207 5208 5209
		r = 0;
		break;
	}
5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222
	case KVM_SET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

		r = -EFAULT;
		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
			goto out;

		r = -EINVAL;
		if (user_ns.flags)
			goto out;

		r = 0;
5223 5224 5225 5226 5227 5228
		/*
		 * TODO: userspace has to take care of races with VCPU_RUN, so
		 * kvm_gen_update_masterclock() can be cut down to locked
		 * pvclock_update_vm_gtod_copy().
		 */
		kvm_gen_update_masterclock(kvm);
5229
		now_ns = get_kvmclock_ns(kvm);
5230
		kvm->arch.kvmclock_offset += user_ns.clock - now_ns;
5231
		kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE);
5232 5233 5234 5235 5236 5237
		break;
	}
	case KVM_GET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

5238
		now_ns = get_kvmclock_ns(kvm);
5239
		user_ns.clock = now_ns;
5240
		user_ns.flags = kvm->arch.use_master_clock ? KVM_CLOCK_TSC_STABLE : 0;
5241
		memset(&user_ns.pad, 0, sizeof(user_ns.pad));
5242 5243 5244 5245 5246 5247 5248

		r = -EFAULT;
		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
			goto out;
		r = 0;
		break;
	}
5249 5250
	case KVM_MEMORY_ENCRYPT_OP: {
		r = -ENOTTY;
5251 5252
		if (kvm_x86_ops.mem_enc_op)
			r = kvm_x86_ops.mem_enc_op(kvm, argp);
5253 5254
		break;
	}
5255 5256 5257 5258 5259 5260 5261 5262
	case KVM_MEMORY_ENCRYPT_REG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
5263 5264
		if (kvm_x86_ops.mem_enc_reg_region)
			r = kvm_x86_ops.mem_enc_reg_region(kvm, &region);
5265 5266 5267 5268 5269 5270 5271 5272 5273 5274
		break;
	}
	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
5275 5276
		if (kvm_x86_ops.mem_enc_unreg_region)
			r = kvm_x86_ops.mem_enc_unreg_region(kvm, &region);
5277 5278
		break;
	}
5279 5280 5281 5282 5283 5284 5285 5286 5287
	case KVM_HYPERV_EVENTFD: {
		struct kvm_hyperv_eventfd hvevfd;

		r = -EFAULT;
		if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
			goto out;
		r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
		break;
	}
Eric Hankland's avatar
Eric Hankland committed
5288 5289 5290
	case KVM_SET_PMU_EVENT_FILTER:
		r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
		break;
5291
	default:
5292
		r = -ENOTTY;
5293 5294 5295 5296 5297
	}
out:
	return r;
}

5298
static void kvm_init_msr_list(void)
5299
{
5300
	struct x86_pmu_capability x86_pmu;
5301
	u32 dummy[2];
5302
	unsigned i;
5303

5304
	BUILD_BUG_ON_MSG(INTEL_PMC_MAX_FIXED != 4,
5305
			 "Please update the fixed PMCs in msrs_to_saved_all[]");
5306 5307

	perf_get_x86_pmu_capability(&x86_pmu);
5308

5309 5310 5311 5312
	num_msrs_to_save = 0;
	num_emulated_msrs = 0;
	num_msr_based_features = 0;

5313 5314
	for (i = 0; i < ARRAY_SIZE(msrs_to_save_all); i++) {
		if (rdmsr_safe(msrs_to_save_all[i], &dummy[0], &dummy[1]) < 0)
5315
			continue;
5316 5317 5318

		/*
		 * Even MSRs that are valid in the host may not be exposed
5319
		 * to the guests in some cases.
5320
		 */
5321
		switch (msrs_to_save_all[i]) {
5322
		case MSR_IA32_BNDCFGS:
5323
			if (!kvm_mpx_supported())
5324 5325
				continue;
			break;
5326
		case MSR_TSC_AUX:
5327
			if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP))
5328 5329
				continue;
			break;
5330 5331 5332 5333
		case MSR_IA32_UMWAIT_CONTROL:
			if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG))
				continue;
			break;
5334 5335
		case MSR_IA32_RTIT_CTL:
		case MSR_IA32_RTIT_STATUS:
5336
			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT))
5337 5338 5339
				continue;
			break;
		case MSR_IA32_RTIT_CR3_MATCH:
5340
			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
5341 5342 5343 5344 5345
			    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
				continue;
			break;
		case MSR_IA32_RTIT_OUTPUT_BASE:
		case MSR_IA32_RTIT_OUTPUT_MASK:
5346
			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
5347 5348 5349 5350
				(!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
				 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
				continue;
			break;
5351
		case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
5352
			if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
5353
				msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >=
5354 5355 5356
				intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
				continue;
			break;
5357
		case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17:
5358
			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
5359 5360 5361
			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
				continue;
			break;
5362
		case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 17:
5363
			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >=
5364 5365
			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
				continue;
5366
			break;
5367 5368 5369 5370
		default:
			break;
		}

5371
		msrs_to_save[num_msrs_to_save++] = msrs_to_save_all[i];
5372
	}
5373

5374
	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
5375
		if (!kvm_x86_ops.has_emulated_msr(emulated_msrs_all[i]))
5376
			continue;
5377

5378
		emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
5379
	}
5380

5381
	for (i = 0; i < ARRAY_SIZE(msr_based_features_all); i++) {
5382 5383
		struct kvm_msr_entry msr;

5384
		msr.index = msr_based_features_all[i];
5385
		if (kvm_get_msr_feature(&msr))
5386 5387
			continue;

5388
		msr_based_features[num_msr_based_features++] = msr_based_features_all[i];
5389
	}
5390 5391
}

5392 5393
static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
			   const void *v)
5394
{
5395 5396 5397 5398 5399
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5400
		if (!(lapic_in_kernel(vcpu) &&
5401 5402
		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
5403 5404 5405 5406 5407 5408
			break;
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5409

5410
	return handled;
5411 5412
}

5413
static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
5414
{
5415 5416 5417 5418 5419
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5420
		if (!(lapic_in_kernel(vcpu) &&
5421 5422 5423
		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
					 addr, n, v))
		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
5424
			break;
5425
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
5426 5427 5428 5429 5430
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5431

5432
	return handled;
5433 5434
}

5435 5436 5437
static void kvm_set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
5438
	kvm_x86_ops.set_segment(vcpu, var, seg);
5439 5440 5441 5442 5443
}

void kvm_get_segment(struct kvm_vcpu *vcpu,
		     struct kvm_segment *var, int seg)
{
5444
	kvm_x86_ops.get_segment(vcpu, var, seg);
5445 5446
}

5447 5448
gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
			   struct x86_exception *exception)
5449 5450 5451 5452 5453 5454 5455
{
	gpa_t t_gpa;

	BUG_ON(!mmu_is_nested(vcpu));

	/* NPT walks are always user-walks */
	access |= PFERR_USER_MASK;
5456
	t_gpa  = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception);
5457 5458 5459 5460

	return t_gpa;
}

5461 5462
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
			      struct x86_exception *exception)
5463
{
5464
	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5465
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5466 5467
}

5468 5469
 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5470
{
5471
	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5472
	access |= PFERR_FETCH_MASK;
5473
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5474 5475
}

5476 5477
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
			       struct x86_exception *exception)
5478
{
5479
	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5480
	access |= PFERR_WRITE_MASK;
5481
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5482 5483 5484
}

/* uses this to access any guest's mapped memory without checking CPL */
5485 5486
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5487
{
5488
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
5489 5490 5491 5492
}

static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
5493
				      struct x86_exception *exception)
5494 5495
{
	void *data = val;
5496
	int r = X86EMUL_CONTINUE;
5497 5498

	while (bytes) {
5499
		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
5500
							    exception);
5501
		unsigned offset = addr & (PAGE_SIZE-1);
5502
		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
5503 5504
		int ret;

5505
		if (gpa == UNMAPPED_GVA)
5506
			return X86EMUL_PROPAGATE_FAULT;
5507 5508
		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
					       offset, toread);
5509
		if (ret < 0) {
5510
			r = X86EMUL_IO_NEEDED;
5511 5512
			goto out;
		}
5513

5514 5515 5516
		bytes -= toread;
		data += toread;
		addr += toread;
5517
	}
5518 5519
out:
	return r;
5520
}
5521

5522
/* used for instruction fetching */
5523 5524
static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
				gva_t addr, void *val, unsigned int bytes,
5525
				struct x86_exception *exception)
5526
{
5527
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5528
	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5529 5530
	unsigned offset;
	int ret;
5531

5532 5533 5534 5535 5536 5537 5538 5539 5540
	/* Inline kvm_read_guest_virt_helper for speed.  */
	gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,
						    exception);
	if (unlikely(gpa == UNMAPPED_GVA))
		return X86EMUL_PROPAGATE_FAULT;

	offset = addr & (PAGE_SIZE-1);
	if (WARN_ON(offset + bytes > PAGE_SIZE))
		bytes = (unsigned)PAGE_SIZE - offset;
5541 5542
	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
				       offset, bytes);
5543 5544 5545 5546
	if (unlikely(ret < 0))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
5547 5548
}

5549
int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
5550
			       gva_t addr, void *val, unsigned int bytes,
5551
			       struct x86_exception *exception)
5552
{
5553
	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5554

5555 5556 5557 5558 5559 5560 5561
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5562
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
5563
					  exception);
5564
}
5565
EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
5566

5567 5568
static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
			     gva_t addr, void *val, unsigned int bytes,
5569
			     struct x86_exception *exception, bool system)
5570
{
5571
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5572 5573
	u32 access = 0;

5574
	if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)
5575 5576 5577
		access |= PFERR_USER_MASK;

	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
5578 5579
}

5580 5581 5582 5583 5584 5585 5586 5587 5588
static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
		unsigned long addr, void *val, unsigned int bytes)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);

	return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
}

5589 5590 5591
static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
				      struct x86_exception *exception)
5592 5593 5594 5595 5596
{
	void *data = val;
	int r = X86EMUL_CONTINUE;

	while (bytes) {
5597
		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
5598
							     access,
5599
							     exception);
5600 5601 5602 5603
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
		int ret;

5604
		if (gpa == UNMAPPED_GVA)
5605
			return X86EMUL_PROPAGATE_FAULT;
5606
		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
5607
		if (ret < 0) {
5608
			r = X86EMUL_IO_NEEDED;
5609 5610 5611 5612 5613 5614 5615 5616 5617 5618
			goto out;
		}

		bytes -= towrite;
		data += towrite;
		addr += towrite;
	}
out:
	return r;
}
5619 5620

static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
5621 5622
			      unsigned int bytes, struct x86_exception *exception,
			      bool system)
5623 5624
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5625 5626
	u32 access = PFERR_WRITE_MASK;

5627
	if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)
5628
		access |= PFERR_USER_MASK;
5629 5630

	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
5631
					   access, exception);
5632 5633 5634 5635 5636
}

int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
				unsigned int bytes, struct x86_exception *exception)
{
5637 5638 5639
	/* kvm_write_guest_virt_system can pull in tons of pages. */
	vcpu->arch.l1tf_flush_l1d = true;

5640 5641 5642 5643 5644 5645 5646
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5647 5648 5649
	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
					   PFERR_WRITE_MASK, exception);
}
Nadav Har'El's avatar
Nadav Har'El committed
5650
EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
5651

5652 5653
int handle_ud(struct kvm_vcpu *vcpu)
{
5654
	static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX };
5655 5656 5657 5658 5659
	int emul_type = EMULTYPE_TRAP_UD;
	char sig[5]; /* ud2; .ascii "kvm" */
	struct x86_exception e;

	if (force_emulation_prefix &&
5660 5661
	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
				sig, sizeof(sig), &e) == 0 &&
5662
	    memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) {
5663
		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
5664
		emul_type = EMULTYPE_TRAP_UD_FORCED;
5665
	}
5666

5667
	return kvm_emulate_instruction(vcpu, emul_type);
5668 5669 5670
}
EXPORT_SYMBOL_GPL(handle_ud);

5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685
static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
			    gpa_t gpa, bool write)
{
	/* For APIC access vmexit */
	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		return 1;

	if (vcpu_match_mmio_gpa(vcpu, gpa)) {
		trace_vcpu_match_mmio(gva, gpa, write, true);
		return 1;
	}

	return 0;
}

5686 5687 5688 5689
static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
				gpa_t *gpa, struct x86_exception *exception,
				bool write)
{
5690
	u32 access = ((kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
5691
		| (write ? PFERR_WRITE_MASK : 0);
5692

5693 5694 5695 5696 5697
	/*
	 * currently PKRU is only applied to ept enabled guest so
	 * there is no pkey in EPT page table for L1 guest or EPT
	 * shadow page table for L2 guest.
	 */
5698
	if (vcpu_match_mmio_gva(vcpu, gva)
5699
	    && !permission_fault(vcpu, vcpu->arch.walk_mmu,
5700
				 vcpu->arch.mmio_access, 0, access)) {
5701 5702
		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
					(gva & (PAGE_SIZE - 1));
5703
		trace_vcpu_match_mmio(gva, *gpa, write, false);
5704 5705 5706
		return 1;
	}

5707 5708 5709 5710 5711
	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);

	if (*gpa == UNMAPPED_GVA)
		return -1;

5712
	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
5713 5714
}

5715
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
5716
			const void *val, int bytes)
5717 5718 5719
{
	int ret;

5720
	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
5721
	if (ret < 0)
5722
		return 0;
5723
	kvm_page_track_write(vcpu, gpa, val, bytes);
5724 5725 5726
	return 1;
}

5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742
struct read_write_emulator_ops {
	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
				  int bytes);
	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
				  void *val, int bytes);
	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
			       int bytes, void *val);
	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
				    void *val, int bytes);
	bool write;
};

static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
{
	if (vcpu->mmio_read_completed) {
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
5743
			       vcpu->mmio_fragments[0].gpa, val);
5744 5745 5746 5747 5748 5749 5750 5751 5752 5753
		vcpu->mmio_read_completed = 0;
		return 1;
	}

	return 0;
}

static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			void *val, int bytes)
{
5754
	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
5755 5756 5757 5758 5759 5760 5761 5762 5763 5764
}

static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			 void *val, int bytes)
{
	return emulator_write_phys(vcpu, gpa, val, bytes);
}

static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
{
5765
	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
5766 5767 5768 5769 5770 5771
	return vcpu_mmio_write(vcpu, gpa, bytes, val);
}

static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			  void *val, int bytes)
{
5772
	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
5773 5774 5775 5776 5777 5778
	return X86EMUL_IO_NEEDED;
}

static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			   void *val, int bytes)
{
Avi Kivity's avatar
Avi Kivity committed
5779 5780
	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];

5781
	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
5782 5783 5784
	return X86EMUL_CONTINUE;
}

5785
static const struct read_write_emulator_ops read_emultor = {
5786 5787 5788 5789 5790 5791
	.read_write_prepare = read_prepare,
	.read_write_emulate = read_emulate,
	.read_write_mmio = vcpu_mmio_read,
	.read_write_exit_mmio = read_exit_mmio,
};

5792
static const struct read_write_emulator_ops write_emultor = {
5793 5794 5795 5796 5797 5798
	.read_write_emulate = write_emulate,
	.read_write_mmio = write_mmio,
	.read_write_exit_mmio = write_exit_mmio,
	.write = true,
};

5799 5800 5801 5802
static int emulator_read_write_onepage(unsigned long addr, void *val,
				       unsigned int bytes,
				       struct x86_exception *exception,
				       struct kvm_vcpu *vcpu,
5803
				       const struct read_write_emulator_ops *ops)
5804
{
5805 5806
	gpa_t gpa;
	int handled, ret;
5807
	bool write = ops->write;
Avi Kivity's avatar
Avi Kivity committed
5808
	struct kvm_mmio_fragment *frag;
5809
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
5810 5811 5812 5813 5814 5815 5816 5817

	/*
	 * If the exit was due to a NPF we may already have a GPA.
	 * If the GPA is present, use it to avoid the GVA to GPA table walk.
	 * Note, this cannot be used on string operations since string
	 * operation using rep will only have the initial GPA from the NPF
	 * occurred.
	 */
5818 5819 5820
	if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) &&
	    (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) {
		gpa = ctxt->gpa_val;
5821 5822 5823 5824 5825
		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
	} else {
		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
		if (ret < 0)
			return X86EMUL_PROPAGATE_FAULT;
5826
	}
5827

5828
	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
5829 5830 5831 5832 5833
		return X86EMUL_CONTINUE;

	/*
	 * Is this MMIO handled locally?
	 */
5834
	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
5835
	if (handled == bytes)
5836 5837
		return X86EMUL_CONTINUE;

5838 5839 5840 5841
	gpa += handled;
	bytes -= handled;
	val += handled;

5842 5843 5844 5845 5846
	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
	frag->gpa = gpa;
	frag->data = val;
	frag->len = bytes;
Avi Kivity's avatar
Avi Kivity committed
5847
	return X86EMUL_CONTINUE;
5848 5849
}

5850 5851
static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
			unsigned long addr,
5852 5853
			void *val, unsigned int bytes,
			struct x86_exception *exception,
5854
			const struct read_write_emulator_ops *ops)
5855
{
5856
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Avi Kivity's avatar
Avi Kivity committed
5857 5858 5859 5860 5861 5862 5863 5864
	gpa_t gpa;
	int rc;

	if (ops->read_write_prepare &&
		  ops->read_write_prepare(vcpu, val, bytes))
		return X86EMUL_CONTINUE;

	vcpu->mmio_nr_fragments = 0;
5865

5866 5867
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
Avi Kivity's avatar
Avi Kivity committed
5868
		int now;
5869 5870

		now = -addr & ~PAGE_MASK;
5871 5872 5873
		rc = emulator_read_write_onepage(addr, val, now, exception,
						 vcpu, ops);

5874 5875 5876
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
5877 5878
		if (ctxt->mode != X86EMUL_MODE_PROT64)
			addr = (u32)addr;
5879 5880 5881
		val += now;
		bytes -= now;
	}
5882

Avi Kivity's avatar
Avi Kivity committed
5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895
	rc = emulator_read_write_onepage(addr, val, bytes, exception,
					 vcpu, ops);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	if (!vcpu->mmio_nr_fragments)
		return rc;

	gpa = vcpu->mmio_fragments[0].gpa;

	vcpu->mmio_needed = 1;
	vcpu->mmio_cur_fragment = 0;

5896
	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
Avi Kivity's avatar
Avi Kivity committed
5897 5898 5899 5900 5901
	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
	vcpu->run->exit_reason = KVM_EXIT_MMIO;
	vcpu->run->mmio.phys_addr = gpa;

	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913
}

static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
				  unsigned long addr,
				  void *val,
				  unsigned int bytes,
				  struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, val, bytes,
				   exception, &read_emultor);
}

5914
static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
5915 5916 5917 5918 5919 5920 5921
			    unsigned long addr,
			    const void *val,
			    unsigned int bytes,
			    struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, (void *)val, bytes,
				   exception, &write_emultor);
5922 5923
}

5924 5925 5926 5927 5928 5929 5930
#define CMPXCHG_TYPE(t, ptr, old, new) \
	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))

#ifdef CONFIG_X86_64
#  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
#else
#  define CMPXCHG64(ptr, old, new) \
5931
	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
5932 5933
#endif

5934 5935
static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
				     unsigned long addr,
5936 5937 5938
				     const void *old,
				     const void *new,
				     unsigned int bytes,
5939
				     struct x86_exception *exception)
5940
{
5941
	struct kvm_host_map map;
5942
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5943
	u64 page_line_mask;
5944 5945 5946
	gpa_t gpa;
	char *kaddr;
	bool exchanged;
5947

5948 5949 5950
	/* guests cmpxchg8b have to be emulated atomically */
	if (bytes > 8 || (bytes & (bytes - 1)))
		goto emul_write;
5951

5952
	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
5953

5954 5955 5956
	if (gpa == UNMAPPED_GVA ||
	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		goto emul_write;
5957

5958 5959 5960 5961 5962 5963 5964 5965 5966 5967
	/*
	 * Emulate the atomic as a straight write to avoid #AC if SLD is
	 * enabled in the host and the access splits a cache line.
	 */
	if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
		page_line_mask = ~(cache_line_size() - 1);
	else
		page_line_mask = PAGE_MASK;

	if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask))
5968
		goto emul_write;
5969

5970
	if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))
5971
		goto emul_write;
5972

5973 5974
	kaddr = map.hva + offset_in_page(gpa);

5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989
	switch (bytes) {
	case 1:
		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
		break;
	case 2:
		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
		break;
	case 4:
		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
		break;
	case 8:
		exchanged = CMPXCHG64(kaddr, old, new);
		break;
	default:
		BUG();
5990
	}
5991 5992

	kvm_vcpu_unmap(vcpu, &map, true);
5993 5994 5995 5996

	if (!exchanged)
		return X86EMUL_CMPXCHG_FAILED;

5997
	kvm_page_track_write(vcpu, gpa, new, bytes);
5998 5999

	return X86EMUL_CONTINUE;
6000

6001
emul_write:
6002
	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
6003

6004
	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
6005 6006
}

6007 6008
static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
{
6009
	int r = 0, i;
6010

6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022
	for (i = 0; i < vcpu->arch.pio.count; i++) {
		if (vcpu->arch.pio.in)
			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
					    vcpu->arch.pio.size, pd);
		else
			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
					     vcpu->arch.pio.port, vcpu->arch.pio.size,
					     pd);
		if (r)
			break;
		pd += vcpu->arch.pio.size;
	}
6023 6024 6025
	return r;
}

6026 6027 6028
static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
			       unsigned short port, void *val,
			       unsigned int count, bool in)
6029 6030
{
	vcpu->arch.pio.port = port;
6031
	vcpu->arch.pio.in = in;
6032
	vcpu->arch.pio.count  = count;
6033 6034 6035
	vcpu->arch.pio.size = size;

	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
6036
		vcpu->arch.pio.count = 0;
6037 6038 6039 6040
		return 1;
	}

	vcpu->run->exit_reason = KVM_EXIT_IO;
6041
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
6042 6043 6044 6045 6046 6047 6048 6049
	vcpu->run->io.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = count;
	vcpu->run->io.port = port;

	return 0;
}

6050 6051
static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
			   unsigned short port, void *val, unsigned int count)
6052
{
6053
	int ret;
6054

6055 6056
	if (vcpu->arch.pio.count)
		goto data_avail;
6057

6058 6059
	memset(vcpu->arch.pio_data, 0, size * count);

6060 6061 6062 6063
	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
	if (ret) {
data_avail:
		memcpy(val, vcpu->arch.pio_data, size * count);
6064
		trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
6065
		vcpu->arch.pio.count = 0;
6066 6067 6068 6069 6070 6071
		return 1;
	}

	return 0;
}

6072 6073 6074
static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
				    int size, unsigned short port, void *val,
				    unsigned int count)
6075
{
6076
	return emulator_pio_in(emul_to_vcpu(ctxt), size, port, val, count);
6077

6078
}
6079

6080 6081 6082 6083
static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,
			    unsigned short port, const void *val,
			    unsigned int count)
{
6084
	memcpy(vcpu->arch.pio_data, val, size * count);
6085
	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
6086 6087 6088
	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
}

6089 6090 6091 6092 6093 6094 6095
static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
				     int size, unsigned short port,
				     const void *val, unsigned int count)
{
	return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count);
}

6096 6097
static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
6098
	return kvm_x86_ops.get_segment_base(vcpu, seg);
6099 6100
}

6101
static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
6102
{
6103
	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
6104 6105
}

6106
static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
6107 6108 6109 6110
{
	if (!need_emulate_wbinvd(vcpu))
		return X86EMUL_CONTINUE;

6111
	if (kvm_x86_ops.has_wbinvd_exit()) {
6112 6113 6114
		int cpu = get_cpu();

		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
6115 6116
		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
				wbinvd_ipi, NULL, 1);
6117
		put_cpu();
6118
		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
6119 6120
	} else
		wbinvd();
6121 6122
	return X86EMUL_CONTINUE;
}
6123 6124 6125

int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
6126 6127
	kvm_emulate_wbinvd_noskip(vcpu);
	return kvm_skip_emulated_instruction(vcpu);
6128
}
6129 6130
EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);

6131 6132


6133 6134
static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
{
6135
	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
6136 6137
}

6138 6139
static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long *dest)
6140
{
6141
	return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
6142 6143
}

6144 6145
static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long value)
6146
{
6147

6148
	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
6149 6150
}

6151
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
6152
{
6153
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
6154 6155
}

6156
static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
6157
{
6158
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6159 6160 6161 6162 6163 6164 6165 6166 6167 6168
	unsigned long value;

	switch (cr) {
	case 0:
		value = kvm_read_cr0(vcpu);
		break;
	case 2:
		value = vcpu->arch.cr2;
		break;
	case 3:
6169
		value = kvm_read_cr3(vcpu);
6170 6171 6172 6173 6174 6175 6176 6177
		break;
	case 4:
		value = kvm_read_cr4(vcpu);
		break;
	case 8:
		value = kvm_get_cr8(vcpu);
		break;
	default:
6178
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
6179 6180 6181 6182 6183 6184
		return 0;
	}

	return value;
}

6185
static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
6186
{
6187
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6188 6189
	int res = 0;

6190 6191
	switch (cr) {
	case 0:
6192
		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
6193 6194 6195 6196 6197
		break;
	case 2:
		vcpu->arch.cr2 = val;
		break;
	case 3:
6198
		res = kvm_set_cr3(vcpu, val);
6199 6200
		break;
	case 4:
6201
		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
6202 6203
		break;
	case 8:
Andre Przywara's avatar
Andre Przywara committed
6204
		res = kvm_set_cr8(vcpu, val);
6205 6206
		break;
	default:
6207
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
6208
		res = -1;
6209
	}
6210 6211

	return res;
6212 6213
}

6214
static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
6215
{
6216
	return kvm_x86_ops.get_cpl(emul_to_vcpu(ctxt));
6217 6218
}

6219
static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6220
{
6221
	kvm_x86_ops.get_gdt(emul_to_vcpu(ctxt), dt);
6222 6223
}

6224
static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6225
{
6226
	kvm_x86_ops.get_idt(emul_to_vcpu(ctxt), dt);
6227 6228
}

6229 6230
static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
6231
	kvm_x86_ops.set_gdt(emul_to_vcpu(ctxt), dt);
6232 6233 6234 6235
}

static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
6236
	kvm_x86_ops.set_idt(emul_to_vcpu(ctxt), dt);
6237 6238
}

6239 6240
static unsigned long emulator_get_cached_segment_base(
	struct x86_emulate_ctxt *ctxt, int seg)
6241
{
6242
	return get_segment_base(emul_to_vcpu(ctxt), seg);
6243 6244
}

6245 6246 6247
static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
				 struct desc_struct *desc, u32 *base3,
				 int seg)
6248 6249 6250
{
	struct kvm_segment var;

6251
	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
6252
	*selector = var.selector;
6253

6254 6255
	if (var.unusable) {
		memset(desc, 0, sizeof(*desc));
6256 6257
		if (base3)
			*base3 = 0;
6258
		return false;
6259
	}
6260 6261 6262 6263 6264

	if (var.g)
		var.limit >>= 12;
	set_desc_limit(desc, var.limit);
	set_desc_base(desc, (unsigned long)var.base);
6265 6266 6267 6268
#ifdef CONFIG_X86_64
	if (base3)
		*base3 = var.base >> 32;
#endif
6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280
	desc->type = var.type;
	desc->s = var.s;
	desc->dpl = var.dpl;
	desc->p = var.present;
	desc->avl = var.avl;
	desc->l = var.l;
	desc->d = var.db;
	desc->g = var.g;

	return true;
}

6281 6282 6283
static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
				 struct desc_struct *desc, u32 base3,
				 int seg)
6284
{
6285
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6286 6287
	struct kvm_segment var;

6288
	var.selector = selector;
6289
	var.base = get_desc_base(desc);
6290 6291 6292
#ifdef CONFIG_X86_64
	var.base |= ((u64)base3) << 32;
#endif
6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310
	var.limit = get_desc_limit(desc);
	if (desc->g)
		var.limit = (var.limit << 12) | 0xfff;
	var.type = desc->type;
	var.dpl = desc->dpl;
	var.db = desc->d;
	var.s = desc->s;
	var.l = desc->l;
	var.g = desc->g;
	var.avl = desc->avl;
	var.present = desc->p;
	var.unusable = !var.present;
	var.padding = 0;

	kvm_set_segment(vcpu, &var, seg);
	return;
}

6311 6312 6313
static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 *pdata)
{
6314
	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
6315 6316 6317 6318 6319
}

static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 data)
{
6320
	return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
6321 6322
}

6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336
static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	return vcpu->arch.smbase;
}

static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	vcpu->arch.smbase = smbase;
}

6337 6338 6339
static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
			      u32 pmc)
{
6340
	return kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc);
6341 6342
}

6343 6344 6345
static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
			     u32 pmc, u64 *pdata)
{
6346
	return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
6347 6348
}

6349 6350 6351 6352 6353
static void emulator_halt(struct x86_emulate_ctxt *ctxt)
{
	emul_to_vcpu(ctxt)->arch.halt_request = 1;
}

6354
static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
6355
			      struct x86_instruction_info *info,
6356 6357
			      enum x86_intercept_stage stage)
{
6358
	return kvm_x86_ops.check_intercept(emul_to_vcpu(ctxt), info, stage,
6359
					    &ctxt->exception);
6360 6361
}

6362
static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
6363 6364
			      u32 *eax, u32 *ebx, u32 *ecx, u32 *edx,
			      bool exact_only)
6365
{
6366
	return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only);
6367 6368
}

6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383
static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_LM);
}

static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE);
}

static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR);
}

6384 6385 6386 6387 6388 6389 6390 6391 6392 6393
static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
{
	return kvm_register_read(emul_to_vcpu(ctxt), reg);
}

static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
{
	kvm_register_write(emul_to_vcpu(ctxt), reg, val);
}

6394 6395
static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
{
6396
	kvm_x86_ops.set_nmi_mask(emul_to_vcpu(ctxt), masked);
6397 6398
}

6399 6400 6401 6402 6403 6404 6405
static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
{
	return emul_to_vcpu(ctxt)->arch.hflags;
}

static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_flags)
{
6406
	emul_to_vcpu(ctxt)->arch.hflags = emul_flags;
6407 6408
}

6409 6410
static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt,
				  const char *smstate)
6411
{
6412
	return kvm_x86_ops.pre_leave_smm(emul_to_vcpu(ctxt), smstate);
6413 6414
}

6415 6416 6417 6418 6419
static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt)
{
	kvm_smm_changed(emul_to_vcpu(ctxt));
}

6420 6421 6422 6423 6424
static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
{
	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
}

6425
static const struct x86_emulate_ops emulate_ops = {
6426 6427
	.read_gpr            = emulator_read_gpr,
	.write_gpr           = emulator_write_gpr,
6428 6429
	.read_std            = emulator_read_std,
	.write_std           = emulator_write_std,
6430
	.read_phys           = kvm_read_guest_phys_system,
6431
	.fetch               = kvm_fetch_guest_virt,
6432 6433 6434
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
6435
	.invlpg              = emulator_invlpg,
6436 6437
	.pio_in_emulated     = emulator_pio_in_emulated,
	.pio_out_emulated    = emulator_pio_out_emulated,
6438 6439
	.get_segment         = emulator_get_segment,
	.set_segment         = emulator_set_segment,
6440
	.get_cached_segment_base = emulator_get_cached_segment_base,
6441
	.get_gdt             = emulator_get_gdt,
6442
	.get_idt	     = emulator_get_idt,
6443 6444
	.set_gdt             = emulator_set_gdt,
	.set_idt	     = emulator_set_idt,
6445 6446
	.get_cr              = emulator_get_cr,
	.set_cr              = emulator_set_cr,
6447
	.cpl                 = emulator_get_cpl,
6448 6449
	.get_dr              = emulator_get_dr,
	.set_dr              = emulator_set_dr,
6450 6451
	.get_smbase          = emulator_get_smbase,
	.set_smbase          = emulator_set_smbase,
6452 6453
	.set_msr             = emulator_set_msr,
	.get_msr             = emulator_get_msr,
6454
	.check_pmc	     = emulator_check_pmc,
6455
	.read_pmc            = emulator_read_pmc,
6456
	.halt                = emulator_halt,
6457
	.wbinvd              = emulator_wbinvd,
6458
	.fix_hypercall       = emulator_fix_hypercall,
6459
	.intercept           = emulator_intercept,
6460
	.get_cpuid           = emulator_get_cpuid,
6461 6462 6463
	.guest_has_long_mode = emulator_guest_has_long_mode,
	.guest_has_movbe     = emulator_guest_has_movbe,
	.guest_has_fxsr      = emulator_guest_has_fxsr,
6464
	.set_nmi_mask        = emulator_set_nmi_mask,
6465 6466
	.get_hflags          = emulator_get_hflags,
	.set_hflags          = emulator_set_hflags,
6467
	.pre_leave_smm       = emulator_pre_leave_smm,
6468
	.post_leave_smm      = emulator_post_leave_smm,
6469
	.set_xcr             = emulator_set_xcr,
6470 6471
};

6472 6473
static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
{
6474
	u32 int_shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);
6475 6476 6477 6478 6479 6480 6481
	/*
	 * an sti; sti; sequence only disable interrupts for the first
	 * instruction. So, if the last instruction, be it emulated or
	 * not, left the system with the INT_STI flag enabled, it
	 * means that the last instruction is an sti. We should not
	 * leave the flag on in this case. The same goes for mov ss
	 */
6482 6483
	if (int_shadow & mask)
		mask = 0;
6484
	if (unlikely(int_shadow || mask)) {
6485
		kvm_x86_ops.set_interrupt_shadow(vcpu, mask);
6486 6487 6488
		if (!mask)
			kvm_make_request(KVM_REQ_EVENT, vcpu);
	}
6489 6490
}

6491
static bool inject_emulated_exception(struct kvm_vcpu *vcpu)
6492
{
6493
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6494
	if (ctxt->exception.vector == PF_VECTOR)
6495
		return kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);
6496 6497

	if (ctxt->exception.error_code_valid)
6498 6499
		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
				      ctxt->exception.error_code);
6500
	else
6501
		kvm_queue_exception(vcpu, ctxt->exception.vector);
6502
	return false;
6503 6504
}

6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521
static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)
{
	struct x86_emulate_ctxt *ctxt;

	ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT);
	if (!ctxt) {
		pr_err("kvm: failed to allocate vcpu's emulator\n");
		return NULL;
	}

	ctxt->vcpu = vcpu;
	ctxt->ops = &emulate_ops;
	vcpu->arch.emulate_ctxt = ctxt;

	return ctxt;
}

6522 6523
static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
{
6524
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6525 6526
	int cs_db, cs_l;

6527
	kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
6528

6529
	ctxt->gpa_available = false;
6530
	ctxt->eflags = kvm_get_rflags(vcpu);
6531 6532
	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;

6533 6534 6535
	ctxt->eip = kvm_rip_read(vcpu);
	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
6536
		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :
6537 6538
		     cs_db				? X86EMUL_MODE_PROT32 :
							  X86EMUL_MODE_PROT16;
6539
	BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK);
6540 6541
	BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK);
	BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK);
6542

6543
	init_decode_cache(ctxt);
6544
	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6545 6546
}

6547
void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
6548
{
6549
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6550 6551 6552 6553
	int ret;

	init_emulate_ctxt(vcpu);

6554 6555 6556
	ctxt->op_bytes = 2;
	ctxt->ad_bytes = 2;
	ctxt->_eip = ctxt->eip + inc_eip;
6557
	ret = emulate_int_real(ctxt, irq);
6558

6559 6560 6561 6562 6563 6564 6565
	if (ret != X86EMUL_CONTINUE) {
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
	} else {
		ctxt->eip = ctxt->_eip;
		kvm_rip_write(vcpu, ctxt->eip);
		kvm_set_rflags(vcpu, ctxt->eflags);
	}
6566 6567 6568
}
EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);

6569
static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
6570 6571 6572
{
	++vcpu->stat.insn_emulation_fail;
	trace_kvm_emulate_insn_failed(vcpu);
6573

6574 6575
	if (emulation_type & EMULTYPE_VMWARE_GP) {
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6576
		return 1;
6577
	}
6578

6579 6580 6581 6582
	if (emulation_type & EMULTYPE_SKIP) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6583
		return 0;
6584 6585
	}

6586 6587
	kvm_queue_exception(vcpu, UD_VECTOR);

6588
	if (!is_guest_mode(vcpu) && kvm_x86_ops.get_cpl(vcpu) == 0) {
6589 6590 6591
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6592
		return 0;
6593
	}
6594

6595
	return 1;
6596 6597
}

6598
static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
6599 6600
				  bool write_fault_to_shadow_pgtable,
				  int emulation_type)
6601
{
6602
	gpa_t gpa = cr2_or_gpa;
6603
	kvm_pfn_t pfn;
6604

6605
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
6606 6607
		return false;

6608 6609
	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
	    WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
6610 6611
		return false;

6612
	if (!vcpu->arch.mmu->direct_map) {
6613 6614 6615 6616
		/*
		 * Write permission should be allowed since only
		 * write access need to be emulated.
		 */
6617
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
6618

6619 6620 6621 6622 6623 6624 6625
		/*
		 * If the mapping is invalid in guest, let cpu retry
		 * it to generate fault.
		 */
		if (gpa == UNMAPPED_GVA)
			return true;
	}
6626

6627 6628 6629 6630 6631 6632 6633
	/*
	 * Do not retry the unhandleable instruction if it faults on the
	 * readonly host memory, otherwise it will goto a infinite loop:
	 * retry instruction -> write #PF -> emulation fail -> retry
	 * instruction -> ...
	 */
	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644

	/*
	 * If the instruction failed on the error pfn, it can not be fixed,
	 * report the error to userspace.
	 */
	if (is_error_noslot_pfn(pfn))
		return false;

	kvm_release_pfn_clean(pfn);

	/* The instructions are well-emulated on direct mmu. */
6645
	if (vcpu->arch.mmu->direct_map) {
6646 6647 6648 6649 6650 6651 6652 6653 6654
		unsigned int indirect_shadow_pages;

		spin_lock(&vcpu->kvm->mmu_lock);
		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
		spin_unlock(&vcpu->kvm->mmu_lock);

		if (indirect_shadow_pages)
			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));

6655
		return true;
6656
	}
6657

6658 6659 6660 6661 6662 6663
	/*
	 * if emulation was due to access to shadowed page table
	 * and it failed try to unshadow page and re-enter the
	 * guest to let CPU execute the instruction.
	 */
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6664 6665 6666 6667 6668 6669 6670

	/*
	 * If the access faults on its page table, it can not
	 * be fixed by unprotecting shadow page and it should
	 * be reported to userspace.
	 */
	return !write_fault_to_shadow_pgtable;
6671 6672
}

6673
static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
6674
			      gpa_t cr2_or_gpa,  int emulation_type)
6675 6676
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6677
	unsigned long last_retry_eip, last_retry_addr, gpa = cr2_or_gpa;
6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696

	last_retry_eip = vcpu->arch.last_retry_eip;
	last_retry_addr = vcpu->arch.last_retry_addr;

	/*
	 * If the emulation is caused by #PF and it is non-page_table
	 * writing instruction, it means the VM-EXIT is caused by shadow
	 * page protected, we can zap the shadow page and retry this
	 * instruction directly.
	 *
	 * Note: if the guest uses a non-page-table modifying instruction
	 * on the PDE that points to the instruction, then we will unmap
	 * the instruction and go to an infinite loop. So, we cache the
	 * last retried eip and the last fault address, if we meet the eip
	 * and the address again, we can break out of the potential infinite
	 * loop.
	 */
	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;

6697
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
6698 6699
		return false;

6700 6701
	if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
	    WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
6702 6703
		return false;

6704 6705 6706
	if (x86_page_table_writing_insn(ctxt))
		return false;

6707
	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa)
6708 6709 6710
		return false;

	vcpu->arch.last_retry_eip = ctxt->eip;
6711
	vcpu->arch.last_retry_addr = cr2_or_gpa;
6712

6713
	if (!vcpu->arch.mmu->direct_map)
6714
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
6715

6716
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6717 6718 6719 6720

	return true;
}

6721 6722 6723
static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
static int complete_emulated_pio(struct kvm_vcpu *vcpu);

6724
static void kvm_smm_changed(struct kvm_vcpu *vcpu)
6725
{
6726
	if (!(vcpu->arch.hflags & HF_SMM_MASK)) {
6727 6728 6729
		/* This is a good place to trace that we are exiting SMM.  */
		trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false);

6730 6731
		/* Process a latched INIT or SMI, if any.  */
		kvm_make_request(KVM_REQ_EVENT, vcpu);
6732
	}
6733 6734

	kvm_mmu_reset_context(vcpu);
6735 6736
}

6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751
static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
				unsigned long *db)
{
	u32 dr6 = 0;
	int i;
	u32 enable, rwlen;

	enable = dr7;
	rwlen = dr7 >> 16;
	for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
		if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
			dr6 |= (1 << i);
	return dr6;
}

6752
static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
6753 6754 6755
{
	struct kvm_run *kvm_run = vcpu->run;

6756 6757
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
		kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1 | DR6_RTM;
6758
		kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
6759 6760
		kvm_run->debug.arch.exception = DB_VECTOR;
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
6761
		return 0;
6762
	}
6763
	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
6764
	return 1;
6765 6766
}

6767 6768
int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
6769
	unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
6770
	int r;
6771

6772
	r = kvm_x86_ops.skip_emulated_instruction(vcpu);
6773
	if (unlikely(!r))
6774
		return 0;
6775 6776 6777 6778 6779 6780 6781 6782 6783 6784

	/*
	 * rflags is the old, "raw" value of the flags.  The new value has
	 * not been saved yet.
	 *
	 * This is correct even for TF set by the guest, because "the
	 * processor will not generate this exception after the instruction
	 * that sets the TF flag".
	 */
	if (unlikely(rflags & X86_EFLAGS_TF))
6785
		r = kvm_vcpu_do_singlestep(vcpu);
6786
	return r;
6787 6788 6789
}
EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);

6790 6791 6792 6793
static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
{
	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
6794 6795 6796
		struct kvm_run *kvm_run = vcpu->run;
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6797 6798 6799 6800
					   vcpu->arch.guest_debug_dr7,
					   vcpu->arch.eff_db);

		if (dr6 != 0) {
6801
			kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
6802
			kvm_run->debug.arch.pc = eip;
6803 6804
			kvm_run->debug.arch.exception = DB_VECTOR;
			kvm_run->exit_reason = KVM_EXIT_DEBUG;
6805
			*r = 0;
6806 6807 6808 6809
			return true;
		}
	}

6810 6811
	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
	    !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {
6812 6813
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6814 6815 6816 6817
					   vcpu->arch.dr7,
					   vcpu->arch.db);

		if (dr6 != 0) {
6818
			kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
6819
			*r = 1;
6820 6821 6822 6823 6824 6825 6826
			return true;
		}
	}

	return false;
}

6827 6828
static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
{
6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849 6850 6851 6852
	switch (ctxt->opcode_len) {
	case 1:
		switch (ctxt->b) {
		case 0xe4:	/* IN */
		case 0xe5:
		case 0xec:
		case 0xed:
		case 0xe6:	/* OUT */
		case 0xe7:
		case 0xee:
		case 0xef:
		case 0x6c:	/* INS */
		case 0x6d:
		case 0x6e:	/* OUTS */
		case 0x6f:
			return true;
		}
		break;
	case 2:
		switch (ctxt->b) {
		case 0x33:	/* RDPMC */
			return true;
		}
		break;
6853 6854 6855 6856 6857
	}

	return false;
}

6858 6859
int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
			    int emulation_type, void *insn, int insn_len)
6860
{
6861
	int r;
6862
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6863
	bool writeback = true;
6864
	bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
6865

6866 6867
	vcpu->arch.l1tf_flush_l1d = true;

6868 6869 6870 6871 6872
	/*
	 * Clear write_fault_to_shadow_pgtable here to ensure it is
	 * never reused.
	 */
	vcpu->arch.write_fault_to_shadow_pgtable = false;
6873
	kvm_clear_exception_queue(vcpu);
6874

6875
	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
6876
		init_emulate_ctxt(vcpu);
6877 6878 6879 6880 6881 6882 6883

		/*
		 * We will reenter on the same instruction since
		 * we do not set complete_userspace_io.  This does not
		 * handle watchpoints yet, those would be handled in
		 * the emulate_ops.
		 */
6884 6885
		if (!(emulation_type & EMULTYPE_SKIP) &&
		    kvm_vcpu_check_breakpoint(vcpu, &r))
6886 6887
			return r;

6888 6889
		ctxt->interruptibility = 0;
		ctxt->have_exception = false;
6890
		ctxt->exception.vector = -1;
6891
		ctxt->perm_ok = false;
6892

6893
		ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
6894

6895
		r = x86_decode_insn(ctxt, insn, insn_len);
6896

6897
		trace_kvm_emulate_insn_start(vcpu);
6898
		++vcpu->stat.insn_emulation;
6899
		if (r != EMULATION_OK)  {
6900
			if ((emulation_type & EMULTYPE_TRAP_UD) ||
6901 6902
			    (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
				kvm_queue_exception(vcpu, UD_VECTOR);
6903
				return 1;
6904
			}
6905 6906 6907
			if (reexecute_instruction(vcpu, cr2_or_gpa,
						  write_fault_to_spt,
						  emulation_type))
6908
				return 1;
6909
			if (ctxt->have_exception) {
6910 6911 6912 6913 6914 6915
				/*
				 * #UD should result in just EMULATION_FAILED, and trap-like
				 * exception should not be encountered during decode.
				 */
				WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
					     exception_type(ctxt->exception.vector) == EXCPT_TRAP);
6916
				inject_emulated_exception(vcpu);
6917
				return 1;
6918
			}
6919
			return handle_emulation_failure(vcpu, emulation_type);
6920 6921 6922
		}
	}

6923 6924 6925
	if ((emulation_type & EMULTYPE_VMWARE_GP) &&
	    !is_vmware_backdoor_opcode(ctxt)) {
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6926
		return 1;
6927
	}
6928

6929 6930 6931 6932 6933
	/*
	 * Note, EMULTYPE_SKIP is intended for use *only* by vendor callbacks
	 * for kvm_skip_emulated_instruction().  The caller is responsible for
	 * updating interruptibility state and injecting single-step #DBs.
	 */
6934
	if (emulation_type & EMULTYPE_SKIP) {
6935
		kvm_rip_write(vcpu, ctxt->_eip);
6936 6937
		if (ctxt->eflags & X86_EFLAGS_RF)
			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
6938
		return 1;
6939 6940
	}

6941
	if (retry_instruction(ctxt, cr2_or_gpa, emulation_type))
6942
		return 1;
6943

6944
	/* this is needed for vmware backdoor interface to work since it
6945
	   changes registers values  during IO operation */
6946 6947
	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6948
		emulator_invalidate_register_cache(ctxt);
6949
	}
6950

6951
restart:
6952 6953 6954 6955 6956 6957
	if (emulation_type & EMULTYPE_PF) {
		/* Save the faulting GPA (cr2) in the address field */
		ctxt->exception.address = cr2_or_gpa;

		/* With shadow page tables, cr2 contains a GVA or nGPA. */
		if (vcpu->arch.mmu->direct_map) {
6958 6959
			ctxt->gpa_available = true;
			ctxt->gpa_val = cr2_or_gpa;
6960 6961 6962 6963 6964
		}
	} else {
		/* Sanitize the address out of an abundance of paranoia. */
		ctxt->exception.address = 0;
	}
6965

6966
	r = x86_emulate_insn(ctxt);
6967

6968
	if (r == EMULATION_INTERCEPTED)
6969
		return 1;
6970

6971
	if (r == EMULATION_FAILED) {
6972
		if (reexecute_instruction(vcpu, cr2_or_gpa, write_fault_to_spt,
6973
					emulation_type))
6974
			return 1;
6975

6976
		return handle_emulation_failure(vcpu, emulation_type);
6977 6978
	}

6979
	if (ctxt->have_exception) {
6980
		r = 1;
6981 6982
		if (inject_emulated_exception(vcpu))
			return r;
6983
	} else if (vcpu->arch.pio.count) {
6984 6985
		if (!vcpu->arch.pio.in) {
			/* FIXME: return into emulator if single-stepping.  */
6986
			vcpu->arch.pio.count = 0;
6987
		} else {
6988
			writeback = false;
6989 6990
			vcpu->arch.complete_userspace_io = complete_emulated_pio;
		}
6991
		r = 0;
6992
	} else if (vcpu->mmio_needed) {
6993 6994
		++vcpu->stat.mmio_exits;

6995 6996
		if (!vcpu->mmio_is_write)
			writeback = false;
6997
		r = 0;
6998
		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
6999
	} else if (r == EMULATION_RESTART)
7000
		goto restart;
7001
	else
7002
		r = 1;
7003

7004
	if (writeback) {
7005
		unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
7006
		toggle_interruptibility(vcpu, ctxt->interruptibility);
7007
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
7008
		if (!ctxt->have_exception ||
7009 7010
		    exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
			kvm_rip_write(vcpu, ctxt->eip);
7011
			if (r && ctxt->tf)
7012
				r = kvm_vcpu_do_singlestep(vcpu);
7013 7014
			if (kvm_x86_ops.update_emulated_instruction)
				kvm_x86_ops.update_emulated_instruction(vcpu);
7015
			__kvm_set_rflags(vcpu, ctxt->eflags);
7016
		}
7017 7018 7019 7020 7021 7022 7023 7024 7025

		/*
		 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
		 * do nothing, and it will be requested again as soon as
		 * the shadow expires.  But we still need to check here,
		 * because POPF has no interrupt shadow.
		 */
		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
			kvm_make_request(KVM_REQ_EVENT, vcpu);
7026 7027
	} else
		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
7028 7029

	return r;
7030
}
7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043

int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
{
	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction);

int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
					void *insn, int insn_len)
{
	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
7044

7045 7046 7047 7048 7049 7050
static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;
	return 1;
}

7051 7052 7053 7054 7055 7056 7057 7058 7059 7060
static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;

	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
		return 1;

	return kvm_skip_emulated_instruction(vcpu);
}

7061 7062
static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
			    unsigned short port)
7063
{
7064
	unsigned long val = kvm_rax_read(vcpu);
7065 7066
	int ret = emulator_pio_out(vcpu, size, port, &val, 1);

7067 7068
	if (ret)
		return ret;
7069

7070 7071 7072 7073 7074 7075 7076 7077 7078 7079
	/*
	 * Workaround userspace that relies on old KVM behavior of %rip being
	 * incremented prior to exiting to userspace to handle "OUT 0x7e".
	 */
	if (port == 0x7e &&
	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
		vcpu->arch.complete_userspace_io =
			complete_fast_pio_out_port_0x7e;
		kvm_skip_emulated_instruction(vcpu);
	} else {
7080 7081 7082
		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
		vcpu->arch.complete_userspace_io = complete_fast_pio_out;
	}
7083
	return 0;
7084 7085
}

7086 7087 7088 7089 7090 7091 7092
static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
{
	unsigned long val;

	/* We should only ever be called with arch.pio.count equal to 1 */
	BUG_ON(vcpu->arch.pio.count != 1);

7093 7094 7095 7096 7097
	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
		vcpu->arch.pio.count = 0;
		return 1;
	}

7098
	/* For size less than 4 we merge, else we zero extend */
7099
	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
7100 7101

	/*
7102
	 * Since vcpu->arch.pio.count == 1 let emulator_pio_in perform
7103 7104
	 * the copy and tracing
	 */
7105
	emulator_pio_in(vcpu, vcpu->arch.pio.size, vcpu->arch.pio.port, &val, 1);
7106
	kvm_rax_write(vcpu, val);
7107

7108
	return kvm_skip_emulated_instruction(vcpu);
7109 7110
}

7111 7112
static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
			   unsigned short port)
7113 7114 7115 7116 7117
{
	unsigned long val;
	int ret;

	/* For size less than 4 we merge, else we zero extend */
7118
	val = (size < 4) ? kvm_rax_read(vcpu) : 0;
7119

7120
	ret = emulator_pio_in(vcpu, size, port, &val, 1);
7121
	if (ret) {
7122
		kvm_rax_write(vcpu, val);
7123 7124 7125
		return ret;
	}

7126
	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
7127 7128 7129 7130
	vcpu->arch.complete_userspace_io = complete_fast_pio_in;

	return 0;
}
7131 7132 7133

int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
{
7134
	int ret;
7135 7136

	if (in)
7137
		ret = kvm_fast_pio_in(vcpu, size, port);
7138
	else
7139 7140
		ret = kvm_fast_pio_out(vcpu, size, port);
	return ret && kvm_skip_emulated_instruction(vcpu);
7141 7142
}
EXPORT_SYMBOL_GPL(kvm_fast_pio);
7143

7144
static int kvmclock_cpu_down_prep(unsigned int cpu)
7145
{
7146
	__this_cpu_write(cpu_tsc_khz, 0);
7147
	return 0;
7148 7149 7150
}

static void tsc_khz_changed(void *data)
7151
{
7152 7153 7154 7155 7156 7157 7158 7159 7160
	struct cpufreq_freqs *freq = data;
	unsigned long khz = 0;

	if (data)
		khz = freq->new;
	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		khz = cpufreq_quick_get(raw_smp_processor_id());
	if (!khz)
		khz = tsc_khz;
7161
	__this_cpu_write(cpu_tsc_khz, khz);
7162 7163
}

7164
#ifdef CONFIG_X86_64
7165 7166 7167 7168 7169 7170
static void kvm_hyperv_tsc_notifier(void)
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int cpu;

7171
	mutex_lock(&kvm_lock);
7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_make_mclock_inprogress_request(kvm);

	hyperv_stop_tsc_emulation();

	/* TSC frequency always matches when on Hyper-V */
	for_each_present_cpu(cpu)
		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
	kvm_max_guest_tsc_khz = tsc_khz;

	list_for_each_entry(kvm, &vm_list, vm_list) {
		struct kvm_arch *ka = &kvm->arch;

		spin_lock(&ka->pvclock_gtod_sync_lock);

		pvclock_update_vm_gtod_copy(kvm);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);

		spin_unlock(&ka->pvclock_gtod_sync_lock);
	}
7197
	mutex_unlock(&kvm_lock);
7198
}
7199
#endif
7200

7201
static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
7202 7203 7204 7205 7206
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i, send_ipi = 0;

7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245
	/*
	 * We allow guests to temporarily run on slowing clocks,
	 * provided we notify them after, or to run on accelerating
	 * clocks, provided we notify them before.  Thus time never
	 * goes backwards.
	 *
	 * However, we have a problem.  We can't atomically update
	 * the frequency of a given CPU from this function; it is
	 * merely a notifier, which can be called from any CPU.
	 * Changing the TSC frequency at arbitrary points in time
	 * requires a recomputation of local variables related to
	 * the TSC for each VCPU.  We must flag these local variables
	 * to be updated and be sure the update takes place with the
	 * new frequency before any guests proceed.
	 *
	 * Unfortunately, the combination of hotplug CPU and frequency
	 * change creates an intractable locking scenario; the order
	 * of when these callouts happen is undefined with respect to
	 * CPU hotplug, and they can race with each other.  As such,
	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
	 * undefined; you can actually have a CPU frequency change take
	 * place in between the computation of X and the setting of the
	 * variable.  To protect against this problem, all updates of
	 * the per_cpu tsc_khz variable are done in an interrupt
	 * protected IPI, and all callers wishing to update the value
	 * must wait for a synchronous IPI to complete (which is trivial
	 * if the caller is on the CPU already).  This establishes the
	 * necessary total order on variable updates.
	 *
	 * Note that because a guest time update may take place
	 * anytime after the setting of the VCPU's request bit, the
	 * correct TSC value must be set before the request.  However,
	 * to ensure the update actually makes it to any guest which
	 * starts running in hardware virtualization between the set
	 * and the acquisition of the spinlock, we must also ping the
	 * CPU after setting the request bit.
	 *
	 */

7246
	smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
7247

7248
	mutex_lock(&kvm_lock);
7249
	list_for_each_entry(kvm, &vm_list, vm_list) {
7250
		kvm_for_each_vcpu(i, vcpu, kvm) {
7251
			if (vcpu->cpu != cpu)
7252
				continue;
Zachary Amsden's avatar
Zachary Amsden committed
7253
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
7254
			if (vcpu->cpu != raw_smp_processor_id())
7255
				send_ipi = 1;
7256 7257
		}
	}
7258
	mutex_unlock(&kvm_lock);
7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272

	if (freq->old < freq->new && send_ipi) {
		/*
		 * We upscale the frequency.  Must make the guest
		 * doesn't see old kvmclock values while running with
		 * the new frequency, otherwise we risk the guest sees
		 * time go backwards.
		 *
		 * In case we update the frequency for another cpu
		 * (which might be in guest context) send an interrupt
		 * to kick the cpu out of guest context.  Next time
		 * guest context is entered kvmclock will be updated,
		 * so the guest will not see stale values.
		 */
7273
		smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
7274
	}
7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290
}

static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
				     void *data)
{
	struct cpufreq_freqs *freq = data;
	int cpu;

	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
		return 0;
	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
		return 0;

	for_each_cpu(cpu, freq->policy->cpus)
		__kvmclock_cpufreq_notifier(freq, cpu);

7291 7292 7293 7294
	return 0;
}

static struct notifier_block kvmclock_cpufreq_notifier_block = {
7295 7296 7297
	.notifier_call  = kvmclock_cpufreq_notifier
};

7298
static int kvmclock_cpu_online(unsigned int cpu)
7299
{
7300 7301
	tsc_khz_changed(NULL);
	return 0;
7302 7303
}

7304 7305
static void kvm_timer_init(void)
{
Zachary Amsden's avatar
Zachary Amsden committed
7306
	max_tsc_khz = tsc_khz;
7307

7308
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
Zachary Amsden's avatar
Zachary Amsden committed
7309
#ifdef CONFIG_CPU_FREQ
7310
		struct cpufreq_policy *policy;
7311 7312
		int cpu;

7313
		cpu = get_cpu();
7314
		policy = cpufreq_cpu_get(cpu);
7315 7316 7317 7318 7319
		if (policy) {
			if (policy->cpuinfo.max_freq)
				max_tsc_khz = policy->cpuinfo.max_freq;
			cpufreq_cpu_put(policy);
		}
7320
		put_cpu();
Zachary Amsden's avatar
Zachary Amsden committed
7321
#endif
7322 7323 7324
		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
					  CPUFREQ_TRANSITION_NOTIFIER);
	}
7325

7326
	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
7327
			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
7328 7329
}

7330 7331
DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu);
7332

7333
int kvm_is_in_guest(void)
7334
{
7335
	return __this_cpu_read(current_vcpu) != NULL;
7336 7337 7338 7339 7340
}

static int kvm_is_user_mode(void)
{
	int user_mode = 3;
7341

7342
	if (__this_cpu_read(current_vcpu))
7343
		user_mode = kvm_x86_ops.get_cpl(__this_cpu_read(current_vcpu));
7344

7345 7346 7347 7348 7349 7350
	return user_mode != 0;
}

static unsigned long kvm_get_guest_ip(void)
{
	unsigned long ip = 0;
7351

7352 7353
	if (__this_cpu_read(current_vcpu))
		ip = kvm_rip_read(__this_cpu_read(current_vcpu));
7354

7355 7356 7357
	return ip;
}

7358 7359 7360 7361 7362 7363 7364 7365 7366
static void kvm_handle_intel_pt_intr(void)
{
	struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);

	kvm_make_request(KVM_REQ_PMI, vcpu);
	__set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
			(unsigned long *)&vcpu->arch.pmu.global_status);
}

7367 7368 7369 7370
static struct perf_guest_info_callbacks kvm_guest_cbs = {
	.is_in_guest		= kvm_is_in_guest,
	.is_user_mode		= kvm_is_user_mode,
	.get_guest_ip		= kvm_get_guest_ip,
7371
	.handle_intel_pt_intr	= kvm_handle_intel_pt_intr,
7372 7373
};

7374 7375 7376
#ifdef CONFIG_X86_64
static void pvclock_gtod_update_fn(struct work_struct *work)
{
7377 7378 7379 7380 7381
	struct kvm *kvm;

	struct kvm_vcpu *vcpu;
	int i;

7382
	mutex_lock(&kvm_lock);
7383 7384
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_for_each_vcpu(i, vcpu, kvm)
7385
			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
7386
	atomic_set(&kvm_guest_has_master_clock, 0);
7387
	mutex_unlock(&kvm_lock);
7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403
}

static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);

/*
 * Notification about pvclock gtod data update.
 */
static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
			       void *priv)
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	struct timekeeper *tk = priv;

	update_pvclock_gtod(tk);

	/* disable master clock if host does not trust, or does not
7404
	 * use, TSC based clocksource.
7405
	 */
7406
	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417
	    atomic_read(&kvm_guest_has_master_clock) != 0)
		queue_work(system_long_wq, &pvclock_gtod_work);

	return 0;
}

static struct notifier_block pvclock_gtod_notifier = {
	.notifier_call = pvclock_gtod_notify,
};
#endif

7418
int kvm_arch_init(void *opaque)
7419
{
7420
	struct kvm_x86_init_ops *ops = opaque;
7421
	int r;
7422

7423
	if (kvm_x86_ops.hardware_enable) {
7424
		printk(KERN_ERR "kvm: already loaded the other module\n");
7425 7426
		r = -EEXIST;
		goto out;
7427 7428 7429
	}

	if (!ops->cpu_has_kvm_support()) {
7430
		pr_err_ratelimited("kvm: no hardware support\n");
7431 7432
		r = -EOPNOTSUPP;
		goto out;
7433 7434
	}
	if (ops->disabled_by_bios()) {
7435
		pr_err_ratelimited("kvm: disabled by bios\n");
7436 7437
		r = -EOPNOTSUPP;
		goto out;
7438 7439
	}

7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450
	/*
	 * KVM explicitly assumes that the guest has an FPU and
	 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
	 * vCPU's FPU state as a fxregs_state struct.
	 */
	if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
		printk(KERN_ERR "kvm: inadequate fpu\n");
		r = -EOPNOTSUPP;
		goto out;
	}

7451
	r = -ENOMEM;
7452
	x86_fpu_cache = kmem_cache_create("x86_fpu", sizeof(struct fpu),
7453 7454 7455 7456 7457 7458 7459
					  __alignof__(struct fpu), SLAB_ACCOUNT,
					  NULL);
	if (!x86_fpu_cache) {
		printk(KERN_ERR "kvm: failed to allocate cache for x86 fpu\n");
		goto out;
	}

7460 7461 7462 7463 7464 7465
	x86_emulator_cache = kvm_alloc_emulator_cache();
	if (!x86_emulator_cache) {
		pr_err("kvm: failed to allocate cache for x86 emulator\n");
		goto out_free_x86_fpu_cache;
	}

7466 7467 7468
	shared_msrs = alloc_percpu(struct kvm_shared_msrs);
	if (!shared_msrs) {
		printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
7469
		goto out_free_x86_emulator_cache;
7470 7471
	}

7472 7473
	r = kvm_mmu_module_init();
	if (r)
7474
		goto out_free_percpu;
7475

Sheng Yang's avatar
Sheng Yang committed
7476
	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
7477
			PT_DIRTY_MASK, PT64_NX_MASK, 0,
7478
			PT_PRESENT_MASK, 0, sme_me_mask);
7479
	kvm_timer_init();
7480

7481 7482
	perf_register_guest_info_callbacks(&kvm_guest_cbs);

7483
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
7484
		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
7485 7486
		supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
	}
7487

7488
	kvm_lapic_init();
7489 7490
	if (pi_inject_timer == -1)
		pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);
7491 7492
#ifdef CONFIG_X86_64
	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
7493

7494
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7495
		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
7496 7497
#endif

7498
	return 0;
7499

7500 7501
out_free_percpu:
	free_percpu(shared_msrs);
7502 7503
out_free_x86_emulator_cache:
	kmem_cache_destroy(x86_emulator_cache);
7504 7505
out_free_x86_fpu_cache:
	kmem_cache_destroy(x86_fpu_cache);
7506 7507
out:
	return r;
7508
}
7509

7510 7511
void kvm_arch_exit(void)
{
7512
#ifdef CONFIG_X86_64
7513
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7514 7515
		clear_hv_tscchange_cb();
#endif
7516
	kvm_lapic_exit();
7517 7518
	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);

7519 7520 7521
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
					    CPUFREQ_TRANSITION_NOTIFIER);
7522
	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
7523 7524 7525
#ifdef CONFIG_X86_64
	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
#endif
7526
	kvm_x86_ops.hardware_enable = NULL;
7527
	kvm_mmu_module_exit();
7528
	free_percpu(shared_msrs);
7529
	kmem_cache_destroy(x86_fpu_cache);
7530
}
7531

7532
int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
7533 7534
{
	++vcpu->stat.halt_exits;
7535
	if (lapic_in_kernel(vcpu)) {
7536
		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
7537 7538 7539 7540 7541 7542
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
}
7543 7544 7545 7546
EXPORT_SYMBOL_GPL(kvm_vcpu_halt);

int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
7547 7548 7549 7550 7551 7552
	int ret = kvm_skip_emulated_instruction(vcpu);
	/*
	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
	 * KVM_EXIT_DEBUG here.
	 */
	return kvm_vcpu_halt(vcpu) && ret;
7553
}
7554 7555
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

7556
#ifdef CONFIG_X86_64
7557 7558 7559 7560
static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
			        unsigned long clock_type)
{
	struct kvm_clock_pairing clock_pairing;
7561
	struct timespec64 ts;
Paolo Bonzini's avatar
Paolo Bonzini committed
7562
	u64 cycle;
7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574
	int ret;

	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
		return -KVM_EOPNOTSUPP;

	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)
		return -KVM_EOPNOTSUPP;

	clock_pairing.sec = ts.tv_sec;
	clock_pairing.nsec = ts.tv_nsec;
	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
	clock_pairing.flags = 0;
7575
	memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
7576 7577 7578 7579 7580 7581 7582 7583

	ret = 0;
	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
			    sizeof(struct kvm_clock_pairing)))
		ret = -KVM_EFAULT;

	return ret;
}
7584
#endif
7585

7586 7587 7588 7589 7590 7591 7592
/*
 * kvm_pv_kick_cpu_op:  Kick a vcpu.
 *
 * @apicid - apicid of vcpu to be kicked.
 */
static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
{
7593
	struct kvm_lapic_irq lapic_irq;
7594

7595
	lapic_irq.shorthand = APIC_DEST_NOSHORT;
7596
	lapic_irq.dest_mode = APIC_DEST_PHYSICAL;
7597
	lapic_irq.level = 0;
7598
	lapic_irq.dest_id = apicid;
7599
	lapic_irq.msi_redir_hint = false;
7600

7601
	lapic_irq.delivery_mode = APIC_DM_REMRD;
7602
	kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
7603 7604
}

7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621
bool kvm_apicv_activated(struct kvm *kvm)
{
	return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0);
}
EXPORT_SYMBOL_GPL(kvm_apicv_activated);

void kvm_apicv_init(struct kvm *kvm, bool enable)
{
	if (enable)
		clear_bit(APICV_INHIBIT_REASON_DISABLE,
			  &kvm->arch.apicv_inhibit_reasons);
	else
		set_bit(APICV_INHIBIT_REASON_DISABLE,
			&kvm->arch.apicv_inhibit_reasons);
}
EXPORT_SYMBOL_GPL(kvm_apicv_init);

7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634
static void kvm_sched_yield(struct kvm *kvm, unsigned long dest_id)
{
	struct kvm_vcpu *target = NULL;
	struct kvm_apic_map *map;

	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);

	if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])
		target = map->phys_map[dest_id]->vcpu;

	rcu_read_unlock();

7635
	if (target && READ_ONCE(target->ready))
7636 7637 7638
		kvm_vcpu_yield_to(target);
}

7639 7640 7641
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
{
	unsigned long nr, a0, a1, a2, a3, ret;
7642
	int op_64_bit;
7643

7644 7645
	if (kvm_hv_hypercall_enabled(vcpu->kvm))
		return kvm_hv_hypercall(vcpu);
7646

7647 7648 7649 7650 7651
	nr = kvm_rax_read(vcpu);
	a0 = kvm_rbx_read(vcpu);
	a1 = kvm_rcx_read(vcpu);
	a2 = kvm_rdx_read(vcpu);
	a3 = kvm_rsi_read(vcpu);
7652

7653
	trace_kvm_hypercall(nr, a0, a1, a2, a3);
Feng (Eric) Liu's avatar
Feng (Eric) Liu committed
7654

7655 7656
	op_64_bit = is_64_bit_mode(vcpu);
	if (!op_64_bit) {
7657 7658 7659 7660 7661 7662 7663
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
	}

7664
	if (kvm_x86_ops.get_cpl(vcpu) != 0) {
7665
		ret = -KVM_EPERM;
7666
		goto out;
7667 7668
	}

7669
	switch (nr) {
Avi Kivity's avatar
Avi Kivity committed
7670 7671 7672
	case KVM_HC_VAPIC_POLL_IRQ:
		ret = 0;
		break;
7673 7674
	case KVM_HC_KICK_CPU:
		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
7675
		kvm_sched_yield(vcpu->kvm, a1);
7676 7677
		ret = 0;
		break;
7678
#ifdef CONFIG_X86_64
7679 7680 7681
	case KVM_HC_CLOCK_PAIRING:
		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
		break;
7682
#endif
7683 7684 7685
	case KVM_HC_SEND_IPI:
		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
		break;
7686 7687 7688 7689
	case KVM_HC_SCHED_YIELD:
		kvm_sched_yield(vcpu->kvm, a0);
		ret = 0;
		break;
7690 7691 7692 7693
	default:
		ret = -KVM_ENOSYS;
		break;
	}
7694
out:
7695 7696
	if (!op_64_bit)
		ret = (u32)ret;
7697
	kvm_rax_write(vcpu, ret);
7698

7699
	++vcpu->stat.hypercalls;
7700
	return kvm_skip_emulated_instruction(vcpu);
7701 7702 7703
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

7704
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
7705
{
7706
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7707
	char instruction[3];
7708
	unsigned long rip = kvm_rip_read(vcpu);
7709

7710
	kvm_x86_ops.patch_hypercall(vcpu, instruction);
7711

7712 7713
	return emulator_write_emulated(ctxt, rip, instruction, 3,
		&ctxt->exception);
7714 7715
}

7716
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
7717
{
7718 7719
	return vcpu->run->request_interrupt_window &&
		likely(!pic_in_kernel(vcpu->kvm));
7720 7721
}

7722
static void post_kvm_run_save(struct kvm_vcpu *vcpu)
7723
{
7724 7725
	struct kvm_run *kvm_run = vcpu->run;

7726
	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
7727
	kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;
7728
	kvm_run->cr8 = kvm_get_cr8(vcpu);
7729
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
7730 7731
	kvm_run->ready_for_interrupt_injection =
		pic_in_kernel(vcpu->kvm) ||
7732
		kvm_vcpu_ready_for_interrupt_injection(vcpu);
7733 7734
}

7735 7736 7737 7738
static void update_cr8_intercept(struct kvm_vcpu *vcpu)
{
	int max_irr, tpr;

7739
	if (!kvm_x86_ops.update_cr8_intercept)
7740 7741
		return;

7742
	if (!lapic_in_kernel(vcpu))
7743 7744
		return;

7745 7746 7747
	if (vcpu->arch.apicv_active)
		return;

7748 7749 7750 7751
	if (!vcpu->arch.apic->vapic_addr)
		max_irr = kvm_lapic_find_highest_irr(vcpu);
	else
		max_irr = -1;
7752 7753 7754 7755 7756 7757

	if (max_irr != -1)
		max_irr >>= 4;

	tpr = kvm_lapic_get_cr8(vcpu);

7758
	kvm_x86_ops.update_cr8_intercept(vcpu, tpr, max_irr);
7759 7760
}

7761
static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit)
7762
{
7763
	int r;
7764
	bool can_inject = true;
7765

7766
	/* try to reinject previous events if any */
7767

7768
	if (vcpu->arch.exception.injected) {
7769
		kvm_x86_ops.queue_exception(vcpu);
7770 7771
		can_inject = false;
	}
7772
	/*
7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784
	 * Do not inject an NMI or interrupt if there is a pending
	 * exception.  Exceptions and interrupts are recognized at
	 * instruction boundaries, i.e. the start of an instruction.
	 * Trap-like exceptions, e.g. #DB, have higher priority than
	 * NMIs and interrupts, i.e. traps are recognized before an
	 * NMI/interrupt that's pending on the same instruction.
	 * Fault-like exceptions, e.g. #GP and #PF, are the lowest
	 * priority, but are only generated (pended) during instruction
	 * execution, i.e. a pending fault-like exception means the
	 * fault occurred on the *previous* instruction and must be
	 * serviced prior to recognizing any new events in order to
	 * fully complete the previous instruction.
7785
	 */
7786
	else if (!vcpu->arch.exception.pending) {
7787
		if (vcpu->arch.nmi_injected) {
7788
			kvm_x86_ops.set_nmi(vcpu);
7789 7790
			can_inject = false;
		} else if (vcpu->arch.interrupt.injected) {
7791
			kvm_x86_ops.set_irq(vcpu);
7792 7793
			can_inject = false;
		}
7794 7795
	}

7796 7797 7798
	WARN_ON_ONCE(vcpu->arch.exception.injected &&
		     vcpu->arch.exception.pending);

7799 7800 7801 7802 7803 7804
	/*
	 * Call check_nested_events() even if we reinjected a previous event
	 * in order for caller to determine if it should require immediate-exit
	 * from L2 to L1 due to pending L1 events which require exit
	 * from L2 to L1.
	 */
7805
	if (is_guest_mode(vcpu)) {
7806
		r = kvm_x86_ops.nested_ops->check_events(vcpu);
7807 7808
		if (r < 0)
			goto busy;
7809 7810 7811
	}

	/* try to inject new event if pending */
7812
	if (vcpu->arch.exception.pending) {
Avi Kivity's avatar
Avi Kivity committed
7813 7814 7815
		trace_kvm_inj_exception(vcpu->arch.exception.nr,
					vcpu->arch.exception.has_error_code,
					vcpu->arch.exception.error_code);
7816

7817 7818 7819
		vcpu->arch.exception.pending = false;
		vcpu->arch.exception.injected = true;

7820 7821 7822 7823
		if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
					     X86_EFLAGS_RF);

7824 7825 7826 7827 7828 7829
		if (vcpu->arch.exception.nr == DB_VECTOR) {
			kvm_deliver_exception_payload(vcpu);
			if (vcpu->arch.dr7 & DR7_GD) {
				vcpu->arch.dr7 &= ~DR7_GD;
				kvm_update_dr7(vcpu);
			}
7830 7831
		}

7832
		kvm_x86_ops.queue_exception(vcpu);
7833
		can_inject = false;
7834 7835
	}

7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873
	/*
	 * Finally, inject interrupt events.  If an event cannot be injected
	 * due to architectural conditions (e.g. IF=0) a window-open exit
	 * will re-request KVM_REQ_EVENT.  Sometimes however an event is pending
	 * and can architecturally be injected, but we cannot do it right now:
	 * an interrupt could have arrived just now and we have to inject it
	 * as a vmexit, or there could already an event in the queue, which is
	 * indicated by can_inject.  In that case we request an immediate exit
	 * in order to make progress and get back here for another iteration.
	 * The kvm_x86_ops hooks communicate this by returning -EBUSY.
	 */
	if (vcpu->arch.smi_pending) {
		r = can_inject ? kvm_x86_ops.smi_allowed(vcpu, true) : -EBUSY;
		if (r < 0)
			goto busy;
		if (r) {
			vcpu->arch.smi_pending = false;
			++vcpu->arch.smi_count;
			enter_smm(vcpu);
			can_inject = false;
		} else
			kvm_x86_ops.enable_smi_window(vcpu);
	}

	if (vcpu->arch.nmi_pending) {
		r = can_inject ? kvm_x86_ops.nmi_allowed(vcpu, true) : -EBUSY;
		if (r < 0)
			goto busy;
		if (r) {
			--vcpu->arch.nmi_pending;
			vcpu->arch.nmi_injected = true;
			kvm_x86_ops.set_nmi(vcpu);
			can_inject = false;
			WARN_ON(kvm_x86_ops.nmi_allowed(vcpu, true) < 0);
		}
		if (vcpu->arch.nmi_pending)
			kvm_x86_ops.enable_nmi_window(vcpu);
	}
7874

7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885
	if (kvm_cpu_has_injectable_intr(vcpu)) {
		r = can_inject ? kvm_x86_ops.interrupt_allowed(vcpu, true) : -EBUSY;
		if (r < 0)
			goto busy;
		if (r) {
			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);
			kvm_x86_ops.set_irq(vcpu);
			WARN_ON(kvm_x86_ops.interrupt_allowed(vcpu, true) < 0);
		}
		if (kvm_cpu_has_injectable_intr(vcpu))
			kvm_x86_ops.enable_irq_window(vcpu);
7886
	}
7887

7888 7889 7890 7891 7892 7893 7894 7895 7896 7897 7898
	if (is_guest_mode(vcpu) &&
	    kvm_x86_ops.nested_ops->hv_timer_pending &&
	    kvm_x86_ops.nested_ops->hv_timer_pending(vcpu))
		*req_immediate_exit = true;

	WARN_ON(vcpu->arch.exception.pending);
	return;

busy:
	*req_immediate_exit = true;
	return;
7899 7900
}

Avi Kivity's avatar
Avi Kivity committed
7901 7902 7903 7904 7905 7906 7907 7908 7909
static void process_nmi(struct kvm_vcpu *vcpu)
{
	unsigned limit = 2;

	/*
	 * x86 is limited to one NMI running, and one NMI pending after it.
	 * If an NMI is already in progress, limit further NMIs to just one.
	 * Otherwise, allow two (and we'll inject the first one immediately).
	 */
7910
	if (kvm_x86_ops.get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
Avi Kivity's avatar
Avi Kivity committed
7911 7912 7913 7914 7915 7916 7917
		limit = 1;

	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

7918
static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931
{
	u32 flags = 0;
	flags |= seg->g       << 23;
	flags |= seg->db      << 22;
	flags |= seg->l       << 21;
	flags |= seg->avl     << 20;
	flags |= seg->present << 15;
	flags |= seg->dpl     << 13;
	flags |= seg->s       << 12;
	flags |= seg->type    << 8;
	return flags;
}

7932
static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946
{
	struct kvm_segment seg;
	int offset;

	kvm_get_segment(vcpu, &seg, n);
	put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector);

	if (n < 3)
		offset = 0x7f84 + n * 12;
	else
		offset = 0x7f2c + (n - 3) * 12;

	put_smstate(u32, buf, offset + 8, seg.base);
	put_smstate(u32, buf, offset + 4, seg.limit);
7947
	put_smstate(u32, buf, offset, enter_smm_get_segment_flags(&seg));
7948 7949
}

7950
#ifdef CONFIG_X86_64
7951
static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
7952 7953 7954 7955 7956 7957 7958 7959
{
	struct kvm_segment seg;
	int offset;
	u16 flags;

	kvm_get_segment(vcpu, &seg, n);
	offset = 0x7e00 + n * 16;

7960
	flags = enter_smm_get_segment_flags(&seg) >> 8;
7961 7962 7963 7964 7965
	put_smstate(u16, buf, offset, seg.selector);
	put_smstate(u16, buf, offset + 2, flags);
	put_smstate(u32, buf, offset + 4, seg.limit);
	put_smstate(u64, buf, offset + 8, seg.base);
}
7966
#endif
7967

7968
static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984 7985 7986 7987 7988 7989 7990 7991
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
	put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
	put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
	put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));

	for (i = 0; i < 8; i++)
		put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read(vcpu, i));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u32, buf, 0x7fcc, (u32)val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u32, buf, 0x7fc8, (u32)val);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u32, buf, 0x7fc4, seg.selector);
	put_smstate(u32, buf, 0x7f64, seg.base);
	put_smstate(u32, buf, 0x7f60, seg.limit);
7992
	put_smstate(u32, buf, 0x7f5c, enter_smm_get_segment_flags(&seg));
7993 7994 7995 7996 7997

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u32, buf, 0x7fc0, seg.selector);
	put_smstate(u32, buf, 0x7f80, seg.base);
	put_smstate(u32, buf, 0x7f7c, seg.limit);
7998
	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
7999

8000
	kvm_x86_ops.get_gdt(vcpu, &dt);
8001 8002 8003
	put_smstate(u32, buf, 0x7f74, dt.address);
	put_smstate(u32, buf, 0x7f70, dt.size);

8004
	kvm_x86_ops.get_idt(vcpu, &dt);
8005 8006 8007 8008
	put_smstate(u32, buf, 0x7f58, dt.address);
	put_smstate(u32, buf, 0x7f54, dt.size);

	for (i = 0; i < 6; i++)
8009
		enter_smm_save_seg_32(vcpu, buf, i);
8010 8011 8012 8013 8014 8015 8016 8017

	put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020000);
	put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
}

8018
#ifdef CONFIG_X86_64
8019
static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032 8033 8034 8035 8036 8037 8038 8039 8040 8041 8042 8043 8044 8045 8046 8047 8048 8049
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	for (i = 0; i < 16; i++)
		put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read(vcpu, i));

	put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
	put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u64, buf, 0x7f68, val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u64, buf, 0x7f60, val);

	put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
	put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
	put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));

	put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020064);

	put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u16, buf, 0x7e90, seg.selector);
8050
	put_smstate(u16, buf, 0x7e92, enter_smm_get_segment_flags(&seg) >> 8);
8051 8052 8053
	put_smstate(u32, buf, 0x7e94, seg.limit);
	put_smstate(u64, buf, 0x7e98, seg.base);

8054
	kvm_x86_ops.get_idt(vcpu, &dt);
8055 8056 8057 8058 8059
	put_smstate(u32, buf, 0x7e84, dt.size);
	put_smstate(u64, buf, 0x7e88, dt.address);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u16, buf, 0x7e70, seg.selector);
8060
	put_smstate(u16, buf, 0x7e72, enter_smm_get_segment_flags(&seg) >> 8);
8061 8062 8063
	put_smstate(u32, buf, 0x7e74, seg.limit);
	put_smstate(u64, buf, 0x7e78, seg.base);

8064
	kvm_x86_ops.get_gdt(vcpu, &dt);
8065 8066 8067 8068
	put_smstate(u32, buf, 0x7e64, dt.size);
	put_smstate(u64, buf, 0x7e68, dt.address);

	for (i = 0; i < 6; i++)
8069
		enter_smm_save_seg_64(vcpu, buf, i);
8070
}
8071
#endif
8072

8073
static void enter_smm(struct kvm_vcpu *vcpu)
8074
{
8075
	struct kvm_segment cs, ds;
8076
	struct desc_ptr dt;
8077 8078 8079 8080 8081
	char buf[512];
	u32 cr0;

	trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true);
	memset(buf, 0, 512);
8082
#ifdef CONFIG_X86_64
8083
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
8084
		enter_smm_save_state_64(vcpu, buf);
8085
	else
8086
#endif
8087
		enter_smm_save_state_32(vcpu, buf);
8088

8089 8090 8091 8092 8093
	/*
	 * Give pre_enter_smm() a chance to make ISA-specific changes to the
	 * vCPU state (e.g. leave guest mode) after we've saved the state into
	 * the SMM state-save area.
	 */
8094
	kvm_x86_ops.pre_enter_smm(vcpu, buf);
8095 8096

	vcpu->arch.hflags |= HF_SMM_MASK;
8097
	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
8098

8099
	if (kvm_x86_ops.get_nmi_mask(vcpu))
8100 8101
		vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
	else
8102
		kvm_x86_ops.set_nmi_mask(vcpu, true);
8103 8104 8105 8106 8107

	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
	kvm_rip_write(vcpu, 0x8000);

	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
8108
	kvm_x86_ops.set_cr0(vcpu, cr0);
8109 8110
	vcpu->arch.cr0 = cr0;

8111
	kvm_x86_ops.set_cr4(vcpu, 0);
8112

8113 8114
	/* Undocumented: IDT limit is set to zero on entry to SMM.  */
	dt.address = dt.size = 0;
8115
	kvm_x86_ops.set_idt(vcpu, &dt);
8116

8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140 8141 8142 8143
	__kvm_set_dr(vcpu, 7, DR7_FIXED_1);

	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
	cs.base = vcpu->arch.smbase;

	ds.selector = 0;
	ds.base = 0;

	cs.limit    = ds.limit = 0xffffffff;
	cs.type     = ds.type = 0x3;
	cs.dpl      = ds.dpl = 0;
	cs.db       = ds.db = 0;
	cs.s        = ds.s = 1;
	cs.l        = ds.l = 0;
	cs.g        = ds.g = 1;
	cs.avl      = ds.avl = 0;
	cs.present  = ds.present = 1;
	cs.unusable = ds.unusable = 0;
	cs.padding  = ds.padding = 0;

	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);

8144
#ifdef CONFIG_X86_64
8145
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
8146
		kvm_x86_ops.set_efer(vcpu, 0);
8147
#endif
8148 8149 8150

	kvm_update_cpuid(vcpu);
	kvm_mmu_reset_context(vcpu);
8151 8152
}

8153
static void process_smi(struct kvm_vcpu *vcpu)
8154 8155 8156 8157 8158
{
	vcpu->arch.smi_pending = true;
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

8159 8160 8161 8162 8163 8164 8165
void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
				       unsigned long *vcpu_bitmap)
{
	cpumask_var_t cpus;

	zalloc_cpumask_var(&cpus, GFP_ATOMIC);

8166
	kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC,
8167
				    NULL, vcpu_bitmap, cpus);
8168 8169 8170 8171

	free_cpumask_var(cpus);
}

8172 8173 8174 8175 8176
void kvm_make_scan_ioapic_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
}

8177 8178 8179 8180 8181 8182 8183
void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
{
	if (!lapic_in_kernel(vcpu))
		return;

	vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm);
	kvm_apic_update_apicv(vcpu);
8184
	kvm_x86_ops.refresh_apicv_exec_ctrl(vcpu);
8185 8186 8187 8188 8189 8190 8191 8192 8193 8194 8195 8196
}
EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv);

/*
 * NOTE: Do not hold any lock prior to calling this.
 *
 * In particular, kvm_request_apicv_update() expects kvm->srcu not to be
 * locked, because it calls __x86_set_memory_region() which does
 * synchronize_srcu(&kvm->srcu).
 */
void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit)
{
8197
	struct kvm_vcpu *except;
8198 8199
	unsigned long old, new, expected;

8200 8201
	if (!kvm_x86_ops.check_apicv_inhibit_reasons ||
	    !kvm_x86_ops.check_apicv_inhibit_reasons(bit))
8202 8203
		return;

8204 8205 8206 8207 8208 8209 8210 8211 8212 8213 8214 8215 8216 8217
	old = READ_ONCE(kvm->arch.apicv_inhibit_reasons);
	do {
		expected = new = old;
		if (activate)
			__clear_bit(bit, &new);
		else
			__set_bit(bit, &new);
		if (new == old)
			break;
		old = cmpxchg(&kvm->arch.apicv_inhibit_reasons, expected, new);
	} while (old != expected);

	if (!!old == !!new)
		return;
8218

8219
	trace_kvm_apicv_update_request(activate, bit);
8220 8221
	if (kvm_x86_ops.pre_update_apicv_exec_ctrl)
		kvm_x86_ops.pre_update_apicv_exec_ctrl(kvm, activate);
8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232

	/*
	 * Sending request to update APICV for all other vcpus,
	 * while update the calling vcpu immediately instead of
	 * waiting for another #VMEXIT to handle the request.
	 */
	except = kvm_get_running_vcpu();
	kvm_make_all_cpus_request_except(kvm, KVM_REQ_APICV_UPDATE,
					 except);
	if (except)
		kvm_vcpu_update_apicv(except);
8233 8234 8235
}
EXPORT_SYMBOL_GPL(kvm_request_apicv_update);

8236
static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
8237
{
8238
	if (!kvm_apic_present(vcpu))
8239
		return;
8240

8241
	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
8242

8243
	if (irqchip_split(vcpu->kvm))
8244
		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
8245
	else {
8246
		if (vcpu->arch.apicv_active)
8247
			kvm_x86_ops.sync_pir_to_irr(vcpu);
8248 8249
		if (ioapic_in_kernel(vcpu->kvm))
			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
8250
	}
8251 8252 8253 8254 8255 8256 8257 8258 8259 8260 8261 8262 8263 8264

	if (is_guest_mode(vcpu))
		vcpu->arch.load_eoi_exitmap_pending = true;
	else
		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
}

static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
{
	u64 eoi_exit_bitmap[4];

	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
		return;

8265 8266
	bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
		  vcpu_to_synic(vcpu)->vec_bitmap, 256);
8267
	kvm_x86_ops.load_eoi_exitmap(vcpu, eoi_exit_bitmap);
8268 8269
}

8270 8271 8272
int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
		unsigned long start, unsigned long end,
		bool blockable)
8273 8274 8275 8276 8277 8278 8279 8280 8281 8282
{
	unsigned long apic_address;

	/*
	 * The physical address of apic access page is stored in the VMCS.
	 * Update it when it becomes invalid.
	 */
	apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
	if (start <= apic_address && apic_address < end)
		kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
8283 8284

	return 0;
8285 8286
}

8287 8288
void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
{
8289
	if (!lapic_in_kernel(vcpu))
8290 8291
		return;

8292
	if (!kvm_x86_ops.set_apic_access_page_addr)
8293 8294
		return;

8295
	kvm_x86_ops.set_apic_access_page_addr(vcpu);
8296 8297
}

8298 8299 8300 8301 8302 8303
void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
{
	smp_send_reschedule(vcpu->cpu);
}
EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);

8304
/*
8305
 * Returns 1 to let vcpu_run() continue the guest execution loop without
8306 8307 8308
 * exiting to the userspace.  Otherwise, the value will be returned to the
 * userspace.
 */
8309
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
8310 8311
{
	int r;
8312 8313 8314
	bool req_int_win =
		dm_request_for_irq_injection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);
8315
	fastpath_t exit_fastpath;
8316

8317
	bool req_immediate_exit = false;
8318

8319
	if (kvm_request_pending(vcpu)) {
8320
		if (kvm_check_request(KVM_REQ_GET_VMCS12_PAGES, vcpu)) {
8321
			if (unlikely(!kvm_x86_ops.nested_ops->get_vmcs12_pages(vcpu))) {
8322 8323 8324 8325
				r = 0;
				goto out;
			}
		}
8326
		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
8327
			kvm_mmu_unload(vcpu);
8328
		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
Marcelo Tosatti's avatar
Marcelo Tosatti committed
8329
			__kvm_migrate_timers(vcpu);
8330 8331
		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
			kvm_gen_update_masterclock(vcpu->kvm);
8332 8333
		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
			kvm_gen_kvmclock_update(vcpu);
8334 8335
		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
			r = kvm_guest_time_update(vcpu);
8336 8337 8338
			if (unlikely(r))
				goto out;
		}
8339
		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
8340
			kvm_mmu_sync_roots(vcpu);
8341 8342
		if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))
			kvm_mmu_load_pgd(vcpu);
8343
		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
8344
			kvm_vcpu_flush_tlb_all(vcpu);
8345 8346 8347 8348 8349 8350

			/* Flushing all ASIDs flushes the current ASID... */
			kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
		}
		if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
			kvm_vcpu_flush_tlb_current(vcpu);
8351 8352
		if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu))
			kvm_vcpu_flush_tlb_guest(vcpu);
8353

8354
		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
8355
			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
Avi Kivity's avatar
Avi Kivity committed
8356 8357 8358
			r = 0;
			goto out;
		}
8359
		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
8360
			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
8361
			vcpu->mmio_needed = 0;
8362 8363 8364
			r = 0;
			goto out;
		}
8365 8366 8367 8368 8369 8370
		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
			/* Page is swapped out. Do synthetic halt */
			vcpu->arch.apf.halted = true;
			r = 1;
			goto out;
		}
8371 8372
		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
			record_steal_time(vcpu);
8373 8374
		if (kvm_check_request(KVM_REQ_SMI, vcpu))
			process_smi(vcpu);
Avi Kivity's avatar
Avi Kivity committed
8375 8376
		if (kvm_check_request(KVM_REQ_NMI, vcpu))
			process_nmi(vcpu);
8377
		if (kvm_check_request(KVM_REQ_PMU, vcpu))
8378
			kvm_pmu_handle_event(vcpu);
8379
		if (kvm_check_request(KVM_REQ_PMI, vcpu))
8380
			kvm_pmu_deliver_pmi(vcpu);
8381 8382 8383
		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
			if (test_bit(vcpu->arch.pending_ioapic_eoi,
8384
				     vcpu->arch.ioapic_handled_vectors)) {
8385 8386 8387 8388 8389 8390 8391
				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
				vcpu->run->eoi.vector =
						vcpu->arch.pending_ioapic_eoi;
				r = 0;
				goto out;
			}
		}
8392 8393
		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
			vcpu_scan_ioapic(vcpu);
8394 8395
		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
			vcpu_load_eoi_exitmap(vcpu);
8396 8397
		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
			kvm_vcpu_reload_apic_access_page(vcpu);
8398 8399 8400 8401 8402 8403
		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
			r = 0;
			goto out;
		}
8404 8405 8406 8407 8408 8409
		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
			r = 0;
			goto out;
		}
8410 8411 8412 8413 8414 8415
		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_HYPERV;
			vcpu->run->hyperv = vcpu->arch.hyperv.exit;
			r = 0;
			goto out;
		}
8416 8417 8418 8419 8420 8421

		/*
		 * KVM_REQ_HV_STIMER has to be processed after
		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
		 * depend on the guest clock being up-to-date
		 */
8422 8423
		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
			kvm_hv_process_stimers(vcpu);
8424 8425
		if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))
			kvm_vcpu_update_apicv(vcpu);
8426 8427
		if (kvm_check_request(KVM_REQ_APF_READY, vcpu))
			kvm_check_async_pf_completion(vcpu);
8428
	}
Avi Kivity's avatar
Avi Kivity committed
8429

8430
	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
8431
		++vcpu->stat.req_event;
8432 8433 8434 8435 8436 8437
		kvm_apic_accept_events(vcpu);
		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
			r = 1;
			goto out;
		}

8438 8439 8440
		inject_pending_event(vcpu, &req_immediate_exit);
		if (req_int_win)
			kvm_x86_ops.enable_irq_window(vcpu);
8441 8442 8443 8444 8445 8446 8447

		if (kvm_lapic_enabled(vcpu)) {
			update_cr8_intercept(vcpu);
			kvm_lapic_sync_to_vapic(vcpu);
		}
	}

8448 8449
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r)) {
8450
		goto cancel_injection;
8451 8452
	}

8453 8454
	preempt_disable();

8455
	kvm_x86_ops.prepare_guest_switch(vcpu);
8456 8457 8458 8459 8460 8461 8462

	/*
	 * Disable IRQs before setting IN_GUEST_MODE.  Posted interrupt
	 * IPI are then delayed after guest entry, which ensures that they
	 * result in virtual interrupt delivery.
	 */
	local_irq_disable();
8463 8464
	vcpu->mode = IN_GUEST_MODE;

8465 8466
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);

8467
	/*
8468
	 * 1) We should set ->mode before checking ->requests.  Please see
8469
	 * the comment in kvm_vcpu_exiting_guest_mode().
8470
	 *
8471
	 * 2) For APICv, we should set ->mode before checking PID.ON. This
8472 8473 8474 8475 8476 8477
	 * pairs with the memory barrier implicit in pi_test_and_set_on
	 * (see vmx_deliver_posted_interrupt).
	 *
	 * 3) This also orders the write to mode from any reads to the page
	 * tables done while the VCPU is running.  Please see the comment
	 * in kvm_flush_remote_tlbs.
8478
	 */
8479
	smp_mb__after_srcu_read_unlock();
8480

8481 8482 8483 8484
	/*
	 * This handles the case where a posted interrupt was
	 * notified with kvm_vcpu_kick.
	 */
8485
	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
8486
		kvm_x86_ops.sync_pir_to_irr(vcpu);
8487

8488
	if (kvm_vcpu_exit_request(vcpu)) {
8489
		vcpu->mode = OUTSIDE_GUEST_MODE;
Avi Kivity's avatar
Avi Kivity committed
8490
		smp_wmb();
8491 8492
		local_irq_enable();
		preempt_enable();
8493
		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8494
		r = 1;
8495
		goto cancel_injection;
8496 8497
	}

8498 8499
	if (req_immediate_exit) {
		kvm_make_request(KVM_REQ_EVENT, vcpu);
8500
		kvm_x86_ops.request_immediate_exit(vcpu);
8501
	}
8502

8503
	trace_kvm_entry(vcpu->vcpu_id);
8504
	guest_enter_irqoff();
8505

8506 8507 8508
	fpregs_assert_state_consistent();
	if (test_thread_flag(TIF_NEED_FPU_LOAD))
		switch_fpu_return();
8509

8510 8511 8512 8513 8514 8515
	if (unlikely(vcpu->arch.switch_db_regs)) {
		set_debugreg(0, 7);
		set_debugreg(vcpu->arch.eff_db[0], 0);
		set_debugreg(vcpu->arch.eff_db[1], 1);
		set_debugreg(vcpu->arch.eff_db[2], 2);
		set_debugreg(vcpu->arch.eff_db[3], 3);
8516
		set_debugreg(vcpu->arch.dr6, 6);
8517
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8518
	}
8519

8520
	exit_fastpath = kvm_x86_ops.run(vcpu);
8521

8522 8523 8524 8525 8526 8527 8528 8529
	/*
	 * Do this here before restoring debug registers on the host.  And
	 * since we do this before handling the vmexit, a DR access vmexit
	 * can (a) read the correct value of the debug registers, (b) set
	 * KVM_DEBUGREG_WONT_EXIT again.
	 */
	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
8530
		kvm_x86_ops.sync_dirty_debug_regs(vcpu);
8531 8532 8533
		kvm_update_dr0123(vcpu);
		kvm_update_dr7(vcpu);
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8534 8535
	}

8536 8537 8538 8539 8540 8541 8542
	/*
	 * If the guest has used debug registers, at least dr7
	 * will be disabled while returning to the host.
	 * If we don't have active breakpoints in the host, we don't
	 * care about the messed up debug address registers. But if
	 * we have some of them active, restore the old state.
	 */
8543
	if (hw_breakpoint_active())
8544
		hw_breakpoint_restore();
8545

8546
	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
8547

8548
	vcpu->mode = OUTSIDE_GUEST_MODE;
Avi Kivity's avatar
Avi Kivity committed
8549
	smp_wmb();
8550

8551
	kvm_x86_ops.handle_exit_irqoff(vcpu);
8552

8553 8554 8555 8556 8557 8558 8559 8560 8561
	/*
	 * Consume any pending interrupts, including the possible source of
	 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
	 * An instruction is required after local_irq_enable() to fully unblock
	 * interrupts on processors that implement an interrupt shadow, the
	 * stat.exits increment will do nicely.
	 */
	kvm_before_interrupt(vcpu);
	local_irq_enable();
8562
	++vcpu->stat.exits;
8563 8564
	local_irq_disable();
	kvm_after_interrupt(vcpu);
8565

8566
	guest_exit_irqoff();
8567 8568 8569 8570 8571 8572 8573
	if (lapic_in_kernel(vcpu)) {
		s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;
		if (delta != S64_MIN) {
			trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta);
			vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;
		}
	}
8574

8575
	local_irq_enable();
8576 8577
	preempt_enable();

8578
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8579

8580 8581 8582 8583
	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
8584 8585
		unsigned long rip = kvm_rip_read(vcpu);
		profile_hit(KVM_PROFILING, (void *)rip);
8586 8587
	}

8588 8589
	if (unlikely(vcpu->arch.tsc_always_catchup))
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
8590

8591 8592
	if (vcpu->arch.apic_attention)
		kvm_lapic_sync_from_vapic(vcpu);
Avi Kivity's avatar
Avi Kivity committed
8593

8594
	r = kvm_x86_ops.handle_exit(vcpu, exit_fastpath);
8595 8596 8597
	return r;

cancel_injection:
8598 8599
	if (req_immediate_exit)
		kvm_make_request(KVM_REQ_EVENT, vcpu);
8600
	kvm_x86_ops.cancel_injection(vcpu);
8601 8602
	if (unlikely(vcpu->arch.apic_attention))
		kvm_lapic_sync_from_vapic(vcpu);
8603 8604 8605
out:
	return r;
}
8606

8607 8608
static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
{
8609
	if (!kvm_arch_vcpu_runnable(vcpu) &&
8610
	    (!kvm_x86_ops.pre_block || kvm_x86_ops.pre_block(vcpu) == 0)) {
8611 8612 8613
		srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
		kvm_vcpu_block(vcpu);
		vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8614

8615 8616
		if (kvm_x86_ops.post_block)
			kvm_x86_ops.post_block(vcpu);
8617

8618 8619 8620
		if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
			return 1;
	}
8621 8622 8623 8624 8625 8626 8627

	kvm_apic_accept_events(vcpu);
	switch(vcpu->arch.mp_state) {
	case KVM_MP_STATE_HALTED:
		vcpu->arch.pv.pv_unhalted = false;
		vcpu->arch.mp_state =
			KVM_MP_STATE_RUNNABLE;
8628
		/* fall through */
8629 8630 8631 8632 8633 8634 8635 8636 8637 8638
	case KVM_MP_STATE_RUNNABLE:
		vcpu->arch.apf.halted = false;
		break;
	case KVM_MP_STATE_INIT_RECEIVED:
		break;
	default:
		return -EINTR;
	}
	return 1;
}
8639

8640 8641
static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
{
8642
	if (is_guest_mode(vcpu))
8643
		kvm_x86_ops.nested_ops->check_events(vcpu);
8644

8645 8646 8647 8648
	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		!vcpu->arch.apf.halted);
}

8649
static int vcpu_run(struct kvm_vcpu *vcpu)
8650 8651
{
	int r;
8652
	struct kvm *kvm = vcpu->kvm;
8653

8654
	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8655
	vcpu->arch.l1tf_flush_l1d = true;
8656

8657
	for (;;) {
8658
		if (kvm_vcpu_running(vcpu)) {
8659
			r = vcpu_enter_guest(vcpu);
8660
		} else {
8661
			r = vcpu_block(kvm, vcpu);
8662 8663
		}

8664 8665 8666
		if (r <= 0)
			break;

8667
		kvm_clear_request(KVM_REQ_PENDING_TIMER, vcpu);
8668 8669 8670
		if (kvm_cpu_has_pending_timer(vcpu))
			kvm_inject_pending_timer_irqs(vcpu);

8671 8672
		if (dm_request_for_irq_injection(vcpu) &&
			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
8673 8674
			r = 0;
			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
8675
			++vcpu->stat.request_irq_exits;
8676
			break;
8677
		}
8678

8679 8680
		if (signal_pending(current)) {
			r = -EINTR;
8681
			vcpu->run->exit_reason = KVM_EXIT_INTR;
8682
			++vcpu->stat.signal_exits;
8683
			break;
8684 8685
		}
		if (need_resched()) {
8686
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8687
			cond_resched();
8688
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8689
		}
8690 8691
	}

8692
	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8693 8694 8695 8696

	return r;
}

8697 8698 8699
static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
{
	int r;
8700

8701
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8702
	r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
8703
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
8704
	return r;
8705 8706 8707 8708 8709 8710 8711 8712 8713
}

static int complete_emulated_pio(struct kvm_vcpu *vcpu)
{
	BUG_ON(!vcpu->arch.pio.count);

	return complete_emulated_io(vcpu);
}

Avi Kivity's avatar
Avi Kivity committed
8714 8715 8716 8717 8718
/*
 * Implements the following, as a state machine:
 *
 * read:
 *   for each fragment
8719 8720 8721 8722
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       exit
 *       copy data
Avi Kivity's avatar
Avi Kivity committed
8723 8724 8725 8726
 *   execute insn
 *
 * write:
 *   for each fragment
8727 8728 8729 8730
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       copy data
 *       exit
Avi Kivity's avatar
Avi Kivity committed
8731
 */
8732
static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
8733 8734
{
	struct kvm_run *run = vcpu->run;
Avi Kivity's avatar
Avi Kivity committed
8735
	struct kvm_mmio_fragment *frag;
8736
	unsigned len;
8737

8738
	BUG_ON(!vcpu->mmio_needed);
8739

8740
	/* Complete previous fragment */
8741 8742
	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
	len = min(8u, frag->len);
8743
	if (!vcpu->mmio_is_write)
8744 8745 8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756
		memcpy(frag->data, run->mmio.data, len);

	if (frag->len <= 8) {
		/* Switch to the next fragment. */
		frag++;
		vcpu->mmio_cur_fragment++;
	} else {
		/* Go forward to the next mmio piece. */
		frag->data += len;
		frag->gpa += len;
		frag->len -= len;
	}

8757
	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
8758
		vcpu->mmio_needed = 0;
8759 8760

		/* FIXME: return into emulator if single-stepping.  */
Avi Kivity's avatar
Avi Kivity committed
8761
		if (vcpu->mmio_is_write)
8762 8763 8764 8765
			return 1;
		vcpu->mmio_read_completed = 1;
		return complete_emulated_io(vcpu);
	}
8766

8767 8768 8769
	run->exit_reason = KVM_EXIT_MMIO;
	run->mmio.phys_addr = frag->gpa;
	if (vcpu->mmio_is_write)
8770 8771
		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
	run->mmio.len = min(8u, frag->len);
8772 8773 8774
	run->mmio.is_write = vcpu->mmio_is_write;
	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
	return 0;
8775 8776
}

8777 8778 8779 8780 8781 8782 8783 8784 8785 8786 8787 8788 8789
static void kvm_save_current_fpu(struct fpu *fpu)
{
	/*
	 * If the target FPU state is not resident in the CPU registers, just
	 * memcpy() from current, else save CPU state directly to the target.
	 */
	if (test_thread_flag(TIF_NEED_FPU_LOAD))
		memcpy(&fpu->state, &current->thread.fpu.state,
		       fpu_kernel_xstate_size);
	else
		copy_fpregs_to_fpstate(fpu);
}

8790 8791 8792
/* Swap (qemu) user FPU context for the guest FPU context. */
static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
8793 8794
	fpregs_lock();

8795 8796
	kvm_save_current_fpu(vcpu->arch.user_fpu);

8797
	/* PKRU is separately restored in kvm_x86_ops.run.  */
8798
	__copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,
8799
				~XFEATURE_MASK_PKRU);
8800 8801 8802 8803

	fpregs_mark_activate();
	fpregs_unlock();

8804 8805 8806 8807 8808 8809
	trace_kvm_fpu(1);
}

/* When vcpu_run ends, restore user space FPU context. */
static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
8810 8811
	fpregs_lock();

8812 8813
	kvm_save_current_fpu(vcpu->arch.guest_fpu);

8814
	copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);
8815 8816 8817 8818

	fpregs_mark_activate();
	fpregs_unlock();

8819 8820 8821 8822
	++vcpu->stat.fpu_reload;
	trace_kvm_fpu(0);
}

8823
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
8824
{
8825
	struct kvm_run *kvm_run = vcpu->run;
8826 8827
	int r;

8828
	vcpu_load(vcpu);
8829
	kvm_sigset_activate(vcpu);
8830 8831
	kvm_load_guest_fpu(vcpu);

8832
	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
8833 8834 8835 8836
		if (kvm_run->immediate_exit) {
			r = -EINTR;
			goto out;
		}
8837
		kvm_vcpu_block(vcpu);
8838
		kvm_apic_accept_events(vcpu);
8839
		kvm_clear_request(KVM_REQ_UNHALT, vcpu);
8840
		r = -EAGAIN;
8841 8842
		if (signal_pending(current)) {
			r = -EINTR;
8843
			kvm_run->exit_reason = KVM_EXIT_INTR;
8844 8845
			++vcpu->stat.signal_exits;
		}
8846
		goto out;
8847 8848
	}

8849
	if (kvm_run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) {
Ken Hofsass's avatar
Ken Hofsass committed
8850 8851 8852 8853
		r = -EINVAL;
		goto out;
	}

8854
	if (kvm_run->kvm_dirty_regs) {
Ken Hofsass's avatar
Ken Hofsass committed
8855 8856 8857 8858 8859
		r = sync_regs(vcpu);
		if (r != 0)
			goto out;
	}

8860
	/* re-sync apic's tpr */
8861
	if (!lapic_in_kernel(vcpu)) {
Andre Przywara's avatar
Andre Przywara committed
8862 8863 8864 8865 8866
		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
			r = -EINVAL;
			goto out;
		}
	}
8867

8868 8869 8870 8871 8872
	if (unlikely(vcpu->arch.complete_userspace_io)) {
		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
		vcpu->arch.complete_userspace_io = NULL;
		r = cui(vcpu);
		if (r <= 0)
8873
			goto out;
8874 8875
	} else
		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
8876

8877 8878 8879 8880
	if (kvm_run->immediate_exit)
		r = -EINTR;
	else
		r = vcpu_run(vcpu);
8881 8882

out:
8883
	kvm_put_guest_fpu(vcpu);
8884
	if (kvm_run->kvm_valid_regs)
Ken Hofsass's avatar
Ken Hofsass committed
8885
		store_regs(vcpu);
8886
	post_kvm_run_save(vcpu);
8887
	kvm_sigset_deactivate(vcpu);
8888

8889
	vcpu_put(vcpu);
8890 8891 8892
	return r;
}

Ken Hofsass's avatar
Ken Hofsass committed
8893
static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8894
{
8895 8896 8897 8898
	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
		/*
		 * We are here if userspace calls get_regs() in the middle of
		 * instruction emulation. Registers state needs to be copied
Guo Chao's avatar
Guo Chao committed
8899
		 * back from emulation context to vcpu. Userspace shouldn't do
8900 8901 8902
		 * that usually, but some bad designed PV devices (vmware
		 * backdoor interface) need this to work
		 */
8903
		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
8904 8905
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
	}
8906 8907 8908 8909 8910 8911
	regs->rax = kvm_rax_read(vcpu);
	regs->rbx = kvm_rbx_read(vcpu);
	regs->rcx = kvm_rcx_read(vcpu);
	regs->rdx = kvm_rdx_read(vcpu);
	regs->rsi = kvm_rsi_read(vcpu);
	regs->rdi = kvm_rdi_read(vcpu);
8912
	regs->rsp = kvm_rsp_read(vcpu);
8913
	regs->rbp = kvm_rbp_read(vcpu);
8914
#ifdef CONFIG_X86_64
8915 8916 8917 8918 8919 8920 8921 8922
	regs->r8 = kvm_r8_read(vcpu);
	regs->r9 = kvm_r9_read(vcpu);
	regs->r10 = kvm_r10_read(vcpu);
	regs->r11 = kvm_r11_read(vcpu);
	regs->r12 = kvm_r12_read(vcpu);
	regs->r13 = kvm_r13_read(vcpu);
	regs->r14 = kvm_r14_read(vcpu);
	regs->r15 = kvm_r15_read(vcpu);
8923 8924
#endif

8925
	regs->rip = kvm_rip_read(vcpu);
8926
	regs->rflags = kvm_get_rflags(vcpu);
Ken Hofsass's avatar
Ken Hofsass committed
8927
}
8928

Ken Hofsass's avatar
Ken Hofsass committed
8929 8930 8931 8932
int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__get_regs(vcpu, regs);
8933
	vcpu_put(vcpu);
8934 8935 8936
	return 0;
}

Ken Hofsass's avatar
Ken Hofsass committed
8937
static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8938
{
8939 8940 8941
	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;

8942 8943 8944 8945 8946 8947
	kvm_rax_write(vcpu, regs->rax);
	kvm_rbx_write(vcpu, regs->rbx);
	kvm_rcx_write(vcpu, regs->rcx);
	kvm_rdx_write(vcpu, regs->rdx);
	kvm_rsi_write(vcpu, regs->rsi);
	kvm_rdi_write(vcpu, regs->rdi);
8948
	kvm_rsp_write(vcpu, regs->rsp);
8949
	kvm_rbp_write(vcpu, regs->rbp);
8950
#ifdef CONFIG_X86_64
8951 8952 8953 8954 8955 8956 8957 8958
	kvm_r8_write(vcpu, regs->r8);
	kvm_r9_write(vcpu, regs->r9);
	kvm_r10_write(vcpu, regs->r10);
	kvm_r11_write(vcpu, regs->r11);
	kvm_r12_write(vcpu, regs->r12);
	kvm_r13_write(vcpu, regs->r13);
	kvm_r14_write(vcpu, regs->r14);
	kvm_r15_write(vcpu, regs->r15);
8959 8960
#endif

8961
	kvm_rip_write(vcpu, regs->rip);
8962
	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
8963

8964 8965
	vcpu->arch.exception.pending = false;

8966
	kvm_make_request(KVM_REQ_EVENT, vcpu);
Ken Hofsass's avatar
Ken Hofsass committed
8967
}
8968

Ken Hofsass's avatar
Ken Hofsass committed
8969 8970 8971 8972
int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__set_regs(vcpu, regs);
8973
	vcpu_put(vcpu);
8974 8975 8976 8977 8978 8979 8980
	return 0;
}

void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

8981
	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
8982 8983 8984 8985 8986
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

Ken Hofsass's avatar
Ken Hofsass committed
8987
static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8988
{
8989
	struct desc_ptr dt;
8990

8991 8992 8993 8994 8995 8996
	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8997

8998 8999
	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
9000

9001
	kvm_x86_ops.get_idt(vcpu, &dt);
9002 9003
	sregs->idt.limit = dt.size;
	sregs->idt.base = dt.address;
9004
	kvm_x86_ops.get_gdt(vcpu, &dt);
9005 9006
	sregs->gdt.limit = dt.size;
	sregs->gdt.base = dt.address;
9007

9008
	sregs->cr0 = kvm_read_cr0(vcpu);
9009
	sregs->cr2 = vcpu->arch.cr2;
9010
	sregs->cr3 = kvm_read_cr3(vcpu);
9011
	sregs->cr4 = kvm_read_cr4(vcpu);
9012
	sregs->cr8 = kvm_get_cr8(vcpu);
9013
	sregs->efer = vcpu->arch.efer;
9014 9015
	sregs->apic_base = kvm_get_apic_base(vcpu);

9016
	memset(sregs->interrupt_bitmap, 0, sizeof(sregs->interrupt_bitmap));
9017

9018
	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
9019 9020
		set_bit(vcpu->arch.interrupt.nr,
			(unsigned long *)sregs->interrupt_bitmap);
Ken Hofsass's avatar
Ken Hofsass committed
9021
}
9022

Ken Hofsass's avatar
Ken Hofsass committed
9023 9024 9025 9026 9027
int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	vcpu_load(vcpu);
	__get_sregs(vcpu, sregs);
9028
	vcpu_put(vcpu);
9029 9030 9031
	return 0;
}

9032 9033 9034
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
9035
	vcpu_load(vcpu);
9036 9037
	if (kvm_mpx_supported())
		kvm_load_guest_fpu(vcpu);
9038

9039
	kvm_apic_accept_events(vcpu);
9040 9041 9042 9043 9044 9045
	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
					vcpu->arch.pv.pv_unhalted)
		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
	else
		mp_state->mp_state = vcpu->arch.mp_state;

9046 9047
	if (kvm_mpx_supported())
		kvm_put_guest_fpu(vcpu);
9048
	vcpu_put(vcpu);
9049 9050 9051 9052 9053 9054
	return 0;
}

int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
9055 9056 9057 9058
	int ret = -EINVAL;

	vcpu_load(vcpu);

9059
	if (!lapic_in_kernel(vcpu) &&
9060
	    mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
9061
		goto out;
9062

9063 9064 9065 9066 9067 9068
	/*
	 * KVM_MP_STATE_INIT_RECEIVED means the processor is in
	 * INIT state; latched init should be reported using
	 * KVM_SET_VCPU_EVENTS, so reject it here.
	 */
	if ((kvm_vcpu_latch_init(vcpu) || vcpu->arch.smi_pending) &&
9069 9070
	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
9071
		goto out;
9072

9073 9074 9075 9076 9077
	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
	} else
		vcpu->arch.mp_state = mp_state->mp_state;
9078
	kvm_make_request(KVM_REQ_EVENT, vcpu);
9079 9080 9081 9082 9083

	ret = 0;
out:
	vcpu_put(vcpu);
	return ret;
9084 9085
}

9086 9087
int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
		    int reason, bool has_error_code, u32 error_code)
9088
{
9089
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
9090
	int ret;
9091

9092
	init_emulate_ctxt(vcpu);
9093

9094
	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
9095
				   has_error_code, error_code);
9096 9097 9098 9099
	if (ret) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
9100
		return 0;
9101
	}
9102

9103 9104
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
9105
	return 1;
9106 9107 9108
}
EXPORT_SYMBOL_GPL(kvm_task_switch);

Peng Hao's avatar
Peng Hao committed
9109
static int kvm_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
9110
{
9111
	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
9112 9113 9114 9115 9116
		/*
		 * When EFER.LME and CR0.PG are set, the processor is in
		 * 64-bit mode (though maybe in a 32-bit code segment).
		 * CR4.PAE and EFER.LMA must be set.
		 */
9117
		if (!(sregs->cr4 & X86_CR4_PAE)
9118 9119 9120 9121 9122 9123 9124 9125 9126 9127 9128
		    || !(sregs->efer & EFER_LMA))
			return -EINVAL;
	} else {
		/*
		 * Not in 64-bit mode: EFER.LMA is clear and the code
		 * segment cannot be 64-bit.
		 */
		if (sregs->efer & EFER_LMA || sregs->cs.l)
			return -EINVAL;
	}

9129
	return kvm_valid_cr4(vcpu, sregs->cr4);
9130 9131
}

Ken Hofsass's avatar
Ken Hofsass committed
9132
static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
9133
{
9134
	struct msr_data apic_base_msr;
9135
	int mmu_reset_needed = 0;
9136
	int cpuid_update_needed = 0;
9137
	int pending_vec, max_bits, idx;
9138
	struct desc_ptr dt;
9139 9140
	int ret = -EINVAL;

9141
	if (kvm_valid_sregs(vcpu, sregs))
9142
		goto out;
9143

9144 9145 9146
	apic_base_msr.data = sregs->apic_base;
	apic_base_msr.host_initiated = true;
	if (kvm_set_apic_base(vcpu, &apic_base_msr))
9147
		goto out;
9148

9149 9150
	dt.size = sregs->idt.limit;
	dt.address = sregs->idt.base;
9151
	kvm_x86_ops.set_idt(vcpu, &dt);
9152 9153
	dt.size = sregs->gdt.limit;
	dt.address = sregs->gdt.base;
9154
	kvm_x86_ops.set_gdt(vcpu, &dt);
9155

9156
	vcpu->arch.cr2 = sregs->cr2;
9157
	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
9158
	vcpu->arch.cr3 = sregs->cr3;
9159
	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
9160

9161
	kvm_set_cr8(vcpu, sregs->cr8);
9162

9163
	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
9164
	kvm_x86_ops.set_efer(vcpu, sregs->efer);
9165

9166
	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
9167
	kvm_x86_ops.set_cr0(vcpu, sregs->cr0);
9168
	vcpu->arch.cr0 = sregs->cr0;
9169

9170
	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
9171 9172
	cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) &
				(X86_CR4_OSXSAVE | X86_CR4_PKE));
9173
	kvm_x86_ops.set_cr4(vcpu, sregs->cr4);
9174
	if (cpuid_update_needed)
9175
		kvm_update_cpuid(vcpu);
9176 9177

	idx = srcu_read_lock(&vcpu->kvm->srcu);
9178
	if (is_pae_paging(vcpu)) {
9179
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
9180 9181
		mmu_reset_needed = 1;
	}
9182
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9183 9184 9185 9186

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

9187
	max_bits = KVM_NR_INTERRUPTS;
9188 9189 9190
	pending_vec = find_first_bit(
		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
	if (pending_vec < max_bits) {
9191
		kvm_queue_interrupt(vcpu, pending_vec, false);
9192
		pr_debug("Set back pending irq %d\n", pending_vec);
9193 9194
	}

9195 9196 9197 9198 9199 9200
	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
9201

9202 9203
	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
9204

9205 9206
	update_cr8_intercept(vcpu);

9207
	/* Older userspace won't unhalt the vcpu on reset. */
9208
	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
9209
	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
9210
	    !is_protmode(vcpu))
9211 9212
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;

9213 9214
	kvm_make_request(KVM_REQ_EVENT, vcpu);

9215 9216
	ret = 0;
out:
Ken Hofsass's avatar
Ken Hofsass committed
9217 9218 9219 9220 9221 9222 9223 9224 9225 9226
	return ret;
}

int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	int ret;

	vcpu_load(vcpu);
	ret = __set_sregs(vcpu, sregs);
9227 9228
	vcpu_put(vcpu);
	return ret;
9229 9230
}

Jan Kiszka's avatar
Jan Kiszka committed
9231 9232
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg)
9233
{
9234
	unsigned long rflags;
9235
	int i, r;
9236

9237 9238
	vcpu_load(vcpu);

9239 9240 9241
	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
		r = -EBUSY;
		if (vcpu->arch.exception.pending)
9242
			goto out;
9243 9244 9245 9246 9247 9248
		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
			kvm_queue_exception(vcpu, DB_VECTOR);
		else
			kvm_queue_exception(vcpu, BP_VECTOR);
	}

9249 9250 9251 9252 9253
	/*
	 * Read rflags as long as potentially injected trace flags are still
	 * filtered out.
	 */
	rflags = kvm_get_rflags(vcpu);
9254 9255 9256 9257 9258 9259

	vcpu->guest_debug = dbg->control;
	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
		vcpu->guest_debug = 0;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
9260 9261
		for (i = 0; i < KVM_NR_DB_REGS; ++i)
			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
9262
		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
9263 9264 9265 9266
	} else {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
	}
9267
	kvm_update_dr7(vcpu);
9268

Jan Kiszka's avatar
Jan Kiszka committed
9269 9270 9271
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
			get_segment_base(vcpu, VCPU_SREG_CS);
9272

9273 9274 9275 9276 9277
	/*
	 * Trigger an rflags update that will inject or remove the trace
	 * flags.
	 */
	kvm_set_rflags(vcpu, rflags);
9278

9279
	kvm_x86_ops.update_bp_intercept(vcpu);
9280

9281
	r = 0;
Jan Kiszka's avatar
Jan Kiszka committed
9282

9283
out:
9284
	vcpu_put(vcpu);
9285 9286 9287
	return r;
}

9288 9289 9290 9291 9292 9293 9294 9295
/*
 * Translate a guest virtual address to a guest physical address.
 */
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;
9296
	int idx;
9297

9298 9299
	vcpu_load(vcpu);

9300
	idx = srcu_read_lock(&vcpu->kvm->srcu);
9301
	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
9302
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9303 9304 9305 9306 9307
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;

9308
	vcpu_put(vcpu);
9309 9310 9311
	return 0;
}

9312 9313
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
9314
	struct fxregs_state *fxsave;
9315

9316
	vcpu_load(vcpu);
9317

9318
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
9319 9320 9321 9322 9323 9324 9325
	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
9326
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
9327

9328
	vcpu_put(vcpu);
9329 9330 9331 9332 9333
	return 0;
}

int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
9334 9335 9336 9337
	struct fxregs_state *fxsave;

	vcpu_load(vcpu);

9338
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
9339 9340 9341 9342 9343 9344 9345 9346

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
9347
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
9348

9349
	vcpu_put(vcpu);
9350 9351 9352
	return 0;
}

Ken Hofsass's avatar
Ken Hofsass committed
9353 9354 9355 9356 9357 9358 9359 9360 9361 9362 9363 9364 9365 9366 9367 9368 9369 9370 9371 9372 9373 9374 9375 9376 9377 9378 9379 9380 9381 9382 9383 9384 9385 9386 9387 9388 9389 9390 9391
static void store_regs(struct kvm_vcpu *vcpu)
{
	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
		__get_regs(vcpu, &vcpu->run->s.regs.regs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
		kvm_vcpu_ioctl_x86_get_vcpu_events(
				vcpu, &vcpu->run->s.regs.events);
}

static int sync_regs(struct kvm_vcpu *vcpu)
{
	if (vcpu->run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)
		return -EINVAL;

	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
		__set_regs(vcpu, &vcpu->run->s.regs.regs);
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
		if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
		if (kvm_vcpu_ioctl_x86_set_vcpu_events(
				vcpu, &vcpu->run->s.regs.events))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
	}

	return 0;
}

9392
static void fx_init(struct kvm_vcpu *vcpu)
9393
{
9394
	fpstate_init(&vcpu->arch.guest_fpu->state);
9395
	if (boot_cpu_has(X86_FEATURE_XSAVES))
9396
		vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =
9397
			host_xcr0 | XSTATE_COMPACTION_ENABLED;
9398

9399 9400 9401
	/*
	 * Ensure guest xcr0 is valid for loading
	 */
Dave Hansen's avatar
Dave Hansen committed
9402
	vcpu->arch.xcr0 = XFEATURE_MASK_FP;
9403

9404
	vcpu->arch.cr0 |= X86_CR0_ET;
9405 9406
}

9407
int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
9408
{
9409 9410 9411
	if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
		pr_warn_once("kvm: SMP vm created on host with unstable TSC; "
			     "guest TSC will not be reliable\n");
9412

9413
	return 0;
9414 9415
}

9416
int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
9417
{
9418 9419
	struct page *page;
	int r;
9420

9421 9422 9423 9424
	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
	else
		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
9425

9426
	kvm_set_tsc_khz(vcpu, max_tsc_khz);
9427

9428 9429 9430 9431 9432 9433 9434 9435
	r = kvm_mmu_create(vcpu);
	if (r < 0)
		return r;

	if (irqchip_in_kernel(vcpu->kvm)) {
		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
		if (r < 0)
			goto fail_mmu_destroy;
9436 9437
		if (kvm_apicv_activated(vcpu->kvm))
			vcpu->arch.apicv_active = true;
9438 9439 9440 9441 9442 9443 9444 9445 9446 9447 9448 9449 9450 9451 9452 9453 9454 9455 9456 9457
	} else
		static_key_slow_inc(&kvm_no_apic_vcpu);

	r = -ENOMEM;

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page)
		goto fail_free_lapic;
	vcpu->arch.pio_data = page_address(page);

	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
				       GFP_KERNEL_ACCOUNT);
	if (!vcpu->arch.mce_banks)
		goto fail_free_pio_data;
	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;

	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
				GFP_KERNEL_ACCOUNT))
		goto fail_free_mce_banks;

9458 9459 9460
	if (!alloc_emulate_ctxt(vcpu))
		goto free_wbinvd_dirty_mask;

9461 9462 9463 9464
	vcpu->arch.user_fpu = kmem_cache_zalloc(x86_fpu_cache,
						GFP_KERNEL_ACCOUNT);
	if (!vcpu->arch.user_fpu) {
		pr_err("kvm: failed to allocate userspace's fpu\n");
9465
		goto free_emulate_ctxt;
9466 9467 9468 9469 9470 9471 9472 9473 9474 9475 9476
	}

	vcpu->arch.guest_fpu = kmem_cache_zalloc(x86_fpu_cache,
						 GFP_KERNEL_ACCOUNT);
	if (!vcpu->arch.guest_fpu) {
		pr_err("kvm: failed to allocate vcpu's fpu\n");
		goto free_user_fpu;
	}
	fx_init(vcpu);

	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
9477
	vcpu->arch.tdp_level = kvm_x86_ops.get_tdp_level(vcpu);
9478 9479 9480 9481 9482 9483 9484 9485 9486 9487 9488

	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;

	kvm_async_pf_hash_reset(vcpu);
	kvm_pmu_init(vcpu);

	vcpu->arch.pending_external_vector = -1;
	vcpu->arch.preempted_in_kernel = false;

	kvm_hv_vcpu_init(vcpu);

9489
	r = kvm_x86_ops.vcpu_create(vcpu);
9490 9491
	if (r)
		goto free_guest_fpu;
9492

9493
	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
9494
	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
9495
	kvm_vcpu_mtrr_init(vcpu);
9496
	vcpu_load(vcpu);
9497
	kvm_vcpu_reset(vcpu, false);
9498
	kvm_init_mmu(vcpu, false);
9499
	vcpu_put(vcpu);
9500
	return 0;
9501 9502 9503 9504 9505

free_guest_fpu:
	kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
free_user_fpu:
	kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
9506 9507
free_emulate_ctxt:
	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518
free_wbinvd_dirty_mask:
	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
fail_free_mce_banks:
	kfree(vcpu->arch.mce_banks);
fail_free_pio_data:
	free_page((unsigned long)vcpu->arch.pio_data);
fail_free_lapic:
	kvm_free_lapic(vcpu);
fail_mmu_destroy:
	kvm_mmu_destroy(vcpu);
	return r;
9519 9520
}

9521
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
9522
{
9523
	struct msr_data msr;
9524
	struct kvm *kvm = vcpu->kvm;
9525

9526 9527
	kvm_hv_vcpu_postcreate(vcpu);

9528
	if (mutex_lock_killable(&vcpu->mutex))
9529
		return;
9530
	vcpu_load(vcpu);
9531 9532 9533 9534
	msr.data = 0x0;
	msr.index = MSR_IA32_TSC;
	msr.host_initiated = true;
	kvm_write_tsc(vcpu, &msr);
9535
	vcpu_put(vcpu);
9536 9537 9538 9539

	/* poll control enabled by default */
	vcpu->arch.msr_kvm_poll_control = 1;

9540
	mutex_unlock(&vcpu->mutex);
9541

9542 9543 9544
	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)
		schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
						KVMCLOCK_SYNC_PERIOD);
9545 9546
}

9547
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
9548
{
9549
	struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache;
9550
	int idx;
9551

9552 9553
	kvm_release_pfn(cache->pfn, cache->dirty, cache);

9554
	kvmclock_reset(vcpu);
9555

9556
	kvm_x86_ops.vcpu_free(vcpu);
9557

9558
	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
9559 9560 9561
	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
	kmem_cache_free(x86_fpu_cache, vcpu->arch.user_fpu);
	kmem_cache_free(x86_fpu_cache, vcpu->arch.guest_fpu);
9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572

	kvm_hv_vcpu_uninit(vcpu);
	kvm_pmu_destroy(vcpu);
	kfree(vcpu->arch.mce_banks);
	kvm_free_lapic(vcpu);
	idx = srcu_read_lock(&vcpu->kvm->srcu);
	kvm_mmu_destroy(vcpu);
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	free_page((unsigned long)vcpu->arch.pio_data);
	if (!lapic_in_kernel(vcpu))
		static_key_slow_dec(&kvm_no_apic_vcpu);
9573 9574
}

9575
void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
9576
{
9577 9578
	kvm_lapic_reset(vcpu, init_event);

9579 9580
	vcpu->arch.hflags = 0;

9581
	vcpu->arch.smi_pending = 0;
9582
	vcpu->arch.smi_count = 0;
Avi Kivity's avatar
Avi Kivity committed
9583 9584
	atomic_set(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = 0;
9585
	vcpu->arch.nmi_injected = false;
9586 9587
	kvm_clear_interrupt_queue(vcpu);
	kvm_clear_exception_queue(vcpu);
9588

9589
	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
9590
	kvm_update_dr0123(vcpu);
9591
	vcpu->arch.dr6 = DR6_INIT;
9592
	vcpu->arch.dr7 = DR7_FIXED_1;
9593
	kvm_update_dr7(vcpu);
9594

9595 9596
	vcpu->arch.cr2 = 0;

9597
	kvm_make_request(KVM_REQ_EVENT, vcpu);
9598 9599
	vcpu->arch.apf.msr_en_val = 0;
	vcpu->arch.apf.msr_int_val = 0;
9600
	vcpu->arch.st.msr_val = 0;
9601

9602 9603
	kvmclock_reset(vcpu);

9604 9605 9606
	kvm_clear_async_pf_completion_queue(vcpu);
	kvm_async_pf_hash_reset(vcpu);
	vcpu->arch.apf.halted = false;
9607

9608 9609 9610 9611 9612 9613 9614
	if (kvm_mpx_supported()) {
		void *mpx_state_buffer;

		/*
		 * To avoid have the INIT path from kvm_apic_has_events() that be
		 * called with loaded FPU and does not let userspace fix the state.
		 */
9615 9616
		if (init_event)
			kvm_put_guest_fpu(vcpu);
9617
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9618
					XFEATURE_BNDREGS);
9619 9620
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndreg_state));
9621
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9622
					XFEATURE_BNDCSR);
9623 9624
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndcsr));
9625 9626
		if (init_event)
			kvm_load_guest_fpu(vcpu);
9627 9628
	}

9629
	if (!init_event) {
9630
		kvm_pmu_reset(vcpu);
9631
		vcpu->arch.smbase = 0x30000;
9632 9633

		vcpu->arch.msr_misc_features_enables = 0;
9634 9635

		vcpu->arch.xcr0 = XFEATURE_MASK_FP;
9636
	}
9637

9638 9639 9640 9641
	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;

9642 9643
	vcpu->arch.ia32_xss = 0;

9644
	kvm_x86_ops.vcpu_reset(vcpu, init_event);
9645 9646
}

9647
void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
9648 9649 9650 9651 9652 9653 9654 9655
{
	struct kvm_segment cs;

	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs.selector = vector << 8;
	cs.base = vector << 12;
	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_rip_write(vcpu, 0);
9656 9657
}

9658
int kvm_arch_hardware_enable(void)
9659
{
9660 9661 9662
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;
9663 9664 9665 9666
	int ret;
	u64 local_tsc;
	u64 max_tsc = 0;
	bool stable, backwards_tsc = false;
9667 9668

	kvm_shared_msr_cpu_online();
9669
	ret = kvm_x86_ops.hardware_enable();
9670 9671 9672
	if (ret != 0)
		return ret;

9673
	local_tsc = rdtsc();
9674
	stable = !kvm_check_tsc_unstable();
9675 9676 9677
	list_for_each_entry(kvm, &vm_list, vm_list) {
		kvm_for_each_vcpu(i, vcpu, kvm) {
			if (!stable && vcpu->cpu == smp_processor_id())
9678
				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
9679 9680 9681 9682 9683 9684 9685 9686 9687 9688 9689 9690 9691 9692 9693 9694
			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
				backwards_tsc = true;
				if (vcpu->arch.last_host_tsc > max_tsc)
					max_tsc = vcpu->arch.last_host_tsc;
			}
		}
	}

	/*
	 * Sometimes, even reliable TSCs go backwards.  This happens on
	 * platforms that reset TSC during suspend or hibernate actions, but
	 * maintain synchronization.  We must compensate.  Fortunately, we can
	 * detect that condition here, which happens early in CPU bringup,
	 * before any KVM threads can be running.  Unfortunately, we can't
	 * bring the TSCs fully up to date with real time, as we aren't yet far
	 * enough into CPU bringup that we know how much real time has actually
9695
	 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot
9696 9697 9698 9699 9700 9701 9702 9703 9704 9705 9706 9707 9708 9709 9710 9711 9712 9713 9714 9715 9716 9717 9718 9719
	 * variables that haven't been updated yet.
	 *
	 * So we simply find the maximum observed TSC above, then record the
	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
	 * the adjustment will be applied.  Note that we accumulate
	 * adjustments, in case multiple suspend cycles happen before some VCPU
	 * gets a chance to run again.  In the event that no KVM threads get a
	 * chance to run, we will miss the entire elapsed period, as we'll have
	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
	 * loose cycle time.  This isn't too big a deal, since the loss will be
	 * uniform across all VCPUs (not to mention the scenario is extremely
	 * unlikely). It is possible that a second hibernate recovery happens
	 * much faster than a first, causing the observed TSC here to be
	 * smaller; this would require additional padding adjustment, which is
	 * why we set last_host_tsc to the local tsc observed here.
	 *
	 * N.B. - this code below runs only on platforms with reliable TSC,
	 * as that is the only way backwards_tsc is set above.  Also note
	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
	 * have the same delta_cyc adjustment applied if backwards_tsc
	 * is detected.  Note further, this adjustment is only done once,
	 * as we reset last_host_tsc on all VCPUs to stop this from being
	 * called multiple times (one for each physical CPU bringup).
	 *
Guo Chao's avatar
Guo Chao committed
9720
	 * Platforms with unreliable TSCs don't have to deal with this, they
9721 9722 9723 9724 9725 9726 9727
	 * will be compensated by the logic in vcpu_load, which sets the TSC to
	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
	 * guarantee that they stay in perfect synchronization.
	 */
	if (backwards_tsc) {
		u64 delta_cyc = max_tsc - local_tsc;
		list_for_each_entry(kvm, &vm_list, vm_list) {
9728
			kvm->arch.backwards_tsc_observed = true;
9729 9730 9731
			kvm_for_each_vcpu(i, vcpu, kvm) {
				vcpu->arch.tsc_offset_adjustment += delta_cyc;
				vcpu->arch.last_host_tsc = local_tsc;
9732
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
9733 9734 9735 9736 9737 9738 9739 9740 9741 9742 9743 9744 9745 9746
			}

			/*
			 * We have to disable TSC offset matching.. if you were
			 * booting a VM while issuing an S4 host suspend....
			 * you may have some problem.  Solving this issue is
			 * left as an exercise to the reader.
			 */
			kvm->arch.last_tsc_nsec = 0;
			kvm->arch.last_tsc_write = 0;
		}

	}
	return 0;
9747 9748
}

9749
void kvm_arch_hardware_disable(void)
9750
{
9751
	kvm_x86_ops.hardware_disable();
9752
	drop_user_return_notifiers();
9753 9754
}

9755
int kvm_arch_hardware_setup(void *opaque)
9756
{
9757
	struct kvm_x86_init_ops *ops = opaque;
9758 9759
	int r;

9760 9761
	rdmsrl_safe(MSR_EFER, &host_efer);

9762 9763 9764
	if (boot_cpu_has(X86_FEATURE_XSAVES))
		rdmsrl(MSR_IA32_XSS, host_xss);

9765
	r = ops->hardware_setup();
9766 9767 9768
	if (r != 0)
		return r;

9769
	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
9770

9771 9772 9773
	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
		supported_xss = 0;

9774 9775 9776
#define __kvm_cpu_cap_has(UNUSED_, f) kvm_cpu_cap_has(f)
	cr4_reserved_bits = __cr4_reserved_bits(__kvm_cpu_cap_has, UNUSED_);
#undef __kvm_cpu_cap_has
9777

9778 9779 9780 9781
	if (kvm_has_tsc_control) {
		/*
		 * Make sure the user can only configure tsc_khz values that
		 * fit into a signed integer.
9782
		 * A min value is not calculated because it will always
9783 9784 9785 9786 9787 9788
		 * be 1 on all machines.
		 */
		u64 max = min(0x7fffffffULL,
			      __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz));
		kvm_max_guest_tsc_khz = max;

9789
		kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits;
9790
	}
9791

9792 9793
	kvm_init_msr_list();
	return 0;
9794 9795 9796 9797
}

void kvm_arch_hardware_unsetup(void)
{
9798
	kvm_x86_ops.hardware_unsetup();
9799 9800
}

9801
int kvm_arch_check_processor_compat(void *opaque)
9802
{
9803
	struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
9804
	struct kvm_x86_init_ops *ops = opaque;
9805 9806 9807

	WARN_ON(!irqs_disabled());

9808 9809
	if (__cr4_reserved_bits(cpu_has, c) !=
	    __cr4_reserved_bits(cpu_has, &boot_cpu_data))
9810 9811
		return -EIO;

9812
	return ops->check_processor_compatibility();
9813 9814 9815 9816 9817 9818 9819 9820 9821 9822 9823
}

bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
{
	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);

bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
9824 9825
}

9826
struct static_key kvm_no_apic_vcpu __read_mostly;
9827
EXPORT_SYMBOL_GPL(kvm_no_apic_vcpu);
9828

Radim Krčmář's avatar
Radim Krčmář committed
9829 9830
void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
{
9831 9832
	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);

9833
	vcpu->arch.l1tf_flush_l1d = true;
9834 9835 9836 9837
	if (pmu->version && unlikely(pmu->event_count)) {
		pmu->need_cleanup = true;
		kvm_make_request(KVM_REQ_PMU, vcpu);
	}
9838
	kvm_x86_ops.sched_in(vcpu, cpu);
Radim Krčmář's avatar
Radim Krčmář committed
9839 9840
}

9841 9842 9843 9844
void kvm_arch_free_vm(struct kvm *kvm)
{
	kfree(kvm->arch.hyperv.hv_pa_pg);
	vfree(kvm);
Radim Krčmář's avatar
Radim Krčmář committed
9845 9846
}

9847

9848
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
9849
{
9850 9851 9852
	if (type)
		return -EINVAL;

9853
	INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
9854
	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
9855
	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
9856
	INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages);
Ben-Ami Yassour's avatar
Ben-Ami Yassour committed
9857
	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
9858
	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
9859

9860 9861
	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
9862 9863 9864
	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
		&kvm->arch.irq_sources_bitmap);
9865

9866
	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
9867
	mutex_init(&kvm->arch.apic_map_lock);
9868 9869
	spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);

9870
	kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
9871
	pvclock_update_vm_gtod_copy(kvm);
9872

9873 9874
	kvm->arch.guest_can_read_msr_platform_info = true;

9875
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
9876
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
9877

9878
	kvm_hv_init_vm(kvm);
9879
	kvm_page_track_init(kvm);
9880
	kvm_mmu_init_vm(kvm);
9881

9882
	return kvm_x86_ops.vm_init(kvm);
9883 9884
}

9885 9886 9887 9888 9889
int kvm_arch_post_init_vm(struct kvm *kvm)
{
	return kvm_mmu_post_init_vm(kvm);
}

9890 9891
static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
9892
	vcpu_load(vcpu);
9893 9894 9895 9896 9897 9898 9899
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;
9900
	struct kvm_vcpu *vcpu;
9901 9902 9903 9904

	/*
	 * Unpin any mmu pages first.
	 */
9905 9906
	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_clear_async_pf_completion_queue(vcpu);
9907
		kvm_unload_vcpu_mmu(vcpu);
9908
	}
9909
	kvm_for_each_vcpu(i, vcpu, kvm)
9910
		kvm_vcpu_destroy(vcpu);
9911 9912 9913 9914

	mutex_lock(&kvm->lock);
	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
		kvm->vcpus[i] = NULL;
9915

9916 9917
	atomic_set(&kvm->online_vcpus, 0);
	mutex_unlock(&kvm->lock);
9918 9919
}

9920 9921
void kvm_arch_sync_events(struct kvm *kvm)
{
9922
	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
9923
	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
9924
	kvm_free_pit(kvm);
9925 9926
}

9927
int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9928 9929
{
	int i, r;
9930
	unsigned long hva, uninitialized_var(old_npages);
9931
	struct kvm_memslots *slots = kvm_memslots(kvm);
9932
	struct kvm_memory_slot *slot;
9933 9934

	/* Called with kvm->slots_lock held.  */
9935 9936
	if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
		return -EINVAL;
9937

9938 9939
	slot = id_to_memslot(slots, id);
	if (size) {
9940
		if (slot && slot->npages)
9941 9942 9943 9944 9945 9946 9947 9948 9949 9950 9951
			return -EEXIST;

		/*
		 * MAP_SHARED to prevent internal slot pages from being moved
		 * by fork()/COW.
		 */
		hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
			      MAP_SHARED | MAP_ANONYMOUS, 0);
		if (IS_ERR((void *)hva))
			return PTR_ERR((void *)hva);
	} else {
9952
		if (!slot || !slot->npages)
9953 9954
			return 0;

9955 9956 9957 9958 9959 9960
		/*
		 * Stuff a non-canonical value to catch use-after-delete.  This
		 * ends up being 0 on 32-bit KVM, but there's no better
		 * alternative.
		 */
		hva = (unsigned long)(0xdeadull << 48);
9961
		old_npages = slot->npages;
9962 9963
	}

9964
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
9965
		struct kvm_userspace_memory_region m;
9966

9967 9968 9969
		m.slot = id | (i << 16);
		m.flags = 0;
		m.guest_phys_addr = gpa;
9970
		m.userspace_addr = hva;
9971
		m.memory_size = size;
9972 9973 9974 9975 9976
		r = __kvm_set_memory_region(kvm, &m);
		if (r < 0)
			return r;
	}

9977
	if (!size)
9978
		vm_munmap(hva, old_npages * PAGE_SIZE);
9979

9980 9981 9982 9983
	return 0;
}
EXPORT_SYMBOL_GPL(__x86_set_memory_region);

9984 9985 9986 9987 9988
void kvm_arch_pre_destroy_vm(struct kvm *kvm)
{
	kvm_mmu_pre_destroy_vm(kvm);
}

9989 9990
void kvm_arch_destroy_vm(struct kvm *kvm)
{
9991 9992 9993 9994 9995 9996
	if (current->mm == kvm->mm) {
		/*
		 * Free memory regions allocated on behalf of userspace,
		 * unless the the memory map has changed due to process exit
		 * or fd copying.
		 */
9997 9998 9999 10000 10001 10002 10003
		mutex_lock(&kvm->slots_lock);
		__x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
					0, 0);
		__x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
					0, 0);
		__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
		mutex_unlock(&kvm->slots_lock);
10004
	}
10005 10006
	if (kvm_x86_ops.vm_destroy)
		kvm_x86_ops.vm_destroy(kvm);
10007 10008
	kvm_pic_destroy(kvm);
	kvm_ioapic_destroy(kvm);
10009
	kvm_free_vcpus(kvm);
10010
	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
Eric Hankland's avatar
Eric Hankland committed
10011
	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
10012
	kvm_mmu_uninit_vm(kvm);
10013
	kvm_page_track_cleanup(kvm);
10014
	kvm_hv_destroy_vm(kvm);
10015
}
10016

10017
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
10018 10019 10020
{
	int i;

10021
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
10022 10023 10024
		kvfree(slot->arch.rmap[i]);
		slot->arch.rmap[i] = NULL;

10025 10026 10027
		if (i == 0)
			continue;

10028 10029
		kvfree(slot->arch.lpage_info[i - 1]);
		slot->arch.lpage_info[i - 1] = NULL;
10030
	}
10031

10032
	kvm_page_track_free_memslot(slot);
10033 10034
}

10035 10036
static int kvm_alloc_memslot_metadata(struct kvm_memory_slot *slot,
				      unsigned long npages)
10037 10038 10039
{
	int i;

10040 10041 10042 10043 10044 10045 10046
	/*
	 * Clear out the previous array pointers for the KVM_MR_MOVE case.  The
	 * old arrays will be freed by __kvm_set_memory_region() if installing
	 * the new memslot is successful.
	 */
	memset(&slot->arch, 0, sizeof(slot->arch));

10047
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
10048
		struct kvm_lpage_info *linfo;
10049 10050
		unsigned long ugfn;
		int lpages;
10051
		int level = i + 1;
10052 10053 10054 10055

		lpages = gfn_to_index(slot->base_gfn + npages - 1,
				      slot->base_gfn, level) + 1;

10056
		slot->arch.rmap[i] =
10057
			kvcalloc(lpages, sizeof(*slot->arch.rmap[i]),
10058
				 GFP_KERNEL_ACCOUNT);
10059
		if (!slot->arch.rmap[i])
10060
			goto out_free;
10061 10062
		if (i == 0)
			continue;
10063

10064
		linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
10065
		if (!linfo)
10066 10067
			goto out_free;

10068 10069
		slot->arch.lpage_info[i - 1] = linfo;

10070
		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
10071
			linfo[0].disallow_lpage = 1;
10072
		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
10073
			linfo[lpages - 1].disallow_lpage = 1;
10074 10075 10076
		ugfn = slot->userspace_addr >> PAGE_SHIFT;
		/*
		 * If the gfn and userspace address are not aligned wrt each
10077
		 * other, disable large page support for this slot.
10078
		 */
10079
		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) {
10080 10081 10082
			unsigned long j;

			for (j = 0; j < lpages; ++j)
10083
				linfo[j].disallow_lpage = 1;
10084 10085 10086
		}
	}

10087 10088 10089
	if (kvm_page_track_create_memslot(slot, npages))
		goto out_free;

10090 10091 10092
	return 0;

out_free:
10093
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
Thomas Huth's avatar
Thomas Huth committed
10094
		kvfree(slot->arch.rmap[i]);
10095 10096 10097 10098
		slot->arch.rmap[i] = NULL;
		if (i == 0)
			continue;

Thomas Huth's avatar
Thomas Huth committed
10099
		kvfree(slot->arch.lpage_info[i - 1]);
10100
		slot->arch.lpage_info[i - 1] = NULL;
10101 10102 10103 10104
	}
	return -ENOMEM;
}

10105
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
10106
{
10107 10108 10109
	struct kvm_vcpu *vcpu;
	int i;

10110 10111 10112 10113
	/*
	 * memslots->generation has been incremented.
	 * mmio generation may have reached its maximum value.
	 */
10114
	kvm_mmu_invalidate_mmio_sptes(kvm, gen);
10115 10116 10117 10118

	/* Force re-initialization of steal_time cache */
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_vcpu_kick(vcpu);
10119 10120
}

10121 10122
int kvm_arch_prepare_memory_region(struct kvm *kvm,
				struct kvm_memory_slot *memslot,
10123
				const struct kvm_userspace_memory_region *mem,
10124
				enum kvm_mr_change change)
10125
{
10126 10127 10128
	if (change == KVM_MR_CREATE || change == KVM_MR_MOVE)
		return kvm_alloc_memslot_metadata(memslot,
						  mem->memory_size >> PAGE_SHIFT);
10129 10130 10131
	return 0;
}

10132 10133 10134 10135 10136
static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
				     struct kvm_memory_slot *new)
{
	/* Still write protect RO slot */
	if (new->flags & KVM_MEM_READONLY) {
10137
		kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K);
10138 10139 10140 10141 10142 10143
		return;
	}

	/*
	 * Call kvm_x86_ops dirty logging hooks when they are valid.
	 *
10144
	 * kvm_x86_ops.slot_disable_log_dirty is called when:
10145 10146 10147 10148 10149 10150
	 *
	 *  - KVM_MR_CREATE with dirty logging is disabled
	 *  - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag
	 *
	 * The reason is, in case of PML, we need to set D-bit for any slots
	 * with dirty logging disabled in order to eliminate unnecessary GPA
10151
	 * logging in PML buffer (and potential PML buffer full VMEXIT). This
10152
	 * guarantees leaving PML enabled during guest's lifetime won't have
Wei Yang's avatar
Wei Yang committed
10153
	 * any additional overhead from PML when guest is running with dirty
10154 10155
	 * logging disabled for memory slots.
	 *
10156
	 * kvm_x86_ops.slot_enable_log_dirty is called when switching new slot
10157 10158 10159 10160 10161 10162 10163 10164 10165 10166 10167 10168 10169 10170 10171
	 * to dirty logging mode.
	 *
	 * If kvm_x86_ops dirty logging hooks are invalid, use write protect.
	 *
	 * In case of write protect:
	 *
	 * Write protect all pages for dirty logging.
	 *
	 * All the sptes including the large sptes which point to this
	 * slot are set to readonly. We can not create any new large
	 * spte on this slot until the end of the logging.
	 *
	 * See the comments in fast_page_fault().
	 */
	if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
10172 10173
		if (kvm_x86_ops.slot_enable_log_dirty) {
			kvm_x86_ops.slot_enable_log_dirty(kvm, new);
10174 10175 10176
		} else {
			int level =
				kvm_dirty_log_manual_protect_and_init_set(kvm) ?
10177
				PG_LEVEL_2M : PG_LEVEL_4K;
10178 10179 10180 10181 10182 10183 10184 10185 10186 10187 10188

			/*
			 * If we're with initial-all-set, we don't need
			 * to write protect any small page because
			 * they're reported as dirty already.  However
			 * we still need to write-protect huge pages
			 * so that the page split can happen lazily on
			 * the first write to the huge page.
			 */
			kvm_mmu_slot_remove_write_access(kvm, new, level);
		}
10189
	} else {
10190 10191
		if (kvm_x86_ops.slot_disable_log_dirty)
			kvm_x86_ops.slot_disable_log_dirty(kvm, new);
10192 10193 10194
	}
}

10195
void kvm_arch_commit_memory_region(struct kvm *kvm,
10196
				const struct kvm_userspace_memory_region *mem,
10197
				struct kvm_memory_slot *old,
10198
				const struct kvm_memory_slot *new,
10199
				enum kvm_mr_change change)
10200
{
10201
	if (!kvm->arch.n_requested_mmu_pages)
10202 10203
		kvm_mmu_change_mmu_pages(kvm,
				kvm_mmu_calculate_default_mmu_pages(kvm));
10204

10205 10206 10207 10208 10209 10210 10211 10212 10213 10214 10215
	/*
	 * Dirty logging tracks sptes in 4k granularity, meaning that large
	 * sptes have to be split.  If live migration is successful, the guest
	 * in the source machine will be destroyed and large sptes will be
	 * created in the destination. However, if the guest continues to run
	 * in the source machine (for example if live migration fails), small
	 * sptes will remain around and cause bad performance.
	 *
	 * Scan sptes if dirty logging has been stopped, dropping those
	 * which can be collapsed into a single large-page spte.  Later
	 * page faults will create the large-page sptes.
10216 10217 10218 10219 10220
	 *
	 * There is no need to do this in any of the following cases:
	 * CREATE:	No dirty mappings will already exist.
	 * MOVE/DELETE:	The old mappings will already have been cleaned up by
	 *		kvm_arch_flush_shadow_memslot()
10221
	 */
10222
	if (change == KVM_MR_FLAGS_ONLY &&
10223 10224 10225 10226
		(old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
		!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
		kvm_mmu_zap_collapsible_sptes(kvm, new);

10227
	/*
10228
	 * Set up write protection and/or dirty logging for the new slot.
10229
	 *
10230 10231 10232 10233
	 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have
	 * been zapped so no dirty logging staff is needed for old slot. For
	 * KVM_MR_FLAGS_ONLY, the old slot is essentially the same one as the
	 * new and it's also covered when dealing with the new slot.
10234 10235
	 *
	 * FIXME: const-ify all uses of struct kvm_memory_slot.
10236
	 */
10237
	if (change != KVM_MR_DELETE)
10238
		kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new);
10239 10240 10241

	/* Free the arrays associated with the old memslot. */
	if (change == KVM_MR_MOVE)
10242
		kvm_arch_free_memslot(kvm, old);
10243
}
10244

10245
void kvm_arch_flush_shadow_all(struct kvm *kvm)
10246
{
10247
	kvm_mmu_zap_all(kvm);
10248 10249
}

10250 10251 10252
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
10253
	kvm_page_track_flush_slot(kvm, slot);
10254 10255
}

10256 10257 10258
static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
{
	return (is_guest_mode(vcpu) &&
10259 10260
			kvm_x86_ops.guest_apic_has_interrupt &&
			kvm_x86_ops.guest_apic_has_interrupt(vcpu));
10261 10262
}

10263 10264 10265 10266 10267 10268 10269 10270 10271 10272 10273
static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
{
	if (!list_empty_careful(&vcpu->async_pf.done))
		return true;

	if (kvm_apic_has_events(vcpu))
		return true;

	if (vcpu->arch.pv.pv_unhalted)
		return true;

10274 10275 10276
	if (vcpu->arch.exception.pending)
		return true;

10277 10278
	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
	    (vcpu->arch.nmi_pending &&
10279
	     kvm_x86_ops.nmi_allowed(vcpu, false)))
10280 10281
		return true;

10282
	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
10283
	    (vcpu->arch.smi_pending &&
10284
	     kvm_x86_ops.smi_allowed(vcpu, false)))
10285 10286
		return true;

10287
	if (kvm_arch_interrupt_allowed(vcpu) &&
10288 10289
	    (kvm_cpu_has_interrupt(vcpu) ||
	    kvm_guest_apic_has_interrupt(vcpu)))
10290 10291
		return true;

10292 10293 10294
	if (kvm_hv_has_stimer_pending(vcpu))
		return true;

10295 10296 10297 10298 10299
	if (is_guest_mode(vcpu) &&
	    kvm_x86_ops.nested_ops->hv_timer_pending &&
	    kvm_x86_ops.nested_ops->hv_timer_pending(vcpu))
		return true;

10300 10301 10302
	return false;
}

10303 10304
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
10305
	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
10306
}
10307

10308 10309 10310 10311 10312 10313 10314 10315 10316 10317
bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
{
	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
		return true;

	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
		kvm_test_request(KVM_REQ_SMI, vcpu) ||
		 kvm_test_request(KVM_REQ_EVENT, vcpu))
		return true;

10318
	if (vcpu->arch.apicv_active && kvm_x86_ops.dy_apicv_has_pending_interrupt(vcpu))
10319 10320 10321 10322 10323
		return true;

	return false;
}

10324 10325
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
{
10326
	return vcpu->arch.preempted_in_kernel;
10327 10328
}

10329
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
10330
{
10331
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
10332
}
10333 10334 10335

int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
{
10336
	return kvm_x86_ops.interrupt_allowed(vcpu, false);
10337
}
10338

10339
unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
Jan Kiszka's avatar
Jan Kiszka committed
10340
{
10341 10342 10343 10344 10345 10346
	if (is_64_bit_mode(vcpu))
		return kvm_rip_read(vcpu);
	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
		     kvm_rip_read(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
Jan Kiszka's avatar
Jan Kiszka committed
10347

10348 10349 10350
bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
{
	return kvm_get_linear_rip(vcpu) == linear_rip;
Jan Kiszka's avatar
Jan Kiszka committed
10351 10352 10353
}
EXPORT_SYMBOL_GPL(kvm_is_linear_rip);

10354 10355 10356 10357
unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
{
	unsigned long rflags;

10358
	rflags = kvm_x86_ops.get_rflags(vcpu);
10359
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
10360
		rflags &= ~X86_EFLAGS_TF;
10361 10362 10363 10364
	return rflags;
}
EXPORT_SYMBOL_GPL(kvm_get_rflags);

10365
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
10366 10367
{
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
Jan Kiszka's avatar
Jan Kiszka committed
10368
	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
10369
		rflags |= X86_EFLAGS_TF;
10370
	kvm_x86_ops.set_rflags(vcpu, rflags);
10371 10372 10373 10374 10375
}

void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
	__kvm_set_rflags(vcpu, rflags);
10376
	kvm_make_request(KVM_REQ_EVENT, vcpu);
10377 10378 10379
}
EXPORT_SYMBOL_GPL(kvm_set_rflags);

10380 10381 10382 10383
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
{
	int r;

10384
	if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||
10385
	      work->wakeup_all)
10386 10387 10388 10389 10390 10391
		return;

	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		return;

10392
	if (!vcpu->arch.mmu->direct_map &&
10393
	      work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu))
10394 10395
		return;

10396
	kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true);
10397 10398
}

10399 10400
static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
{
10401 10402
	BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU));

10403 10404 10405 10406 10407
	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
}

static inline u32 kvm_async_pf_next_probe(u32 key)
{
10408
	return (key + 1) & (ASYNC_PF_PER_VCPU - 1);
10409 10410 10411 10412 10413 10414 10415 10416 10417 10418 10419 10420 10421 10422 10423 10424 10425
}

static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 key = kvm_async_pf_hash_fn(gfn);

	while (vcpu->arch.apf.gfns[key] != ~0)
		key = kvm_async_pf_next_probe(key);

	vcpu->arch.apf.gfns[key] = gfn;
}

static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	int i;
	u32 key = kvm_async_pf_hash_fn(gfn);

10426
	for (i = 0; i < ASYNC_PF_PER_VCPU &&
10427 10428
		     (vcpu->arch.apf.gfns[key] != gfn &&
		      vcpu->arch.apf.gfns[key] != ~0); i++)
10429 10430 10431 10432 10433 10434 10435 10436 10437 10438 10439 10440 10441 10442 10443
		key = kvm_async_pf_next_probe(key);

	return key;
}

bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
}

static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 i, j, k;

	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
10444 10445 10446 10447

	if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))
		return;

10448 10449 10450 10451 10452 10453 10454 10455 10456 10457 10458 10459 10460 10461 10462 10463 10464 10465
	while (true) {
		vcpu->arch.apf.gfns[i] = ~0;
		do {
			j = kvm_async_pf_next_probe(j);
			if (vcpu->arch.apf.gfns[j] == ~0)
				return;
			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
			/*
			 * k lies cyclically in ]i,j]
			 * |    i.k.j |
			 * |....j i.k.| or  |.k..j i...|
			 */
		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
		i = j;
	}
}

10466
static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)
10467
{
10468 10469 10470 10471 10472 10473 10474 10475
	u32 reason = KVM_PV_REASON_PAGE_NOT_PRESENT;

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,
				      sizeof(reason));
}

static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)
{
10476
	unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
10477

10478 10479 10480 10481 10482 10483 10484 10485 10486 10487 10488 10489 10490 10491
	return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
					     &token, offset, sizeof(token));
}

static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)
{
	unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
	u32 val;

	if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
					 &val, offset, sizeof(val)))
		return false;

	return !val;
10492 10493
}

10494 10495 10496 10497 10498
static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
{
	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
		return false;

10499 10500
	if (!kvm_pv_async_pf_enabled(vcpu) ||
	    (vcpu->arch.apf.send_user_only && kvm_x86_ops.get_cpl(vcpu) == 0))
10501 10502 10503 10504 10505 10506 10507 10508 10509 10510 10511 10512 10513 10514 10515 10516 10517 10518 10519
		return false;

	return true;
}

bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
{
	if (unlikely(!lapic_in_kernel(vcpu) ||
		     kvm_event_needs_reinjection(vcpu) ||
		     vcpu->arch.exception.pending))
		return false;

	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
		return false;

	/*
	 * If interrupts are off we cannot even use an artificial
	 * halt state.
	 */
10520
	return kvm_arch_interrupt_allowed(vcpu);
10521 10522
}

10523 10524 10525
void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
				     struct kvm_async_pf *work)
{
10526 10527
	struct x86_exception fault;

10528
	trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa);
10529
	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
10530

10531
	if (kvm_can_deliver_async_pf(vcpu) &&
10532
	    !apf_put_user_notpresent(vcpu)) {
10533 10534 10535 10536 10537
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
10538
		fault.async_page_fault = true;
10539
		kvm_inject_page_fault(vcpu, &fault);
10540 10541 10542 10543 10544 10545 10546 10547 10548 10549
	} else {
		/*
		 * It is not possible to deliver a paravirtualized asynchronous
		 * page fault, but putting the guest in an artificial halt state
		 * can be beneficial nevertheless: if an interrupt arrives, we
		 * can deliver it timely and perhaps the guest will schedule
		 * another process.  When the instruction that triggered a page
		 * fault is retried, hopefully the page will be ready in the host.
		 */
		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
10550
	}
10551 10552 10553 10554 10555
}

void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
				 struct kvm_async_pf *work)
{
10556 10557 10558 10559
	struct kvm_lapic_irq irq = {
		.delivery_mode = APIC_DM_FIXED,
		.vector = vcpu->arch.apf.vec
	};
10560

10561
	if (work->wakeup_all)
10562 10563 10564
		work->arch.token = ~0; /* broadcast wakeup */
	else
		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
10565
	trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);
10566

10567
	if (kvm_pv_async_pf_enabled(vcpu) &&
10568 10569
	    !apf_put_user_ready(vcpu, work->arch.token)) {
		vcpu->arch.apf.pageready_pending = true;
10570
		kvm_apic_set_irq(vcpu, &irq, NULL);
10571
	}
10572

10573
	vcpu->arch.apf.halted = false;
10574
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10575 10576
}

10577 10578 10579 10580 10581 10582 10583
void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)
{
	kvm_make_request(KVM_REQ_APF_READY, vcpu);
	if (!vcpu->arch.apf.pageready_pending)
		kvm_vcpu_kick(vcpu);
}

10584
bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
10585
{
10586
	if (!kvm_pv_async_pf_enabled(vcpu))
10587 10588
		return true;
	else
10589
		return apf_pageready_slot_free(vcpu);
10590 10591
}

10592 10593 10594 10595 10596 10597 10598 10599 10600 10601 10602 10603 10604 10605 10606 10607 10608 10609
void kvm_arch_start_assignment(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);

void kvm_arch_end_assignment(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);

bool kvm_arch_has_assigned_device(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);

10610 10611 10612 10613 10614 10615 10616 10617 10618 10619 10620 10621 10622 10623 10624 10625 10626 10627
void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);

void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);

bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);

10628 10629
bool kvm_arch_has_irq_bypass(void)
{
10630
	return true;
10631 10632
}

10633 10634 10635 10636 10637 10638
int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

10639
	irqfd->producer = prod;
10640

10641
	return kvm_x86_ops.update_pi_irte(irqfd->kvm,
10642
					   prod->irq, irqfd->gsi, 1);
10643 10644 10645 10646 10647 10648 10649 10650 10651 10652 10653 10654 10655 10656 10657
}

void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	int ret;
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

	WARN_ON(irqfd->producer != prod);
	irqfd->producer = NULL;

	/*
	 * When producer of consumer is unregistered, we change back to
	 * remapped mode, so we can re-use the current implementation
Andrea Gelmini's avatar
Andrea Gelmini committed
10658
	 * when the irq is masked/disabled or the consumer side (KVM
10659 10660
	 * int this case doesn't want to receive the interrupts.
	*/
10661
	ret = kvm_x86_ops.update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0);
10662 10663 10664 10665 10666 10667 10668 10669
	if (ret)
		printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
		       " fails: %d\n", irqfd->consumer.token, ret);
}

int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
				   uint32_t guest_irq, bool set)
{
10670
	return kvm_x86_ops.update_pi_irte(kvm, host_irq, guest_irq, set);
10671 10672
}

10673 10674 10675 10676 10677
bool kvm_vector_hashing_enabled(void)
{
	return vector_hashing;
}

10678 10679 10680 10681 10682 10683
bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
}
EXPORT_SYMBOL_GPL(kvm_arch_no_poll);

10684 10685 10686 10687 10688 10689 10690 10691 10692 10693 10694 10695 10696 10697 10698 10699 10700 10701 10702 10703 10704 10705
u64 kvm_spec_ctrl_valid_bits(struct kvm_vcpu *vcpu)
{
	uint64_t bits = SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD;

	/* The STIBP bit doesn't fault even if it's not advertised */
	if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS))
		bits &= ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP);
	if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL) &&
	    !boot_cpu_has(X86_FEATURE_AMD_IBRS))
		bits &= ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP);

	if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL_SSBD) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
		bits &= ~SPEC_CTRL_SSBD;
	if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
	    !boot_cpu_has(X86_FEATURE_AMD_SSBD))
		bits &= ~SPEC_CTRL_SSBD;

	return bits;
}
EXPORT_SYMBOL_GPL(kvm_spec_ctrl_valid_bits);
10706

10707
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
10708
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
10709 10710 10711 10712
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
10713
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
10714
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
10715
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
10716
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
10717
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
10718
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
10719
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
10720
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
10721
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
10722
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
10723
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
10724
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
10725 10726
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);
10727
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log);
10728
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_update_request);