x86.c 266 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
Ben-Ami Yassour's avatar
Ben-Ami Yassour committed
8 9
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
10
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11 12 13 14
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
Ben-Ami Yassour's avatar
Ben-Ami Yassour committed
15 16
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
17 18
 */

19
#include <linux/kvm_host.h>
20
#include "irq.h"
21
#include "mmu.h"
Sheng Yang's avatar
Sheng Yang committed
22
#include "i8254.h"
23
#include "tss.h"
24
#include "kvm_cache_regs.h"
25
#include "x86.h"
26
#include "cpuid.h"
27
#include "pmu.h"
28
#include "hyperv.h"
29

30
#include <linux/clocksource.h>
Ben-Ami Yassour's avatar
Ben-Ami Yassour committed
31
#include <linux/interrupt.h>
32 33 34
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
35 36
#include <linux/export.h>
#include <linux/moduleparam.h>
37
#include <linux/mman.h>
38
#include <linux/highmem.h>
39
#include <linux/iommu.h>
40
#include <linux/intel-iommu.h>
41
#include <linux/cpufreq.h>
42
#include <linux/user-return-notifier.h>
43
#include <linux/srcu.h>
44
#include <linux/slab.h>
45
#include <linux/perf_event.h>
46
#include <linux/uaccess.h>
47
#include <linux/hash.h>
48
#include <linux/pci.h>
49 50
#include <linux/timekeeper_internal.h>
#include <linux/pvclock_gtod.h>
51 52
#include <linux/kvm_irqfd.h>
#include <linux/irqbypass.h>
53
#include <linux/sched/stat.h>
54
#include <linux/sched/isolation.h>
55
#include <linux/mem_encrypt.h>
56

Avi Kivity's avatar
Avi Kivity committed
57
#include <trace/events/kvm.h>
Xiao Guangrong's avatar
Xiao Guangrong committed
58

59
#include <asm/debugreg.h>
60
#include <asm/msr.h>
61
#include <asm/desc.h>
Huang Ying's avatar
Huang Ying committed
62
#include <asm/mce.h>
63
#include <linux/kernel_stat.h>
64
#include <asm/fpu/internal.h> /* Ugh! */
65
#include <asm/pvclock.h>
66
#include <asm/div64.h>
67
#include <asm/irq_remapping.h>
68
#include <asm/mshyperv.h>
69
#include <asm/hypervisor.h>
70
#include <asm/intel_pt.h>
71
#include <clocksource/hyperv_timer.h>
72

73 74 75
#define CREATE_TRACE_POINTS
#include "trace.h"

76
#define MAX_IO_MSRS 256
Huang Ying's avatar
Huang Ying committed
77
#define KVM_MAX_MCE_BANKS 32
78 79
u64 __read_mostly kvm_mce_cap_supported = MCG_CTL_P | MCG_SER_P;
EXPORT_SYMBOL_GPL(kvm_mce_cap_supported);
Huang Ying's avatar
Huang Ying committed
80

81 82 83
#define emul_to_vcpu(ctxt) \
	container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)

84 85 86 87 88
/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
89 90
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
91
#else
92
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
93
#endif
94

95 96
#define VM_STAT(x, ...) offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__
#define VCPU_STAT(x, ...) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__
97

98 99
#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
                                    KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
100

101
static void update_cr8_intercept(struct kvm_vcpu *vcpu);
Avi Kivity's avatar
Avi Kivity committed
102
static void process_nmi(struct kvm_vcpu *vcpu);
103
static void enter_smm(struct kvm_vcpu *vcpu);
104
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
Ken Hofsass's avatar
Ken Hofsass committed
105 106
static void store_regs(struct kvm_vcpu *vcpu);
static int sync_regs(struct kvm_vcpu *vcpu);
107

108
struct kvm_x86_ops *kvm_x86_ops __read_mostly;
109
EXPORT_SYMBOL_GPL(kvm_x86_ops);
110

111
static bool __read_mostly ignore_msrs = 0;
112
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
113

114 115 116
static bool __read_mostly report_ignored_msrs = true;
module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);

117
unsigned int min_timer_period_us = 200;
118 119
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

120 121 122
static bool __read_mostly kvmclock_periodic_sync = true;
module_param(kvmclock_periodic_sync, bool, S_IRUGO);

123
bool __read_mostly kvm_has_tsc_control;
124
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
125
u32  __read_mostly kvm_max_guest_tsc_khz;
126
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
127 128 129 130
u8   __read_mostly kvm_tsc_scaling_ratio_frac_bits;
EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits);
u64  __read_mostly kvm_max_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
131 132
u64 __read_mostly kvm_default_tsc_scaling_ratio;
EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
133

134
/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
135
static u32 __read_mostly tsc_tolerance_ppm = 250;
136 137
module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);

138 139 140 141 142 143 144
/*
 * lapic timer advance (tscdeadline mode only) in nanoseconds.  '-1' enables
 * adaptive tuning starting from default advancment of 1000ns.  '0' disables
 * advancement entirely.  Any other value is used as-is and disables adaptive
 * tuning, i.e. allows priveleged userspace to set an exact advancement time.
 */
static int __read_mostly lapic_timer_advance_ns = -1;
145
module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
146

147 148 149
static bool __read_mostly vector_hashing = true;
module_param(vector_hashing, bool, S_IRUGO);

150 151 152 153
bool __read_mostly enable_vmware_backdoor = false;
module_param(enable_vmware_backdoor, bool, S_IRUGO);
EXPORT_SYMBOL_GPL(enable_vmware_backdoor);

154 155 156
static bool __read_mostly force_emulation_prefix = false;
module_param(force_emulation_prefix, bool, S_IRUGO);

157 158 159
int __read_mostly pi_inject_timer = -1;
module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR);

160 161 162 163
#define KVM_NR_SHARED_MSRS 16

struct kvm_shared_msrs_global {
	int nr;
164
	u32 msrs[KVM_NR_SHARED_MSRS];
165 166 167 168 169
};

struct kvm_shared_msrs {
	struct user_return_notifier urn;
	bool registered;
170 171 172 173
	struct kvm_shared_msr_values {
		u64 host;
		u64 curr;
	} values[KVM_NR_SHARED_MSRS];
174 175 176
};

static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
177
static struct kvm_shared_msrs __percpu *shared_msrs;
178

179
struct kvm_stats_debugfs_item debugfs_entries[] = {
180 181 182 183 184 185 186 187 188
	{ "pf_fixed", VCPU_STAT(pf_fixed) },
	{ "pf_guest", VCPU_STAT(pf_guest) },
	{ "tlb_flush", VCPU_STAT(tlb_flush) },
	{ "invlpg", VCPU_STAT(invlpg) },
	{ "exits", VCPU_STAT(exits) },
	{ "io_exits", VCPU_STAT(io_exits) },
	{ "mmio_exits", VCPU_STAT(mmio_exits) },
	{ "signal_exits", VCPU_STAT(signal_exits) },
	{ "irq_window", VCPU_STAT(irq_window_exits) },
189
	{ "nmi_window", VCPU_STAT(nmi_window_exits) },
190
	{ "halt_exits", VCPU_STAT(halt_exits) },
191
	{ "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
192
	{ "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
193
	{ "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
194
	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
195
	{ "hypercalls", VCPU_STAT(hypercalls) },
196 197 198 199 200 201
	{ "request_irq", VCPU_STAT(request_irq_exits) },
	{ "irq_exits", VCPU_STAT(irq_exits) },
	{ "host_state_reload", VCPU_STAT(host_state_reload) },
	{ "fpu_reload", VCPU_STAT(fpu_reload) },
	{ "insn_emulation", VCPU_STAT(insn_emulation) },
	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
202
	{ "irq_injections", VCPU_STAT(irq_injections) },
203
	{ "nmi_injections", VCPU_STAT(nmi_injections) },
204
	{ "req_event", VCPU_STAT(req_event) },
205
	{ "l1d_flush", VCPU_STAT(l1d_flush) },
206 207 208 209 210 211
	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
	{ "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
	{ "mmu_flooded", VM_STAT(mmu_flooded) },
	{ "mmu_recycled", VM_STAT(mmu_recycled) },
212
	{ "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
213
	{ "mmu_unsync", VM_STAT(mmu_unsync) },
214
	{ "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
215
	{ "largepages", VM_STAT(lpages, .mode = 0444) },
216
	{ "nx_largepages_splitted", VM_STAT(nx_lpage_splits, .mode = 0444) },
217 218
	{ "max_mmu_page_hash_collisions",
		VM_STAT(max_mmu_page_hash_collisions) },
219 220 221
	{ NULL }
};

222 223
u64 __read_mostly host_xcr0;

224 225 226
struct kmem_cache *x86_fpu_cache;
EXPORT_SYMBOL_GPL(x86_fpu_cache);

227
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
228

229 230 231 232 233 234 235
static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
	int i;
	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
		vcpu->arch.apf.gfns[i] = ~0;
}

236 237 238 239 240
static void kvm_on_user_return(struct user_return_notifier *urn)
{
	unsigned slot;
	struct kvm_shared_msrs *locals
		= container_of(urn, struct kvm_shared_msrs, urn);
241
	struct kvm_shared_msr_values *values;
242 243 244 245 246 247 248 249 250 251 252 253
	unsigned long flags;

	/*
	 * Disabling irqs at this point since the following code could be
	 * interrupted and executed through kvm_arch_hardware_disable()
	 */
	local_irq_save(flags);
	if (locals->registered) {
		locals->registered = false;
		user_return_notifier_unregister(urn);
	}
	local_irq_restore(flags);
254
	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
255 256 257 258
		values = &locals->values[slot];
		if (values->host != values->curr) {
			wrmsrl(shared_msrs_global.msrs[slot], values->host);
			values->curr = values->host;
259 260 261 262
		}
	}
}

263
static void shared_msr_update(unsigned slot, u32 msr)
264 265
{
	u64 value;
266 267
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
268

269 270 271 272 273 274 275 276 277 278 279 280 281
	/* only read, and nobody should modify it at this time,
	 * so don't need lock */
	if (slot >= shared_msrs_global.nr) {
		printk(KERN_ERR "kvm: invalid MSR slot!");
		return;
	}
	rdmsrl_safe(msr, &value);
	smsr->values[slot].host = value;
	smsr->values[slot].curr = value;
}

void kvm_define_shared_msr(unsigned slot, u32 msr)
{
282
	BUG_ON(slot >= KVM_NR_SHARED_MSRS);
283
	shared_msrs_global.msrs[slot] = msr;
284 285 286 287 288 289 290 291 292 293
	if (slot >= shared_msrs_global.nr)
		shared_msrs_global.nr = slot + 1;
}
EXPORT_SYMBOL_GPL(kvm_define_shared_msr);

static void kvm_shared_msr_cpu_online(void)
{
	unsigned i;

	for (i = 0; i < shared_msrs_global.nr; ++i)
294
		shared_msr_update(i, shared_msrs_global.msrs[i]);
295 296
}

297
int kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
298
{
299 300
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
301
	int err;
302

303
	if (((value ^ smsr->values[slot].curr) & mask) == 0)
304
		return 0;
305
	smsr->values[slot].curr = value;
306 307 308 309
	err = wrmsrl_safe(shared_msrs_global.msrs[slot], value);
	if (err)
		return 1;

310 311 312 313 314
	if (!smsr->registered) {
		smsr->urn.on_user_return = kvm_on_user_return;
		user_return_notifier_register(&smsr->urn);
		smsr->registered = true;
	}
315
	return 0;
316 317 318
}
EXPORT_SYMBOL_GPL(kvm_set_shared_msr);

319
static void drop_user_return_notifiers(void)
320
{
321 322
	unsigned int cpu = smp_processor_id();
	struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
323 324 325 326 327

	if (smsr->registered)
		kvm_on_user_return(&smsr->urn);
}

328 329
u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
330
	return vcpu->arch.apic_base;
331 332 333
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

334 335 336 337 338 339
enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
{
	return kvm_apic_mode(kvm_get_apic_base(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_apic_mode);

340 341
int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
342 343
	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
344 345
	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 0x2ff |
		(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
346

347
	if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
348
		return 1;
349 350 351 352 353 354
	if (!msr_info->host_initiated) {
		if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
			return 1;
		if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
			return 1;
	}
355 356 357

	kvm_lapic_set_base(vcpu, msr_info->data);
	return 0;
358 359 360
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

361
asmlinkage __visible void kvm_spurious_fault(void)
362 363
{
	/* Fault while not rebooting.  We want the trace. */
364
	BUG_ON(!kvm_rebooting);
365 366 367
}
EXPORT_SYMBOL_GPL(kvm_spurious_fault);

368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
#define EXCPT_BENIGN		0
#define EXCPT_CONTRIBUTORY	1
#define EXCPT_PF		2

static int exception_class(int vector)
{
	switch (vector) {
	case PF_VECTOR:
		return EXCPT_PF;
	case DE_VECTOR:
	case TS_VECTOR:
	case NP_VECTOR:
	case SS_VECTOR:
	case GP_VECTOR:
		return EXCPT_CONTRIBUTORY;
	default:
		break;
	}
	return EXCPT_BENIGN;
}

389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
#define EXCPT_FAULT		0
#define EXCPT_TRAP		1
#define EXCPT_ABORT		2
#define EXCPT_INTERRUPT		3

static int exception_type(int vector)
{
	unsigned int mask;

	if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
		return EXCPT_INTERRUPT;

	mask = 1 << vector;

	/* #DB is trap, as instruction watchpoints are handled elsewhere */
	if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR)))
		return EXCPT_TRAP;

	if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
		return EXCPT_ABORT;

	/* Reserved exceptions will result in fault */
	return EXCPT_FAULT;
}

414 415 416 417 418 419 420 421 422 423
void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu)
{
	unsigned nr = vcpu->arch.exception.nr;
	bool has_payload = vcpu->arch.exception.has_payload;
	unsigned long payload = vcpu->arch.exception.payload;

	if (!has_payload)
		return;

	switch (nr) {
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
	case DB_VECTOR:
		/*
		 * "Certain debug exceptions may clear bit 0-3.  The
		 * remaining contents of the DR6 register are never
		 * cleared by the processor".
		 */
		vcpu->arch.dr6 &= ~DR_TRAP_BITS;
		/*
		 * DR6.RTM is set by all #DB exceptions that don't clear it.
		 */
		vcpu->arch.dr6 |= DR6_RTM;
		vcpu->arch.dr6 |= payload;
		/*
		 * Bit 16 should be set in the payload whenever the #DB
		 * exception should clear DR6.RTM. This makes the payload
		 * compatible with the pending debug exceptions under VMX.
		 * Though not currently documented in the SDM, this also
		 * makes the payload compatible with the exit qualification
		 * for #DB exceptions under VMX.
		 */
		vcpu->arch.dr6 ^= payload & DR6_RTM;
		break;
446 447 448 449 450 451 452 453 454 455
	case PF_VECTOR:
		vcpu->arch.cr2 = payload;
		break;
	}

	vcpu->arch.exception.has_payload = false;
	vcpu->arch.exception.payload = 0;
}
EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);

456
static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
457
		unsigned nr, bool has_error, u32 error_code,
458
	        bool has_payload, unsigned long payload, bool reinject)
459 460 461 462
{
	u32 prev_nr;
	int class1, class2;

463 464
	kvm_make_request(KVM_REQ_EVENT, vcpu);

465
	if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
466
	queue:
467 468
		if (has_error && !is_protmode(vcpu))
			has_error = false;
469 470 471 472 473 474 475 476 477 478 479
		if (reinject) {
			/*
			 * On vmentry, vcpu->arch.exception.pending is only
			 * true if an event injection was blocked by
			 * nested_run_pending.  In that case, however,
			 * vcpu_enter_guest requests an immediate exit,
			 * and the guest shouldn't proceed far enough to
			 * need reinjection.
			 */
			WARN_ON_ONCE(vcpu->arch.exception.pending);
			vcpu->arch.exception.injected = true;
480 481 482 483 484 485 486 487
			if (WARN_ON_ONCE(has_payload)) {
				/*
				 * A reinjected event has already
				 * delivered its payload.
				 */
				has_payload = false;
				payload = 0;
			}
488 489 490 491
		} else {
			vcpu->arch.exception.pending = true;
			vcpu->arch.exception.injected = false;
		}
492 493 494
		vcpu->arch.exception.has_error_code = has_error;
		vcpu->arch.exception.nr = nr;
		vcpu->arch.exception.error_code = error_code;
495 496
		vcpu->arch.exception.has_payload = has_payload;
		vcpu->arch.exception.payload = payload;
497 498 499
		/*
		 * In guest mode, payload delivery should be deferred,
		 * so that the L1 hypervisor can intercept #PF before
500 501 502 503 504 505 506
		 * CR2 is modified (or intercept #DB before DR6 is
		 * modified under nVMX).  However, for ABI
		 * compatibility with KVM_GET_VCPU_EVENTS and
		 * KVM_SET_VCPU_EVENTS, we can't delay payload
		 * delivery unless userspace has enabled this
		 * functionality via the per-VM capability,
		 * KVM_CAP_EXCEPTION_PAYLOAD.
507 508 509 510
		 */
		if (!vcpu->kvm->arch.exception_payload_enabled ||
		    !is_guest_mode(vcpu))
			kvm_deliver_exception_payload(vcpu);
511 512 513 514 515 516 517
		return;
	}

	/* to check exception */
	prev_nr = vcpu->arch.exception.nr;
	if (prev_nr == DF_VECTOR) {
		/* triple fault -> shutdown */
518
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
519 520 521 522 523 524
		return;
	}
	class1 = exception_class(prev_nr);
	class2 = exception_class(nr);
	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
525 526 527 528 529
		/*
		 * Generate double fault per SDM Table 5-5.  Set
		 * exception.pending = true so that the double fault
		 * can trigger a nested vmexit.
		 */
530
		vcpu->arch.exception.pending = true;
531
		vcpu->arch.exception.injected = false;
532 533 534
		vcpu->arch.exception.has_error_code = true;
		vcpu->arch.exception.nr = DF_VECTOR;
		vcpu->arch.exception.error_code = 0;
535 536
		vcpu->arch.exception.has_payload = false;
		vcpu->arch.exception.payload = 0;
537 538 539 540 541 542 543
	} else
		/* replace previous exception with a new one in a hope
		   that instruction re-execution will regenerate lost
		   exception */
		goto queue;
}

544 545
void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
546
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
547 548 549
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

550 551
void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
552
	kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
553 554 555
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

556 557 558 559 560 561
static void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
				  unsigned long payload)
{
	kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
}

562 563 564 565 566 567 568
static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
				    u32 error_code, unsigned long payload)
{
	kvm_multiple_exception(vcpu, nr, true, error_code,
			       true, payload, false);
}

569
int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
570
{
571 572 573
	if (err)
		kvm_inject_gp(vcpu, 0);
	else
574 575 576
		return kvm_skip_emulated_instruction(vcpu);

	return 1;
577 578
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
579

580
void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
581 582
{
	++vcpu->stat.pf_guest;
583 584
	vcpu->arch.exception.nested_apf =
		is_guest_mode(vcpu) && fault->async_page_fault;
585
	if (vcpu->arch.exception.nested_apf) {
586
		vcpu->arch.apf.nested_apf_token = fault->address;
587 588 589 590 591
		kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
	} else {
		kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
					fault->address);
	}
592
}
Nadav Har'El's avatar
Nadav Har'El committed
593
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
594

595
static bool kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
596
{
597 598
	if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
		vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
599
	else
600
		vcpu->arch.mmu->inject_page_fault(vcpu, fault);
601 602

	return fault->nested_page_fault;
603 604
}

605 606
void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
Avi Kivity's avatar
Avi Kivity committed
607 608
	atomic_inc(&vcpu->arch.nmi_queued);
	kvm_make_request(KVM_REQ_NMI, vcpu);
609 610 611
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

612 613
void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
614
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
615 616 617
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

618 619
void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
620
	kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
621 622 623
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

624 625 626 627 628
/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
629
{
630 631 632 633
	if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
		return true;
	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
	return false;
634
}
635
EXPORT_SYMBOL_GPL(kvm_require_cpl);
636

637 638 639 640 641 642 643 644 645 646
bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
{
	if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
		return true;

	kvm_queue_exception(vcpu, UD_VECTOR);
	return false;
}
EXPORT_SYMBOL_GPL(kvm_require_dr);

647 648
/*
 * This function will be used to read from the physical memory of the currently
649
 * running guest. The difference to kvm_vcpu_read_guest_page is that this function
650 651 652 653 654 655
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gfn_t ngfn, void *data, int offset, int len,
			    u32 access)
{
656
	struct x86_exception exception;
657 658 659 660
	gfn_t real_gfn;
	gpa_t ngpa;

	ngpa     = gfn_to_gpa(ngfn);
661
	real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception);
662 663 664 665 666
	if (real_gfn == UNMAPPED_GVA)
		return -EFAULT;

	real_gfn = gpa_to_gfn(real_gfn);

667
	return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len);
668 669 670
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

671
static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
672 673 674 675 676 677
			       void *data, int offset, int len, u32 access)
{
	return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
				       data, offset, len, access);
}

678 679 680 681 682 683
static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
{
	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) | rsvd_bits(5, 8) |
	       rsvd_bits(1, 2);
}

684
/*
685
 * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
686
 */
687
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
688 689 690 691 692
{
	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
	int i;
	int ret;
693
	u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
694

695 696 697
	ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
				      offset * sizeof(u64), sizeof(pdpte),
				      PFERR_USER_MASK|PFERR_WRITE_MASK);
698 699 700 701 702
	if (ret < 0) {
		ret = 0;
		goto out;
	}
	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
703
		if ((pdpte[i] & PT_PRESENT_MASK) &&
704
		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
705 706 707 708 709 710
			ret = 0;
			goto out;
		}
	}
	ret = 1;

711
	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
Avi Kivity's avatar
Avi Kivity committed
712 713 714 715
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_avail);
	__set_bit(VCPU_EXREG_PDPTR,
		  (unsigned long *)&vcpu->arch.regs_dirty);
716 717 718 719
out:

	return ret;
}
720
EXPORT_SYMBOL_GPL(load_pdptrs);
721

722
bool pdptrs_changed(struct kvm_vcpu *vcpu)
723
{
724
	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
725
	bool changed = true;
726 727
	int offset;
	gfn_t gfn;
728 729
	int r;

730
	if (!is_pae_paging(vcpu))
731 732
		return false;

Avi Kivity's avatar
Avi Kivity committed
733 734 735 736
	if (!test_bit(VCPU_EXREG_PDPTR,
		      (unsigned long *)&vcpu->arch.regs_avail))
		return true;

737 738
	gfn = (kvm_read_cr3(vcpu) & 0xffffffe0ul) >> PAGE_SHIFT;
	offset = (kvm_read_cr3(vcpu) & 0xffffffe0ul) & (PAGE_SIZE - 1);
739 740
	r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
				       PFERR_USER_MASK | PFERR_WRITE_MASK);
741 742
	if (r < 0)
		goto out;
743
	changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
744 745 746 747
out:

	return changed;
}
748
EXPORT_SYMBOL_GPL(pdptrs_changed);
749

750
int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
751
{
752
	unsigned long old_cr0 = kvm_read_cr0(vcpu);
753
	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP;
754

755 756
	cr0 |= X86_CR0_ET;

757
#ifdef CONFIG_X86_64
758 759
	if (cr0 & 0xffffffff00000000UL)
		return 1;
760 761 762
#endif

	cr0 &= ~CR0_RESERVED_BITS;
763

764 765
	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
		return 1;
766

767 768
	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
		return 1;
769 770 771

	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
#ifdef CONFIG_X86_64
772
		if ((vcpu->arch.efer & EFER_LME)) {
773 774
			int cs_db, cs_l;

775 776
			if (!is_pae(vcpu))
				return 1;
777
			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
778 779
			if (cs_l)
				return 1;
780 781
		} else
#endif
782
		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
783
						 kvm_read_cr3(vcpu)))
784
			return 1;
785 786
	}

787 788 789
	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
		return 1;

790 791
	kvm_x86_ops->set_cr0(vcpu, cr0);

792
	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
793
		kvm_clear_async_pf_completion_queue(vcpu);
794 795
		kvm_async_pf_hash_reset(vcpu);
	}
796

797 798
	if ((cr0 ^ old_cr0) & update_bits)
		kvm_mmu_reset_context(vcpu);
799

800 801 802
	if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
803 804
		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);

805 806
	return 0;
}
807
EXPORT_SYMBOL_GPL(kvm_set_cr0);
808

809
void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
810
{
811
	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
812
}
813
EXPORT_SYMBOL_GPL(kvm_lmsw);
814

815
void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
816 817 818 819
{
	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
			!vcpu->guest_xcr0_loaded) {
		/* kvm_set_xcr() also depends on this */
820 821
		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
822 823 824
		vcpu->guest_xcr0_loaded = 1;
	}
}
825
EXPORT_SYMBOL_GPL(kvm_load_guest_xcr0);
826

827
void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
828 829 830 831 832 833 834
{
	if (vcpu->guest_xcr0_loaded) {
		if (vcpu->arch.xcr0 != host_xcr0)
			xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
		vcpu->guest_xcr0_loaded = 0;
	}
}
835
EXPORT_SYMBOL_GPL(kvm_put_guest_xcr0);
836

837
static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
838
{
839 840
	u64 xcr0 = xcr;
	u64 old_xcr0 = vcpu->arch.xcr0;
841
	u64 valid_bits;
842 843 844 845

	/* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
	if (index != XCR_XFEATURE_ENABLED_MASK)
		return 1;
Dave Hansen's avatar
Dave Hansen committed
846
	if (!(xcr0 & XFEATURE_MASK_FP))
847
		return 1;
Dave Hansen's avatar
Dave Hansen committed
848
	if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
849
		return 1;
850 851 852 853 854 855

	/*
	 * Do not allow the guest to set bits that we do not support
	 * saving.  However, xcr0 bit 0 is always set, even if the
	 * emulated CPU does not support XSAVE (see fx_init).
	 */
Dave Hansen's avatar
Dave Hansen committed
856
	valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
857
	if (xcr0 & ~valid_bits)
858
		return 1;
859

Dave Hansen's avatar
Dave Hansen committed
860 861
	if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
	    (!(xcr0 & XFEATURE_MASK_BNDCSR)))
862 863
		return 1;

Dave Hansen's avatar
Dave Hansen committed
864 865
	if (xcr0 & XFEATURE_MASK_AVX512) {
		if (!(xcr0 & XFEATURE_MASK_YMM))
866
			return 1;
Dave Hansen's avatar
Dave Hansen committed
867
		if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
868 869
			return 1;
	}
870
	vcpu->arch.xcr0 = xcr0;
871

Dave Hansen's avatar
Dave Hansen committed
872
	if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
873
		kvm_update_cpuid(vcpu);
874 875 876 877 878
	return 0;
}

int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
879 880
	if (kvm_x86_ops->get_cpl(vcpu) != 0 ||
	    __kvm_set_xcr(vcpu, index, xcr)) {
881 882 883 884 885 886 887
		kvm_inject_gp(vcpu, 0);
		return 1;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_xcr);

888
static int kvm_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
889
{
890
	if (cr4 & CR4_RESERVED_BITS)
891
		return -EINVAL;
892

893
	if (!guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) && (cr4 & X86_CR4_OSXSAVE))
894
		return -EINVAL;
895

896
	if (!guest_cpuid_has(vcpu, X86_FEATURE_SMEP) && (cr4 & X86_CR4_SMEP))
897
		return -EINVAL;
898

899
	if (!guest_cpuid_has(vcpu, X86_FEATURE_SMAP) && (cr4 & X86_CR4_SMAP))
900
		return -EINVAL;
901

902
	if (!guest_cpuid_has(vcpu, X86_FEATURE_FSGSBASE) && (cr4 & X86_CR4_FSGSBASE))
903
		return -EINVAL;
904

905
	if (!guest_cpuid_has(vcpu, X86_FEATURE_PKU) && (cr4 & X86_CR4_PKE))
906
		return -EINVAL;
907

908
	if (!guest_cpuid_has(vcpu, X86_FEATURE_LA57) && (cr4 & X86_CR4_LA57))
909
		return -EINVAL;
910

911
	if (!guest_cpuid_has(vcpu, X86_FEATURE_UMIP) && (cr4 & X86_CR4_UMIP))
912 913 914 915 916 917 918 919 920 921 922 923
		return -EINVAL;

	return 0;
}

int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
	unsigned long old_cr4 = kvm_read_cr4(vcpu);
	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE |
				   X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE;

	if (kvm_valid_cr4(vcpu, cr4))
924 925
		return 1;

926
	if (is_long_mode(vcpu)) {
927 928
		if (!(cr4 & X86_CR4_PAE))
			return 1;
929 930
	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
		   && ((cr4 ^ old_cr4) & pdptr_bits)
931 932
		   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
				   kvm_read_cr3(vcpu)))
933 934
		return 1;

935
	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
936
		if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
937 938 939 940 941 942 943
			return 1;

		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
			return 1;
	}

944
	if (kvm_x86_ops->set_cr4(vcpu, cr4))
945
		return 1;
946

947 948
	if (((cr4 ^ old_cr4) & pdptr_bits) ||
	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
949
		kvm_mmu_reset_context(vcpu);
950

951
	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
952
		kvm_update_cpuid(vcpu);
953

954 955
	return 0;
}
956
EXPORT_SYMBOL_GPL(kvm_set_cr4);
957

958
int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
959
{
960
	bool skip_tlb_flush = false;
961
#ifdef CONFIG_X86_64
962 963
	bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);

964
	if (pcid_enabled) {
965 966
		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
		cr3 &= ~X86_CR3_PCID_NOFLUSH;
967
	}
968
#endif
969

970
	if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
971 972
		if (!skip_tlb_flush) {
			kvm_mmu_sync_roots(vcpu);
973
			kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
974
		}
975
		return 0;
976 977
	}

978
	if (is_long_mode(vcpu) &&
979
	    (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63)))
980
		return 1;
981 982
	else if (is_pae_paging(vcpu) &&
		 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
Nadav Amit's avatar
Nadav Amit committed
983
		return 1;
984

985
	kvm_mmu_new_cr3(vcpu, cr3, skip_tlb_flush);
986
	vcpu->arch.cr3 = cr3;
987
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
988

989 990
	return 0;
}
991
EXPORT_SYMBOL_GPL(kvm_set_cr3);
992

Andre Przywara's avatar
Andre Przywara committed
993
int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
994
{
995 996
	if (cr8 & CR8_RESERVED_BITS)
		return 1;
997
	if (lapic_in_kernel(vcpu))
998 999
		kvm_lapic_set_tpr(vcpu, cr8);
	else
1000
		vcpu->arch.cr8 = cr8;
1001 1002
	return 0;
}
1003
EXPORT_SYMBOL_GPL(kvm_set_cr8);
1004

1005
unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1006
{
1007
	if (lapic_in_kernel(vcpu))
1008 1009
		return kvm_lapic_get_cr8(vcpu);
	else
1010
		return vcpu->arch.cr8;
1011
}
1012
EXPORT_SYMBOL_GPL(kvm_get_cr8);
1013

1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
{
	int i;

	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD;
	}
}

Jan Kiszka's avatar
Jan Kiszka committed
1025 1026 1027 1028 1029 1030
static void kvm_update_dr6(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
		kvm_x86_ops->set_dr6(vcpu, vcpu->arch.dr6);
}

1031 1032 1033 1034 1035 1036 1037 1038 1039
static void kvm_update_dr7(struct kvm_vcpu *vcpu)
{
	unsigned long dr7;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
		dr7 = vcpu->arch.guest_debug_dr7;
	else
		dr7 = vcpu->arch.dr7;
	kvm_x86_ops->set_dr7(vcpu, dr7);
1040 1041 1042
	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
	if (dr7 & DR7_BP_EN_MASK)
		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1043 1044
}

1045 1046 1047 1048
static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
{
	u64 fixed = DR6_FIXED_1;

1049
	if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1050 1051 1052 1053
		fixed |= DR6_RTM;
	return fixed;
}

1054
static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
{
	switch (dr) {
	case 0 ... 3:
		vcpu->arch.db[dr] = val;
		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
			vcpu->arch.eff_db[dr] = val;
		break;
	case 4:
		/* fall through */
	case 6:
1065 1066
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
1067
		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
Jan Kiszka's avatar
Jan Kiszka committed
1068
		kvm_update_dr6(vcpu);
1069 1070 1071 1072
		break;
	case 5:
		/* fall through */
	default: /* 7 */
1073 1074
		if (val & 0xffffffff00000000ULL)
			return -1; /* #GP */
1075
		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1076
		kvm_update_dr7(vcpu);
1077 1078 1079 1080 1081
		break;
	}

	return 0;
}
1082 1083 1084

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
1085
	if (__kvm_set_dr(vcpu, dr, val)) {
1086
		kvm_inject_gp(vcpu, 0);
1087 1088 1089
		return 1;
	}
	return 0;
1090
}
1091 1092
EXPORT_SYMBOL_GPL(kvm_set_dr);

1093
int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
1094 1095 1096 1097 1098 1099 1100 1101
{
	switch (dr) {
	case 0 ... 3:
		*val = vcpu->arch.db[dr];
		break;
	case 4:
		/* fall through */
	case 6:
Jan Kiszka's avatar
Jan Kiszka committed
1102 1103 1104 1105
		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
			*val = vcpu->arch.dr6;
		else
			*val = kvm_x86_ops->get_dr6(vcpu);
1106 1107 1108 1109 1110 1111 1112
		break;
	case 5:
		/* fall through */
	default: /* 7 */
		*val = vcpu->arch.dr7;
		break;
	}
1113 1114
	return 0;
}
1115 1116
EXPORT_SYMBOL_GPL(kvm_get_dr);

Avi Kivity's avatar
Avi Kivity committed
1117 1118
bool kvm_rdpmc(struct kvm_vcpu *vcpu)
{
1119
	u32 ecx = kvm_rcx_read(vcpu);
Avi Kivity's avatar
Avi Kivity committed
1120 1121 1122
	u64 data;
	int err;

1123
	err = kvm_pmu_rdpmc(vcpu, ecx, &data);
Avi Kivity's avatar
Avi Kivity committed
1124 1125
	if (err)
		return err;
1126 1127
	kvm_rax_write(vcpu, (u32)data);
	kvm_rdx_write(vcpu, data >> 32);
Avi Kivity's avatar
Avi Kivity committed
1128 1129 1130 1131
	return err;
}
EXPORT_SYMBOL_GPL(kvm_rdpmc);

1132 1133 1134 1135
/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
1136 1137 1138
 * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features)
 * extract the supported MSRs from the related const lists.
 * msrs_to_save is selected from the msrs_to_save_all to reflect the
1139
 * capabilities of the host cpu. This capabilities test skips MSRs that are
1140
 * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs
1141
 * may depend on host virtualization features rather than host cpu features.
1142
 */
1143

1144
static const u32 msrs_to_save_all[] = {
1145
	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
Brian Gerst's avatar
Brian Gerst committed
1146
	MSR_STAR,
1147 1148 1149
#ifdef CONFIG_X86_64
	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
1150
	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
1151
	MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
1152
	MSR_IA32_SPEC_CTRL,
1153 1154 1155 1156 1157 1158
	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
1159 1160
	MSR_IA32_UMWAIT_CONTROL,

1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
	MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
	MSR_ARCH_PERFMON_FIXED_CTR0 + 2, MSR_ARCH_PERFMON_FIXED_CTR0 + 3,
	MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
	MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
	MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
	MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
	MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
	MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
	MSR_ARCH_PERFMON_PERFCTR0 + 8, MSR_ARCH_PERFMON_PERFCTR0 + 9,
	MSR_ARCH_PERFMON_PERFCTR0 + 10, MSR_ARCH_PERFMON_PERFCTR0 + 11,
	MSR_ARCH_PERFMON_PERFCTR0 + 12, MSR_ARCH_PERFMON_PERFCTR0 + 13,
	MSR_ARCH_PERFMON_PERFCTR0 + 14, MSR_ARCH_PERFMON_PERFCTR0 + 15,
	MSR_ARCH_PERFMON_PERFCTR0 + 16, MSR_ARCH_PERFMON_PERFCTR0 + 17,
	MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
	MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
	MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
	MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
	MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
	MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
	MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
	MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
	MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
1183 1184
};

1185
static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_all)];
1186 1187
static unsigned num_msrs_to_save;

1188
static const u32 emulated_msrs_all[] = {
1189 1190 1191 1192
	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
1193
	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
1194 1195
	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
1196
	HV_X64_MSR_RESET,
1197
	HV_X64_MSR_VP_INDEX,
1198
	HV_X64_MSR_VP_RUNTIME,
1199
	HV_X64_MSR_SCONTROL,
1200
	HV_X64_MSR_STIMER0_CONFIG,
1201
	HV_X64_MSR_VP_ASSIST_PAGE,
1202 1203 1204 1205
	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
	HV_X64_MSR_TSC_EMULATION_STATUS,

	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
1206 1207
	MSR_KVM_PV_EOI_EN,

1208
	MSR_IA32_TSC_ADJUST,
1209
	MSR_IA32_TSCDEADLINE,
1210
	MSR_IA32_ARCH_CAPABILITIES,
1211
	MSR_IA32_MISC_ENABLE,
1212 1213
	MSR_IA32_MCG_STATUS,
	MSR_IA32_MCG_CTL,
1214
	MSR_IA32_MCG_EXT_CTL,
1215
	MSR_IA32_SMBASE,
1216
	MSR_SMI_COUNT,
1217 1218
	MSR_PLATFORM_INFO,
	MSR_MISC_FEATURES_ENABLES,
1219
	MSR_AMD64_VIRT_SPEC_CTRL,
1220
	MSR_IA32_POWER_CTL,
1221

1222 1223 1224 1225 1226
	/*
	 * The following list leaves out MSRs whose values are determined
	 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
	 * We always support the "true" VMX control MSRs, even if the host
	 * processor does not, so I am putting these registers here rather
1227
	 * than in msrs_to_save_all.
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241
	 */
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1242
	MSR_K7_HWCR,
1243
	MSR_KVM_POLL_CONTROL,
1244 1245
};

1246
static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
1247 1248
static unsigned num_emulated_msrs;

1249 1250 1251 1252
/*
 * List of msr numbers which are used to expose MSR-based features that
 * can be used by a hypervisor to validate requested CPU features.
 */
1253
static const u32 msr_based_features_all[] = {
1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
	MSR_IA32_VMX_BASIC,
	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
	MSR_IA32_VMX_PINBASED_CTLS,
	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
	MSR_IA32_VMX_PROCBASED_CTLS,
	MSR_IA32_VMX_TRUE_EXIT_CTLS,
	MSR_IA32_VMX_EXIT_CTLS,
	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
	MSR_IA32_VMX_ENTRY_CTLS,
	MSR_IA32_VMX_MISC,
	MSR_IA32_VMX_CR0_FIXED0,
	MSR_IA32_VMX_CR0_FIXED1,
	MSR_IA32_VMX_CR4_FIXED0,
	MSR_IA32_VMX_CR4_FIXED1,
	MSR_IA32_VMX_VMCS_ENUM,
	MSR_IA32_VMX_PROCBASED_CTLS2,
	MSR_IA32_VMX_EPT_VPID_CAP,
	MSR_IA32_VMX_VMFUNC,

1273
	MSR_F10H_DECFG,
1274
	MSR_IA32_UCODE_REV,
1275
	MSR_IA32_ARCH_CAPABILITIES,
1276 1277
};

1278
static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)];
1279 1280
static unsigned int num_msr_based_features;

1281
static u64 kvm_get_arch_capabilities(void)
1282
{
1283
	u64 data = 0;
1284

1285 1286
	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);
1287

1288 1289 1290 1291 1292 1293 1294 1295
	/*
	 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
	 * the nested hypervisor runs with NX huge pages.  If it is not,
	 * L1 is anyway vulnerable to ITLB_MULTIHIT explots from other
	 * L1 guests, so it need not worry about its own (L2) guests.
	 */
	data |= ARCH_CAP_PSCHANGE_MC_NO;

1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
	/*
	 * If we're doing cache flushes (either "always" or "cond")
	 * we will do one whenever the guest does a vmlaunch/vmresume.
	 * If an outer hypervisor is doing the cache flush for us
	 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
	 * capability to the guest too, and if EPT is disabled we're not
	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
	 * require a nested hypervisor to do a flush of its own.
	 */
	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;

1308 1309 1310 1311 1312 1313 1314
	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
		data |= ARCH_CAP_RDCL_NO;
	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
		data |= ARCH_CAP_SSB_NO;
	if (!boot_cpu_has_bug(X86_BUG_MDS))
		data |= ARCH_CAP_MDS_NO;

1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
	/*
	 * On TAA affected systems, export MDS_NO=0 when:
	 *	- TSX is enabled on the host, i.e. X86_FEATURE_RTM=1.
	 *	- Updated microcode is present. This is detected by
	 *	  the presence of ARCH_CAP_TSX_CTRL_MSR and ensures
	 *	  that VERW clears CPU buffers.
	 *
	 * When MDS_NO=0 is exported, guests deploy clear CPU buffer
	 * mitigation and don't complain:
	 *
	 *	"Vulnerable: Clear CPU buffers attempted, no microcode"
	 *
	 * If TSX is disabled on the system, guests are also mitigated against
	 * TAA and clear CPU buffer mitigation is not required for guests.
	 */
	if (boot_cpu_has_bug(X86_BUG_TAA) && boot_cpu_has(X86_FEATURE_RTM) &&
	    (data & ARCH_CAP_TSX_CTRL_MSR))
		data &= ~ARCH_CAP_MDS_NO;

1334 1335 1336
	return data;
}

1337 1338 1339
static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
{
	switch (msr->index) {
1340
	case MSR_IA32_ARCH_CAPABILITIES:
1341 1342 1343
		msr->data = kvm_get_arch_capabilities();
		break;
	case MSR_IA32_UCODE_REV:
1344
		rdmsrl_safe(msr->index, &msr->data);
1345
		break;
1346 1347 1348 1349 1350 1351 1352
	default:
		if (kvm_x86_ops->get_msr_feature(msr))
			return 1;
	}
	return 0;
}

1353 1354 1355
static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	struct kvm_msr_entry msr;
1356
	int r;
1357 1358

	msr.index = index;
1359 1360 1361
	r = kvm_get_msr_feature(&msr);
	if (r)
		return r;
1362 1363 1364 1365 1366 1367

	*data = msr.data;

	return 0;
}

1368
static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1369
{
1370
	if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1371
		return false;
Alexander Graf's avatar
Alexander Graf committed
1372

1373
	if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1374
		return false;
1375

1376 1377 1378 1379 1380 1381
	if (efer & (EFER_LME | EFER_LMA) &&
	    !guest_cpuid_has(vcpu, X86_FEATURE_LM))
		return false;

	if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
		return false;
1382

1383
	return true;
1384 1385 1386 1387 1388 1389 1390 1391

}
bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
{
	if (efer & efer_reserved_bits)
		return false;

	return __kvm_valid_efer(vcpu, efer);
1392 1393 1394
}
EXPORT_SYMBOL_GPL(kvm_valid_efer);

1395
static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1396 1397
{
	u64 old_efer = vcpu->arch.efer;
1398
	u64 efer = msr_info->data;
1399

1400
	if (efer & efer_reserved_bits)
1401
		return 1;
1402

1403 1404 1405 1406 1407 1408 1409 1410
	if (!msr_info->host_initiated) {
		if (!__kvm_valid_efer(vcpu, efer))
			return 1;

		if (is_paging(vcpu) &&
		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
			return 1;
	}
1411

1412
	efer &= ~EFER_LMA;
1413
	efer |= vcpu->arch.efer & EFER_LMA;
1414

1415 1416
	kvm_x86_ops->set_efer(vcpu, efer);

1417 1418 1419 1420
	/* Update reserved bits */
	if ((efer ^ old_efer) & EFER_NX)
		kvm_mmu_reset_context(vcpu);

1421
	return 0;
1422 1423
}

1424 1425 1426 1427 1428 1429
void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);

1430
/*
1431 1432
 * Write @data into the MSR specified by @index.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
1433 1434 1435
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
1436 1437
static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
			 bool host_initiated)
1438
{
1439 1440 1441
	struct msr_data msr;

	switch (index) {
1442 1443 1444 1445 1446
	case MSR_FS_BASE:
	case MSR_GS_BASE:
	case MSR_KERNEL_GS_BASE:
	case MSR_CSTAR:
	case MSR_LSTAR:
1447
		if (is_noncanonical_address(data, vcpu))
1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463
			return 1;
		break;
	case MSR_IA32_SYSENTER_EIP:
	case MSR_IA32_SYSENTER_ESP:
		/*
		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
		 * non-canonical address is written on Intel but not on
		 * AMD (which ignores the top 32-bits, because it does
		 * not implement 64-bit SYSENTER).
		 *
		 * 64-bit code should hence be able to write a non-canonical
		 * value on AMD.  Making the address canonical ensures that
		 * vmentry does not fail on Intel after writing a non-canonical
		 * value, and that something deterministic happens if the guest
		 * invokes 64-bit SYSENTER.
		 */
1464
		data = get_canonical(data, vcpu_virt_addr_bits(vcpu));
1465
	}
1466 1467 1468 1469 1470 1471

	msr.data = data;
	msr.index = index;
	msr.host_initiated = host_initiated;

	return kvm_x86_ops->set_msr(vcpu, &msr);
1472 1473
}

1474
/*
1475 1476 1477 1478
 * Read the MSR specified by @index into @data.  Select MSR specific fault
 * checks are bypassed if @host_initiated is %true.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
1479
 */
1480 1481
static int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
			 bool host_initiated)
1482 1483
{
	struct msr_data msr;
1484
	int ret;
1485 1486

	msr.index = index;
1487
	msr.host_initiated = host_initiated;
1488

1489 1490 1491 1492
	ret = kvm_x86_ops->get_msr(vcpu, &msr);
	if (!ret)
		*data = msr.data;
	return ret;
1493 1494
}

1495
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1496
{
1497 1498 1499
	return __kvm_get_msr(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_get_msr);
1500

1501 1502 1503 1504 1505 1506
int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
{
	return __kvm_set_msr(vcpu, index, data, false);
}
EXPORT_SYMBOL_GPL(kvm_set_msr);

1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541
int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data;

	if (kvm_get_msr(vcpu, ecx, &data)) {
		trace_kvm_msr_read_ex(ecx);
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

	trace_kvm_msr_read(ecx, data);

	kvm_rax_write(vcpu, data & -1u);
	kvm_rdx_write(vcpu, (data >> 32) & -1u);
	return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);

int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
{
	u32 ecx = kvm_rcx_read(vcpu);
	u64 data = kvm_read_edx_eax(vcpu);

	if (kvm_set_msr(vcpu, ecx, data)) {
		trace_kvm_msr_write_ex(ecx, data);
		kvm_inject_gp(vcpu, 0);
		return 1;
	}

	trace_kvm_msr_write(ecx, data);
	return kvm_skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);

1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552
/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return __kvm_get_msr(vcpu, index, data, true);
}

static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
	return __kvm_set_msr(vcpu, index, *data, true);
1553 1554
}

1555 1556 1557 1558 1559 1560
#ifdef CONFIG_X86_64
struct pvclock_gtod_data {
	seqcount_t	seq;

	struct { /* extract of a clocksource struct */
		int vclock_mode;
1561 1562
		u64	cycle_last;
		u64	mask;
1563 1564 1565 1566
		u32	mult;
		u32	shift;
	} clock;

1567 1568
	u64		boot_ns;
	u64		nsec_base;
1569
	u64		wall_time_sec;
1570 1571 1572 1573 1574 1575 1576
};

static struct pvclock_gtod_data pvclock_gtod_data;

static void update_pvclock_gtod(struct timekeeper *tk)
{
	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
1577 1578
	u64 boot_ns;

1579
	boot_ns = ktime_to_ns(ktime_add(tk->tkr_mono.base, tk->offs_boot));
1580 1581 1582 1583

	write_seqcount_begin(&vdata->seq);

	/* copy pvclock gtod data */
1584 1585 1586 1587 1588
	vdata->clock.vclock_mode	= tk->tkr_mono.clock->archdata.vclock_mode;
	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
	vdata->clock.mask		= tk->tkr_mono.mask;
	vdata->clock.mult		= tk->tkr_mono.mult;
	vdata->clock.shift		= tk->tkr_mono.shift;
1589

1590
	vdata->boot_ns			= boot_ns;
1591
	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
1592

1593 1594
	vdata->wall_time_sec            = tk->xtime_sec;

1595 1596 1597 1598
	write_seqcount_end(&vdata->seq);
}
#endif

1599 1600 1601
void kvm_set_pending_timer(struct kvm_vcpu *vcpu)
{
	kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
1602
	kvm_vcpu_kick(vcpu);
1603
}
1604

1605 1606
static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
{
1607 1608
	int version;
	int r;
1609
	struct pvclock_wall_clock wc;
1610
	struct timespec64 boot;
1611 1612 1613 1614

	if (!wall_clock)
		return;

1615 1616 1617 1618 1619 1620 1621 1622
	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
	if (r)
		return;

	if (version & 1)
		++version;  /* first time write, random junk */

	++version;
1623

1624 1625
	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
		return;
1626

1627 1628
	/*
	 * The guest calculates current wall clock time by adding
1629
	 * system time (updated by kvm_guest_time_update below) to the
1630 1631 1632
	 * wall clock specified here.  guest system time equals host
	 * system time for us, thus we must fill in host boot time here.
	 */
1633
	getboottime64(&boot);
1634

1635
	if (kvm->arch.kvmclock_offset) {
1636 1637
		struct timespec64 ts = ns_to_timespec64(kvm->arch.kvmclock_offset);
		boot = timespec64_sub(boot, ts);
1638
	}
1639
	wc.sec = (u32)boot.tv_sec; /* overflow in 2106 guest time */
1640 1641
	wc.nsec = boot.tv_nsec;
	wc.version = version;
1642 1643 1644 1645 1646 1647 1648

	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));

	version++;
	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
}

1649 1650
static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
1651 1652
	do_shl32_div32(dividend, divisor);
	return dividend;
1653 1654
}

1655
static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
1656
			       s8 *pshift, u32 *pmultiplier)
1657
{
1658
	uint64_t scaled64;
1659 1660 1661 1662
	int32_t  shift = 0;
	uint64_t tps64;
	uint32_t tps32;

1663 1664
	tps64 = base_hz;
	scaled64 = scaled_hz;
1665
	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
1666 1667 1668 1669 1670
		tps64 >>= 1;
		shift--;
	}

	tps32 = (uint32_t)tps64;
1671 1672
	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
1673 1674 1675
			scaled64 >>= 1;
		else
			tps32 <<= 1;
1676 1677 1678
		shift++;
	}

1679 1680
	*pshift = shift;
	*pmultiplier = div_frac(scaled64, tps32);
1681 1682
}

1683
#ifdef CONFIG_X86_64
1684
static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
1685
#endif
1686

1687
static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
1688
static unsigned long max_tsc_khz;
1689

1690
static u32 adjust_tsc_khz(u32 khz, s32 ppm)
1691
{
1692 1693 1694
	u64 v = (u64)khz * (1000000 + ppm);
	do_div(v, 1000000);
	return v;
1695 1696
}

1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713
static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
{
	u64 ratio;

	/* Guest TSC same frequency as host TSC? */
	if (!scale) {
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
		return 0;
	}

	/* TSC scaling supported? */
	if (!kvm_has_tsc_control) {
		if (user_tsc_khz > tsc_khz) {
			vcpu->arch.tsc_catchup = 1;
			vcpu->arch.tsc_always_catchup = 1;
			return 0;
		} else {
1714
			pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
1715 1716 1717 1718 1719 1720 1721 1722 1723
			return -1;
		}
	}

	/* TSC scaling required  - calculate ratio */
	ratio = mul_u64_u32_div(1ULL << kvm_tsc_scaling_ratio_frac_bits,
				user_tsc_khz, tsc_khz);

	if (ratio == 0 || ratio >= kvm_max_tsc_scaling_ratio) {
1724 1725
		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
			            user_tsc_khz);
1726 1727 1728 1729 1730 1731 1732
		return -1;
	}

	vcpu->arch.tsc_scaling_ratio = ratio;
	return 0;
}

1733
static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
1734
{
1735 1736
	u32 thresh_lo, thresh_hi;
	int use_scaling = 0;
1737

1738
	/* tsc_khz can be zero if TSC calibration fails */
1739
	if (user_tsc_khz == 0) {
1740 1741
		/* set tsc_scaling_ratio to a safe value */
		vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio;
1742
		return -1;
1743
	}
1744

Zachary Amsden's avatar
Zachary Amsden committed
1745
	/* Compute a scale to convert nanoseconds in TSC cycles */
1746
	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
1747 1748
			   &vcpu->arch.virtual_tsc_shift,
			   &vcpu->arch.virtual_tsc_mult);
1749
	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
1750 1751 1752 1753 1754 1755 1756 1757 1758

	/*
	 * Compute the variation in TSC rate which is acceptable
	 * within the range of tolerance and decide if the
	 * rate being applied is within that bounds of the hardware
	 * rate.  If so, no scaling or compensation need be done.
	 */
	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
1759 1760
	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
		pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", user_tsc_khz, thresh_lo, thresh_hi);
1761 1762
		use_scaling = 1;
	}
1763
	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
Zachary Amsden's avatar
Zachary Amsden committed
1764 1765 1766 1767
}

static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
1768
	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
1769 1770
				      vcpu->arch.virtual_tsc_mult,
				      vcpu->arch.virtual_tsc_shift);
1771
	tsc += vcpu->arch.this_tsc_write;
Zachary Amsden's avatar
Zachary Amsden committed
1772 1773 1774
	return tsc;
}

1775 1776 1777 1778 1779
static inline int gtod_is_based_on_tsc(int mode)
{
	return mode == VCLOCK_TSC || mode == VCLOCK_HVCLOCK;
}

1780
static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
1781 1782 1783 1784 1785 1786 1787 1788 1789
{
#ifdef CONFIG_X86_64
	bool vcpus_matched;
	struct kvm_arch *ka = &vcpu->kvm->arch;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			 atomic_read(&vcpu->kvm->online_vcpus));

1790 1791 1792 1793 1794 1795 1796 1797 1798
	/*
	 * Once the masterclock is enabled, always perform request in
	 * order to update it.
	 *
	 * In order to enable masterclock, the host clocksource must be TSC
	 * and the vcpus need to have matched TSCs.  When that happens,
	 * perform request to enable masterclock.
	 */
	if (ka->use_master_clock ||
1799
	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
1800 1801 1802 1803 1804 1805 1806 1807
		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);

	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
			    atomic_read(&vcpu->kvm->online_vcpus),
		            ka->use_master_clock, gtod->clock.vclock_mode);
#endif
}

1808 1809
static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
{
1810
	u64 curr_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
1811 1812 1813
	vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
}

1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840
/*
 * Multiply tsc by a fixed point number represented by ratio.
 *
 * The most significant 64-N bits (mult) of ratio represent the
 * integral part of the fixed point number; the remaining N bits
 * (frac) represent the fractional part, ie. ratio represents a fixed
 * point number (mult + frac * 2^(-N)).
 *
 * N equals to kvm_tsc_scaling_ratio_frac_bits.
 */
static inline u64 __scale_tsc(u64 ratio, u64 tsc)
{
	return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits);
}

u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc)
{
	u64 _tsc = tsc;
	u64 ratio = vcpu->arch.tsc_scaling_ratio;

	if (ratio != kvm_default_tsc_scaling_ratio)
		_tsc = __scale_tsc(ratio, tsc);

	return _tsc;
}
EXPORT_SYMBOL_GPL(kvm_scale_tsc);

1841 1842 1843 1844 1845 1846 1847 1848 1849
static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
{
	u64 tsc;

	tsc = kvm_scale_tsc(vcpu, rdtsc());

	return target_tsc - tsc;
}

1850 1851
u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
{
1852 1853 1854
	u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);

	return tsc_offset + kvm_scale_tsc(vcpu, host_tsc);
1855 1856 1857
}
EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);

1858 1859
static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
{
1860
	vcpu->arch.tsc_offset = kvm_x86_ops->write_l1_tsc_offset(vcpu, offset);
1861 1862
}

1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875
static inline bool kvm_check_tsc_unstable(void)
{
#ifdef CONFIG_X86_64
	/*
	 * TSC is marked unstable when we're running on Hyper-V,
	 * 'TSC page' clocksource is good.
	 */
	if (pvclock_gtod_data.clock.vclock_mode == VCLOCK_HVCLOCK)
		return false;
#endif
	return check_tsc_unstable();
}

1876
void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
1877 1878
{
	struct kvm *kvm = vcpu->kvm;
1879
	u64 offset, ns, elapsed;
1880
	unsigned long flags;
1881
	bool matched;
Tomasz Grabiec's avatar
Tomasz Grabiec committed
1882
	bool already_matched;
1883
	u64 data = msr->data;
1884
	bool synchronizing = false;
1885

1886
	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
1887
	offset = kvm_compute_tsc_offset(vcpu, data);
1888
	ns = ktime_get_boottime_ns();
1889
	elapsed = ns - kvm->arch.last_tsc_nsec;
1890

1891
	if (vcpu->arch.virtual_tsc_khz) {
1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910
		if (data == 0 && msr->host_initiated) {
			/*
			 * detection of vcpu initialization -- need to sync
			 * with other vCPUs. This particularly helps to keep
			 * kvm_clock stable after CPU hotplug
			 */
			synchronizing = true;
		} else {
			u64 tsc_exp = kvm->arch.last_tsc_write +
						nsec_to_cycles(vcpu, elapsed);
			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
			/*
			 * Special case: TSC write with a small delta (1 second)
			 * of virtual cycle time against real time is
			 * interpreted as an attempt to synchronize the CPU.
			 */
			synchronizing = data < tsc_exp + tsc_hz &&
					data + tsc_hz > tsc_exp;
		}
1911
	}
1912 1913

	/*
1914 1915 1916 1917 1918
	 * For a reliable TSC, we can match TSC offsets, and for an unstable
	 * TSC, we add elapsed time in this computation.  We could let the
	 * compensation code attempt to catch up if we fall behind, but
	 * it's better to try to match offsets from the beginning.
         */
1919
	if (synchronizing &&
1920
	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
1921
		if (!kvm_check_tsc_unstable()) {
1922
			offset = kvm->arch.cur_tsc_offset;
1923
		} else {
1924
			u64 delta = nsec_to_cycles(vcpu, elapsed);
1925
			data += delta;
1926
			offset = kvm_compute_tsc_offset(vcpu, data);
1927
		}
1928
		matched = true;
Tomasz Grabiec's avatar
Tomasz Grabiec committed
1929
		already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation);
1930 1931 1932 1933 1934 1935
	} else {
		/*
		 * We split periods of matched TSC writes into generations.
		 * For each generation, we track the original measured
		 * nanosecond time, offset, and write, so if TSCs are in
		 * sync, we can match exact offset, and if not, we can match
Guo Chao's avatar
Guo Chao committed
1936
		 * exact software computation in compute_guest_tsc()
1937 1938 1939 1940 1941 1942 1943
		 *
		 * These values are tracked in kvm->arch.cur_xxx variables.
		 */
		kvm->arch.cur_tsc_generation++;
		kvm->arch.cur_tsc_nsec = ns;
		kvm->arch.cur_tsc_write = data;
		kvm->arch.cur_tsc_offset = offset;
1944
		matched = false;
1945
	}
1946 1947 1948 1949 1950

	/*
	 * We also track th most recent recorded KHZ, write and time to
	 * allow the matching interval to be extended at each write.
	 */
1951 1952
	kvm->arch.last_tsc_nsec = ns;
	kvm->arch.last_tsc_write = data;
1953
	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
1954

1955
	vcpu->arch.last_guest_tsc = data;
1956 1957 1958 1959 1960 1961

	/* Keep track of which generation this VCPU has synchronized to */
	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;

1962
	if (!msr->host_initiated && guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST))
1963
		update_ia32_tsc_adjust_msr(vcpu, offset);
1964

1965
	kvm_vcpu_write_tsc_offset(vcpu, offset);
1966
	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
1967 1968

	spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
Tomasz Grabiec's avatar
Tomasz Grabiec committed
1969
	if (!matched) {
1970
		kvm->arch.nr_vcpus_matched_tsc = 0;
Tomasz Grabiec's avatar
Tomasz Grabiec committed
1971 1972 1973
	} else if (!already_matched) {
		kvm->arch.nr_vcpus_matched_tsc++;
	}
1974 1975 1976

	kvm_track_tsc_matching(vcpu);
	spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
1977
}
1978

1979 1980
EXPORT_SYMBOL_GPL(kvm_write_tsc);

1981 1982 1983
static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
					   s64 adjustment)
{
1984 1985
	u64 tsc_offset = kvm_x86_ops->read_l1_tsc_offset(vcpu);
	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
1986 1987 1988 1989 1990 1991 1992
}

static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
{
	if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio)
		WARN_ON(adjustment < 0);
	adjustment = kvm_scale_tsc(vcpu, (u64) adjustment);
1993
	adjust_tsc_offset_guest(vcpu, adjustment);
1994 1995
}

1996 1997
#ifdef CONFIG_X86_64

1998
static u64 read_tsc(void)
1999
{
2000
	u64 ret = (u64)rdtsc_ordered();
2001
	u64 last = pvclock_gtod_data.clock.cycle_last;
2002 2003 2004 2005 2006 2007

	if (likely(ret >= last))
		return ret;

	/*
	 * GCC likes to generate cmov here, but this branch is extremely
2008
	 * predictable (it's just a function of time and the likely is
2009 2010 2011 2012 2013 2014 2015 2016 2017
	 * very likely) and there's a data dependence, so force GCC
	 * to generate a branch instead.  I don't barrier() because
	 * we don't actually need a barrier, and if this function
	 * ever gets inlined it will generate worse code.
	 */
	asm volatile ("");
	return last;
}

2018
static inline u64 vgettsc(u64 *tsc_timestamp, int *mode)
2019 2020 2021
{
	long v;
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046
	u64 tsc_pg_val;

	switch (gtod->clock.vclock_mode) {
	case VCLOCK_HVCLOCK:
		tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
						  tsc_timestamp);
		if (tsc_pg_val != U64_MAX) {
			/* TSC page valid */
			*mode = VCLOCK_HVCLOCK;
			v = (tsc_pg_val - gtod->clock.cycle_last) &
				gtod->clock.mask;
		} else {
			/* TSC page invalid */
			*mode = VCLOCK_NONE;
		}
		break;
	case VCLOCK_TSC:
		*mode = VCLOCK_TSC;
		*tsc_timestamp = read_tsc();
		v = (*tsc_timestamp - gtod->clock.cycle_last) &
			gtod->clock.mask;
		break;
	default:
		*mode = VCLOCK_NONE;
	}
2047

2048 2049
	if (*mode == VCLOCK_NONE)
		*tsc_timestamp = v = 0;
2050 2051 2052 2053

	return v * gtod->clock.mult;
}

2054
static int do_monotonic_boot(s64 *t, u64 *tsc_timestamp)
2055
{
2056
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2057 2058
	unsigned long seq;
	int mode;
2059
	u64 ns;
2060 2061 2062

	do {
		seq = read_seqcount_begin(&gtod->seq);
2063
		ns = gtod->nsec_base;
2064
		ns += vgettsc(tsc_timestamp, &mode);
2065
		ns >>= gtod->clock.shift;
2066
		ns += gtod->boot_ns;
2067
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2068
	*t = ns;
2069 2070 2071 2072

	return mode;
}

2073
static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
2074 2075 2076 2077 2078 2079 2080 2081 2082 2083
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	unsigned long seq;
	int mode;
	u64 ns;

	do {
		seq = read_seqcount_begin(&gtod->seq);
		ts->tv_sec = gtod->wall_time_sec;
		ns = gtod->nsec_base;
2084
		ns += vgettsc(tsc_timestamp, &mode);
2085 2086 2087 2088 2089 2090 2091 2092 2093
		ns >>= gtod->clock.shift;
	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));

	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return mode;
}

2094 2095
/* returns true if host is using TSC based clocksource */
static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
2096 2097
{
	/* checked again under seqlock below */
2098
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2099 2100
		return false;

2101 2102
	return gtod_is_based_on_tsc(do_monotonic_boot(kernel_ns,
						      tsc_timestamp));
2103
}
2104

2105
/* returns true if host is using TSC based clocksource */
2106
static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
2107
					   u64 *tsc_timestamp)
2108 2109
{
	/* checked again under seqlock below */
2110
	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2111 2112
		return false;

2113
	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
2114
}
2115 2116 2117 2118
#endif

/*
 *
2119 2120 2121
 * Assuming a stable TSC across physical CPUS, and a stable TSC
 * across virtual CPUs, the following condition is possible.
 * Each numbered line represents an event visible to both
2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153
 * CPUs at the next numbered event.
 *
 * "timespecX" represents host monotonic time. "tscX" represents
 * RDTSC value.
 *
 * 		VCPU0 on CPU0		|	VCPU1 on CPU1
 *
 * 1.  read timespec0,tsc0
 * 2.					| timespec1 = timespec0 + N
 * 					| tsc1 = tsc0 + M
 * 3. transition to guest		| transition to guest
 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
 * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
 * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
 *
 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
 *
 * 	- ret0 < ret1
 *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
 *		...
 *	- 0 < N - M => M < N
 *
 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
 * always the case (the difference between two distinct xtime instances
 * might be smaller then the difference between corresponding TSC reads,
 * when updating guest vcpus pvclock areas).
 *
 * To avoid that problem, do not allow visibility of distinct
 * system_timestamp/tsc_timestamp values simultaneously: use a master
 * copy of host monotonic time values. Update that master copy
 * in lockstep.
 *
2154
 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
2155 2156 2157 2158 2159 2160 2161 2162
 *
 */

static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	struct kvm_arch *ka = &kvm->arch;
	int vclock_mode;
2163 2164 2165 2166
	bool host_tsc_clocksource, vcpus_matched;

	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
			atomic_read(&kvm->online_vcpus));
2167 2168 2169 2170 2171

	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
2172
	host_tsc_clocksource = kvm_get_time_and_clockread(
2173 2174 2175
					&ka->master_kernel_ns,
					&ka->master_cycle_now);

2176
	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
2177
				&& !ka->backwards_tsc_observed
2178
				&& !ka->boot_vcpu_runs_old_kvmclock;
2179

2180 2181 2182 2183
	if (ka->use_master_clock)
		atomic_set(&kvm_guest_has_master_clock, 1);

	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
2184 2185
	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
					vcpus_matched);
2186 2187 2188
#endif
}

2189 2190 2191 2192 2193
void kvm_make_mclock_inprogress_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
}

2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206
static void kvm_gen_update_masterclock(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
	int i;
	struct kvm_vcpu *vcpu;
	struct kvm_arch *ka = &kvm->arch;

	spin_lock(&ka->pvclock_gtod_sync_lock);
	kvm_make_mclock_inprogress_request(kvm);
	/* no guest entries from this point */
	pvclock_update_vm_gtod_copy(kvm);

	kvm_for_each_vcpu(i, vcpu, kvm)
2207
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2208 2209 2210

	/* guest entries allowed */
	kvm_for_each_vcpu(i, vcpu, kvm)
2211
		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
2212 2213 2214 2215 2216

	spin_unlock(&ka->pvclock_gtod_sync_lock);
#endif
}

2217
u64 get_kvmclock_ns(struct kvm *kvm)
2218 2219
{
	struct kvm_arch *ka = &kvm->arch;
2220
	struct pvclock_vcpu_time_info hv_clock;
2221
	u64 ret;
2222

2223 2224 2225
	spin_lock(&ka->pvclock_gtod_sync_lock);
	if (!ka->use_master_clock) {
		spin_unlock(&ka->pvclock_gtod_sync_lock);
2226
		return ktime_get_boottime_ns() + ka->kvmclock_offset;
2227 2228
	}

2229 2230 2231 2232
	hv_clock.tsc_timestamp = ka->master_cycle_now;
	hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
	spin_unlock(&ka->pvclock_gtod_sync_lock);

2233 2234 2235
	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
	get_cpu();

2236 2237 2238 2239 2240 2241
	if (__this_cpu_read(cpu_tsc_khz)) {
		kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL,
				   &hv_clock.tsc_shift,
				   &hv_clock.tsc_to_system_mul);
		ret = __pvclock_read_cycles(&hv_clock, rdtsc());
	} else
2242
		ret = ktime_get_boottime_ns() + ka->kvmclock_offset;
2243 2244 2245 2246

	put_cpu();

	return ret;
2247 2248
}

2249 2250 2251 2252 2253
static void kvm_setup_pvclock_page(struct kvm_vcpu *v)
{
	struct kvm_vcpu_arch *vcpu = &v->arch;
	struct pvclock_vcpu_time_info guest_hv_clock;

2254
	if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273
		&guest_hv_clock, sizeof(guest_hv_clock))))
		return;

	/* This VCPU is paused, but it's legal for a guest to read another
	 * VCPU's kvmclock, so we really have to follow the specification where
	 * it says that version is odd if data is being modified, and even after
	 * it is consistent.
	 *
	 * Version field updates must be kept separate.  This is because
	 * kvm_write_guest_cached might use a "rep movs" instruction, and
	 * writes within a string instruction are weakly ordered.  So there
	 * are three writes overall.
	 *
	 * As a small optimization, only write the version field in the first
	 * and third write.  The vcpu->pv_time cache is still valid, because the
	 * version field is the first in the struct.
	 */
	BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);

2274 2275 2276
	if (guest_hv_clock.version & 1)
		++guest_hv_clock.version;  /* first time write, random junk */

2277
	vcpu->hv_clock.version = guest_hv_clock.version + 1;
2278 2279 2280
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293

	smp_wmb();

	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
	vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);

	if (vcpu->pvclock_set_guest_stopped_request) {
		vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
		vcpu->pvclock_set_guest_stopped_request = false;
	}

	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);

2294 2295 2296
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock));
2297 2298 2299 2300

	smp_wmb();

	vcpu->hv_clock.version++;
2301 2302 2303
	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
				&vcpu->hv_clock,
				sizeof(vcpu->hv_clock.version));
2304 2305
}

2306
static int kvm_guest_time_update(struct kvm_vcpu *v)
2307
{
2308
	unsigned long flags, tgt_tsc_khz;
2309
	struct kvm_vcpu_arch *vcpu = &v->arch;
2310
	struct kvm_arch *ka = &v->kvm->arch;
2311
	s64 kernel_ns;
2312
	u64 tsc_timestamp, host_tsc;
2313
	u8 pvclock_flags;
2314 2315 2316 2317
	bool use_master_clock;

	kernel_ns = 0;
	host_tsc = 0;
2318

2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329
	/*
	 * If the host uses TSC clock, then passthrough TSC as stable
	 * to the guest.
	 */
	spin_lock(&ka->pvclock_gtod_sync_lock);
	use_master_clock = ka->use_master_clock;
	if (use_master_clock) {
		host_tsc = ka->master_cycle_now;
		kernel_ns = ka->master_kernel_ns;
	}
	spin_unlock(&ka->pvclock_gtod_sync_lock);
2330 2331 2332

	/* Keep irq disabled to prevent changes to the clock */
	local_irq_save(flags);
2333 2334
	tgt_tsc_khz = __this_cpu_read(cpu_tsc_khz);
	if (unlikely(tgt_tsc_khz == 0)) {
2335 2336 2337 2338
		local_irq_restore(flags);
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
		return 1;
	}
2339
	if (!use_master_clock) {
2340
		host_tsc = rdtsc();
2341
		kernel_ns = ktime_get_boottime_ns();
2342 2343
	}

2344
	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
2345

Zachary Amsden's avatar
Zachary Amsden committed
2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358
	/*
	 * We may have to catch up the TSC to match elapsed wall clock
	 * time for two reasons, even if kvmclock is used.
	 *   1) CPU could have been running below the maximum TSC rate
	 *   2) Broken TSC compensation resets the base at each VCPU
	 *      entry to avoid unknown leaps of TSC even when running
	 *      again on the same CPU.  This may cause apparent elapsed
	 *      time to disappear, and the guest to stand still or run
	 *	very slowly.
	 */
	if (vcpu->tsc_catchup) {
		u64 tsc = compute_guest_tsc(v, kernel_ns);
		if (tsc > tsc_timestamp) {
2359
			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
Zachary Amsden's avatar
Zachary Amsden committed
2360 2361
			tsc_timestamp = tsc;
		}
2362 2363
	}

2364 2365
	local_irq_restore(flags);

2366
	/* With all the info we got, fill in the values */
2367

2368 2369 2370 2371
	if (kvm_has_tsc_control)
		tgt_tsc_khz = kvm_scale_tsc(v, tgt_tsc_khz);

	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
2372
		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
2373 2374
				   &vcpu->hv_clock.tsc_shift,
				   &vcpu->hv_clock.tsc_to_system_mul);
2375
		vcpu->hw_tsc_khz = tgt_tsc_khz;
2376 2377
	}

2378
	vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
2379
	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
Zachary Amsden's avatar
Zachary Amsden committed
2380
	vcpu->last_guest_tsc = tsc_timestamp;
2381

2382
	/* If the host uses TSC clocksource, then it is stable */
2383
	pvclock_flags = 0;
2384 2385 2386
	if (use_master_clock)
		pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;

2387 2388
	vcpu->hv_clock.flags = pvclock_flags;

2389 2390 2391 2392
	if (vcpu->pv_time_enabled)
		kvm_setup_pvclock_page(v);
	if (v == kvm_get_vcpu(v->kvm, 0))
		kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
2393
	return 0;
2394 2395
}

2396 2397 2398 2399 2400 2401 2402 2403
/*
 * kvmclock updates which are isolated to a given vcpu, such as
 * vcpu->cpu migration, should not allow system_timestamp from
 * the rest of the vcpus to remain static. Otherwise ntp frequency
 * correction applies to one vcpu's system_timestamp but not
 * the others.
 *
 * So in those cases, request a kvmclock update for all vcpus.
2404 2405 2406 2407
 * We need to rate-limit these requests though, as they can
 * considerably slow guests that have a large number of vcpus.
 * The time for a remote vcpu to update its kvmclock is bound
 * by the delay we use to rate-limit the updates.
2408 2409
 */

2410 2411 2412
#define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)

static void kvmclock_update_fn(struct work_struct *work)
2413 2414
{
	int i;
2415 2416 2417 2418
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_update_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);
2419 2420 2421
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm) {
2422
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2423 2424 2425 2426
		kvm_vcpu_kick(vcpu);
	}
}

2427 2428 2429 2430
static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
{
	struct kvm *kvm = v->kvm;

2431
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
2432 2433 2434 2435
	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
					KVMCLOCK_UPDATE_DELAY);
}

2436 2437 2438 2439 2440 2441 2442 2443 2444
#define KVMCLOCK_SYNC_PERIOD (300 * HZ)

static void kvmclock_sync_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_sync_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);

2445 2446 2447
	if (!kvmclock_periodic_sync)
		return;

2448 2449 2450 2451 2452
	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
}

2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464
/*
 * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
 */
static bool can_set_mci_status(struct kvm_vcpu *vcpu)
{
	/* McStatusWrEn enabled? */
	if (guest_cpuid_is_amd(vcpu))
		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));

	return false;
}

2465
static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2466
{
Huang Ying's avatar
Huang Ying committed
2467 2468
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2469 2470
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
Huang Ying's avatar
Huang Ying committed
2471

2472 2473
	switch (msr) {
	case MSR_IA32_MCG_STATUS:
Huang Ying's avatar
Huang Ying committed
2474
		vcpu->arch.mcg_status = data;
2475
		break;
2476
	case MSR_IA32_MCG_CTL:
2477 2478
		if (!(mcg_cap & MCG_CTL_P) &&
		    (data || !msr_info->host_initiated))
Huang Ying's avatar
Huang Ying committed
2479 2480
			return 1;
		if (data != 0 && data != ~(u64)0)
2481
			return 1;
Huang Ying's avatar
Huang Ying committed
2482 2483 2484 2485
		vcpu->arch.mcg_ctl = data;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2486
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
Huang Ying's avatar
Huang Ying committed
2487
			u32 offset = msr - MSR_IA32_MC0_CTL;
2488 2489 2490 2491 2492
			/* only 0 or all 1s can be written to IA32_MCi_CTL
			 * some Linux kernels though clear bit 10 in bank 4 to
			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
			 * this to avoid an uncatched #GP in the guest
			 */
Huang Ying's avatar
Huang Ying committed
2493
			if ((offset & 0x3) == 0 &&
2494
			    data != 0 && (data | (1 << 10)) != ~(u64)0)
Huang Ying's avatar
Huang Ying committed
2495
				return -1;
2496 2497

			/* MCi_STATUS */
2498
			if (!msr_info->host_initiated &&
2499 2500 2501 2502 2503
			    (offset & 0x3) == 1 && data != 0) {
				if (!can_set_mci_status(vcpu))
					return -1;
			}

Huang Ying's avatar
Huang Ying committed
2504 2505 2506 2507 2508 2509 2510 2511
			vcpu->arch.mce_banks[offset] = data;
			break;
		}
		return 1;
	}
	return 0;
}

Ed Swierk's avatar
Ed Swierk committed
2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528
static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
	int lm = is_long_mode(vcpu);
	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
		: kvm->arch.xen_hvm_config.blob_size_32;
	u32 page_num = data & ~PAGE_MASK;
	u64 page_addr = data & PAGE_MASK;
	u8 *page;
	int r;

	r = -E2BIG;
	if (page_num >= blob_size)
		goto out;
	r = -ENOMEM;
2529 2530 2531
	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
	if (IS_ERR(page)) {
		r = PTR_ERR(page);
Ed Swierk's avatar
Ed Swierk committed
2532
		goto out;
2533
	}
2534
	if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE))
Ed Swierk's avatar
Ed Swierk committed
2535 2536 2537 2538 2539 2540 2541 2542
		goto out_free;
	r = 0;
out_free:
	kfree(page);
out:
	return r;
}

2543 2544 2545 2546
static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
{
	gpa_t gpa = data & ~0x3f;

2547 2548
	/* Bits 3:5 are reserved, Should be zero */
	if (data & 0x38)
2549 2550 2551 2552 2553 2554 2555 2556 2557 2558
		return 1;

	vcpu->arch.apf.msr_val = data;

	if (!(data & KVM_ASYNC_PF_ENABLED)) {
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_async_pf_hash_reset(vcpu);
		return 0;
	}

2559
	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
2560
					sizeof(u32)))
2561 2562
		return 1;

2563
	vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
2564
	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
2565 2566 2567 2568
	kvm_async_pf_wakeup_all(vcpu);
	return 0;
}

2569 2570
static void kvmclock_reset(struct kvm_vcpu *vcpu)
{
2571
	vcpu->arch.pv_time_enabled = false;
2572
	vcpu->arch.time = 0;
2573 2574
}

2575 2576 2577 2578 2579 2580
static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
{
	++vcpu->stat.tlb_flush;
	kvm_x86_ops->tlb_flush(vcpu, invalidate_gpa);
}

2581 2582 2583 2584 2585
static void record_steal_time(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

2586
	if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
2587 2588 2589
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
		return;

2590 2591 2592 2593
	/*
	 * Doing a TLB flush here, on the guest's behalf, can avoid
	 * expensive IPIs.
	 */
2594 2595
	trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
		vcpu->arch.st.steal.preempted & KVM_VCPU_FLUSH_TLB);
2596 2597
	if (xchg(&vcpu->arch.st.steal.preempted, 0) & KVM_VCPU_FLUSH_TLB)
		kvm_vcpu_flush_tlb(vcpu, false);
2598

2599 2600 2601 2602 2603
	if (vcpu->arch.st.steal.version & 1)
		vcpu->arch.st.steal.version += 1;  /* first time write, random junk */

	vcpu->arch.st.steal.version += 1;

2604
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
2605 2606 2607 2608
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));

	smp_wmb();

2609 2610 2611
	vcpu->arch.st.steal.steal += current->sched_info.run_delay -
		vcpu->arch.st.last_steal;
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
2612

2613
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
2614 2615 2616 2617 2618
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));

	smp_wmb();

	vcpu->arch.st.steal.version += 1;
2619

2620
	kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
2621 2622 2623
		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
}

2624
int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2625
{
2626
	bool pr = false;
2627 2628
	u32 msr = msr_info->index;
	u64 data = msr_info->data;
2629

2630
	switch (msr) {
2631 2632 2633 2634 2635
	case MSR_AMD64_NB_CFG:
	case MSR_IA32_UCODE_WRITE:
	case MSR_VM_HSAVE_PA:
	case MSR_AMD64_PATCH_LOADER:
	case MSR_AMD64_BU_CFG2:
2636
	case MSR_AMD64_DC_CFG:
2637
	case MSR_F15H_EX_CFG:
2638 2639
		break;

2640 2641 2642 2643
	case MSR_IA32_UCODE_REV:
		if (msr_info->host_initiated)
			vcpu->arch.microcode_version = data;
		break;
2644 2645 2646 2647 2648
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.arch_capabilities = data;
		break;
2649
	case MSR_EFER:
2650
		return set_efer(vcpu, msr_info);
2651 2652
	case MSR_K7_HWCR:
		data &= ~(u64)0x40;	/* ignore flush filter disable */
2653
		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
2654
		data &= ~(u64)0x8;	/* ignore TLB cache disable */
2655 2656 2657 2658 2659

		/* Handle McStatusWrEn */
		if (data == BIT_ULL(18)) {
			vcpu->arch.msr_hwcr = data;
		} else if (data != 0) {
2660 2661
			vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
				    data);
2662 2663
			return 1;
		}
2664
		break;
2665 2666
	case MSR_FAM10H_MMIO_CONF_BASE:
		if (data != 0) {
2667 2668
			vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
				    "0x%llx\n", data);
2669 2670
			return 1;
		}
2671
		break;
2672 2673 2674 2675 2676 2677 2678 2679 2680
	case MSR_IA32_DEBUGCTLMSR:
		if (!data) {
			/* We support the non-activated case already */
			break;
		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
			/* Values other than LBR and BTF are vendor-specific,
			   thus reserved and should throw a #GP */
			return 1;
		}
2681 2682
		vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
			    __func__, data);
2683
		break;
Avi Kivity's avatar
Avi Kivity committed
2684
	case 0x200 ... 0x2ff:
2685
		return kvm_mtrr_set_msr(vcpu, msr, data);
2686
	case MSR_IA32_APICBASE:
2687
		return kvm_set_apic_base(vcpu, msr_info);
2688 2689
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
		return kvm_x2apic_msr_write(vcpu, msr, data);
2690 2691 2692
	case MSR_IA32_TSCDEADLINE:
		kvm_set_lapic_tscdeadline_msr(vcpu, data);
		break;
2693
	case MSR_IA32_TSC_ADJUST:
2694
		if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
2695
			if (!msr_info->host_initiated) {
2696
				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
2697
				adjust_tsc_offset_guest(vcpu, adj);
2698 2699 2700 2701
			}
			vcpu->arch.ia32_tsc_adjust_msr = data;
		}
		break;
2702
	case MSR_IA32_MISC_ENABLE:
2703 2704 2705 2706 2707 2708 2709 2710 2711
		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
		    ((vcpu->arch.ia32_misc_enable_msr ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
			if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
				return 1;
			vcpu->arch.ia32_misc_enable_msr = data;
			kvm_update_cpuid(vcpu);
		} else {
			vcpu->arch.ia32_misc_enable_msr = data;
		}
2712
		break;
2713 2714 2715 2716 2717
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smbase = data;
		break;
2718 2719 2720
	case MSR_IA32_POWER_CTL:
		vcpu->arch.msr_ia32_power_ctl = data;
		break;
2721 2722 2723
	case MSR_IA32_TSC:
		kvm_write_tsc(vcpu, msr_info);
		break;
2724 2725 2726 2727 2728
	case MSR_SMI_COUNT:
		if (!msr_info->host_initiated)
			return 1;
		vcpu->arch.smi_count = data;
		break;
2729
	case MSR_KVM_WALL_CLOCK_NEW:
2730 2731 2732 2733
	case MSR_KVM_WALL_CLOCK:
		vcpu->kvm->arch.wall_clock = data;
		kvm_write_wall_clock(vcpu->kvm, data);
		break;
2734
	case MSR_KVM_SYSTEM_TIME_NEW:
2735
	case MSR_KVM_SYSTEM_TIME: {
2736 2737 2738 2739 2740 2741
		struct kvm_arch *ka = &vcpu->kvm->arch;

		if (vcpu->vcpu_id == 0 && !msr_info->host_initiated) {
			bool tmp = (msr == MSR_KVM_SYSTEM_TIME);

			if (ka->boot_vcpu_runs_old_kvmclock != tmp)
2742
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2743 2744 2745 2746

			ka->boot_vcpu_runs_old_kvmclock = tmp;
		}

2747
		vcpu->arch.time = data;
2748
		kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2749 2750

		/* we verify if the enable bit is set... */
2751
		vcpu->arch.pv_time_enabled = false;
2752 2753 2754
		if (!(data & 1))
			break;

2755
		if (!kvm_gfn_to_hva_cache_init(vcpu->kvm,
2756 2757
		     &vcpu->arch.pv_time, data & ~1ULL,
		     sizeof(struct pvclock_vcpu_time_info)))
2758
			vcpu->arch.pv_time_enabled = true;
2759

2760 2761
		break;
	}
2762 2763 2764 2765
	case MSR_KVM_ASYNC_PF_EN:
		if (kvm_pv_enable_async_pf(vcpu, data))
			return 1;
		break;
2766 2767 2768 2769 2770 2771 2772 2773
	case MSR_KVM_STEAL_TIME:

		if (unlikely(!sched_info_on()))
			return 1;

		if (data & KVM_STEAL_RESERVED_MASK)
			return 1;

2774
		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
2775 2776
						data & KVM_STEAL_VALID_BITS,
						sizeof(struct kvm_steal_time)))
2777 2778 2779 2780 2781 2782 2783 2784 2785 2786
			return 1;

		vcpu->arch.st.msr_val = data;

		if (!(data & KVM_MSR_ENABLED))
			break;

		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);

		break;
2787
	case MSR_KVM_PV_EOI_EN:
2788
		if (kvm_lapic_enable_pv_eoi(vcpu, data, sizeof(u8)))
2789 2790
			return 1;
		break;
2791

2792 2793 2794 2795 2796 2797 2798 2799
	case MSR_KVM_POLL_CONTROL:
		/* only enable bit supported */
		if (data & (-1ULL << 1))
			return 1;

		vcpu->arch.msr_kvm_poll_control = data;
		break;

Huang Ying's avatar
Huang Ying committed
2800 2801
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
2802
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
2803
		return set_msr_mce(vcpu, msr_info);
2804

2805 2806 2807 2808 2809
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
		pr = true; /* fall through */
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
2810
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2811
			return kvm_pmu_set_msr(vcpu, msr_info);
2812 2813

		if (pr || data != 0)
2814 2815
			vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
				    "0x%x data 0x%llx\n", msr, data);
2816
		break;
2817 2818 2819 2820 2821
	case MSR_K7_CLK_CTL:
		/*
		 * Ignore all writes to this no longer documented MSR.
		 * Writes are only relevant for old K7 processors,
		 * all pre-dating SVM, but a recommended workaround from
Guo Chao's avatar
Guo Chao committed
2822
		 * AMD for these chips. It is possible to specify the
2823 2824 2825 2826
		 * affected processor models on the command line, hence
		 * the need to ignore the workaround.
		 */
		break;
2827
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
2828 2829
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
2830
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
2831 2832 2833
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
2834 2835
		return kvm_hv_set_msr_common(vcpu, msr, data,
					     msr_info->host_initiated);
2836 2837 2838 2839
	case MSR_IA32_BBL_CR_CTL3:
		/* Drop writes to this legacy MSR -- see rdmsr
		 * counterpart for further detail.
		 */
2840 2841 2842
		if (report_ignored_msrs)
			vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
				msr, data);
2843
		break;
2844
	case MSR_AMD64_OSVW_ID_LENGTH:
2845
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2846 2847 2848 2849
			return 1;
		vcpu->arch.osvw.length = data;
		break;
	case MSR_AMD64_OSVW_STATUS:
2850
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
2851 2852 2853
			return 1;
		vcpu->arch.osvw.status = data;
		break;
2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867
	case MSR_PLATFORM_INFO:
		if (!msr_info->host_initiated ||
		    (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
		     cpuid_fault_enabled(vcpu)))
			return 1;
		vcpu->arch.msr_platform_info = data;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
		     !supports_cpuid_fault(vcpu)))
			return 1;
		vcpu->arch.msr_misc_features_enables = data;
		break;
2868
	default:
Ed Swierk's avatar
Ed Swierk committed
2869 2870
		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
			return xen_hvm_config(vcpu, data);
2871
		if (kvm_pmu_is_valid_msr(vcpu, msr))
2872
			return kvm_pmu_set_msr(vcpu, msr_info);
2873
		if (!ignore_msrs) {
2874
			vcpu_debug_ratelimited(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n",
2875
				    msr, data);
2876 2877
			return 1;
		} else {
2878 2879 2880 2881
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu,
					"ignored wrmsr: 0x%x data 0x%llx\n",
					msr, data);
2882 2883
			break;
		}
2884 2885 2886 2887 2888
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);

2889
static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
2890 2891
{
	u64 data;
Huang Ying's avatar
Huang Ying committed
2892 2893
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
2894 2895 2896 2897

	switch (msr) {
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
Huang Ying's avatar
Huang Ying committed
2898 2899
		data = 0;
		break;
2900
	case MSR_IA32_MCG_CAP:
Huang Ying's avatar
Huang Ying committed
2901 2902
		data = vcpu->arch.mcg_cap;
		break;
2903
	case MSR_IA32_MCG_CTL:
2904
		if (!(mcg_cap & MCG_CTL_P) && !host)
Huang Ying's avatar
Huang Ying committed
2905 2906 2907 2908 2909 2910 2911 2912
			return 1;
		data = vcpu->arch.mcg_ctl;
		break;
	case MSR_IA32_MCG_STATUS:
		data = vcpu->arch.mcg_status;
		break;
	default:
		if (msr >= MSR_IA32_MC0_CTL &&
2913
		    msr < MSR_IA32_MCx_CTL(bank_num)) {
Huang Ying's avatar
Huang Ying committed
2914 2915 2916 2917 2918 2919 2920 2921 2922 2923
			u32 offset = msr - MSR_IA32_MC0_CTL;
			data = vcpu->arch.mce_banks[offset];
			break;
		}
		return 1;
	}
	*pdata = data;
	return 0;
}

2924
int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
Huang Ying's avatar
Huang Ying committed
2925
{
2926
	switch (msr_info->index) {
Huang Ying's avatar
Huang Ying committed
2927
	case MSR_IA32_PLATFORM_ID:
2928
	case MSR_IA32_EBL_CR_POWERON:
2929 2930 2931 2932 2933
	case MSR_IA32_DEBUGCTLMSR:
	case MSR_IA32_LASTBRANCHFROMIP:
	case MSR_IA32_LASTBRANCHTOIP:
	case MSR_IA32_LASTINTFROMIP:
	case MSR_IA32_LASTINTTOIP:
2934
	case MSR_K8_SYSCFG:
2935 2936
	case MSR_K8_TSEG_ADDR:
	case MSR_K8_TSEG_MASK:
2937
	case MSR_VM_HSAVE_PA:
2938
	case MSR_K8_INT_PENDING_MSG:
2939
	case MSR_AMD64_NB_CFG:
2940
	case MSR_FAM10H_MMIO_CONF_BASE:
2941
	case MSR_AMD64_BU_CFG2:
2942
	case MSR_IA32_PERF_CTL:
2943
	case MSR_AMD64_DC_CFG:
2944
	case MSR_F15H_EX_CFG:
2945
		msr_info->data = 0;
2946
		break;
2947
	case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
2948 2949 2950 2951
	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
2952
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
2953 2954
			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
		msr_info->data = 0;
2955
		break;
2956
	case MSR_IA32_UCODE_REV:
2957
		msr_info->data = vcpu->arch.microcode_version;
2958
		break;
2959 2960 2961 2962 2963 2964
	case MSR_IA32_ARCH_CAPABILITIES:
		if (!msr_info->host_initiated &&
		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
			return 1;
		msr_info->data = vcpu->arch.arch_capabilities;
		break;
2965 2966 2967
	case MSR_IA32_POWER_CTL:
		msr_info->data = vcpu->arch.msr_ia32_power_ctl;
		break;
2968 2969 2970
	case MSR_IA32_TSC:
		msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + vcpu->arch.tsc_offset;
		break;
Avi Kivity's avatar
Avi Kivity committed
2971 2972
	case MSR_MTRRcap:
	case 0x200 ... 0x2ff:
2973
		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
2974
	case 0xcd: /* fsb frequency */
2975
		msr_info->data = 3;
2976
		break;
2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988
		/*
		 * MSR_EBC_FREQUENCY_ID
		 * Conservative value valid for even the basic CPU models.
		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
		 * and 266MHz for model 3, or 4. Set Core Clock
		 * Frequency to System Bus Frequency Ratio to 1 (bits
		 * 31:24) even though these are only valid for CPU
		 * models > 2, however guests may end up dividing or
		 * multiplying by zero otherwise.
		 */
	case MSR_EBC_FREQUENCY_ID:
2989
		msr_info->data = 1 << 24;
2990
		break;
2991
	case MSR_IA32_APICBASE:
2992
		msr_info->data = kvm_get_apic_base(vcpu);
2993
		break;
2994
	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
2995
		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
2996
		break;
2997
	case MSR_IA32_TSCDEADLINE:
2998
		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
2999
		break;
3000
	case MSR_IA32_TSC_ADJUST:
3001
		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
3002
		break;
3003
	case MSR_IA32_MISC_ENABLE:
3004
		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
3005
		break;
3006 3007 3008 3009
	case MSR_IA32_SMBASE:
		if (!msr_info->host_initiated)
			return 1;
		msr_info->data = vcpu->arch.smbase;
3010
		break;
3011 3012 3013
	case MSR_SMI_COUNT:
		msr_info->data = vcpu->arch.smi_count;
		break;
3014 3015
	case MSR_IA32_PERF_STATUS:
		/* TSC increment by tick */
3016
		msr_info->data = 1000ULL;
3017
		/* CPU multiplier */
3018
		msr_info->data |= (((uint64_t)4ULL) << 40);
3019
		break;
3020
	case MSR_EFER:
3021
		msr_info->data = vcpu->arch.efer;
3022
		break;
3023
	case MSR_KVM_WALL_CLOCK:
3024
	case MSR_KVM_WALL_CLOCK_NEW:
3025
		msr_info->data = vcpu->kvm->arch.wall_clock;
3026 3027
		break;
	case MSR_KVM_SYSTEM_TIME:
3028
	case MSR_KVM_SYSTEM_TIME_NEW:
3029
		msr_info->data = vcpu->arch.time;
3030
		break;
3031
	case MSR_KVM_ASYNC_PF_EN:
3032
		msr_info->data = vcpu->arch.apf.msr_val;
3033
		break;
3034
	case MSR_KVM_STEAL_TIME:
3035
		msr_info->data = vcpu->arch.st.msr_val;
3036
		break;
3037
	case MSR_KVM_PV_EOI_EN:
3038
		msr_info->data = vcpu->arch.pv_eoi.msr_val;
3039
		break;
3040 3041 3042
	case MSR_KVM_POLL_CONTROL:
		msr_info->data = vcpu->arch.msr_kvm_poll_control;
		break;
Huang Ying's avatar
Huang Ying committed
3043 3044 3045 3046 3047
	case MSR_IA32_P5_MC_ADDR:
	case MSR_IA32_P5_MC_TYPE:
	case MSR_IA32_MCG_CAP:
	case MSR_IA32_MCG_CTL:
	case MSR_IA32_MCG_STATUS:
3048
	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3049 3050
		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
				   msr_info->host_initiated);
3051 3052 3053 3054 3055 3056 3057 3058 3059 3060
	case MSR_K7_CLK_CTL:
		/*
		 * Provide expected ramp-up count for K7. All other
		 * are set to zero, indicating minimum divisors for
		 * every field.
		 *
		 * This prevents guest kernels on AMD host with CPU
		 * type 6, model 8 and higher from exploding due to
		 * the rdmsr failing.
		 */
3061
		msr_info->data = 0x20000000;
3062
		break;
3063
	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
3064 3065
	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
	case HV_X64_MSR_CRASH_CTL:
3066
	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
3067 3068 3069
	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_CONTROL:
	case HV_X64_MSR_TSC_EMULATION_STATUS:
3070
		return kvm_hv_get_msr_common(vcpu,
3071 3072
					     msr_info->index, &msr_info->data,
					     msr_info->host_initiated);
3073
		break;
3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084
	case MSR_IA32_BBL_CR_CTL3:
		/* This legacy MSR exists but isn't fully documented in current
		 * silicon.  It is however accessed by winxp in very narrow
		 * scenarios where it sets bit #19, itself documented as
		 * a "reserved" bit.  Best effort attempt to source coherent
		 * read data here should the balance of the register be
		 * interpreted by the guest:
		 *
		 * L2 cache control register 3: 64GB range, 256KB size,
		 * enabled, latency 0x1, configured
		 */
3085
		msr_info->data = 0xbe702111;
3086
		break;
3087
	case MSR_AMD64_OSVW_ID_LENGTH:
3088
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3089
			return 1;
3090
		msr_info->data = vcpu->arch.osvw.length;
3091 3092
		break;
	case MSR_AMD64_OSVW_STATUS:
3093
		if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
3094
			return 1;
3095
		msr_info->data = vcpu->arch.osvw.status;
3096
		break;
3097
	case MSR_PLATFORM_INFO:
3098 3099 3100
		if (!msr_info->host_initiated &&
		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
			return 1;
3101 3102 3103 3104 3105
		msr_info->data = vcpu->arch.msr_platform_info;
		break;
	case MSR_MISC_FEATURES_ENABLES:
		msr_info->data = vcpu->arch.msr_misc_features_enables;
		break;
3106 3107 3108
	case MSR_K7_HWCR:
		msr_info->data = vcpu->arch.msr_hwcr;
		break;
3109
	default:
3110
		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3111
			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
3112
		if (!ignore_msrs) {
3113 3114
			vcpu_debug_ratelimited(vcpu, "unhandled rdmsr: 0x%x\n",
					       msr_info->index);
3115 3116
			return 1;
		} else {
3117 3118 3119
			if (report_ignored_msrs)
				vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n",
					msr_info->index);
3120
			msr_info->data = 0;
3121 3122
		}
		break;
3123 3124 3125 3126 3127
	}
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

3128 3129 3130 3131 3132 3133 3134 3135 3136 3137
/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
		    struct kvm_msr_entry *entries,
		    int (*do_msr)(struct kvm_vcpu *vcpu,
				  unsigned index, u64 *data))
{
3138
	int i;
3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162

	for (i = 0; i < msrs->nmsrs; ++i)
		if (do_msr(vcpu, entries[i].index, &entries[i].data))
			break;

	return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
		  int (*do_msr)(struct kvm_vcpu *vcpu,
				unsigned index, u64 *data),
		  int writeback)
{
	struct kvm_msrs msrs;
	struct kvm_msr_entry *entries;
	int r, n;
	unsigned size;

	r = -EFAULT;
3163
	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
3164 3165 3166 3167 3168 3169 3170
		goto out;

	r = -E2BIG;
	if (msrs.nmsrs >= MAX_IO_MSRS)
		goto out;

	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
3171 3172 3173
	entries = memdup_user(user_msrs->entries, size);
	if (IS_ERR(entries)) {
		r = PTR_ERR(entries);
3174
		goto out;
3175
	}
3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187

	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
	if (r < 0)
		goto out_free;

	r = -EFAULT;
	if (writeback && copy_to_user(user_msrs->entries, entries, size))
		goto out_free;

	r = n;

out_free:
3188
	kfree(entries);
3189 3190 3191 3192
out:
	return r;
}

3193 3194 3195
static inline bool kvm_can_mwait_in_guest(void)
{
	return boot_cpu_has(X86_FEATURE_MWAIT) &&
3196 3197
		!boot_cpu_has_bug(X86_BUG_MONITOR) &&
		boot_cpu_has(X86_FEATURE_ARAT);
3198 3199
}

3200
int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
3201
{
3202
	int r = 0;
3203 3204 3205 3206 3207 3208

	switch (ext) {
	case KVM_CAP_IRQCHIP:
	case KVM_CAP_HLT:
	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
	case KVM_CAP_SET_TSS_ADDR:
3209
	case KVM_CAP_EXT_CPUID:
3210
	case KVM_CAP_EXT_EMUL_CPUID:
3211
	case KVM_CAP_CLOCKSOURCE:
Sheng Yang's avatar
Sheng Yang committed
3212
	case KVM_CAP_PIT:
3213
	case KVM_CAP_NOP_IO_DELAY:
3214
	case KVM_CAP_MP_STATE:
3215
	case KVM_CAP_SYNC_MMU:
3216
	case KVM_CAP_USER_NMI:
3217
	case KVM_CAP_REINJECT_CONTROL:
3218
	case KVM_CAP_IRQ_INJECT_STATUS:
Gregory Haskins's avatar
Gregory Haskins committed
3219
	case KVM_CAP_IOEVENTFD:
3220
	case KVM_CAP_IOEVENTFD_NO_LENGTH:
3221
	case KVM_CAP_PIT2:
3222
	case KVM_CAP_PIT_STATE2:
3223
	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
Ed Swierk's avatar
Ed Swierk committed
3224
	case KVM_CAP_XEN_HVM:
3225
	case KVM_CAP_VCPU_EVENTS:
3226
	case KVM_CAP_HYPERV:
3227
	case KVM_CAP_HYPERV_VAPIC:
3228
	case KVM_CAP_HYPERV_SPIN:
3229
	case KVM_CAP_HYPERV_SYNIC:
3230
	case KVM_CAP_HYPERV_SYNIC2:
3231
	case KVM_CAP_HYPERV_VP_INDEX:
3232
	case KVM_CAP_HYPERV_EVENTFD:
3233
	case KVM_CAP_HYPERV_TLBFLUSH:
3234
	case KVM_CAP_HYPERV_SEND_IPI:
3235
	case KVM_CAP_HYPERV_CPUID:
3236
	case KVM_CAP_PCI_SEGMENT:
3237
	case KVM_CAP_DEBUGREGS:
3238
	case KVM_CAP_X86_ROBUST_SINGLESTEP:
3239
	case KVM_CAP_XSAVE:
3240
	case KVM_CAP_ASYNC_PF:
3241
	case KVM_CAP_GET_TSC_KHZ:
3242
	case KVM_CAP_KVMCLOCK_CTRL:
3243
	case KVM_CAP_READONLY_MEM:
3244
	case KVM_CAP_HYPERV_TIME:
3245
	case KVM_CAP_IOAPIC_POLARITY_IGNORED:
3246
	case KVM_CAP_TSC_DEADLINE_TIMER:
3247
	case KVM_CAP_DISABLE_QUIRKS:
3248
	case KVM_CAP_SET_BOOT_CPU_ID:
3249
 	case KVM_CAP_SPLIT_IRQCHIP:
3250
	case KVM_CAP_IMMEDIATE_EXIT:
Eric Hankland's avatar
Eric Hankland committed
3251
	case KVM_CAP_PMU_EVENT_FILTER:
3252
	case KVM_CAP_GET_MSR_FEATURES:
3253
	case KVM_CAP_MSR_PLATFORM_INFO:
3254
	case KVM_CAP_EXCEPTION_PAYLOAD:
3255 3256
		r = 1;
		break;
Ken Hofsass's avatar
Ken Hofsass committed
3257 3258 3259
	case KVM_CAP_SYNC_REGS:
		r = KVM_SYNC_X86_VALID_FIELDS;
		break;
3260 3261 3262
	case KVM_CAP_ADJUST_CLOCK:
		r = KVM_CLOCK_TSC_STABLE;
		break;
3263
	case KVM_CAP_X86_DISABLE_EXITS:
3264 3265
		r |=  KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE |
		      KVM_X86_DISABLE_EXITS_CSTATE;
3266 3267
		if(kvm_can_mwait_in_guest())
			r |= KVM_X86_DISABLE_EXITS_MWAIT;
3268
		break;
3269 3270 3271 3272 3273 3274 3275 3276 3277
	case KVM_CAP_X86_SMM:
		/* SMBASE is usually relocated above 1M on modern chipsets,
		 * and SMM handlers might indeed rely on 4G segment limits,
		 * so do not report SMM to be available if real mode is
		 * emulated via vm86 mode.  Still, do not go to great lengths
		 * to avoid userspace's usage of the feature, because it is a
		 * fringe case that is not enabled except via specific settings
		 * of the module parameters.
		 */
3278
		r = kvm_x86_ops->has_emulated_msr(MSR_IA32_SMBASE);
3279
		break;
3280 3281 3282
	case KVM_CAP_VAPIC:
		r = !kvm_x86_ops->cpu_has_accelerated_tpr();
		break;
3283
	case KVM_CAP_NR_VCPUS:
3284 3285 3286
		r = KVM_SOFT_MAX_VCPUS;
		break;
	case KVM_CAP_MAX_VCPUS:
3287 3288
		r = KVM_MAX_VCPUS;
		break;
3289 3290 3291
	case KVM_CAP_MAX_VCPU_ID:
		r = KVM_MAX_VCPU_ID;
		break;
3292 3293
	case KVM_CAP_PV_MMU:	/* obsolete */
		r = 0;
3294
		break;
Huang Ying's avatar
Huang Ying committed
3295 3296 3297
	case KVM_CAP_MCE:
		r = KVM_MAX_MCE_BANKS;
		break;
3298
	case KVM_CAP_XCRS:
3299
		r = boot_cpu_has(X86_FEATURE_XSAVE);
3300
		break;
3301 3302 3303
	case KVM_CAP_TSC_CONTROL:
		r = kvm_has_tsc_control;
		break;
3304 3305 3306
	case KVM_CAP_X2APIC_API:
		r = KVM_X2APIC_API_VALID_FLAGS;
		break;
3307 3308
	case KVM_CAP_NESTED_STATE:
		r = kvm_x86_ops->get_nested_state ?
3309
			kvm_x86_ops->get_nested_state(NULL, NULL, 0) : 0;
3310
		break;
3311
	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
3312 3313 3314 3315
		r = kvm_x86_ops->enable_direct_tlbflush != NULL;
		break;
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
		r = kvm_x86_ops->nested_enable_evmcs != NULL;
3316
		break;
3317 3318 3319 3320 3321 3322 3323
	default:
		break;
	}
	return r;

}

3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336
long kvm_arch_dev_ioctl(struct file *filp,
			unsigned int ioctl, unsigned long arg)
{
	void __user *argp = (void __user *)arg;
	long r;

	switch (ioctl) {
	case KVM_GET_MSR_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned n;

		r = -EFAULT;
3337
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
3338 3339
			goto out;
		n = msr_list.nmsrs;
3340
		msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
3341
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
3342 3343
			goto out;
		r = -E2BIG;
Jan Kiszka's avatar
Jan Kiszka committed
3344
		if (n < msr_list.nmsrs)
3345 3346 3347 3348 3349
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
				 num_msrs_to_save * sizeof(u32)))
			goto out;
Jan Kiszka's avatar
Jan Kiszka committed
3350
		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
3351
				 &emulated_msrs,
3352
				 num_emulated_msrs * sizeof(u32)))
3353 3354 3355 3356
			goto out;
		r = 0;
		break;
	}
3357 3358
	case KVM_GET_SUPPORTED_CPUID:
	case KVM_GET_EMULATED_CPUID: {
3359 3360 3361 3362
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
3363
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
3364
			goto out;
3365 3366 3367

		r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
					    ioctl);
3368 3369 3370 3371
		if (r)
			goto out;

		r = -EFAULT;
3372
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
3373 3374 3375 3376
			goto out;
		r = 0;
		break;
	}
Huang Ying's avatar
Huang Ying committed
3377 3378
	case KVM_X86_GET_MCE_CAP_SUPPORTED: {
		r = -EFAULT;
3379 3380
		if (copy_to_user(argp, &kvm_mce_cap_supported,
				 sizeof(kvm_mce_cap_supported)))
Huang Ying's avatar
Huang Ying committed
3381 3382 3383
			goto out;
		r = 0;
		break;
3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408
	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
		struct kvm_msr_list __user *user_msr_list = argp;
		struct kvm_msr_list msr_list;
		unsigned int n;

		r = -EFAULT;
		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
			goto out;
		n = msr_list.nmsrs;
		msr_list.nmsrs = num_msr_based_features;
		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
			goto out;
		r = -E2BIG;
		if (n < msr_list.nmsrs)
			goto out;
		r = -EFAULT;
		if (copy_to_user(user_msr_list->indices, &msr_based_features,
				 num_msr_based_features * sizeof(u32)))
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_MSRS:
		r = msr_io(NULL, argp, do_get_msr_feature, 1);
		break;
Huang Ying's avatar
Huang Ying committed
3409
	}
3410 3411 3412 3413 3414 3415 3416
	default:
		r = -EINVAL;
	}
out:
	return r;
}

3417 3418 3419 3420 3421 3422 3423
static void wbinvd_ipi(void *garbage)
{
	wbinvd();
}

static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
3424
	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
3425 3426
}

3427 3428
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
3429 3430 3431 3432 3433 3434 3435 3436 3437
	/* Address WBINVD may be executed by guest */
	if (need_emulate_wbinvd(vcpu)) {
		if (kvm_x86_ops->has_wbinvd_exit())
			cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
		else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
			smp_call_function_single(vcpu->cpu,
					wbinvd_ipi, NULL, 1);
	}

3438
	kvm_x86_ops->vcpu_load(vcpu, cpu);
3439

3440 3441 3442 3443
	fpregs_assert_state_consistent();
	if (test_thread_flag(TIF_NEED_FPU_LOAD))
		switch_fpu_return();

3444 3445 3446 3447
	/* Apply any externally detected TSC adjustments (due to suspend) */
	if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
		adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
		vcpu->arch.tsc_offset_adjustment = 0;
3448
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3449
	}
3450

3451
	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
3452
		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
3453
				rdtsc() - vcpu->arch.last_host_tsc;
Zachary Amsden's avatar
Zachary Amsden committed
3454 3455
		if (tsc_delta < 0)
			mark_tsc_unstable("KVM discovered backwards TSC");
3456

3457
		if (kvm_check_tsc_unstable()) {
3458
			u64 offset = kvm_compute_tsc_offset(vcpu,
3459
						vcpu->arch.last_guest_tsc);
3460
			kvm_vcpu_write_tsc_offset(vcpu, offset);
Zachary Amsden's avatar
Zachary Amsden committed
3461 3462
			vcpu->arch.tsc_catchup = 1;
		}
3463 3464 3465 3466

		if (kvm_lapic_hv_timer_in_use(vcpu))
			kvm_lapic_restart_hv_timer(vcpu);

3467 3468 3469 3470 3471
		/*
		 * On a host with synchronized TSC, there is no need to update
		 * kvmclock on vcpu->cpu migration
		 */
		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
3472
			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
Zachary Amsden's avatar
Zachary Amsden committed
3473
		if (vcpu->cpu != cpu)
3474
			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
Zachary Amsden's avatar
Zachary Amsden committed
3475
		vcpu->cpu = cpu;
3476
	}
3477 3478

	kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3479 3480
}

3481 3482 3483 3484 3485
static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

3486
	vcpu->arch.st.steal.preempted = KVM_VCPU_PREEMPTED;
3487

3488
	kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.st.stime,
3489 3490 3491 3492 3493
			&vcpu->arch.st.steal.preempted,
			offsetof(struct kvm_steal_time, preempted),
			sizeof(vcpu->arch.st.steal.preempted));
}

3494 3495
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
3496
	int idx;
3497 3498 3499 3500

	if (vcpu->preempted)
		vcpu->arch.preempted_in_kernel = !kvm_x86_ops->get_cpl(vcpu);

3501 3502 3503 3504 3505 3506 3507 3508 3509
	/*
	 * Disable page faults because we're in atomic context here.
	 * kvm_write_guest_offset_cached() would call might_fault()
	 * that relies on pagefault_disable() to tell if there's a
	 * bug. NOTE: the write to guest memory may not go through if
	 * during postcopy live migration or if there's heavy guest
	 * paging.
	 */
	pagefault_disable();
3510 3511 3512 3513 3514
	/*
	 * kvm_memslots() will be called by
	 * kvm_write_guest_offset_cached() so take the srcu lock.
	 */
	idx = srcu_read_lock(&vcpu->kvm->srcu);
3515
	kvm_steal_time_set_preempted(vcpu);
3516
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
3517
	pagefault_enable();
3518
	kvm_x86_ops->vcpu_put(vcpu);
3519
	vcpu->arch.last_host_tsc = rdtsc();
3520
	/*
3521 3522 3523
	 * If userspace has set any breakpoints or watchpoints, dr6 is restored
	 * on every vmexit, but if not, we might have a stale dr6 from the
	 * guest. do_debug expects dr6 to be cleared after it runs, do the same.
3524
	 */
3525
	set_debugreg(0, 6);
3526 3527 3528 3529 3530
}

static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3531
	if (vcpu->arch.apicv_active)
3532 3533
		kvm_x86_ops->sync_pir_to_irr(vcpu);

3534
	return kvm_apic_get_state(vcpu, s);
3535 3536 3537 3538 3539
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
				    struct kvm_lapic_state *s)
{
3540 3541 3542 3543 3544
	int r;

	r = kvm_apic_set_state(vcpu, s);
	if (r)
		return r;
3545
	update_cr8_intercept(vcpu);
3546 3547 3548 3549

	return 0;
}

3550 3551 3552 3553 3554 3555
static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
{
	return (!lapic_in_kernel(vcpu) ||
		kvm_apic_accept_pic_intr(vcpu));
}

3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569
/*
 * if userspace requested an interrupt window, check that the
 * interrupt window is open.
 *
 * No need to exit to userspace if we already have an interrupt queued.
 */
static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
{
	return kvm_arch_interrupt_allowed(vcpu) &&
		!kvm_cpu_has_interrupt(vcpu) &&
		!kvm_event_needs_reinjection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);
}

3570 3571 3572
static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
				    struct kvm_interrupt *irq)
{
3573
	if (irq->irq >= KVM_NR_INTERRUPTS)
3574
		return -EINVAL;
3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586

	if (!irqchip_in_kernel(vcpu->kvm)) {
		kvm_queue_interrupt(vcpu, irq->irq, false);
		kvm_make_request(KVM_REQ_EVENT, vcpu);
		return 0;
	}

	/*
	 * With in-kernel LAPIC, we only use this to inject EXTINT, so
	 * fail for in-kernel 8259.
	 */
	if (pic_in_kernel(vcpu->kvm))
3587 3588
		return -ENXIO;

3589 3590
	if (vcpu->arch.pending_external_vector != -1)
		return -EEXIST;
3591

3592
	vcpu->arch.pending_external_vector = irq->irq;
3593
	kvm_make_request(KVM_REQ_EVENT, vcpu);
3594 3595 3596
	return 0;
}

3597 3598 3599 3600 3601 3602 3603
static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
{
	kvm_inject_nmi(vcpu);

	return 0;
}

3604 3605
static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
{
3606 3607
	kvm_make_request(KVM_REQ_SMI, vcpu);

3608 3609 3610
	return 0;
}

3611 3612 3613 3614 3615 3616 3617 3618 3619
static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
					   struct kvm_tpr_access_ctl *tac)
{
	if (tac->flags)
		return -EINVAL;
	vcpu->arch.tpr_access_reporting = !!tac->enabled;
	return 0;
}

Huang Ying's avatar
Huang Ying committed
3620 3621 3622 3623 3624 3625 3626
static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
					u64 mcg_cap)
{
	int r;
	unsigned bank_num = mcg_cap & 0xff, bank;

	r = -EINVAL;
3627
	if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
Huang Ying's avatar
Huang Ying committed
3628
		goto out;
3629
	if (mcg_cap & ~(kvm_mce_cap_supported | 0xff | 0xff0000))
Huang Ying's avatar
Huang Ying committed
3630 3631 3632 3633 3634 3635 3636 3637 3638
		goto out;
	r = 0;
	vcpu->arch.mcg_cap = mcg_cap;
	/* Init IA32_MCG_CTL to all 1s */
	if (mcg_cap & MCG_CTL_P)
		vcpu->arch.mcg_ctl = ~(u64)0;
	/* Init IA32_MCi_CTL to all 1s */
	for (bank = 0; bank < bank_num; bank++)
		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
3639

3640
	kvm_x86_ops->setup_mce(vcpu);
Huang Ying's avatar
Huang Ying committed
3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669
out:
	return r;
}

static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
				      struct kvm_x86_mce *mce)
{
	u64 mcg_cap = vcpu->arch.mcg_cap;
	unsigned bank_num = mcg_cap & 0xff;
	u64 *banks = vcpu->arch.mce_banks;

	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
		return -EINVAL;
	/*
	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
	 * reporting is disabled
	 */
	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
	    vcpu->arch.mcg_ctl != ~(u64)0)
		return 0;
	banks += 4 * mce->bank;
	/*
	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
	 * reporting is disabled for the bank
	 */
	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
		return 0;
	if (mce->status & MCI_STATUS_UC) {
		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
3670
		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
3671
			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
Huang Ying's avatar
Huang Ying committed
3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692
			return 0;
		}
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		vcpu->arch.mcg_status = mce->mcg_status;
		banks[1] = mce->status;
		kvm_queue_exception(vcpu, MC_VECTOR);
	} else if (!(banks[1] & MCI_STATUS_VAL)
		   || !(banks[1] & MCI_STATUS_UC)) {
		if (banks[1] & MCI_STATUS_VAL)
			mce->status |= MCI_STATUS_OVER;
		banks[2] = mce->addr;
		banks[3] = mce->misc;
		banks[1] = mce->status;
	} else
		banks[1] |= MCI_STATUS_OVER;
	return 0;
}

3693 3694 3695
static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
					       struct kvm_vcpu_events *events)
{
Avi Kivity's avatar
Avi Kivity committed
3696
	process_nmi(vcpu);
3697

3698
	/*
3699 3700 3701 3702
	 * The API doesn't provide the instruction length for software
	 * exceptions, so don't report them. As long as the guest RIP
	 * isn't advanced, we should expect to encounter the exception
	 * again.
3703
	 */
3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718
	if (kvm_exception_is_soft(vcpu->arch.exception.nr)) {
		events->exception.injected = 0;
		events->exception.pending = 0;
	} else {
		events->exception.injected = vcpu->arch.exception.injected;
		events->exception.pending = vcpu->arch.exception.pending;
		/*
		 * For ABI compatibility, deliberately conflate
		 * pending and injected exceptions when
		 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
		 */
		if (!vcpu->kvm->arch.exception_payload_enabled)
			events->exception.injected |=
				vcpu->arch.exception.pending;
	}
3719 3720 3721
	events->exception.nr = vcpu->arch.exception.nr;
	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
	events->exception.error_code = vcpu->arch.exception.error_code;
3722 3723
	events->exception_has_payload = vcpu->arch.exception.has_payload;
	events->exception_payload = vcpu->arch.exception.payload;
3724

3725
	events->interrupt.injected =
3726
		vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
3727
	events->interrupt.nr = vcpu->arch.interrupt.nr;
3728
	events->interrupt.soft = 0;
3729
	events->interrupt.shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
3730 3731

	events->nmi.injected = vcpu->arch.nmi_injected;
Avi Kivity's avatar
Avi Kivity committed
3732
	events->nmi.pending = vcpu->arch.nmi_pending != 0;
3733
	events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
3734
	events->nmi.pad = 0;
3735

3736
	events->sipi_vector = 0; /* never valid when reporting to user space */
3737

3738 3739 3740 3741 3742 3743
	events->smi.smm = is_smm(vcpu);
	events->smi.pending = vcpu->arch.smi_pending;
	events->smi.smm_inside_nmi =
		!!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
	events->smi.latched_init = kvm_lapic_latched_init(vcpu);

3744
	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
3745 3746
			 | KVM_VCPUEVENT_VALID_SHADOW
			 | KVM_VCPUEVENT_VALID_SMM);
3747 3748 3749
	if (vcpu->kvm->arch.exception_payload_enabled)
		events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;

3750
	memset(&events->reserved, 0, sizeof(events->reserved));
3751 3752
}

3753
static void kvm_smm_changed(struct kvm_vcpu *vcpu);
3754

3755 3756 3757
static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
					      struct kvm_vcpu_events *events)
{
3758
	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
3759
			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
3760
			      | KVM_VCPUEVENT_VALID_SHADOW
3761 3762
			      | KVM_VCPUEVENT_VALID_SMM
			      | KVM_VCPUEVENT_VALID_PAYLOAD))
3763 3764
		return -EINVAL;

3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778
	if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
		if (!vcpu->kvm->arch.exception_payload_enabled)
			return -EINVAL;
		if (events->exception.pending)
			events->exception.injected = 0;
		else
			events->exception_has_payload = 0;
	} else {
		events->exception.pending = 0;
		events->exception_has_payload = 0;
	}

	if ((events->exception.injected || events->exception.pending) &&
	    (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
3779 3780
		return -EINVAL;

3781 3782 3783 3784 3785 3786
	/* INITs are latched while in SMM */
	if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
	    (events->smi.smm || events->smi.pending) &&
	    vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
		return -EINVAL;

Avi Kivity's avatar
Avi Kivity committed
3787
	process_nmi(vcpu);
3788 3789
	vcpu->arch.exception.injected = events->exception.injected;
	vcpu->arch.exception.pending = events->exception.pending;
3790 3791 3792
	vcpu->arch.exception.nr = events->exception.nr;
	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
	vcpu->arch.exception.error_code = events->exception.error_code;
3793 3794
	vcpu->arch.exception.has_payload = events->exception_has_payload;
	vcpu->arch.exception.payload = events->exception_payload;
3795

3796
	vcpu->arch.interrupt.injected = events->interrupt.injected;
3797 3798
	vcpu->arch.interrupt.nr = events->interrupt.nr;
	vcpu->arch.interrupt.soft = events->interrupt.soft;
3799 3800 3801
	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
		kvm_x86_ops->set_interrupt_shadow(vcpu,
						  events->interrupt.shadow);
3802 3803

	vcpu->arch.nmi_injected = events->nmi.injected;
3804 3805
	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
		vcpu->arch.nmi_pending = events->nmi.pending;
3806 3807
	kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);

3808
	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
3809
	    lapic_in_kernel(vcpu))
3810
		vcpu->arch.apic->sipi_vector = events->sipi_vector;
3811

3812
	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
3813 3814 3815 3816 3817 3818 3819
		if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
			if (events->smi.smm)
				vcpu->arch.hflags |= HF_SMM_MASK;
			else
				vcpu->arch.hflags &= ~HF_SMM_MASK;
			kvm_smm_changed(vcpu);
		}
3820

3821
		vcpu->arch.smi_pending = events->smi.pending;
3822 3823 3824 3825

		if (events->smi.smm) {
			if (events->smi.smm_inside_nmi)
				vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
3826
			else
3827 3828 3829 3830 3831 3832 3833
				vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
			if (lapic_in_kernel(vcpu)) {
				if (events->smi.latched_init)
					set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
				else
					clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
			}
3834 3835 3836
		}
	}

3837 3838
	kvm_make_request(KVM_REQ_EVENT, vcpu);

3839 3840 3841
	return 0;
}

3842 3843 3844
static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
					     struct kvm_debugregs *dbgregs)
{
Jan Kiszka's avatar
Jan Kiszka committed
3845 3846
	unsigned long val;

3847
	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
3848
	kvm_get_dr(vcpu, 6, &val);
Jan Kiszka's avatar
Jan Kiszka committed
3849
	dbgregs->dr6 = val;
3850 3851
	dbgregs->dr7 = vcpu->arch.dr7;
	dbgregs->flags = 0;
3852
	memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
3853 3854 3855 3856 3857 3858 3859 3860
}

static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
					    struct kvm_debugregs *dbgregs)
{
	if (dbgregs->flags)
		return -EINVAL;

3861 3862 3863 3864 3865
	if (dbgregs->dr6 & ~0xffffffffull)
		return -EINVAL;
	if (dbgregs->dr7 & ~0xffffffffull)
		return -EINVAL;

3866
	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
3867
	kvm_update_dr0123(vcpu);
3868
	vcpu->arch.dr6 = dbgregs->dr6;
Jan Kiszka's avatar
Jan Kiszka committed
3869
	kvm_update_dr6(vcpu);
3870
	vcpu->arch.dr7 = dbgregs->dr7;
3871
	kvm_update_dr7(vcpu);
3872 3873 3874 3875

	return 0;
}

3876 3877 3878 3879
#define XSTATE_COMPACTION_ENABLED (1ULL << 63)

static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu)
{
3880
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
3881
	u64 xstate_bv = xsave->header.xfeatures;
3882 3883 3884 3885 3886 3887 3888 3889 3890
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(dest, xsave, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV */
3891
	xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE;
3892 3893 3894 3895 3896 3897
	*(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv;

	/*
	 * Copy each region from the possibly compacted offset to the
	 * non-compacted offset.
	 */
Dave Hansen's avatar
Dave Hansen committed
3898
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
3899
	while (valid) {
3900 3901 3902
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *src = get_xsave_addr(xsave, xfeature_nr);
3903 3904 3905

		if (src) {
			u32 size, offset, ecx, edx;
3906
			cpuid_count(XSTATE_CPUID, xfeature_nr,
3907
				    &size, &offset, &ecx, &edx);
3908
			if (xfeature_nr == XFEATURE_PKRU)
3909 3910 3911 3912 3913
				memcpy(dest + offset, &vcpu->arch.pkru,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest + offset, src, size);

3914 3915
		}

3916
		valid -= xfeature_mask;
3917 3918 3919 3920 3921
	}
}

static void load_xsave(struct kvm_vcpu *vcpu, u8 *src)
{
3922
	struct xregs_state *xsave = &vcpu->arch.guest_fpu->state.xsave;
3923 3924 3925 3926 3927 3928 3929 3930 3931 3932
	u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET);
	u64 valid;

	/*
	 * Copy legacy XSAVE area, to avoid complications with CPUID
	 * leaves 0 and 1 in the loop below.
	 */
	memcpy(xsave, src, XSAVE_HDR_OFFSET);

	/* Set XSTATE_BV and possibly XCOMP_BV.  */
3933
	xsave->header.xfeatures = xstate_bv;
3934
	if (boot_cpu_has(X86_FEATURE_XSAVES))
3935
		xsave->header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED;
3936 3937 3938 3939 3940

	/*
	 * Copy each region from the non-compacted offset to the
	 * possibly compacted offset.
	 */
Dave Hansen's avatar
Dave Hansen committed
3941
	valid = xstate_bv & ~XFEATURE_MASK_FPSSE;
3942
	while (valid) {
3943 3944 3945
		u64 xfeature_mask = valid & -valid;
		int xfeature_nr = fls64(xfeature_mask) - 1;
		void *dest = get_xsave_addr(xsave, xfeature_nr);
3946 3947 3948

		if (dest) {
			u32 size, offset, ecx, edx;
3949
			cpuid_count(XSTATE_CPUID, xfeature_nr,
3950
				    &size, &offset, &ecx, &edx);
3951
			if (xfeature_nr == XFEATURE_PKRU)
3952 3953 3954 3955
				memcpy(&vcpu->arch.pkru, src + offset,
				       sizeof(vcpu->arch.pkru));
			else
				memcpy(dest, src + offset, size);
3956
		}
3957

3958
		valid -= xfeature_mask;
3959 3960 3961
	}
}

3962 3963 3964
static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
					 struct kvm_xsave *guest_xsave)
{
3965
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
3966 3967
		memset(guest_xsave, 0, sizeof(struct kvm_xsave));
		fill_xsave((u8 *) guest_xsave->region, vcpu);
3968
	} else {
3969
		memcpy(guest_xsave->region,
3970
			&vcpu->arch.guest_fpu->state.fxsave,
3971
			sizeof(struct fxregs_state));
3972
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
Dave Hansen's avatar
Dave Hansen committed
3973
			XFEATURE_MASK_FPSSE;
3974 3975 3976
	}
}

3977 3978
#define XSAVE_MXCSR_OFFSET 24

3979 3980 3981 3982 3983
static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
					struct kvm_xsave *guest_xsave)
{
	u64 xstate_bv =
		*(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
3984
	u32 mxcsr = *(u32 *)&guest_xsave->region[XSAVE_MXCSR_OFFSET / sizeof(u32)];
3985

3986
	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
3987 3988 3989 3990 3991
		/*
		 * Here we allow setting states that are not present in
		 * CPUID leaf 0xD, index 0, EDX:EAX.  This is for compatibility
		 * with old userspace.
		 */
3992 3993
		if (xstate_bv & ~kvm_supported_xcr0() ||
			mxcsr & ~mxcsr_feature_mask)
3994
			return -EINVAL;
3995
		load_xsave(vcpu, (u8 *)guest_xsave->region);
3996
	} else {
3997 3998
		if (xstate_bv & ~XFEATURE_MASK_FPSSE ||
			mxcsr & ~mxcsr_feature_mask)
3999
			return -EINVAL;
4000
		memcpy(&vcpu->arch.guest_fpu->state.fxsave,
4001
			guest_xsave->region, sizeof(struct fxregs_state));
4002 4003 4004 4005 4006 4007 4008
	}
	return 0;
}

static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
					struct kvm_xcrs *guest_xcrs)
{
4009
	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024
		guest_xcrs->nr_xcrs = 0;
		return;
	}

	guest_xcrs->nr_xcrs = 1;
	guest_xcrs->flags = 0;
	guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
	guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
}

static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
				       struct kvm_xcrs *guest_xcrs)
{
	int i, r = 0;

4025
	if (!boot_cpu_has(X86_FEATURE_XSAVE))
4026 4027 4028 4029 4030 4031 4032
		return -EINVAL;

	if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
		return -EINVAL;

	for (i = 0; i < guest_xcrs->nr_xcrs; i++)
		/* Only support XCR0 currently */
4033
		if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
4034
			r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
4035
				guest_xcrs->xcrs[i].value);
4036 4037 4038 4039 4040 4041 4042
			break;
		}
	if (r)
		r = -EINVAL;
	return r;
}

4043 4044 4045 4046 4047 4048 4049 4050
/*
 * kvm_set_guest_paused() indicates to the guest kernel that it has been
 * stopped by the hypervisor.  This function will be called from the host only.
 * EINVAL is returned when the host attempts to set the flag for a guest that
 * does not support pv clocks.
 */
static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
{
4051
	if (!vcpu->arch.pv_time_enabled)
4052
		return -EINVAL;
4053
	vcpu->arch.pvclock_set_guest_stopped_request = true;
4054 4055 4056 4057
	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	return 0;
}

4058 4059 4060
static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
				     struct kvm_enable_cap *cap)
{
4061 4062 4063 4064
	int r;
	uint16_t vmcs_version;
	void __user *user_ptr;

4065 4066 4067 4068
	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
4069 4070 4071
	case KVM_CAP_HYPERV_SYNIC2:
		if (cap->args[0])
			return -EINVAL;
4072 4073
		/* fall through */

4074
	case KVM_CAP_HYPERV_SYNIC:
4075 4076
		if (!irqchip_in_kernel(vcpu->kvm))
			return -EINVAL;
4077 4078
		return kvm_hv_activate_synic(vcpu, cap->cap ==
					     KVM_CAP_HYPERV_SYNIC2);
4079
	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
4080 4081
		if (!kvm_x86_ops->nested_enable_evmcs)
			return -ENOTTY;
4082 4083 4084 4085 4086 4087 4088 4089
		r = kvm_x86_ops->nested_enable_evmcs(vcpu, &vmcs_version);
		if (!r) {
			user_ptr = (void __user *)(uintptr_t)cap->args[0];
			if (copy_to_user(user_ptr, &vmcs_version,
					 sizeof(vmcs_version)))
				r = -EFAULT;
		}
		return r;
4090 4091 4092 4093 4094
	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
		if (!kvm_x86_ops->enable_direct_tlbflush)
			return -ENOTTY;

		return kvm_x86_ops->enable_direct_tlbflush(vcpu);
4095

4096 4097 4098 4099 4100
	default:
		return -EINVAL;
	}
}

4101 4102 4103 4104 4105 4106
long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r;
4107 4108 4109 4110 4111 4112 4113
	union {
		struct kvm_lapic_state *lapic;
		struct kvm_xsave *xsave;
		struct kvm_xcrs *xcrs;
		void *buffer;
	} u;

4114 4115
	vcpu_load(vcpu);

4116
	u.buffer = NULL;
4117 4118
	switch (ioctl) {
	case KVM_GET_LAPIC: {
4119
		r = -EINVAL;
4120
		if (!lapic_in_kernel(vcpu))
4121
			goto out;
4122 4123
		u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
				GFP_KERNEL_ACCOUNT);
4124

4125
		r = -ENOMEM;
4126
		if (!u.lapic)
4127
			goto out;
4128
		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
4129 4130 4131
		if (r)
			goto out;
		r = -EFAULT;
4132
		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
4133 4134 4135 4136 4137
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
4138
		r = -EINVAL;
4139
		if (!lapic_in_kernel(vcpu))
4140
			goto out;
4141
		u.lapic = memdup_user(argp, sizeof(*u.lapic));
4142 4143 4144 4145
		if (IS_ERR(u.lapic)) {
			r = PTR_ERR(u.lapic);
			goto out_nofree;
		}
4146

4147
		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
4148 4149
		break;
	}
4150 4151 4152 4153
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
4154
		if (copy_from_user(&irq, argp, sizeof(irq)))
4155 4156 4157 4158
			goto out;
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
		break;
	}
4159 4160 4161 4162
	case KVM_NMI: {
		r = kvm_vcpu_ioctl_nmi(vcpu);
		break;
	}
4163 4164 4165 4166
	case KVM_SMI: {
		r = kvm_vcpu_ioctl_smi(vcpu);
		break;
	}
4167 4168 4169 4170 4171
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		r = -EFAULT;
4172
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4173 4174 4175 4176
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		break;
	}
4177 4178 4179 4180 4181
	case KVM_SET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4182
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4183 4184
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
4185
					      cpuid_arg->entries);
4186 4187 4188 4189 4190 4191 4192
		break;
	}
	case KVM_GET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
4193
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4194 4195
			goto out;
		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
4196
					      cpuid_arg->entries);
4197 4198 4199
		if (r)
			goto out;
		r = -EFAULT;
4200
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4201 4202 4203 4204
			goto out;
		r = 0;
		break;
	}
4205 4206
	case KVM_GET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4207
		r = msr_io(vcpu, argp, do_get_msr, 1);
4208
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4209
		break;
4210 4211 4212
	}
	case KVM_SET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
4213
		r = msr_io(vcpu, argp, do_set_msr, 0);
4214
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
4215
		break;
4216
	}
4217 4218 4219 4220
	case KVM_TPR_ACCESS_REPORTING: {
		struct kvm_tpr_access_ctl tac;

		r = -EFAULT;
4221
		if (copy_from_user(&tac, argp, sizeof(tac)))
4222 4223 4224 4225 4226
			goto out;
		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
		if (r)
			goto out;
		r = -EFAULT;
4227
		if (copy_to_user(argp, &tac, sizeof(tac)))
4228 4229 4230 4231
			goto out;
		r = 0;
		break;
	};
Avi Kivity's avatar
Avi Kivity committed
4232 4233
	case KVM_SET_VAPIC_ADDR: {
		struct kvm_vapic_addr va;
4234
		int idx;
Avi Kivity's avatar
Avi Kivity committed
4235 4236

		r = -EINVAL;
4237
		if (!lapic_in_kernel(vcpu))
Avi Kivity's avatar
Avi Kivity committed
4238 4239
			goto out;
		r = -EFAULT;
4240
		if (copy_from_user(&va, argp, sizeof(va)))
Avi Kivity's avatar
Avi Kivity committed
4241
			goto out;
4242
		idx = srcu_read_lock(&vcpu->kvm->srcu);
4243
		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
4244
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
Avi Kivity's avatar
Avi Kivity committed
4245 4246
		break;
	}
Huang Ying's avatar
Huang Ying committed
4247 4248 4249 4250
	case KVM_X86_SETUP_MCE: {
		u64 mcg_cap;

		r = -EFAULT;
4251
		if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
Huang Ying's avatar
Huang Ying committed
4252 4253 4254 4255 4256 4257 4258 4259
			goto out;
		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
		break;
	}
	case KVM_X86_SET_MCE: {
		struct kvm_x86_mce mce;

		r = -EFAULT;
4260
		if (copy_from_user(&mce, argp, sizeof(mce)))
Huang Ying's avatar
Huang Ying committed
4261 4262 4263 4264
			goto out;
		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
		break;
	}
4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);

		r = -EFAULT;
		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		r = -EFAULT;
		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
			break;

		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
		break;
	}
4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308
	case KVM_GET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);

		r = -EFAULT;
		if (copy_to_user(argp, &dbgregs,
				 sizeof(struct kvm_debugregs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		r = -EFAULT;
		if (copy_from_user(&dbgregs, argp,
				   sizeof(struct kvm_debugregs)))
			break;

		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
		break;
	}
4309
	case KVM_GET_XSAVE: {
4310
		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
4311
		r = -ENOMEM;
4312
		if (!u.xsave)
4313 4314
			break;

4315
		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
4316 4317

		r = -EFAULT;
4318
		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
4319 4320 4321 4322 4323
			break;
		r = 0;
		break;
	}
	case KVM_SET_XSAVE: {
4324
		u.xsave = memdup_user(argp, sizeof(*u.xsave));
4325 4326 4327 4328
		if (IS_ERR(u.xsave)) {
			r = PTR_ERR(u.xsave);
			goto out_nofree;
		}
4329

4330
		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
4331 4332 4333
		break;
	}
	case KVM_GET_XCRS: {
4334
		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
4335
		r = -ENOMEM;
4336
		if (!u.xcrs)
4337 4338
			break;

4339
		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
4340 4341

		r = -EFAULT;
4342
		if (copy_to_user(argp, u.xcrs,
4343 4344 4345 4346 4347 4348
				 sizeof(struct kvm_xcrs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XCRS: {
4349
		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
4350 4351 4352 4353
		if (IS_ERR(u.xcrs)) {
			r = PTR_ERR(u.xcrs);
			goto out_nofree;
		}
4354

4355
		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
4356 4357
		break;
	}
4358 4359 4360 4361 4362 4363 4364 4365 4366
	case KVM_SET_TSC_KHZ: {
		u32 user_tsc_khz;

		r = -EINVAL;
		user_tsc_khz = (u32)arg;

		if (user_tsc_khz >= kvm_max_guest_tsc_khz)
			goto out;

4367 4368 4369
		if (user_tsc_khz == 0)
			user_tsc_khz = tsc_khz;

4370 4371
		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
			r = 0;
4372 4373 4374 4375

		goto out;
	}
	case KVM_GET_TSC_KHZ: {
4376
		r = vcpu->arch.virtual_tsc_khz;
4377 4378
		goto out;
	}
4379 4380 4381 4382
	case KVM_KVMCLOCK_CTRL: {
		r = kvm_set_guest_paused(vcpu);
		goto out;
	}
4383 4384 4385 4386 4387 4388 4389 4390 4391
	case KVM_ENABLE_CAP: {
		struct kvm_enable_cap cap;

		r = -EFAULT;
		if (copy_from_user(&cap, argp, sizeof(cap)))
			goto out;
		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
		break;
	}
4392 4393 4394 4395 4396 4397 4398 4399 4400
	case KVM_GET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		u32 user_data_size;

		r = -EINVAL;
		if (!kvm_x86_ops->get_nested_state)
			break;

		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
4401
		r = -EFAULT;
4402
		if (get_user(user_data_size, &user_kvm_nested_state->size))
4403
			break;
4404 4405 4406 4407

		r = kvm_x86_ops->get_nested_state(vcpu, user_kvm_nested_state,
						  user_data_size);
		if (r < 0)
4408
			break;
4409 4410 4411

		if (r > user_data_size) {
			if (put_user(r, &user_kvm_nested_state->size))
4412 4413 4414 4415
				r = -EFAULT;
			else
				r = -E2BIG;
			break;
4416
		}
4417

4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428
		r = 0;
		break;
	}
	case KVM_SET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		struct kvm_nested_state kvm_state;

		r = -EINVAL;
		if (!kvm_x86_ops->set_nested_state)
			break;

4429
		r = -EFAULT;
4430
		if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
4431
			break;
4432

4433
		r = -EINVAL;
4434
		if (kvm_state.size < sizeof(kvm_state))
4435
			break;
4436 4437

		if (kvm_state.flags &
4438 4439
		    ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
		      | KVM_STATE_NESTED_EVMCS))
4440
			break;
4441 4442

		/* nested_run_pending implies guest_mode.  */
4443 4444
		if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
		    && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
4445
			break;
4446 4447 4448 4449

		r = kvm_x86_ops->set_nested_state(vcpu, user_kvm_nested_state, &kvm_state);
		break;
	}
4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468
	case KVM_GET_SUPPORTED_HV_CPUID: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
			goto out;

		r = kvm_vcpu_ioctl_get_hv_cpuid(vcpu, &cpuid,
						cpuid_arg->entries);
		if (r)
			goto out;

		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
			goto out;
		r = 0;
		break;
	}
4469 4470 4471 4472
	default:
		r = -EINVAL;
	}
out:
4473
	kfree(u.buffer);
4474 4475
out_nofree:
	vcpu_put(vcpu);
4476 4477 4478
	return r;
}

4479
vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4480 4481 4482 4483
{
	return VM_FAULT_SIGBUS;
}

4484 4485 4486 4487 4488
static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
{
	int ret;

	if (addr > (unsigned int)(-3 * PAGE_SIZE))
4489
		return -EINVAL;
4490 4491 4492 4493
	ret = kvm_x86_ops->set_tss_addr(kvm, addr);
	return ret;
}

4494 4495 4496
static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
					      u64 ident_addr)
{
4497
	return kvm_x86_ops->set_identity_map_addr(kvm, ident_addr);
4498 4499
}

4500
static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
4501
					 unsigned long kvm_nr_mmu_pages)
4502 4503 4504 4505
{
	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
		return -EINVAL;

4506
	mutex_lock(&kvm->slots_lock);
4507 4508

	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
4509
	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
4510

4511
	mutex_unlock(&kvm->slots_lock);
4512 4513 4514
	return 0;
}

4515
static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
4516
{
4517
	return kvm->arch.n_max_mmu_pages;
4518 4519 4520 4521
}

static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4522
	struct kvm_pic *pic = kvm->arch.vpic;
4523 4524 4525 4526 4527
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4528
		memcpy(&chip->chip.pic, &pic->pics[0],
4529 4530 4531
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4532
		memcpy(&chip->chip.pic, &pic->pics[1],
4533 4534 4535
			sizeof(struct kvm_pic_state));
		break;
	case KVM_IRQCHIP_IOAPIC:
4536
		kvm_get_ioapic(kvm, &chip->chip.ioapic);
4537 4538 4539 4540 4541 4542 4543 4544 4545 4546
		break;
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
{
4547
	struct kvm_pic *pic = kvm->arch.vpic;
4548 4549 4550 4551 4552
	int r;

	r = 0;
	switch (chip->chip_id) {
	case KVM_IRQCHIP_PIC_MASTER:
4553 4554
		spin_lock(&pic->lock);
		memcpy(&pic->pics[0], &chip->chip.pic,
4555
			sizeof(struct kvm_pic_state));
4556
		spin_unlock(&pic->lock);
4557 4558
		break;
	case KVM_IRQCHIP_PIC_SLAVE:
4559 4560
		spin_lock(&pic->lock);
		memcpy(&pic->pics[1], &chip->chip.pic,
4561
			sizeof(struct kvm_pic_state));
4562
		spin_unlock(&pic->lock);
4563 4564
		break;
	case KVM_IRQCHIP_IOAPIC:
4565
		kvm_set_ioapic(kvm, &chip->chip.ioapic);
4566 4567 4568 4569 4570
		break;
	default:
		r = -EINVAL;
		break;
	}
4571
	kvm_pic_update_irq(pic);
4572 4573 4574
	return r;
}

4575 4576
static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4577 4578 4579 4580 4581 4582 4583
	struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;

	BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));

	mutex_lock(&kps->lock);
	memcpy(ps, &kps->channels, sizeof(*ps));
	mutex_unlock(&kps->lock);
4584
	return 0;
4585 4586 4587 4588
}

static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
{
4589
	int i;
4590 4591 4592
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
4593
	memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
4594
	for (i = 0; i < 3; i++)
4595 4596
		kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
	mutex_unlock(&pit->pit_state.lock);
4597
	return 0;
4598 4599 4600 4601 4602 4603 4604 4605 4606
}

static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
	mutex_lock(&kvm->arch.vpit->pit_state.lock);
	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
		sizeof(ps->channels));
	ps->flags = kvm->arch.vpit->pit_state.flags;
	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
4607
	memset(&ps->reserved, 0, sizeof(ps->reserved));
4608
	return 0;
4609 4610 4611 4612
}

static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
{
4613
	int start = 0;
4614
	int i;
4615
	u32 prev_legacy, cur_legacy;
4616 4617 4618 4619
	struct kvm_pit *pit = kvm->arch.vpit;

	mutex_lock(&pit->pit_state.lock);
	prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
4620 4621 4622
	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
	if (!prev_legacy && cur_legacy)
		start = 1;
4623 4624 4625
	memcpy(&pit->pit_state.channels, &ps->channels,
	       sizeof(pit->pit_state.channels));
	pit->pit_state.flags = ps->flags;
4626
	for (i = 0; i < 3; i++)
4627
		kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
4628
				   start && i == 0);
4629
	mutex_unlock(&pit->pit_state.lock);
4630
	return 0;
4631 4632
}

4633 4634 4635
static int kvm_vm_ioctl_reinject(struct kvm *kvm,
				 struct kvm_reinject_control *control)
{
4636 4637 4638
	struct kvm_pit *pit = kvm->arch.vpit;

	if (!pit)
4639
		return -ENXIO;
4640

4641 4642 4643 4644 4645 4646 4647
	/* pit->pit_state.lock was overloaded to prevent userspace from getting
	 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
	 * ioctls in parallel.  Use a separate lock if that ioctl isn't rare.
	 */
	mutex_lock(&pit->pit_state.lock);
	kvm_pit_set_reinject(pit, control->pit_reinject);
	mutex_unlock(&pit->pit_state.lock);
4648

4649 4650 4651
	return 0;
}

4652
/**
4653 4654 4655
 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
 * @kvm: kvm instance
 * @log: slot id and address to which we copy the log
4656
 *
4657 4658 4659 4660 4661 4662 4663 4664
 * Steps 1-4 below provide general overview of dirty page logging. See
 * kvm_get_dirty_log_protect() function description for additional details.
 *
 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
 * always flush the TLB (step 4) even if previous step failed  and the dirty
 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
 * does not preclude user space subsequent dirty log read. Flushing TLB ensures
 * writes will be marked dirty for next log read.
4665
 *
4666 4667
 *   1. Take a snapshot of the bit and clear it if needed.
 *   2. Write protect the corresponding page.
4668 4669
 *   3. Copy the snapshot to the userspace.
 *   4. Flush TLB's if needed.
4670
 */
4671
int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
4672
{
4673
	bool flush = false;
4674
	int r;
4675

4676
	mutex_lock(&kvm->slots_lock);
4677

4678 4679 4680 4681 4682 4683
	/*
	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
	 */
	if (kvm_x86_ops->flush_log_dirty)
		kvm_x86_ops->flush_log_dirty(kvm);

4684
	r = kvm_get_dirty_log_protect(kvm, log, &flush);
4685 4686 4687 4688 4689

	/*
	 * All the TLBs can be flushed out of mmu lock, see the comments in
	 * kvm_mmu_slot_remove_write_access().
	 */
4690
	lockdep_assert_held(&kvm->slots_lock);
4691
	if (flush)
4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718
		kvm_flush_remote_tlbs(kvm);

	mutex_unlock(&kvm->slots_lock);
	return r;
}

int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log)
{
	bool flush = false;
	int r;

	mutex_lock(&kvm->slots_lock);

	/*
	 * Flush potentially hardware-cached dirty pages to dirty_bitmap.
	 */
	if (kvm_x86_ops->flush_log_dirty)
		kvm_x86_ops->flush_log_dirty(kvm);

	r = kvm_clear_dirty_log_protect(kvm, log, &flush);

	/*
	 * All the TLBs can be flushed out of mmu lock, see the comments in
	 * kvm_mmu_slot_remove_write_access().
	 */
	lockdep_assert_held(&kvm->slots_lock);
	if (flush)
4719 4720
		kvm_flush_remote_tlbs(kvm);

4721
	mutex_unlock(&kvm->slots_lock);
4722 4723 4724
	return r;
}

4725 4726
int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
			bool line_status)
4727 4728 4729 4730 4731
{
	if (!irqchip_in_kernel(kvm))
		return -ENXIO;

	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
4732 4733
					irq_event->irq, irq_event->level,
					line_status);
4734 4735 4736
	return 0;
}

4737 4738
int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
			    struct kvm_enable_cap *cap)
4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749
{
	int r;

	if (cap->flags)
		return -EINVAL;

	switch (cap->cap) {
	case KVM_CAP_DISABLE_QUIRKS:
		kvm->arch.disabled_quirks = cap->args[0];
		r = 0;
		break;
4750 4751
	case KVM_CAP_SPLIT_IRQCHIP: {
		mutex_lock(&kvm->lock);
4752 4753 4754
		r = -EINVAL;
		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
			goto split_irqchip_unlock;
4755 4756 4757
		r = -EEXIST;
		if (irqchip_in_kernel(kvm))
			goto split_irqchip_unlock;
4758
		if (kvm->created_vcpus)
4759 4760
			goto split_irqchip_unlock;
		r = kvm_setup_empty_irq_routing(kvm);
4761
		if (r)
4762 4763 4764
			goto split_irqchip_unlock;
		/* Pairs with irqchip_in_kernel. */
		smp_wmb();
4765
		kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
4766
		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
4767 4768 4769 4770 4771
		r = 0;
split_irqchip_unlock:
		mutex_unlock(&kvm->lock);
		break;
	}
4772 4773 4774 4775 4776 4777 4778
	case KVM_CAP_X2APIC_API:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
			break;

		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
			kvm->arch.x2apic_format = true;
4779 4780
		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
			kvm->arch.x2apic_broadcast_quirk_disabled = true;
4781 4782 4783

		r = 0;
		break;
4784 4785 4786 4787 4788 4789 4790 4791
	case KVM_CAP_X86_DISABLE_EXITS:
		r = -EINVAL;
		if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
			break;

		if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
			kvm_can_mwait_in_guest())
			kvm->arch.mwait_in_guest = true;
4792
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
4793
			kvm->arch.hlt_in_guest = true;
4794 4795
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
			kvm->arch.pause_in_guest = true;
4796 4797
		if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
			kvm->arch.cstate_in_guest = true;
4798 4799
		r = 0;
		break;
4800 4801 4802
	case KVM_CAP_MSR_PLATFORM_INFO:
		kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
		r = 0;
4803 4804 4805 4806
		break;
	case KVM_CAP_EXCEPTION_PAYLOAD:
		kvm->arch.exception_payload_enabled = cap->args[0];
		r = 0;
4807
		break;
4808 4809 4810 4811 4812 4813 4814
	default:
		r = -EINVAL;
		break;
	}
	return r;
}

4815 4816 4817 4818 4819
long kvm_arch_vm_ioctl(struct file *filp,
		       unsigned int ioctl, unsigned long arg)
{
	struct kvm *kvm = filp->private_data;
	void __user *argp = (void __user *)arg;
4820
	int r = -ENOTTY;
4821 4822 4823 4824 4825 4826 4827
	/*
	 * This union makes it completely explicit to gcc-3.x
	 * that these two variables' stack usage should be
	 * combined, not added together.
	 */
	union {
		struct kvm_pit_state ps;
4828
		struct kvm_pit_state2 ps2;
4829
		struct kvm_pit_config pit_config;
4830
	} u;
4831 4832 4833 4834 4835

	switch (ioctl) {
	case KVM_SET_TSS_ADDR:
		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
		break;
4836 4837 4838
	case KVM_SET_IDENTITY_MAP_ADDR: {
		u64 ident_addr;

4839 4840 4841 4842
		mutex_lock(&kvm->lock);
		r = -EINVAL;
		if (kvm->created_vcpus)
			goto set_identity_unlock;
4843
		r = -EFAULT;
4844
		if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
4845
			goto set_identity_unlock;
4846
		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
4847 4848
set_identity_unlock:
		mutex_unlock(&kvm->lock);
4849 4850
		break;
	}
4851 4852 4853 4854 4855 4856
	case KVM_SET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
		break;
	case KVM_GET_NR_MMU_PAGES:
		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
		break;
4857 4858
	case KVM_CREATE_IRQCHIP: {
		mutex_lock(&kvm->lock);
4859

4860
		r = -EEXIST;
4861
		if (irqchip_in_kernel(kvm))
4862
			goto create_irqchip_unlock;
4863

4864
		r = -EINVAL;
4865
		if (kvm->created_vcpus)
4866
			goto create_irqchip_unlock;
4867 4868 4869

		r = kvm_pic_init(kvm);
		if (r)
4870
			goto create_irqchip_unlock;
4871 4872 4873 4874

		r = kvm_ioapic_init(kvm);
		if (r) {
			kvm_pic_destroy(kvm);
4875
			goto create_irqchip_unlock;
4876 4877
		}

4878 4879
		r = kvm_setup_default_irq_routing(kvm);
		if (r) {
4880
			kvm_ioapic_destroy(kvm);
4881
			kvm_pic_destroy(kvm);
4882
			goto create_irqchip_unlock;
4883
		}
4884
		/* Write kvm->irq_routing before enabling irqchip_in_kernel. */
4885
		smp_wmb();
4886
		kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
4887 4888
	create_irqchip_unlock:
		mutex_unlock(&kvm->lock);
4889
		break;
4890
	}
Sheng Yang's avatar
Sheng Yang committed
4891
	case KVM_CREATE_PIT:
4892 4893 4894 4895 4896 4897 4898 4899
		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
		goto create_pit;
	case KVM_CREATE_PIT2:
		r = -EFAULT;
		if (copy_from_user(&u.pit_config, argp,
				   sizeof(struct kvm_pit_config)))
			goto out;
	create_pit:
4900
		mutex_lock(&kvm->lock);
Avi Kivity's avatar
Avi Kivity committed
4901 4902 4903
		r = -EEXIST;
		if (kvm->arch.vpit)
			goto create_pit_unlock;
Sheng Yang's avatar
Sheng Yang committed
4904
		r = -ENOMEM;
4905
		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
Sheng Yang's avatar
Sheng Yang committed
4906 4907
		if (kvm->arch.vpit)
			r = 0;
Avi Kivity's avatar
Avi Kivity committed
4908
	create_pit_unlock:
4909
		mutex_unlock(&kvm->lock);
Sheng Yang's avatar
Sheng Yang committed
4910
		break;
4911 4912
	case KVM_GET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
4913
		struct kvm_irqchip *chip;
4914

4915 4916 4917
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
4918
			goto out;
4919 4920
		}

4921
		r = -ENXIO;
4922
		if (!irqchip_kernel(kvm))
4923 4924
			goto get_irqchip_out;
		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
4925
		if (r)
4926
			goto get_irqchip_out;
4927
		r = -EFAULT;
4928
		if (copy_to_user(argp, chip, sizeof(*chip)))
4929
			goto get_irqchip_out;
4930
		r = 0;
4931 4932
	get_irqchip_out:
		kfree(chip);
4933 4934 4935 4936
		break;
	}
	case KVM_SET_IRQCHIP: {
		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
4937
		struct kvm_irqchip *chip;
4938

4939 4940 4941
		chip = memdup_user(argp, sizeof(*chip));
		if (IS_ERR(chip)) {
			r = PTR_ERR(chip);
4942
			goto out;
4943 4944
		}

4945
		r = -ENXIO;
4946
		if (!irqchip_kernel(kvm))
4947 4948
			goto set_irqchip_out;
		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
4949
		if (r)
4950
			goto set_irqchip_out;
4951
		r = 0;
4952 4953
	set_irqchip_out:
		kfree(chip);
4954 4955
		break;
	}
4956 4957
	case KVM_GET_PIT: {
		r = -EFAULT;
4958
		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
4959 4960 4961 4962
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
4963
		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
4964 4965 4966
		if (r)
			goto out;
		r = -EFAULT;
4967
		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
4968 4969 4970 4971 4972 4973
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT: {
		r = -EFAULT;
4974
		if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
4975 4976 4977 4978
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
4979
		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
4980 4981
		break;
	}
4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004
	case KVM_GET_PIT2: {
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_PIT2: {
		r = -EFAULT;
		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
			goto out;
		r = -ENXIO;
		if (!kvm->arch.vpit)
			goto out;
		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
		break;
	}
5005 5006 5007 5008 5009 5010 5011 5012
	case KVM_REINJECT_CONTROL: {
		struct kvm_reinject_control control;
		r =  -EFAULT;
		if (copy_from_user(&control, argp, sizeof(control)))
			goto out;
		r = kvm_vm_ioctl_reinject(kvm, &control);
		break;
	}
5013 5014 5015
	case KVM_SET_BOOT_CPU_ID:
		r = 0;
		mutex_lock(&kvm->lock);
5016
		if (kvm->created_vcpus)
5017 5018 5019 5020 5021
			r = -EBUSY;
		else
			kvm->arch.bsp_vcpu_id = arg;
		mutex_unlock(&kvm->lock);
		break;
Ed Swierk's avatar
Ed Swierk committed
5022
	case KVM_XEN_HVM_CONFIG: {
5023
		struct kvm_xen_hvm_config xhc;
Ed Swierk's avatar
Ed Swierk committed
5024
		r = -EFAULT;
5025
		if (copy_from_user(&xhc, argp, sizeof(xhc)))
Ed Swierk's avatar
Ed Swierk committed
5026 5027
			goto out;
		r = -EINVAL;
5028
		if (xhc.flags)
Ed Swierk's avatar
Ed Swierk committed
5029
			goto out;
5030
		memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));
Ed Swierk's avatar
Ed Swierk committed
5031 5032 5033
		r = 0;
		break;
	}
5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046
	case KVM_SET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

		r = -EFAULT;
		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
			goto out;

		r = -EINVAL;
		if (user_ns.flags)
			goto out;

		r = 0;
5047 5048 5049 5050 5051 5052
		/*
		 * TODO: userspace has to take care of races with VCPU_RUN, so
		 * kvm_gen_update_masterclock() can be cut down to locked
		 * pvclock_update_vm_gtod_copy().
		 */
		kvm_gen_update_masterclock(kvm);
5053
		now_ns = get_kvmclock_ns(kvm);
5054
		kvm->arch.kvmclock_offset += user_ns.clock - now_ns;
5055
		kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE);
5056 5057 5058 5059 5060 5061
		break;
	}
	case KVM_GET_CLOCK: {
		struct kvm_clock_data user_ns;
		u64 now_ns;

5062
		now_ns = get_kvmclock_ns(kvm);
5063
		user_ns.clock = now_ns;
5064
		user_ns.flags = kvm->arch.use_master_clock ? KVM_CLOCK_TSC_STABLE : 0;
5065
		memset(&user_ns.pad, 0, sizeof(user_ns.pad));
5066 5067 5068 5069 5070 5071 5072

		r = -EFAULT;
		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
			goto out;
		r = 0;
		break;
	}
5073 5074 5075 5076 5077 5078
	case KVM_MEMORY_ENCRYPT_OP: {
		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_op)
			r = kvm_x86_ops->mem_enc_op(kvm, argp);
		break;
	}
5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102
	case KVM_MEMORY_ENCRYPT_REG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_reg_region)
			r = kvm_x86_ops->mem_enc_reg_region(kvm, &region);
		break;
	}
	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
		struct kvm_enc_region region;

		r = -EFAULT;
		if (copy_from_user(&region, argp, sizeof(region)))
			goto out;

		r = -ENOTTY;
		if (kvm_x86_ops->mem_enc_unreg_region)
			r = kvm_x86_ops->mem_enc_unreg_region(kvm, &region);
		break;
	}
5103 5104 5105 5106 5107 5108 5109 5110 5111
	case KVM_HYPERV_EVENTFD: {
		struct kvm_hyperv_eventfd hvevfd;

		r = -EFAULT;
		if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
			goto out;
		r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
		break;
	}
Eric Hankland's avatar
Eric Hankland committed
5112 5113 5114
	case KVM_SET_PMU_EVENT_FILTER:
		r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
		break;
5115
	default:
5116
		r = -ENOTTY;
5117 5118 5119 5120 5121
	}
out:
	return r;
}

5122
static void kvm_init_msr_list(void)
5123
{
5124
	struct x86_pmu_capability x86_pmu;
5125
	u32 dummy[2];
5126
	unsigned i;
5127

5128
	BUILD_BUG_ON_MSG(INTEL_PMC_MAX_FIXED != 4,
5129
			 "Please update the fixed PMCs in msrs_to_saved_all[]");
5130 5131

	perf_get_x86_pmu_capability(&x86_pmu);
5132

5133 5134 5135 5136
	num_msrs_to_save = 0;
	num_emulated_msrs = 0;
	num_msr_based_features = 0;

5137 5138
	for (i = 0; i < ARRAY_SIZE(msrs_to_save_all); i++) {
		if (rdmsr_safe(msrs_to_save_all[i], &dummy[0], &dummy[1]) < 0)
5139
			continue;
5140 5141 5142

		/*
		 * Even MSRs that are valid in the host may not be exposed
5143
		 * to the guests in some cases.
5144
		 */
5145
		switch (msrs_to_save_all[i]) {
5146
		case MSR_IA32_BNDCFGS:
5147
			if (!kvm_mpx_supported())
5148 5149
				continue;
			break;
5150 5151 5152 5153
		case MSR_TSC_AUX:
			if (!kvm_x86_ops->rdtscp_supported())
				continue;
			break;
5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172
		case MSR_IA32_RTIT_CTL:
		case MSR_IA32_RTIT_STATUS:
			if (!kvm_x86_ops->pt_supported())
				continue;
			break;
		case MSR_IA32_RTIT_CR3_MATCH:
			if (!kvm_x86_ops->pt_supported() ||
			    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
				continue;
			break;
		case MSR_IA32_RTIT_OUTPUT_BASE:
		case MSR_IA32_RTIT_OUTPUT_MASK:
			if (!kvm_x86_ops->pt_supported() ||
				(!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
				 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
				continue;
			break;
		case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: {
			if (!kvm_x86_ops->pt_supported() ||
5173
				msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >=
5174 5175 5176
				intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
				continue;
			break;
5177
		case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17:
5178
			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
5179 5180 5181
			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
				continue;
			break;
5182
		case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 17:
5183
			if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >=
5184 5185
			    min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp))
				continue;
5186
		}
5187 5188 5189 5190
		default:
			break;
		}

5191
		msrs_to_save[num_msrs_to_save++] = msrs_to_save_all[i];
5192
	}
5193

5194 5195
	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
		if (!kvm_x86_ops->has_emulated_msr(emulated_msrs_all[i]))
5196
			continue;
5197

5198
		emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
5199
	}
5200

5201
	for (i = 0; i < ARRAY_SIZE(msr_based_features_all); i++) {
5202 5203
		struct kvm_msr_entry msr;

5204
		msr.index = msr_based_features_all[i];
5205
		if (kvm_get_msr_feature(&msr))
5206 5207
			continue;

5208
		msr_based_features[num_msr_based_features++] = msr_based_features_all[i];
5209
	}
5210 5211
}

5212 5213
static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
			   const void *v)
5214
{
5215 5216 5217 5218 5219
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5220
		if (!(lapic_in_kernel(vcpu) &&
5221 5222
		      !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
		    && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
5223 5224 5225 5226 5227 5228
			break;
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5229

5230
	return handled;
5231 5232
}

5233
static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
5234
{
5235 5236 5237 5238 5239
	int handled = 0;
	int n;

	do {
		n = min(len, 8);
5240
		if (!(lapic_in_kernel(vcpu) &&
5241 5242 5243
		      !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
					 addr, n, v))
		    && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
5244
			break;
5245
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
5246 5247 5248 5249 5250
		handled += n;
		addr += n;
		len -= n;
		v += n;
	} while (len);
5251

5252
	return handled;
5253 5254
}

5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266
static void kvm_set_segment(struct kvm_vcpu *vcpu,
			struct kvm_segment *var, int seg)
{
	kvm_x86_ops->set_segment(vcpu, var, seg);
}

void kvm_get_segment(struct kvm_vcpu *vcpu,
		     struct kvm_segment *var, int seg)
{
	kvm_x86_ops->get_segment(vcpu, var, seg);
}

5267 5268
gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
			   struct x86_exception *exception)
5269 5270 5271 5272 5273 5274 5275
{
	gpa_t t_gpa;

	BUG_ON(!mmu_is_nested(vcpu));

	/* NPT walks are always user-walks */
	access |= PFERR_USER_MASK;
5276
	t_gpa  = vcpu->arch.mmu->gva_to_gpa(vcpu, gpa, access, exception);
5277 5278 5279 5280

	return t_gpa;
}

5281 5282
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
			      struct x86_exception *exception)
5283 5284
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5285
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5286 5287
}

5288 5289
 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5290 5291 5292
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_FETCH_MASK;
5293
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5294 5295
}

5296 5297
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
			       struct x86_exception *exception)
5298 5299 5300
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	access |= PFERR_WRITE_MASK;
5301
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5302 5303 5304
}

/* uses this to access any guest's mapped memory without checking CPL */
5305 5306
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
				struct x86_exception *exception)
5307
{
5308
	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
5309 5310 5311 5312
}

static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
5313
				      struct x86_exception *exception)
5314 5315
{
	void *data = val;
5316
	int r = X86EMUL_CONTINUE;
5317 5318

	while (bytes) {
5319
		gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
5320
							    exception);
5321
		unsigned offset = addr & (PAGE_SIZE-1);
5322
		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
5323 5324
		int ret;

5325
		if (gpa == UNMAPPED_GVA)
5326
			return X86EMUL_PROPAGATE_FAULT;
5327 5328
		ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
					       offset, toread);
5329
		if (ret < 0) {
5330
			r = X86EMUL_IO_NEEDED;
5331 5332
			goto out;
		}
5333

5334 5335 5336
		bytes -= toread;
		data += toread;
		addr += toread;
5337
	}
5338 5339
out:
	return r;
5340
}
5341

5342
/* used for instruction fetching */
5343 5344
static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
				gva_t addr, void *val, unsigned int bytes,
5345
				struct x86_exception *exception)
5346
{
5347
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5348
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5349 5350
	unsigned offset;
	int ret;
5351

5352 5353 5354 5355 5356 5357 5358 5359 5360
	/* Inline kvm_read_guest_virt_helper for speed.  */
	gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK,
						    exception);
	if (unlikely(gpa == UNMAPPED_GVA))
		return X86EMUL_PROPAGATE_FAULT;

	offset = addr & (PAGE_SIZE-1);
	if (WARN_ON(offset + bytes > PAGE_SIZE))
		bytes = (unsigned)PAGE_SIZE - offset;
5361 5362
	ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
				       offset, bytes);
5363 5364 5365 5366
	if (unlikely(ret < 0))
		return X86EMUL_IO_NEEDED;

	return X86EMUL_CONTINUE;
5367 5368
}

5369
int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
5370
			       gva_t addr, void *val, unsigned int bytes,
5371
			       struct x86_exception *exception)
5372 5373
{
	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
5374

5375 5376 5377 5378 5379 5380 5381
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5382
	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
5383
					  exception);
5384
}
5385
EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
5386

5387 5388
static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
			     gva_t addr, void *val, unsigned int bytes,
5389
			     struct x86_exception *exception, bool system)
5390
{
5391
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5392 5393 5394 5395 5396 5397
	u32 access = 0;

	if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
		access |= PFERR_USER_MASK;

	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
5398 5399
}

5400 5401 5402 5403 5404 5405 5406 5407 5408
static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
		unsigned long addr, void *val, unsigned int bytes)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);

	return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
}

5409 5410 5411
static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
				      struct x86_exception *exception)
5412 5413 5414 5415 5416
{
	void *data = val;
	int r = X86EMUL_CONTINUE;

	while (bytes) {
5417
		gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
5418
							     access,
5419
							     exception);
5420 5421 5422 5423
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
		int ret;

5424
		if (gpa == UNMAPPED_GVA)
5425
			return X86EMUL_PROPAGATE_FAULT;
5426
		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
5427
		if (ret < 0) {
5428
			r = X86EMUL_IO_NEEDED;
5429 5430 5431 5432 5433 5434 5435 5436 5437 5438
			goto out;
		}

		bytes -= towrite;
		data += towrite;
		addr += towrite;
	}
out:
	return r;
}
5439 5440

static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
5441 5442
			      unsigned int bytes, struct x86_exception *exception,
			      bool system)
5443 5444
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5445 5446 5447 5448
	u32 access = PFERR_WRITE_MASK;

	if (!system && kvm_x86_ops->get_cpl(vcpu) == 3)
		access |= PFERR_USER_MASK;
5449 5450

	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
5451
					   access, exception);
5452 5453 5454 5455 5456
}

int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
				unsigned int bytes, struct x86_exception *exception)
{
5457 5458 5459
	/* kvm_write_guest_virt_system can pull in tons of pages. */
	vcpu->arch.l1tf_flush_l1d = true;

5460 5461 5462 5463 5464 5465 5466
	/*
	 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
	 * is returned, but our callers are not ready for that and they blindly
	 * call kvm_inject_page_fault.  Ensure that they at least do not leak
	 * uninitialized kernel stack memory into cr2 and error code.
	 */
	memset(exception, 0, sizeof(*exception));
5467 5468 5469
	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
					   PFERR_WRITE_MASK, exception);
}
Nadav Har'El's avatar
Nadav Har'El committed
5470
EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
5471

5472 5473
int handle_ud(struct kvm_vcpu *vcpu)
{
5474 5475 5476 5477 5478
	int emul_type = EMULTYPE_TRAP_UD;
	char sig[5]; /* ud2; .ascii "kvm" */
	struct x86_exception e;

	if (force_emulation_prefix &&
5479 5480
	    kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
				sig, sizeof(sig), &e) == 0 &&
5481 5482
	    memcmp(sig, "\xf\xbkvm", sizeof(sig)) == 0) {
		kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
5483
		emul_type = EMULTYPE_TRAP_UD_FORCED;
5484
	}
5485

5486
	return kvm_emulate_instruction(vcpu, emul_type);
5487 5488 5489
}
EXPORT_SYMBOL_GPL(handle_ud);

5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504
static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
			    gpa_t gpa, bool write)
{
	/* For APIC access vmexit */
	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		return 1;

	if (vcpu_match_mmio_gpa(vcpu, gpa)) {
		trace_vcpu_match_mmio(gva, gpa, write, true);
		return 1;
	}

	return 0;
}

5505 5506 5507 5508
static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
				gpa_t *gpa, struct x86_exception *exception,
				bool write)
{
5509 5510
	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
		| (write ? PFERR_WRITE_MASK : 0);
5511

5512 5513 5514 5515 5516
	/*
	 * currently PKRU is only applied to ept enabled guest so
	 * there is no pkey in EPT page table for L1 guest or EPT
	 * shadow page table for L2 guest.
	 */
5517
	if (vcpu_match_mmio_gva(vcpu, gva)
5518
	    && !permission_fault(vcpu, vcpu->arch.walk_mmu,
5519
				 vcpu->arch.mmio_access, 0, access)) {
5520 5521
		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
					(gva & (PAGE_SIZE - 1));
5522
		trace_vcpu_match_mmio(gva, *gpa, write, false);
5523 5524 5525
		return 1;
	}

5526 5527 5528 5529 5530
	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);

	if (*gpa == UNMAPPED_GVA)
		return -1;

5531
	return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
5532 5533
}

5534
int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
5535
			const void *val, int bytes)
5536 5537 5538
{
	int ret;

5539
	ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
5540
	if (ret < 0)
5541
		return 0;
5542
	kvm_page_track_write(vcpu, gpa, val, bytes);
5543 5544 5545
	return 1;
}

5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561
struct read_write_emulator_ops {
	int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
				  int bytes);
	int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
				  void *val, int bytes);
	int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
			       int bytes, void *val);
	int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
				    void *val, int bytes);
	bool write;
};

static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
{
	if (vcpu->mmio_read_completed) {
		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
5562
			       vcpu->mmio_fragments[0].gpa, val);
5563 5564 5565 5566 5567 5568 5569 5570 5571 5572
		vcpu->mmio_read_completed = 0;
		return 1;
	}

	return 0;
}

static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			void *val, int bytes)
{
5573
	return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
5574 5575 5576 5577 5578 5579 5580 5581 5582 5583
}

static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
			 void *val, int bytes)
{
	return emulator_write_phys(vcpu, gpa, val, bytes);
}

static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
{
5584
	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
5585 5586 5587 5588 5589 5590
	return vcpu_mmio_write(vcpu, gpa, bytes, val);
}

static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			  void *val, int bytes)
{
5591
	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
5592 5593 5594 5595 5596 5597
	return X86EMUL_IO_NEEDED;
}

static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
			   void *val, int bytes)
{
Avi Kivity's avatar
Avi Kivity committed
5598 5599
	struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];

5600
	memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
5601 5602 5603
	return X86EMUL_CONTINUE;
}

5604
static const struct read_write_emulator_ops read_emultor = {
5605 5606 5607 5608 5609 5610
	.read_write_prepare = read_prepare,
	.read_write_emulate = read_emulate,
	.read_write_mmio = vcpu_mmio_read,
	.read_write_exit_mmio = read_exit_mmio,
};

5611
static const struct read_write_emulator_ops write_emultor = {
5612 5613 5614 5615 5616 5617
	.read_write_emulate = write_emulate,
	.read_write_mmio = write_mmio,
	.read_write_exit_mmio = write_exit_mmio,
	.write = true,
};

5618 5619 5620 5621
static int emulator_read_write_onepage(unsigned long addr, void *val,
				       unsigned int bytes,
				       struct x86_exception *exception,
				       struct kvm_vcpu *vcpu,
5622
				       const struct read_write_emulator_ops *ops)
5623
{
5624 5625
	gpa_t gpa;
	int handled, ret;
5626
	bool write = ops->write;
Avi Kivity's avatar
Avi Kivity committed
5627
	struct kvm_mmio_fragment *frag;
5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;

	/*
	 * If the exit was due to a NPF we may already have a GPA.
	 * If the GPA is present, use it to avoid the GVA to GPA table walk.
	 * Note, this cannot be used on string operations since string
	 * operation using rep will only have the initial GPA from the NPF
	 * occurred.
	 */
	if (vcpu->arch.gpa_available &&
	    emulator_can_use_gpa(ctxt) &&
5639 5640 5641 5642 5643 5644 5645
	    (addr & ~PAGE_MASK) == (vcpu->arch.gpa_val & ~PAGE_MASK)) {
		gpa = vcpu->arch.gpa_val;
		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
	} else {
		ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
		if (ret < 0)
			return X86EMUL_PROPAGATE_FAULT;
5646
	}
5647

5648
	if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
5649 5650 5651 5652 5653
		return X86EMUL_CONTINUE;

	/*
	 * Is this MMIO handled locally?
	 */
5654
	handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
5655
	if (handled == bytes)
5656 5657
		return X86EMUL_CONTINUE;

5658 5659 5660 5661
	gpa += handled;
	bytes -= handled;
	val += handled;

5662 5663 5664 5665 5666
	WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
	frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
	frag->gpa = gpa;
	frag->data = val;
	frag->len = bytes;
Avi Kivity's avatar
Avi Kivity committed
5667
	return X86EMUL_CONTINUE;
5668 5669
}

5670 5671
static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
			unsigned long addr,
5672 5673
			void *val, unsigned int bytes,
			struct x86_exception *exception,
5674
			const struct read_write_emulator_ops *ops)
5675
{
5676
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Avi Kivity's avatar
Avi Kivity committed
5677 5678 5679 5680 5681 5682 5683 5684
	gpa_t gpa;
	int rc;

	if (ops->read_write_prepare &&
		  ops->read_write_prepare(vcpu, val, bytes))
		return X86EMUL_CONTINUE;

	vcpu->mmio_nr_fragments = 0;
5685

5686 5687
	/* Crossing a page boundary? */
	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
Avi Kivity's avatar
Avi Kivity committed
5688
		int now;
5689 5690

		now = -addr & ~PAGE_MASK;
5691 5692 5693
		rc = emulator_read_write_onepage(addr, val, now, exception,
						 vcpu, ops);

5694 5695 5696
		if (rc != X86EMUL_CONTINUE)
			return rc;
		addr += now;
5697 5698
		if (ctxt->mode != X86EMUL_MODE_PROT64)
			addr = (u32)addr;
5699 5700 5701
		val += now;
		bytes -= now;
	}
5702

Avi Kivity's avatar
Avi Kivity committed
5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715
	rc = emulator_read_write_onepage(addr, val, bytes, exception,
					 vcpu, ops);
	if (rc != X86EMUL_CONTINUE)
		return rc;

	if (!vcpu->mmio_nr_fragments)
		return rc;

	gpa = vcpu->mmio_fragments[0].gpa;

	vcpu->mmio_needed = 1;
	vcpu->mmio_cur_fragment = 0;

5716
	vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
Avi Kivity's avatar
Avi Kivity committed
5717 5718 5719 5720 5721
	vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
	vcpu->run->exit_reason = KVM_EXIT_MMIO;
	vcpu->run->mmio.phys_addr = gpa;

	return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733
}

static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
				  unsigned long addr,
				  void *val,
				  unsigned int bytes,
				  struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, val, bytes,
				   exception, &read_emultor);
}

5734
static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
5735 5736 5737 5738 5739 5740 5741
			    unsigned long addr,
			    const void *val,
			    unsigned int bytes,
			    struct x86_exception *exception)
{
	return emulator_read_write(ctxt, addr, (void *)val, bytes,
				   exception, &write_emultor);
5742 5743
}

5744 5745 5746 5747 5748 5749 5750
#define CMPXCHG_TYPE(t, ptr, old, new) \
	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))

#ifdef CONFIG_X86_64
#  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
#else
#  define CMPXCHG64(ptr, old, new) \
5751
	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
5752 5753
#endif

5754 5755
static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
				     unsigned long addr,
5756 5757 5758
				     const void *old,
				     const void *new,
				     unsigned int bytes,
5759
				     struct x86_exception *exception)
5760
{
5761
	struct kvm_host_map map;
5762
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5763 5764 5765
	gpa_t gpa;
	char *kaddr;
	bool exchanged;
5766

5767 5768 5769
	/* guests cmpxchg8b have to be emulated atomically */
	if (bytes > 8 || (bytes & (bytes - 1)))
		goto emul_write;
5770

5771
	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
5772

5773 5774 5775
	if (gpa == UNMAPPED_GVA ||
	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		goto emul_write;
5776

5777 5778
	if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
		goto emul_write;
5779

5780
	if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))
5781
		goto emul_write;
5782

5783 5784
	kaddr = map.hva + offset_in_page(gpa);

5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799
	switch (bytes) {
	case 1:
		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
		break;
	case 2:
		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
		break;
	case 4:
		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
		break;
	case 8:
		exchanged = CMPXCHG64(kaddr, old, new);
		break;
	default:
		BUG();
5800
	}
5801 5802

	kvm_vcpu_unmap(vcpu, &map, true);
5803 5804 5805 5806

	if (!exchanged)
		return X86EMUL_CMPXCHG_FAILED;

5807
	kvm_page_track_write(vcpu, gpa, new, bytes);
5808 5809

	return X86EMUL_CONTINUE;
5810

5811
emul_write:
5812
	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
5813

5814
	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
5815 5816
}

5817 5818
static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
{
5819
	int r = 0, i;
5820

5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832
	for (i = 0; i < vcpu->arch.pio.count; i++) {
		if (vcpu->arch.pio.in)
			r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port,
					    vcpu->arch.pio.size, pd);
		else
			r = kvm_io_bus_write(vcpu, KVM_PIO_BUS,
					     vcpu->arch.pio.port, vcpu->arch.pio.size,
					     pd);
		if (r)
			break;
		pd += vcpu->arch.pio.size;
	}
5833 5834 5835
	return r;
}

5836 5837 5838
static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
			       unsigned short port, void *val,
			       unsigned int count, bool in)
5839 5840
{
	vcpu->arch.pio.port = port;
5841
	vcpu->arch.pio.in = in;
5842
	vcpu->arch.pio.count  = count;
5843 5844 5845
	vcpu->arch.pio.size = size;

	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
5846
		vcpu->arch.pio.count = 0;
5847 5848 5849 5850
		return 1;
	}

	vcpu->run->exit_reason = KVM_EXIT_IO;
5851
	vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
5852 5853 5854 5855 5856 5857 5858 5859
	vcpu->run->io.size = size;
	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
	vcpu->run->io.count = count;
	vcpu->run->io.port = port;

	return 0;
}

5860 5861 5862
static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
				    int size, unsigned short port, void *val,
				    unsigned int count)
5863
{
5864
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5865
	int ret;
5866

5867 5868
	if (vcpu->arch.pio.count)
		goto data_avail;
5869

5870 5871
	memset(vcpu->arch.pio_data, 0, size * count);

5872 5873 5874 5875
	ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
	if (ret) {
data_avail:
		memcpy(val, vcpu->arch.pio_data, size * count);
5876
		trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
5877
		vcpu->arch.pio.count = 0;
5878 5879 5880 5881 5882 5883
		return 1;
	}

	return 0;
}

5884 5885 5886 5887 5888 5889 5890
static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
				     int size, unsigned short port,
				     const void *val, unsigned int count)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	memcpy(vcpu->arch.pio_data, val, size * count);
5891
	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
5892 5893 5894
	return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
}

5895 5896 5897 5898 5899
static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
{
	return kvm_x86_ops->get_segment_base(vcpu, seg);
}

5900
static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
5901
{
5902
	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
5903 5904
}

5905
static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
5906 5907 5908 5909 5910
{
	if (!need_emulate_wbinvd(vcpu))
		return X86EMUL_CONTINUE;

	if (kvm_x86_ops->has_wbinvd_exit()) {
5911 5912 5913
		int cpu = get_cpu();

		cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
5914 5915
		smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
				wbinvd_ipi, NULL, 1);
5916
		put_cpu();
5917
		cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
5918 5919
	} else
		wbinvd();
5920 5921
	return X86EMUL_CONTINUE;
}
5922 5923 5924

int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
5925 5926
	kvm_emulate_wbinvd_noskip(vcpu);
	return kvm_skip_emulated_instruction(vcpu);
5927
}
5928 5929
EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);

5930 5931


5932 5933
static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
{
5934
	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
5935 5936
}

5937 5938
static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long *dest)
5939
{
5940
	return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
5941 5942
}

5943 5944
static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
			   unsigned long value)
5945
{
5946

5947
	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
5948 5949
}

5950
static u64 mk_cr_64(u64 curr_cr, u32 new_val)
5951
{
5952
	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
5953 5954
}

5955
static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
5956
{
5957
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5958 5959 5960 5961 5962 5963 5964 5965 5966 5967
	unsigned long value;

	switch (cr) {
	case 0:
		value = kvm_read_cr0(vcpu);
		break;
	case 2:
		value = vcpu->arch.cr2;
		break;
	case 3:
5968
		value = kvm_read_cr3(vcpu);
5969 5970 5971 5972 5973 5974 5975 5976
		break;
	case 4:
		value = kvm_read_cr4(vcpu);
		break;
	case 8:
		value = kvm_get_cr8(vcpu);
		break;
	default:
5977
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
5978 5979 5980 5981 5982 5983
		return 0;
	}

	return value;
}

5984
static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
5985
{
5986
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5987 5988
	int res = 0;

5989 5990
	switch (cr) {
	case 0:
5991
		res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
5992 5993 5994 5995 5996
		break;
	case 2:
		vcpu->arch.cr2 = val;
		break;
	case 3:
5997
		res = kvm_set_cr3(vcpu, val);
5998 5999
		break;
	case 4:
6000
		res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
6001 6002
		break;
	case 8:
Andre Przywara's avatar
Andre Przywara committed
6003
		res = kvm_set_cr8(vcpu, val);
6004 6005
		break;
	default:
6006
		kvm_err("%s: unexpected cr %u\n", __func__, cr);
6007
		res = -1;
6008
	}
6009 6010

	return res;
6011 6012
}

6013
static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
6014
{
6015
	return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
6016 6017
}

6018
static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6019
{
6020
	kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
6021 6022
}

6023
static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
6024
{
6025
	kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
6026 6027
}

6028 6029 6030 6031 6032 6033 6034 6035 6036 6037
static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
}

static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
}

6038 6039
static unsigned long emulator_get_cached_segment_base(
	struct x86_emulate_ctxt *ctxt, int seg)
6040
{
6041
	return get_segment_base(emul_to_vcpu(ctxt), seg);
6042 6043
}

6044 6045 6046
static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
				 struct desc_struct *desc, u32 *base3,
				 int seg)
6047 6048 6049
{
	struct kvm_segment var;

6050
	kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
6051
	*selector = var.selector;
6052

6053 6054
	if (var.unusable) {
		memset(desc, 0, sizeof(*desc));
6055 6056
		if (base3)
			*base3 = 0;
6057
		return false;
6058
	}
6059 6060 6061 6062 6063

	if (var.g)
		var.limit >>= 12;
	set_desc_limit(desc, var.limit);
	set_desc_base(desc, (unsigned long)var.base);
6064 6065 6066 6067
#ifdef CONFIG_X86_64
	if (base3)
		*base3 = var.base >> 32;
#endif
6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079
	desc->type = var.type;
	desc->s = var.s;
	desc->dpl = var.dpl;
	desc->p = var.present;
	desc->avl = var.avl;
	desc->l = var.l;
	desc->d = var.db;
	desc->g = var.g;

	return true;
}

6080 6081 6082
static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
				 struct desc_struct *desc, u32 base3,
				 int seg)
6083
{
6084
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6085 6086
	struct kvm_segment var;

6087
	var.selector = selector;
6088
	var.base = get_desc_base(desc);
6089 6090 6091
#ifdef CONFIG_X86_64
	var.base |= ((u64)base3) << 32;
#endif
6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109
	var.limit = get_desc_limit(desc);
	if (desc->g)
		var.limit = (var.limit << 12) | 0xfff;
	var.type = desc->type;
	var.dpl = desc->dpl;
	var.db = desc->d;
	var.s = desc->s;
	var.l = desc->l;
	var.g = desc->g;
	var.avl = desc->avl;
	var.present = desc->p;
	var.unusable = !var.present;
	var.padding = 0;

	kvm_set_segment(vcpu, &var, seg);
	return;
}

6110 6111 6112
static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 *pdata)
{
6113
	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
6114 6115 6116 6117 6118
}

static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 data)
{
6119
	return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
6120 6121
}

6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135
static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	return vcpu->arch.smbase;
}

static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);

	vcpu->arch.smbase = smbase;
}

6136 6137 6138
static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
			      u32 pmc)
{
6139
	return kvm_pmu_is_valid_msr_idx(emul_to_vcpu(ctxt), pmc);
6140 6141
}

6142 6143 6144
static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
			     u32 pmc, u64 *pdata)
{
6145
	return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
6146 6147
}

6148 6149 6150 6151 6152
static void emulator_halt(struct x86_emulate_ctxt *ctxt)
{
	emul_to_vcpu(ctxt)->arch.halt_request = 1;
}

6153
static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
6154
			      struct x86_instruction_info *info,
6155 6156
			      enum x86_intercept_stage stage)
{
6157
	return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
6158 6159
}

6160 6161
static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
			u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, bool check_limit)
6162
{
6163
	return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, check_limit);
6164 6165
}

6166 6167 6168 6169 6170 6171 6172 6173 6174 6175
static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
{
	return kvm_register_read(emul_to_vcpu(ctxt), reg);
}

static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
{
	kvm_register_write(emul_to_vcpu(ctxt), reg, val);
}

6176 6177 6178 6179 6180
static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
{
	kvm_x86_ops->set_nmi_mask(emul_to_vcpu(ctxt), masked);
}

6181 6182 6183 6184 6185 6186 6187
static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
{
	return emul_to_vcpu(ctxt)->arch.hflags;
}

static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_flags)
{
6188
	emul_to_vcpu(ctxt)->arch.hflags = emul_flags;
6189 6190
}

6191 6192
static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt,
				  const char *smstate)
6193
{
6194
	return kvm_x86_ops->pre_leave_smm(emul_to_vcpu(ctxt), smstate);
6195 6196
}

6197 6198 6199 6200 6201
static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt)
{
	kvm_smm_changed(emul_to_vcpu(ctxt));
}

6202 6203 6204 6205 6206
static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
{
	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
}

6207
static const struct x86_emulate_ops emulate_ops = {
6208 6209
	.read_gpr            = emulator_read_gpr,
	.write_gpr           = emulator_write_gpr,
6210 6211
	.read_std            = emulator_read_std,
	.write_std           = emulator_write_std,
6212
	.read_phys           = kvm_read_guest_phys_system,
6213
	.fetch               = kvm_fetch_guest_virt,
6214 6215 6216
	.read_emulated       = emulator_read_emulated,
	.write_emulated      = emulator_write_emulated,
	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
6217
	.invlpg              = emulator_invlpg,
6218 6219
	.pio_in_emulated     = emulator_pio_in_emulated,
	.pio_out_emulated    = emulator_pio_out_emulated,
6220 6221
	.get_segment         = emulator_get_segment,
	.set_segment         = emulator_set_segment,
6222
	.get_cached_segment_base = emulator_get_cached_segment_base,
6223
	.get_gdt             = emulator_get_gdt,
6224
	.get_idt	     = emulator_get_idt,
6225 6226
	.set_gdt             = emulator_set_gdt,
	.set_idt	     = emulator_set_idt,
6227 6228
	.get_cr              = emulator_get_cr,
	.set_cr              = emulator_set_cr,
6229
	.cpl                 = emulator_get_cpl,
6230 6231
	.get_dr              = emulator_get_dr,
	.set_dr              = emulator_set_dr,
6232 6233
	.get_smbase          = emulator_get_smbase,
	.set_smbase          = emulator_set_smbase,
6234 6235
	.set_msr             = emulator_set_msr,
	.get_msr             = emulator_get_msr,
6236
	.check_pmc	     = emulator_check_pmc,
6237
	.read_pmc            = emulator_read_pmc,
6238
	.halt                = emulator_halt,
6239
	.wbinvd              = emulator_wbinvd,
6240
	.fix_hypercall       = emulator_fix_hypercall,
6241
	.intercept           = emulator_intercept,
6242
	.get_cpuid           = emulator_get_cpuid,
6243
	.set_nmi_mask        = emulator_set_nmi_mask,
6244 6245
	.get_hflags          = emulator_get_hflags,
	.set_hflags          = emulator_set_hflags,
6246
	.pre_leave_smm       = emulator_pre_leave_smm,
6247
	.post_leave_smm      = emulator_post_leave_smm,
6248
	.set_xcr             = emulator_set_xcr,
6249 6250
};

6251 6252
static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
{
6253
	u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu);
6254 6255 6256 6257 6258 6259 6260
	/*
	 * an sti; sti; sequence only disable interrupts for the first
	 * instruction. So, if the last instruction, be it emulated or
	 * not, left the system with the INT_STI flag enabled, it
	 * means that the last instruction is an sti. We should not
	 * leave the flag on in this case. The same goes for mov ss
	 */
6261 6262
	if (int_shadow & mask)
		mask = 0;
6263
	if (unlikely(int_shadow || mask)) {
6264
		kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
6265 6266 6267
		if (!mask)
			kvm_make_request(KVM_REQ_EVENT, vcpu);
	}
6268 6269
}

6270
static bool inject_emulated_exception(struct kvm_vcpu *vcpu)
6271 6272
{
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6273
	if (ctxt->exception.vector == PF_VECTOR)
6274 6275 6276
		return kvm_propagate_fault(vcpu, &ctxt->exception);

	if (ctxt->exception.error_code_valid)
6277 6278
		kvm_queue_exception_e(vcpu, ctxt->exception.vector,
				      ctxt->exception.error_code);
6279
	else
6280
		kvm_queue_exception(vcpu, ctxt->exception.vector);
6281
	return false;
6282 6283
}

6284 6285
static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
{
6286
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6287 6288 6289 6290
	int cs_db, cs_l;

	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);

6291
	ctxt->eflags = kvm_get_rflags(vcpu);
6292 6293
	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;

6294 6295 6296
	ctxt->eip = kvm_rip_read(vcpu);
	ctxt->mode = (!is_protmode(vcpu))		? X86EMUL_MODE_REAL :
		     (ctxt->eflags & X86_EFLAGS_VM)	? X86EMUL_MODE_VM86 :
6297
		     (cs_l && is_long_mode(vcpu))	? X86EMUL_MODE_PROT64 :
6298 6299
		     cs_db				? X86EMUL_MODE_PROT32 :
							  X86EMUL_MODE_PROT16;
6300
	BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK);
6301 6302
	BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK);
	BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK);
6303

6304
	init_decode_cache(ctxt);
6305
	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6306 6307
}

6308
void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
6309
{
6310
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6311 6312 6313 6314
	int ret;

	init_emulate_ctxt(vcpu);

6315 6316 6317
	ctxt->op_bytes = 2;
	ctxt->ad_bytes = 2;
	ctxt->_eip = ctxt->eip + inc_eip;
6318
	ret = emulate_int_real(ctxt, irq);
6319

6320 6321 6322 6323 6324 6325 6326
	if (ret != X86EMUL_CONTINUE) {
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
	} else {
		ctxt->eip = ctxt->_eip;
		kvm_rip_write(vcpu, ctxt->eip);
		kvm_set_rflags(vcpu, ctxt->eflags);
	}
6327 6328 6329
}
EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);

6330
static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
6331 6332 6333
{
	++vcpu->stat.insn_emulation_fail;
	trace_kvm_emulate_insn_failed(vcpu);
6334

6335 6336
	if (emulation_type & EMULTYPE_VMWARE_GP) {
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6337
		return 1;
6338
	}
6339

6340 6341 6342 6343
	if (emulation_type & EMULTYPE_SKIP) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6344
		return 0;
6345 6346
	}

6347 6348
	kvm_queue_exception(vcpu, UD_VECTOR);

6349
	if (!is_guest_mode(vcpu) && kvm_x86_ops->get_cpl(vcpu) == 0) {
6350 6351 6352
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
6353
		return 0;
6354
	}
6355

6356
	return 1;
6357 6358
}

6359
static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,
6360 6361
				  bool write_fault_to_shadow_pgtable,
				  int emulation_type)
6362
{
6363
	gpa_t gpa = cr2;
6364
	kvm_pfn_t pfn;
6365

6366
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
6367 6368
		return false;

6369 6370 6371
	if (WARN_ON_ONCE(is_guest_mode(vcpu)))
		return false;

6372
	if (!vcpu->arch.mmu->direct_map) {
6373 6374 6375 6376 6377
		/*
		 * Write permission should be allowed since only
		 * write access need to be emulated.
		 */
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
6378

6379 6380 6381 6382 6383 6384 6385
		/*
		 * If the mapping is invalid in guest, let cpu retry
		 * it to generate fault.
		 */
		if (gpa == UNMAPPED_GVA)
			return true;
	}
6386

6387 6388 6389 6390 6391 6392 6393
	/*
	 * Do not retry the unhandleable instruction if it faults on the
	 * readonly host memory, otherwise it will goto a infinite loop:
	 * retry instruction -> write #PF -> emulation fail -> retry
	 * instruction -> ...
	 */
	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404

	/*
	 * If the instruction failed on the error pfn, it can not be fixed,
	 * report the error to userspace.
	 */
	if (is_error_noslot_pfn(pfn))
		return false;

	kvm_release_pfn_clean(pfn);

	/* The instructions are well-emulated on direct mmu. */
6405
	if (vcpu->arch.mmu->direct_map) {
6406 6407 6408 6409 6410 6411 6412 6413 6414
		unsigned int indirect_shadow_pages;

		spin_lock(&vcpu->kvm->mmu_lock);
		indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
		spin_unlock(&vcpu->kvm->mmu_lock);

		if (indirect_shadow_pages)
			kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));

6415
		return true;
6416
	}
6417

6418 6419 6420 6421 6422 6423
	/*
	 * if emulation was due to access to shadowed page table
	 * and it failed try to unshadow page and re-enter the
	 * guest to let CPU execute the instruction.
	 */
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6424 6425 6426 6427 6428 6429 6430

	/*
	 * If the access faults on its page table, it can not
	 * be fixed by unprotecting shadow page and it should
	 * be reported to userspace.
	 */
	return !write_fault_to_shadow_pgtable;
6431 6432
}

6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456
static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
			      unsigned long cr2,  int emulation_type)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	unsigned long last_retry_eip, last_retry_addr, gpa = cr2;

	last_retry_eip = vcpu->arch.last_retry_eip;
	last_retry_addr = vcpu->arch.last_retry_addr;

	/*
	 * If the emulation is caused by #PF and it is non-page_table
	 * writing instruction, it means the VM-EXIT is caused by shadow
	 * page protected, we can zap the shadow page and retry this
	 * instruction directly.
	 *
	 * Note: if the guest uses a non-page-table modifying instruction
	 * on the PDE that points to the instruction, then we will unmap
	 * the instruction and go to an infinite loop. So, we cache the
	 * last retried eip and the last fault address, if we meet the eip
	 * and the address again, we can break out of the potential infinite
	 * loop.
	 */
	vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;

6457
	if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
6458 6459
		return false;

6460 6461 6462
	if (WARN_ON_ONCE(is_guest_mode(vcpu)))
		return false;

6463 6464 6465 6466 6467 6468 6469 6470 6471
	if (x86_page_table_writing_insn(ctxt))
		return false;

	if (ctxt->eip == last_retry_eip && last_retry_addr == cr2)
		return false;

	vcpu->arch.last_retry_eip = ctxt->eip;
	vcpu->arch.last_retry_addr = cr2;

6472
	if (!vcpu->arch.mmu->direct_map)
6473 6474
		gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);

6475
	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
6476 6477 6478 6479

	return true;
}

6480 6481 6482
static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
static int complete_emulated_pio(struct kvm_vcpu *vcpu);

6483
static void kvm_smm_changed(struct kvm_vcpu *vcpu)
6484
{
6485
	if (!(vcpu->arch.hflags & HF_SMM_MASK)) {
6486 6487 6488
		/* This is a good place to trace that we are exiting SMM.  */
		trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false);

6489 6490
		/* Process a latched INIT or SMI, if any.  */
		kvm_make_request(KVM_REQ_EVENT, vcpu);
6491
	}
6492 6493

	kvm_mmu_reset_context(vcpu);
6494 6495
}

6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510
static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
				unsigned long *db)
{
	u32 dr6 = 0;
	int i;
	u32 enable, rwlen;

	enable = dr7;
	rwlen = dr7 >> 16;
	for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
		if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
			dr6 |= (1 << i);
	return dr6;
}

6511
static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
6512 6513 6514
{
	struct kvm_run *kvm_run = vcpu->run;

6515 6516 6517 6518 6519
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
		kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1 | DR6_RTM;
		kvm_run->debug.arch.pc = vcpu->arch.singlestep_rip;
		kvm_run->debug.arch.exception = DB_VECTOR;
		kvm_run->exit_reason = KVM_EXIT_DEBUG;
6520
		return 0;
6521
	}
6522
	kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
6523
	return 1;
6524 6525
}

6526 6527 6528
int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
	unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6529
	int r;
6530

6531
	r = kvm_x86_ops->skip_emulated_instruction(vcpu);
6532
	if (unlikely(!r))
6533
		return 0;
6534 6535 6536 6537 6538 6539 6540 6541 6542 6543

	/*
	 * rflags is the old, "raw" value of the flags.  The new value has
	 * not been saved yet.
	 *
	 * This is correct even for TF set by the guest, because "the
	 * processor will not generate this exception after the instruction
	 * that sets the TF flag".
	 */
	if (unlikely(rflags & X86_EFLAGS_TF))
6544
		r = kvm_vcpu_do_singlestep(vcpu);
6545
	return r;
6546 6547 6548
}
EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);

6549 6550 6551 6552
static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
{
	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
	    (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
6553 6554 6555
		struct kvm_run *kvm_run = vcpu->run;
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6556 6557 6558 6559
					   vcpu->arch.guest_debug_dr7,
					   vcpu->arch.eff_db);

		if (dr6 != 0) {
6560
			kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
6561
			kvm_run->debug.arch.pc = eip;
6562 6563
			kvm_run->debug.arch.exception = DB_VECTOR;
			kvm_run->exit_reason = KVM_EXIT_DEBUG;
6564
			*r = 0;
6565 6566 6567 6568
			return true;
		}
	}

6569 6570
	if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
	    !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) {
6571 6572
		unsigned long eip = kvm_get_linear_rip(vcpu);
		u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
6573 6574 6575 6576
					   vcpu->arch.dr7,
					   vcpu->arch.db);

		if (dr6 != 0) {
6577
			vcpu->arch.dr6 &= ~DR_TRAP_BITS;
6578
			vcpu->arch.dr6 |= dr6 | DR6_RTM;
6579
			kvm_queue_exception(vcpu, DB_VECTOR);
6580
			*r = 1;
6581 6582 6583 6584 6585 6586 6587
			return true;
		}
	}

	return false;
}

6588 6589
static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
{
6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613
	switch (ctxt->opcode_len) {
	case 1:
		switch (ctxt->b) {
		case 0xe4:	/* IN */
		case 0xe5:
		case 0xec:
		case 0xed:
		case 0xe6:	/* OUT */
		case 0xe7:
		case 0xee:
		case 0xef:
		case 0x6c:	/* INS */
		case 0x6d:
		case 0x6e:	/* OUTS */
		case 0x6f:
			return true;
		}
		break;
	case 2:
		switch (ctxt->b) {
		case 0x33:	/* RDPMC */
			return true;
		}
		break;
6614 6615 6616 6617 6618
	}

	return false;
}

6619 6620
int x86_emulate_instruction(struct kvm_vcpu *vcpu,
			    unsigned long cr2,
6621 6622 6623
			    int emulation_type,
			    void *insn,
			    int insn_len)
6624
{
6625
	int r;
6626
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6627
	bool writeback = true;
6628
	bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
6629

6630 6631
	vcpu->arch.l1tf_flush_l1d = true;

6632 6633 6634 6635 6636
	/*
	 * Clear write_fault_to_shadow_pgtable here to ensure it is
	 * never reused.
	 */
	vcpu->arch.write_fault_to_shadow_pgtable = false;
6637
	kvm_clear_exception_queue(vcpu);
6638

6639
	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
6640
		init_emulate_ctxt(vcpu);
6641 6642 6643 6644 6645 6646 6647

		/*
		 * We will reenter on the same instruction since
		 * we do not set complete_userspace_io.  This does not
		 * handle watchpoints yet, those would be handled in
		 * the emulate_ops.
		 */
6648 6649
		if (!(emulation_type & EMULTYPE_SKIP) &&
		    kvm_vcpu_check_breakpoint(vcpu, &r))
6650 6651
			return r;

6652 6653
		ctxt->interruptibility = 0;
		ctxt->have_exception = false;
6654
		ctxt->exception.vector = -1;
6655
		ctxt->perm_ok = false;
6656

6657
		ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
6658

6659
		r = x86_decode_insn(ctxt, insn, insn_len);
6660

6661
		trace_kvm_emulate_insn_start(vcpu);
6662
		++vcpu->stat.insn_emulation;
6663
		if (r != EMULATION_OK)  {
6664
			if ((emulation_type & EMULTYPE_TRAP_UD) ||
6665 6666
			    (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
				kvm_queue_exception(vcpu, UD_VECTOR);
6667
				return 1;
6668
			}
6669 6670
			if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
						emulation_type))
6671
				return 1;
6672
			if (ctxt->have_exception) {
6673 6674 6675 6676 6677 6678
				/*
				 * #UD should result in just EMULATION_FAILED, and trap-like
				 * exception should not be encountered during decode.
				 */
				WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
					     exception_type(ctxt->exception.vector) == EXCPT_TRAP);
6679
				inject_emulated_exception(vcpu);
6680
				return 1;
6681
			}
6682
			return handle_emulation_failure(vcpu, emulation_type);
6683 6684 6685
		}
	}

6686 6687 6688
	if ((emulation_type & EMULTYPE_VMWARE_GP) &&
	    !is_vmware_backdoor_opcode(ctxt)) {
		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
6689
		return 1;
6690
	}
6691

6692 6693 6694 6695 6696
	/*
	 * Note, EMULTYPE_SKIP is intended for use *only* by vendor callbacks
	 * for kvm_skip_emulated_instruction().  The caller is responsible for
	 * updating interruptibility state and injecting single-step #DBs.
	 */
6697
	if (emulation_type & EMULTYPE_SKIP) {
6698
		kvm_rip_write(vcpu, ctxt->_eip);
6699 6700
		if (ctxt->eflags & X86_EFLAGS_RF)
			kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
6701
		return 1;
6702 6703
	}

6704
	if (retry_instruction(ctxt, cr2, emulation_type))
6705
		return 1;
6706

6707
	/* this is needed for vmware backdoor interface to work since it
6708
	   changes registers values  during IO operation */
6709 6710
	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
6711
		emulator_invalidate_register_cache(ctxt);
6712
	}
6713

6714
restart:
6715 6716 6717
	/* Save the faulting GPA (cr2) in the address field */
	ctxt->exception.address = cr2;

6718
	r = x86_emulate_insn(ctxt);
6719

6720
	if (r == EMULATION_INTERCEPTED)
6721
		return 1;
6722

6723
	if (r == EMULATION_FAILED) {
6724 6725
		if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
					emulation_type))
6726
			return 1;
6727

6728
		return handle_emulation_failure(vcpu, emulation_type);
6729 6730
	}

6731
	if (ctxt->have_exception) {
6732
		r = 1;
6733 6734
		if (inject_emulated_exception(vcpu))
			return r;
6735
	} else if (vcpu->arch.pio.count) {
6736 6737
		if (!vcpu->arch.pio.in) {
			/* FIXME: return into emulator if single-stepping.  */
6738
			vcpu->arch.pio.count = 0;
6739
		} else {
6740
			writeback = false;
6741 6742
			vcpu->arch.complete_userspace_io = complete_emulated_pio;
		}
6743
		r = 0;
6744
	} else if (vcpu->mmio_needed) {
6745 6746
		++vcpu->stat.mmio_exits;

6747 6748
		if (!vcpu->mmio_is_write)
			writeback = false;
6749
		r = 0;
6750
		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
6751
	} else if (r == EMULATION_RESTART)
6752
		goto restart;
6753
	else
6754
		r = 1;
6755

6756
	if (writeback) {
6757
		unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6758
		toggle_interruptibility(vcpu, ctxt->interruptibility);
6759
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
6760
		if (!ctxt->have_exception ||
6761 6762
		    exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
			kvm_rip_write(vcpu, ctxt->eip);
6763
			if (r && ctxt->tf)
6764
				r = kvm_vcpu_do_singlestep(vcpu);
6765
			__kvm_set_rflags(vcpu, ctxt->eflags);
6766
		}
6767 6768 6769 6770 6771 6772 6773 6774 6775

		/*
		 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
		 * do nothing, and it will be requested again as soon as
		 * the shadow expires.  But we still need to check here,
		 * because POPF has no interrupt shadow.
		 */
		if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
			kvm_make_request(KVM_REQ_EVENT, vcpu);
6776 6777
	} else
		vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
6778 6779

	return r;
6780
}
6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793

int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
{
	return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction);

int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
					void *insn, int insn_len)
{
	return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
}
EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
6794

6795 6796 6797 6798 6799 6800
static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;
	return 1;
}

6801 6802 6803 6804 6805 6806 6807 6808 6809 6810
static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;

	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
		return 1;

	return kvm_skip_emulated_instruction(vcpu);
}

6811 6812
static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
			    unsigned short port)
6813
{
6814
	unsigned long val = kvm_rax_read(vcpu);
6815 6816
	int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
					    size, port, &val, 1);
6817 6818
	if (ret)
		return ret;
6819

6820 6821 6822 6823 6824 6825 6826 6827 6828 6829
	/*
	 * Workaround userspace that relies on old KVM behavior of %rip being
	 * incremented prior to exiting to userspace to handle "OUT 0x7e".
	 */
	if (port == 0x7e &&
	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
		vcpu->arch.complete_userspace_io =
			complete_fast_pio_out_port_0x7e;
		kvm_skip_emulated_instruction(vcpu);
	} else {
6830 6831 6832
		vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
		vcpu->arch.complete_userspace_io = complete_fast_pio_out;
	}
6833
	return 0;
6834 6835
}

6836 6837 6838 6839 6840 6841 6842
static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
{
	unsigned long val;

	/* We should only ever be called with arch.pio.count equal to 1 */
	BUG_ON(vcpu->arch.pio.count != 1);

6843 6844 6845 6846 6847
	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
		vcpu->arch.pio.count = 0;
		return 1;
	}

6848
	/* For size less than 4 we merge, else we zero extend */
6849
	val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
6850 6851 6852 6853 6854 6855 6856

	/*
	 * Since vcpu->arch.pio.count == 1 let emulator_pio_in_emulated perform
	 * the copy and tracing
	 */
	emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, vcpu->arch.pio.size,
				 vcpu->arch.pio.port, &val, 1);
6857
	kvm_rax_write(vcpu, val);
6858

6859
	return kvm_skip_emulated_instruction(vcpu);
6860 6861
}

6862 6863
static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
			   unsigned short port)
6864 6865 6866 6867 6868
{
	unsigned long val;
	int ret;

	/* For size less than 4 we merge, else we zero extend */
6869
	val = (size < 4) ? kvm_rax_read(vcpu) : 0;
6870 6871 6872 6873

	ret = emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, size, port,
				       &val, 1);
	if (ret) {
6874
		kvm_rax_write(vcpu, val);
6875 6876 6877
		return ret;
	}

6878
	vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
6879 6880 6881 6882
	vcpu->arch.complete_userspace_io = complete_fast_pio_in;

	return 0;
}
6883 6884 6885

int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
{
6886
	int ret;
6887 6888

	if (in)
6889
		ret = kvm_fast_pio_in(vcpu, size, port);
6890
	else
6891 6892
		ret = kvm_fast_pio_out(vcpu, size, port);
	return ret && kvm_skip_emulated_instruction(vcpu);
6893 6894
}
EXPORT_SYMBOL_GPL(kvm_fast_pio);
6895

6896
static int kvmclock_cpu_down_prep(unsigned int cpu)
6897
{
6898
	__this_cpu_write(cpu_tsc_khz, 0);
6899
	return 0;
6900 6901 6902
}

static void tsc_khz_changed(void *data)
6903
{
6904 6905 6906 6907 6908 6909 6910 6911 6912
	struct cpufreq_freqs *freq = data;
	unsigned long khz = 0;

	if (data)
		khz = freq->new;
	else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		khz = cpufreq_quick_get(raw_smp_processor_id());
	if (!khz)
		khz = tsc_khz;
6913
	__this_cpu_write(cpu_tsc_khz, khz);
6914 6915
}

6916
#ifdef CONFIG_X86_64
6917 6918 6919 6920 6921 6922
static void kvm_hyperv_tsc_notifier(void)
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int cpu;

6923
	mutex_lock(&kvm_lock);
6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_make_mclock_inprogress_request(kvm);

	hyperv_stop_tsc_emulation();

	/* TSC frequency always matches when on Hyper-V */
	for_each_present_cpu(cpu)
		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
	kvm_max_guest_tsc_khz = tsc_khz;

	list_for_each_entry(kvm, &vm_list, vm_list) {
		struct kvm_arch *ka = &kvm->arch;

		spin_lock(&ka->pvclock_gtod_sync_lock);

		pvclock_update_vm_gtod_copy(kvm);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);

		kvm_for_each_vcpu(cpu, vcpu, kvm)
			kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);

		spin_unlock(&ka->pvclock_gtod_sync_lock);
	}
6949
	mutex_unlock(&kvm_lock);
6950
}
6951
#endif
6952

6953
static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
6954 6955 6956 6957 6958
{
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i, send_ipi = 0;

6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997
	/*
	 * We allow guests to temporarily run on slowing clocks,
	 * provided we notify them after, or to run on accelerating
	 * clocks, provided we notify them before.  Thus time never
	 * goes backwards.
	 *
	 * However, we have a problem.  We can't atomically update
	 * the frequency of a given CPU from this function; it is
	 * merely a notifier, which can be called from any CPU.
	 * Changing the TSC frequency at arbitrary points in time
	 * requires a recomputation of local variables related to
	 * the TSC for each VCPU.  We must flag these local variables
	 * to be updated and be sure the update takes place with the
	 * new frequency before any guests proceed.
	 *
	 * Unfortunately, the combination of hotplug CPU and frequency
	 * change creates an intractable locking scenario; the order
	 * of when these callouts happen is undefined with respect to
	 * CPU hotplug, and they can race with each other.  As such,
	 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
	 * undefined; you can actually have a CPU frequency change take
	 * place in between the computation of X and the setting of the
	 * variable.  To protect against this problem, all updates of
	 * the per_cpu tsc_khz variable are done in an interrupt
	 * protected IPI, and all callers wishing to update the value
	 * must wait for a synchronous IPI to complete (which is trivial
	 * if the caller is on the CPU already).  This establishes the
	 * necessary total order on variable updates.
	 *
	 * Note that because a guest time update may take place
	 * anytime after the setting of the VCPU's request bit, the
	 * correct TSC value must be set before the request.  However,
	 * to ensure the update actually makes it to any guest which
	 * starts running in hardware virtualization between the set
	 * and the acquisition of the spinlock, we must also ping the
	 * CPU after setting the request bit.
	 *
	 */

6998
	smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
6999

7000
	mutex_lock(&kvm_lock);
7001
	list_for_each_entry(kvm, &vm_list, vm_list) {
7002
		kvm_for_each_vcpu(i, vcpu, kvm) {
7003
			if (vcpu->cpu != cpu)
7004
				continue;
Zachary Amsden's avatar
Zachary Amsden committed
7005
			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
7006
			if (vcpu->cpu != raw_smp_processor_id())
7007
				send_ipi = 1;
7008 7009
		}
	}
7010
	mutex_unlock(&kvm_lock);
7011 7012 7013 7014 7015 7016 7017 7018 7019 7020 7021 7022 7023 7024

	if (freq->old < freq->new && send_ipi) {
		/*
		 * We upscale the frequency.  Must make the guest
		 * doesn't see old kvmclock values while running with
		 * the new frequency, otherwise we risk the guest sees
		 * time go backwards.
		 *
		 * In case we update the frequency for another cpu
		 * (which might be in guest context) send an interrupt
		 * to kick the cpu out of guest context.  Next time
		 * guest context is entered kvmclock will be updated,
		 * so the guest will not see stale values.
		 */
7025
		smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
7026
	}
7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042
}

static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
				     void *data)
{
	struct cpufreq_freqs *freq = data;
	int cpu;

	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
		return 0;
	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
		return 0;

	for_each_cpu(cpu, freq->policy->cpus)
		__kvmclock_cpufreq_notifier(freq, cpu);

7043 7044 7045 7046
	return 0;
}

static struct notifier_block kvmclock_cpufreq_notifier_block = {
7047 7048 7049
	.notifier_call  = kvmclock_cpufreq_notifier
};

7050
static int kvmclock_cpu_online(unsigned int cpu)
7051
{
7052 7053
	tsc_khz_changed(NULL);
	return 0;
7054 7055
}

7056 7057
static void kvm_timer_init(void)
{
Zachary Amsden's avatar
Zachary Amsden committed
7058
	max_tsc_khz = tsc_khz;
7059

7060
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
Zachary Amsden's avatar
Zachary Amsden committed
7061 7062
#ifdef CONFIG_CPU_FREQ
		struct cpufreq_policy policy;
7063 7064
		int cpu;

Zachary Amsden's avatar
Zachary Amsden committed
7065
		memset(&policy, 0, sizeof(policy));
7066 7067
		cpu = get_cpu();
		cpufreq_get_policy(&policy, cpu);
Zachary Amsden's avatar
Zachary Amsden committed
7068 7069
		if (policy.cpuinfo.max_freq)
			max_tsc_khz = policy.cpuinfo.max_freq;
7070
		put_cpu();
Zachary Amsden's avatar
Zachary Amsden committed
7071
#endif
7072 7073 7074
		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
					  CPUFREQ_TRANSITION_NOTIFIER);
	}
7075

7076
	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
7077
			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
7078 7079
}

7080 7081
DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu);
7082

7083
int kvm_is_in_guest(void)
7084
{
7085
	return __this_cpu_read(current_vcpu) != NULL;
7086 7087 7088 7089 7090
}

static int kvm_is_user_mode(void)
{
	int user_mode = 3;
7091

7092 7093
	if (__this_cpu_read(current_vcpu))
		user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
7094

7095 7096 7097 7098 7099 7100
	return user_mode != 0;
}

static unsigned long kvm_get_guest_ip(void)
{
	unsigned long ip = 0;
7101

7102 7103
	if (__this_cpu_read(current_vcpu))
		ip = kvm_rip_read(__this_cpu_read(current_vcpu));
7104

7105 7106 7107
	return ip;
}

7108 7109 7110 7111 7112 7113 7114 7115 7116
static void kvm_handle_intel_pt_intr(void)
{
	struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);

	kvm_make_request(KVM_REQ_PMI, vcpu);
	__set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
			(unsigned long *)&vcpu->arch.pmu.global_status);
}

7117 7118 7119 7120
static struct perf_guest_info_callbacks kvm_guest_cbs = {
	.is_in_guest		= kvm_is_in_guest,
	.is_user_mode		= kvm_is_user_mode,
	.get_guest_ip		= kvm_get_guest_ip,
7121
	.handle_intel_pt_intr	= kvm_handle_intel_pt_intr,
7122 7123
};

7124 7125 7126
#ifdef CONFIG_X86_64
static void pvclock_gtod_update_fn(struct work_struct *work)
{
7127 7128 7129 7130 7131
	struct kvm *kvm;

	struct kvm_vcpu *vcpu;
	int i;

7132
	mutex_lock(&kvm_lock);
7133 7134
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_for_each_vcpu(i, vcpu, kvm)
7135
			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
7136
	atomic_set(&kvm_guest_has_master_clock, 0);
7137
	mutex_unlock(&kvm_lock);
7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153
}

static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);

/*
 * Notification about pvclock gtod data update.
 */
static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
			       void *priv)
{
	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
	struct timekeeper *tk = priv;

	update_pvclock_gtod(tk);

	/* disable master clock if host does not trust, or does not
7154
	 * use, TSC based clocksource.
7155
	 */
7156
	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167
	    atomic_read(&kvm_guest_has_master_clock) != 0)
		queue_work(system_long_wq, &pvclock_gtod_work);

	return 0;
}

static struct notifier_block pvclock_gtod_notifier = {
	.notifier_call = pvclock_gtod_notify,
};
#endif

7168
int kvm_arch_init(void *opaque)
7169
{
7170
	int r;
Mathias Krause's avatar
Mathias Krause committed
7171
	struct kvm_x86_ops *ops = opaque;
7172 7173 7174

	if (kvm_x86_ops) {
		printk(KERN_ERR "kvm: already loaded the other module\n");
7175 7176
		r = -EEXIST;
		goto out;
7177 7178 7179 7180
	}

	if (!ops->cpu_has_kvm_support()) {
		printk(KERN_ERR "kvm: no hardware support\n");
7181 7182
		r = -EOPNOTSUPP;
		goto out;
7183 7184 7185
	}
	if (ops->disabled_by_bios()) {
		printk(KERN_ERR "kvm: disabled by bios\n");
7186 7187
		r = -EOPNOTSUPP;
		goto out;
7188 7189
	}

7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200
	/*
	 * KVM explicitly assumes that the guest has an FPU and
	 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
	 * vCPU's FPU state as a fxregs_state struct.
	 */
	if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
		printk(KERN_ERR "kvm: inadequate fpu\n");
		r = -EOPNOTSUPP;
		goto out;
	}

7201
	r = -ENOMEM;
7202
	x86_fpu_cache = kmem_cache_create("x86_fpu", sizeof(struct fpu),
7203 7204 7205 7206 7207 7208 7209
					  __alignof__(struct fpu), SLAB_ACCOUNT,
					  NULL);
	if (!x86_fpu_cache) {
		printk(KERN_ERR "kvm: failed to allocate cache for x86 fpu\n");
		goto out;
	}

7210 7211 7212
	shared_msrs = alloc_percpu(struct kvm_shared_msrs);
	if (!shared_msrs) {
		printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
7213
		goto out_free_x86_fpu_cache;
7214 7215
	}

7216 7217
	r = kvm_mmu_module_init();
	if (r)
7218
		goto out_free_percpu;
7219

7220
	kvm_x86_ops = ops;
Paolo Bonzini's avatar
Paolo Bonzini committed
7221

Sheng Yang's avatar
Sheng Yang committed
7222
	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
7223
			PT_DIRTY_MASK, PT64_NX_MASK, 0,
7224
			PT_PRESENT_MASK, 0, sme_me_mask);
7225
	kvm_timer_init();
7226

7227 7228
	perf_register_guest_info_callbacks(&kvm_guest_cbs);

7229
	if (boot_cpu_has(X86_FEATURE_XSAVE))
7230 7231
		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);

7232
	kvm_lapic_init();
7233 7234
	if (pi_inject_timer == -1)
		pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);
7235 7236
#ifdef CONFIG_X86_64
	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
7237

7238
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7239
		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
7240 7241
#endif

7242
	return 0;
7243

7244 7245
out_free_percpu:
	free_percpu(shared_msrs);
7246 7247
out_free_x86_fpu_cache:
	kmem_cache_destroy(x86_fpu_cache);
7248 7249
out:
	return r;
7250
}
7251

7252 7253
void kvm_arch_exit(void)
{
7254
#ifdef CONFIG_X86_64
7255
	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
7256 7257
		clear_hv_tscchange_cb();
#endif
7258
	kvm_lapic_exit();
7259 7260
	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);

7261 7262 7263
	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
					    CPUFREQ_TRANSITION_NOTIFIER);
7264
	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
7265 7266 7267
#ifdef CONFIG_X86_64
	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
#endif
7268
	kvm_x86_ops = NULL;
7269
	kvm_mmu_module_exit();
7270
	free_percpu(shared_msrs);
7271
	kmem_cache_destroy(x86_fpu_cache);
7272
}
7273

7274
int kvm_vcpu_halt(struct kvm_vcpu *vcpu)
7275 7276
{
	++vcpu->stat.halt_exits;
7277
	if (lapic_in_kernel(vcpu)) {
7278
		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
7279 7280 7281 7282 7283 7284
		return 1;
	} else {
		vcpu->run->exit_reason = KVM_EXIT_HLT;
		return 0;
	}
}
7285 7286 7287 7288
EXPORT_SYMBOL_GPL(kvm_vcpu_halt);

int kvm_emulate_halt(struct kvm_vcpu *vcpu)
{
7289 7290 7291 7292 7293 7294
	int ret = kvm_skip_emulated_instruction(vcpu);
	/*
	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
	 * KVM_EXIT_DEBUG here.
	 */
	return kvm_vcpu_halt(vcpu) && ret;
7295
}
7296 7297
EXPORT_SYMBOL_GPL(kvm_emulate_halt);

7298
#ifdef CONFIG_X86_64
7299 7300 7301 7302
static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
			        unsigned long clock_type)
{
	struct kvm_clock_pairing clock_pairing;
7303
	struct timespec64 ts;
Paolo Bonzini's avatar
Paolo Bonzini committed
7304
	u64 cycle;
7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316
	int ret;

	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
		return -KVM_EOPNOTSUPP;

	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)
		return -KVM_EOPNOTSUPP;

	clock_pairing.sec = ts.tv_sec;
	clock_pairing.nsec = ts.tv_nsec;
	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
	clock_pairing.flags = 0;
7317
	memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
7318 7319 7320 7321 7322 7323 7324 7325

	ret = 0;
	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
			    sizeof(struct kvm_clock_pairing)))
		ret = -KVM_EFAULT;

	return ret;
}
7326
#endif
7327

7328 7329 7330 7331 7332 7333 7334
/*
 * kvm_pv_kick_cpu_op:  Kick a vcpu.
 *
 * @apicid - apicid of vcpu to be kicked.
 */
static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
{
7335
	struct kvm_lapic_irq lapic_irq;
7336

7337 7338
	lapic_irq.shorthand = 0;
	lapic_irq.dest_mode = 0;
7339
	lapic_irq.level = 0;
7340
	lapic_irq.dest_id = apicid;
7341
	lapic_irq.msi_redir_hint = false;
7342

7343
	lapic_irq.delivery_mode = APIC_DM_REMRD;
7344
	kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
7345 7346
}

7347 7348
void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu)
{
7349 7350 7351 7352 7353 7354 7355
	if (!lapic_in_kernel(vcpu)) {
		WARN_ON_ONCE(vcpu->arch.apicv_active);
		return;
	}
	if (!vcpu->arch.apicv_active)
		return;

7356 7357 7358 7359
	vcpu->arch.apicv_active = false;
	kvm_x86_ops->refresh_apicv_exec_ctrl(vcpu);
}

7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372
static void kvm_sched_yield(struct kvm *kvm, unsigned long dest_id)
{
	struct kvm_vcpu *target = NULL;
	struct kvm_apic_map *map;

	rcu_read_lock();
	map = rcu_dereference(kvm->arch.apic_map);

	if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])
		target = map->phys_map[dest_id]->vcpu;

	rcu_read_unlock();

7373
	if (target && READ_ONCE(target->ready))
7374 7375 7376
		kvm_vcpu_yield_to(target);
}

7377 7378 7379
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
{
	unsigned long nr, a0, a1, a2, a3, ret;
7380
	int op_64_bit;
7381

7382 7383
	if (kvm_hv_hypercall_enabled(vcpu->kvm))
		return kvm_hv_hypercall(vcpu);
7384

7385 7386 7387 7388 7389
	nr = kvm_rax_read(vcpu);
	a0 = kvm_rbx_read(vcpu);
	a1 = kvm_rcx_read(vcpu);
	a2 = kvm_rdx_read(vcpu);
	a3 = kvm_rsi_read(vcpu);
7390

7391
	trace_kvm_hypercall(nr, a0, a1, a2, a3);
Feng (Eric) Liu's avatar
Feng (Eric) Liu committed
7392

7393 7394
	op_64_bit = is_64_bit_mode(vcpu);
	if (!op_64_bit) {
7395 7396 7397 7398 7399 7400 7401
		nr &= 0xFFFFFFFF;
		a0 &= 0xFFFFFFFF;
		a1 &= 0xFFFFFFFF;
		a2 &= 0xFFFFFFFF;
		a3 &= 0xFFFFFFFF;
	}

7402 7403
	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
		ret = -KVM_EPERM;
7404
		goto out;
7405 7406
	}

7407
	switch (nr) {
Avi Kivity's avatar
Avi Kivity committed
7408 7409 7410
	case KVM_HC_VAPIC_POLL_IRQ:
		ret = 0;
		break;
7411 7412
	case KVM_HC_KICK_CPU:
		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
7413
		kvm_sched_yield(vcpu->kvm, a1);
7414 7415
		ret = 0;
		break;
7416
#ifdef CONFIG_X86_64
7417 7418 7419
	case KVM_HC_CLOCK_PAIRING:
		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
		break;
7420
#endif
7421 7422 7423
	case KVM_HC_SEND_IPI:
		ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
		break;
7424 7425 7426 7427
	case KVM_HC_SCHED_YIELD:
		kvm_sched_yield(vcpu->kvm, a0);
		ret = 0;
		break;
7428 7429 7430 7431
	default:
		ret = -KVM_ENOSYS;
		break;
	}
7432
out:
7433 7434
	if (!op_64_bit)
		ret = (u32)ret;
7435
	kvm_rax_write(vcpu, ret);
7436

7437
	++vcpu->stat.hypercalls;
7438
	return kvm_skip_emulated_instruction(vcpu);
7439 7440 7441
}
EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);

7442
static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
7443
{
7444
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7445
	char instruction[3];
7446
	unsigned long rip = kvm_rip_read(vcpu);
7447 7448 7449

	kvm_x86_ops->patch_hypercall(vcpu, instruction);

7450 7451
	return emulator_write_emulated(ctxt, rip, instruction, 3,
		&ctxt->exception);
7452 7453
}

7454
static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
7455
{
7456 7457
	return vcpu->run->request_interrupt_window &&
		likely(!pic_in_kernel(vcpu->kvm));
7458 7459
}

7460
static void post_kvm_run_save(struct kvm_vcpu *vcpu)
7461
{
7462 7463
	struct kvm_run *kvm_run = vcpu->run;

7464
	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
7465
	kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;
7466
	kvm_run->cr8 = kvm_get_cr8(vcpu);
7467
	kvm_run->apic_base = kvm_get_apic_base(vcpu);
7468 7469
	kvm_run->ready_for_interrupt_injection =
		pic_in_kernel(vcpu->kvm) ||
7470
		kvm_vcpu_ready_for_interrupt_injection(vcpu);
7471 7472
}

7473 7474 7475 7476 7477 7478 7479
static void update_cr8_intercept(struct kvm_vcpu *vcpu)
{
	int max_irr, tpr;

	if (!kvm_x86_ops->update_cr8_intercept)
		return;

7480
	if (!lapic_in_kernel(vcpu))
7481 7482
		return;

7483 7484 7485
	if (vcpu->arch.apicv_active)
		return;

7486 7487 7488 7489
	if (!vcpu->arch.apic->vapic_addr)
		max_irr = kvm_lapic_find_highest_irr(vcpu);
	else
		max_irr = -1;
7490 7491 7492 7493 7494 7495 7496 7497 7498

	if (max_irr != -1)
		max_irr >>= 4;

	tpr = kvm_lapic_get_cr8(vcpu);

	kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
}

7499
static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win)
7500
{
7501 7502
	int r;

7503
	/* try to reinject previous events if any */
7504

7505 7506
	if (vcpu->arch.exception.injected)
		kvm_x86_ops->queue_exception(vcpu);
7507
	/*
7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519
	 * Do not inject an NMI or interrupt if there is a pending
	 * exception.  Exceptions and interrupts are recognized at
	 * instruction boundaries, i.e. the start of an instruction.
	 * Trap-like exceptions, e.g. #DB, have higher priority than
	 * NMIs and interrupts, i.e. traps are recognized before an
	 * NMI/interrupt that's pending on the same instruction.
	 * Fault-like exceptions, e.g. #GP and #PF, are the lowest
	 * priority, but are only generated (pended) during instruction
	 * execution, i.e. a pending fault-like exception means the
	 * fault occurred on the *previous* instruction and must be
	 * serviced prior to recognizing any new events in order to
	 * fully complete the previous instruction.
7520
	 */
7521 7522
	else if (!vcpu->arch.exception.pending) {
		if (vcpu->arch.nmi_injected)
7523
			kvm_x86_ops->set_nmi(vcpu);
7524
		else if (vcpu->arch.interrupt.injected)
7525 7526 7527
			kvm_x86_ops->set_irq(vcpu);
	}

7528 7529 7530 7531 7532 7533
	/*
	 * Call check_nested_events() even if we reinjected a previous event
	 * in order for caller to determine if it should require immediate-exit
	 * from L2 to L1 due to pending L1 events which require exit
	 * from L2 to L1.
	 */
7534 7535 7536 7537 7538 7539 7540
	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
		r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
		if (r != 0)
			return r;
	}

	/* try to inject new event if pending */
7541
	if (vcpu->arch.exception.pending) {
Avi Kivity's avatar
Avi Kivity committed
7542 7543 7544
		trace_kvm_inj_exception(vcpu->arch.exception.nr,
					vcpu->arch.exception.has_error_code,
					vcpu->arch.exception.error_code);
7545

7546
		WARN_ON_ONCE(vcpu->arch.exception.injected);
7547 7548 7549
		vcpu->arch.exception.pending = false;
		vcpu->arch.exception.injected = true;

7550 7551 7552 7553
		if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT)
			__kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
					     X86_EFLAGS_RF);

7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569
		if (vcpu->arch.exception.nr == DB_VECTOR) {
			/*
			 * This code assumes that nSVM doesn't use
			 * check_nested_events(). If it does, the
			 * DR6/DR7 changes should happen before L1
			 * gets a #VMEXIT for an intercepted #DB in
			 * L2.  (Under VMX, on the other hand, the
			 * DR6/DR7 changes should not happen in the
			 * event of a VM-exit to L1 for an intercepted
			 * #DB in L2.)
			 */
			kvm_deliver_exception_payload(vcpu);
			if (vcpu->arch.dr7 & DR7_GD) {
				vcpu->arch.dr7 &= ~DR7_GD;
				kvm_update_dr7(vcpu);
			}
7570 7571
		}

7572
		kvm_x86_ops->queue_exception(vcpu);
7573 7574 7575 7576 7577 7578 7579 7580
	}

	/* Don't consider new event if we re-injected an event */
	if (kvm_event_needs_reinjection(vcpu))
		return 0;

	if (vcpu->arch.smi_pending && !is_smm(vcpu) &&
	    kvm_x86_ops->smi_allowed(vcpu)) {
7581
		vcpu->arch.smi_pending = false;
7582
		++vcpu->arch.smi_count;
7583
		enter_smm(vcpu);
7584
	} else if (vcpu->arch.nmi_pending && kvm_x86_ops->nmi_allowed(vcpu)) {
7585 7586 7587
		--vcpu->arch.nmi_pending;
		vcpu->arch.nmi_injected = true;
		kvm_x86_ops->set_nmi(vcpu);
7588
	} else if (kvm_cpu_has_injectable_intr(vcpu)) {
7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600
		/*
		 * Because interrupts can be injected asynchronously, we are
		 * calling check_nested_events again here to avoid a race condition.
		 * See https://lkml.org/lkml/2014/7/2/60 for discussion about this
		 * proposal and current concerns.  Perhaps we should be setting
		 * KVM_REQ_EVENT only on certain events and not unconditionally?
		 */
		if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
			r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
			if (r != 0)
				return r;
		}
7601
		if (kvm_x86_ops->interrupt_allowed(vcpu)) {
7602 7603 7604
			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
					    false);
			kvm_x86_ops->set_irq(vcpu);
7605 7606
		}
	}
7607

7608
	return 0;
7609 7610
}

Avi Kivity's avatar
Avi Kivity committed
7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627
static void process_nmi(struct kvm_vcpu *vcpu)
{
	unsigned limit = 2;

	/*
	 * x86 is limited to one NMI running, and one NMI pending after it.
	 * If an NMI is already in progress, limit further NMIs to just one.
	 * Otherwise, allow two (and we'll inject the first one immediately).
	 */
	if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
		limit = 1;

	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

7628
static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641
{
	u32 flags = 0;
	flags |= seg->g       << 23;
	flags |= seg->db      << 22;
	flags |= seg->l       << 21;
	flags |= seg->avl     << 20;
	flags |= seg->present << 15;
	flags |= seg->dpl     << 13;
	flags |= seg->s       << 12;
	flags |= seg->type    << 8;
	return flags;
}

7642
static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656
{
	struct kvm_segment seg;
	int offset;

	kvm_get_segment(vcpu, &seg, n);
	put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector);

	if (n < 3)
		offset = 0x7f84 + n * 12;
	else
		offset = 0x7f2c + (n - 3) * 12;

	put_smstate(u32, buf, offset + 8, seg.base);
	put_smstate(u32, buf, offset + 4, seg.limit);
7657
	put_smstate(u32, buf, offset, enter_smm_get_segment_flags(&seg));
7658 7659
}

7660
#ifdef CONFIG_X86_64
7661
static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
7662 7663 7664 7665 7666 7667 7668 7669
{
	struct kvm_segment seg;
	int offset;
	u16 flags;

	kvm_get_segment(vcpu, &seg, n);
	offset = 0x7e00 + n * 16;

7670
	flags = enter_smm_get_segment_flags(&seg) >> 8;
7671 7672 7673 7674 7675
	put_smstate(u16, buf, offset, seg.selector);
	put_smstate(u16, buf, offset + 2, flags);
	put_smstate(u32, buf, offset + 4, seg.limit);
	put_smstate(u64, buf, offset + 8, seg.base);
}
7676
#endif
7677

7678
static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
7679 7680 7681 7682 7683 7684 7685 7686 7687 7688 7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699 7700 7701
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
	put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
	put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
	put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));

	for (i = 0; i < 8; i++)
		put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read(vcpu, i));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u32, buf, 0x7fcc, (u32)val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u32, buf, 0x7fc8, (u32)val);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u32, buf, 0x7fc4, seg.selector);
	put_smstate(u32, buf, 0x7f64, seg.base);
	put_smstate(u32, buf, 0x7f60, seg.limit);
7702
	put_smstate(u32, buf, 0x7f5c, enter_smm_get_segment_flags(&seg));
7703 7704 7705 7706 7707

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u32, buf, 0x7fc0, seg.selector);
	put_smstate(u32, buf, 0x7f80, seg.base);
	put_smstate(u32, buf, 0x7f7c, seg.limit);
7708
	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
7709 7710 7711 7712 7713 7714 7715 7716 7717 7718

	kvm_x86_ops->get_gdt(vcpu, &dt);
	put_smstate(u32, buf, 0x7f74, dt.address);
	put_smstate(u32, buf, 0x7f70, dt.size);

	kvm_x86_ops->get_idt(vcpu, &dt);
	put_smstate(u32, buf, 0x7f58, dt.address);
	put_smstate(u32, buf, 0x7f54, dt.size);

	for (i = 0; i < 6; i++)
7719
		enter_smm_save_seg_32(vcpu, buf, i);
7720 7721 7722 7723 7724 7725 7726 7727

	put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020000);
	put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
}

7728
#ifdef CONFIG_X86_64
7729
static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
7730 7731 7732 7733 7734 7735 7736 7737 7738 7739 7740 7741 7742 7743 7744 7745 7746 7747 7748 7749 7750 7751 7752 7753 7754 7755 7756 7757 7758 7759
{
	struct desc_ptr dt;
	struct kvm_segment seg;
	unsigned long val;
	int i;

	for (i = 0; i < 16; i++)
		put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read(vcpu, i));

	put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
	put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));

	kvm_get_dr(vcpu, 6, &val);
	put_smstate(u64, buf, 0x7f68, val);
	kvm_get_dr(vcpu, 7, &val);
	put_smstate(u64, buf, 0x7f60, val);

	put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
	put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
	put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));

	put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);

	/* revision id */
	put_smstate(u32, buf, 0x7efc, 0x00020064);

	put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
	put_smstate(u16, buf, 0x7e90, seg.selector);
7760
	put_smstate(u16, buf, 0x7e92, enter_smm_get_segment_flags(&seg) >> 8);
7761 7762 7763 7764 7765 7766 7767 7768 7769
	put_smstate(u32, buf, 0x7e94, seg.limit);
	put_smstate(u64, buf, 0x7e98, seg.base);

	kvm_x86_ops->get_idt(vcpu, &dt);
	put_smstate(u32, buf, 0x7e84, dt.size);
	put_smstate(u64, buf, 0x7e88, dt.address);

	kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
	put_smstate(u16, buf, 0x7e70, seg.selector);
7770
	put_smstate(u16, buf, 0x7e72, enter_smm_get_segment_flags(&seg) >> 8);
7771 7772 7773 7774 7775 7776 7777 7778
	put_smstate(u32, buf, 0x7e74, seg.limit);
	put_smstate(u64, buf, 0x7e78, seg.base);

	kvm_x86_ops->get_gdt(vcpu, &dt);
	put_smstate(u32, buf, 0x7e64, dt.size);
	put_smstate(u64, buf, 0x7e68, dt.address);

	for (i = 0; i < 6; i++)
7779
		enter_smm_save_seg_64(vcpu, buf, i);
7780
}
7781
#endif
7782

7783
static void enter_smm(struct kvm_vcpu *vcpu)
7784
{
7785
	struct kvm_segment cs, ds;
7786
	struct desc_ptr dt;
7787 7788 7789 7790 7791
	char buf[512];
	u32 cr0;

	trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true);
	memset(buf, 0, 512);
7792
#ifdef CONFIG_X86_64
7793
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7794
		enter_smm_save_state_64(vcpu, buf);
7795
	else
7796
#endif
7797
		enter_smm_save_state_32(vcpu, buf);
7798

7799 7800 7801 7802 7803 7804 7805 7806
	/*
	 * Give pre_enter_smm() a chance to make ISA-specific changes to the
	 * vCPU state (e.g. leave guest mode) after we've saved the state into
	 * the SMM state-save area.
	 */
	kvm_x86_ops->pre_enter_smm(vcpu, buf);

	vcpu->arch.hflags |= HF_SMM_MASK;
7807
	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822

	if (kvm_x86_ops->get_nmi_mask(vcpu))
		vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
	else
		kvm_x86_ops->set_nmi_mask(vcpu, true);

	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
	kvm_rip_write(vcpu, 0x8000);

	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
	kvm_x86_ops->set_cr0(vcpu, cr0);
	vcpu->arch.cr0 = cr0;

	kvm_x86_ops->set_cr4(vcpu, 0);

7823 7824 7825 7826
	/* Undocumented: IDT limit is set to zero on entry to SMM.  */
	dt.address = dt.size = 0;
	kvm_x86_ops->set_idt(vcpu, &dt);

7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853
	__kvm_set_dr(vcpu, 7, DR7_FIXED_1);

	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
	cs.base = vcpu->arch.smbase;

	ds.selector = 0;
	ds.base = 0;

	cs.limit    = ds.limit = 0xffffffff;
	cs.type     = ds.type = 0x3;
	cs.dpl      = ds.dpl = 0;
	cs.db       = ds.db = 0;
	cs.s        = ds.s = 1;
	cs.l        = ds.l = 0;
	cs.g        = ds.g = 1;
	cs.avl      = ds.avl = 0;
	cs.present  = ds.present = 1;
	cs.unusable = ds.unusable = 0;
	cs.padding  = ds.padding = 0;

	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);

7854
#ifdef CONFIG_X86_64
7855
	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
7856
		kvm_x86_ops->set_efer(vcpu, 0);
7857
#endif
7858 7859 7860

	kvm_update_cpuid(vcpu);
	kvm_mmu_reset_context(vcpu);
7861 7862
}

7863
static void process_smi(struct kvm_vcpu *vcpu)
7864 7865 7866 7867 7868
{
	vcpu->arch.smi_pending = true;
	kvm_make_request(KVM_REQ_EVENT, vcpu);
}

7869 7870 7871 7872 7873
void kvm_make_scan_ioapic_request(struct kvm *kvm)
{
	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
}

7874
static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
7875
{
7876
	if (!kvm_apic_present(vcpu))
7877
		return;
7878

7879
	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
7880

7881
	if (irqchip_split(vcpu->kvm))
7882
		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
7883
	else {
7884
		if (vcpu->arch.apicv_active)
7885
			kvm_x86_ops->sync_pir_to_irr(vcpu);
7886 7887
		if (ioapic_in_kernel(vcpu->kvm))
			kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
7888
	}
7889 7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902

	if (is_guest_mode(vcpu))
		vcpu->arch.load_eoi_exitmap_pending = true;
	else
		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
}

static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
{
	u64 eoi_exit_bitmap[4];

	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
		return;

7903 7904 7905
	bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
		  vcpu_to_synic(vcpu)->vec_bitmap, 256);
	kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
7906 7907
}

7908 7909 7910
int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
		unsigned long start, unsigned long end,
		bool blockable)
7911 7912 7913 7914 7915 7916 7917 7918 7919 7920
{
	unsigned long apic_address;

	/*
	 * The physical address of apic access page is stored in the VMCS.
	 * Update it when it becomes invalid.
	 */
	apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
	if (start <= apic_address && apic_address < end)
		kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
7921 7922

	return 0;
7923 7924
}

7925 7926
void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
{
7927 7928
	struct page *page = NULL;

7929
	if (!lapic_in_kernel(vcpu))
7930 7931
		return;

7932 7933 7934
	if (!kvm_x86_ops->set_apic_access_page_addr)
		return;

7935
	page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
7936 7937
	if (is_error_page(page))
		return;
7938 7939 7940 7941 7942 7943 7944
	kvm_x86_ops->set_apic_access_page_addr(vcpu, page_to_phys(page));

	/*
	 * Do not pin apic access page in memory, the MMU notifier
	 * will call us again if it is migrated or swapped out.
	 */
	put_page(page);
7945 7946 7947
}
EXPORT_SYMBOL_GPL(kvm_vcpu_reload_apic_access_page);

7948 7949 7950 7951 7952 7953
void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
{
	smp_send_reschedule(vcpu->cpu);
}
EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);

7954
/*
7955
 * Returns 1 to let vcpu_run() continue the guest execution loop without
7956 7957 7958
 * exiting to the userspace.  Otherwise, the value will be returned to the
 * userspace.
 */
7959
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
7960 7961
{
	int r;
7962 7963 7964 7965
	bool req_int_win =
		dm_request_for_irq_injection(vcpu) &&
		kvm_cpu_accept_dm_intr(vcpu);

7966
	bool req_immediate_exit = false;
7967

7968
	if (kvm_request_pending(vcpu)) {
7969 7970 7971 7972 7973 7974
		if (kvm_check_request(KVM_REQ_GET_VMCS12_PAGES, vcpu)) {
			if (unlikely(!kvm_x86_ops->get_vmcs12_pages(vcpu))) {
				r = 0;
				goto out;
			}
		}
7975
		if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
7976
			kvm_mmu_unload(vcpu);
7977
		if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
Marcelo Tosatti's avatar
Marcelo Tosatti committed
7978
			__kvm_migrate_timers(vcpu);
7979 7980
		if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
			kvm_gen_update_masterclock(vcpu->kvm);
7981 7982
		if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
			kvm_gen_kvmclock_update(vcpu);
7983 7984
		if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
			r = kvm_guest_time_update(vcpu);
7985 7986 7987
			if (unlikely(r))
				goto out;
		}
7988
		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
7989
			kvm_mmu_sync_roots(vcpu);
7990 7991
		if (kvm_check_request(KVM_REQ_LOAD_CR3, vcpu))
			kvm_mmu_load_cr3(vcpu);
7992
		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
7993
			kvm_vcpu_flush_tlb(vcpu, true);
7994
		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
7995
			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
Avi Kivity's avatar
Avi Kivity committed
7996 7997 7998
			r = 0;
			goto out;
		}
7999
		if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
8000
			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
8001
			vcpu->mmio_needed = 0;
8002 8003 8004
			r = 0;
			goto out;
		}
8005 8006 8007 8008 8009 8010
		if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
			/* Page is swapped out. Do synthetic halt */
			vcpu->arch.apf.halted = true;
			r = 1;
			goto out;
		}
8011 8012
		if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
			record_steal_time(vcpu);
8013 8014
		if (kvm_check_request(KVM_REQ_SMI, vcpu))
			process_smi(vcpu);
Avi Kivity's avatar
Avi Kivity committed
8015 8016
		if (kvm_check_request(KVM_REQ_NMI, vcpu))
			process_nmi(vcpu);
8017
		if (kvm_check_request(KVM_REQ_PMU, vcpu))
8018
			kvm_pmu_handle_event(vcpu);
8019
		if (kvm_check_request(KVM_REQ_PMI, vcpu))
8020
			kvm_pmu_deliver_pmi(vcpu);
8021 8022 8023
		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
			if (test_bit(vcpu->arch.pending_ioapic_eoi,
8024
				     vcpu->arch.ioapic_handled_vectors)) {
8025 8026 8027 8028 8029 8030 8031
				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
				vcpu->run->eoi.vector =
						vcpu->arch.pending_ioapic_eoi;
				r = 0;
				goto out;
			}
		}
8032 8033
		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
			vcpu_scan_ioapic(vcpu);
8034 8035
		if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
			vcpu_load_eoi_exitmap(vcpu);
8036 8037
		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
			kvm_vcpu_reload_apic_access_page(vcpu);
8038 8039 8040 8041 8042 8043
		if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
			r = 0;
			goto out;
		}
8044 8045 8046 8047 8048 8049
		if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
			vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
			r = 0;
			goto out;
		}
8050 8051 8052 8053 8054 8055
		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
			vcpu->run->exit_reason = KVM_EXIT_HYPERV;
			vcpu->run->hyperv = vcpu->arch.hyperv.exit;
			r = 0;
			goto out;
		}
8056 8057 8058 8059 8060 8061

		/*
		 * KVM_REQ_HV_STIMER has to be processed after
		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
		 * depend on the guest clock being up-to-date
		 */
8062 8063
		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
			kvm_hv_process_stimers(vcpu);
8064
	}
Avi Kivity's avatar
Avi Kivity committed
8065

8066
	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
8067
		++vcpu->stat.req_event;
8068 8069 8070 8071 8072 8073
		kvm_apic_accept_events(vcpu);
		if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
			r = 1;
			goto out;
		}

8074 8075
		if (inject_pending_event(vcpu, req_int_win) != 0)
			req_immediate_exit = true;
8076
		else {
8077
			/* Enable SMI/NMI/IRQ window open exits if needed.
8078
			 *
8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089
			 * SMIs have three cases:
			 * 1) They can be nested, and then there is nothing to
			 *    do here because RSM will cause a vmexit anyway.
			 * 2) There is an ISA-specific reason why SMI cannot be
			 *    injected, and the moment when this changes can be
			 *    intercepted.
			 * 3) Or the SMI can be pending because
			 *    inject_pending_event has completed the injection
			 *    of an IRQ or NMI from the previous vmexit, and
			 *    then we request an immediate exit to inject the
			 *    SMI.
8090 8091
			 */
			if (vcpu->arch.smi_pending && !is_smm(vcpu))
8092 8093
				if (!kvm_x86_ops->enable_smi_window(vcpu))
					req_immediate_exit = true;
8094 8095 8096 8097
			if (vcpu->arch.nmi_pending)
				kvm_x86_ops->enable_nmi_window(vcpu);
			if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
				kvm_x86_ops->enable_irq_window(vcpu);
8098
			WARN_ON(vcpu->arch.exception.pending);
8099
		}
8100 8101 8102 8103 8104 8105 8106

		if (kvm_lapic_enabled(vcpu)) {
			update_cr8_intercept(vcpu);
			kvm_lapic_sync_to_vapic(vcpu);
		}
	}

8107 8108
	r = kvm_mmu_reload(vcpu);
	if (unlikely(r)) {
8109
		goto cancel_injection;
8110 8111
	}

8112 8113 8114
	preempt_disable();

	kvm_x86_ops->prepare_guest_switch(vcpu);
8115 8116 8117 8118 8119 8120 8121

	/*
	 * Disable IRQs before setting IN_GUEST_MODE.  Posted interrupt
	 * IPI are then delayed after guest entry, which ensures that they
	 * result in virtual interrupt delivery.
	 */
	local_irq_disable();
8122 8123
	vcpu->mode = IN_GUEST_MODE;

8124 8125
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);

8126
	/*
8127
	 * 1) We should set ->mode before checking ->requests.  Please see
8128
	 * the comment in kvm_vcpu_exiting_guest_mode().
8129
	 *
8130
	 * 2) For APICv, we should set ->mode before checking PID.ON. This
8131 8132 8133 8134 8135 8136
	 * pairs with the memory barrier implicit in pi_test_and_set_on
	 * (see vmx_deliver_posted_interrupt).
	 *
	 * 3) This also orders the write to mode from any reads to the page
	 * tables done while the VCPU is running.  Please see the comment
	 * in kvm_flush_remote_tlbs.
8137
	 */
8138
	smp_mb__after_srcu_read_unlock();
8139

8140 8141 8142 8143
	/*
	 * This handles the case where a posted interrupt was
	 * notified with kvm_vcpu_kick.
	 */
8144 8145
	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
		kvm_x86_ops->sync_pir_to_irr(vcpu);
8146

8147
	if (vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu)
Avi Kivity's avatar
Avi Kivity committed
8148
	    || need_resched() || signal_pending(current)) {
8149
		vcpu->mode = OUTSIDE_GUEST_MODE;
Avi Kivity's avatar
Avi Kivity committed
8150
		smp_wmb();
8151 8152
		local_irq_enable();
		preempt_enable();
8153
		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8154
		r = 1;
8155
		goto cancel_injection;
8156 8157
	}

8158 8159
	if (req_immediate_exit) {
		kvm_make_request(KVM_REQ_EVENT, vcpu);
8160
		kvm_x86_ops->request_immediate_exit(vcpu);
8161
	}
8162

8163
	trace_kvm_entry(vcpu->vcpu_id);
8164
	guest_enter_irqoff();
8165

8166 8167
	/* The preempt notifier should have taken care of the FPU already.  */
	WARN_ON_ONCE(test_thread_flag(TIF_NEED_FPU_LOAD));
8168

8169 8170 8171 8172 8173 8174
	if (unlikely(vcpu->arch.switch_db_regs)) {
		set_debugreg(0, 7);
		set_debugreg(vcpu->arch.eff_db[0], 0);
		set_debugreg(vcpu->arch.eff_db[1], 1);
		set_debugreg(vcpu->arch.eff_db[2], 2);
		set_debugreg(vcpu->arch.eff_db[3], 3);
8175
		set_debugreg(vcpu->arch.dr6, 6);
8176
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8177
	}
8178

8179
	kvm_x86_ops->run(vcpu);
8180

8181 8182 8183 8184 8185 8186 8187 8188 8189
	/*
	 * Do this here before restoring debug registers on the host.  And
	 * since we do this before handling the vmexit, a DR access vmexit
	 * can (a) read the correct value of the debug registers, (b) set
	 * KVM_DEBUGREG_WONT_EXIT again.
	 */
	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
		kvm_x86_ops->sync_dirty_debug_regs(vcpu);
8190 8191 8192 8193
		kvm_update_dr0123(vcpu);
		kvm_update_dr6(vcpu);
		kvm_update_dr7(vcpu);
		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
8194 8195
	}

8196 8197 8198 8199 8200 8201 8202
	/*
	 * If the guest has used debug registers, at least dr7
	 * will be disabled while returning to the host.
	 * If we don't have active breakpoints in the host, we don't
	 * care about the messed up debug address registers. But if
	 * we have some of them active, restore the old state.
	 */
8203
	if (hw_breakpoint_active())
8204
		hw_breakpoint_restore();
8205

8206
	vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
8207

8208
	vcpu->mode = OUTSIDE_GUEST_MODE;
Avi Kivity's avatar
Avi Kivity committed
8209
	smp_wmb();
8210

8211
	kvm_x86_ops->handle_exit_irqoff(vcpu);
8212

8213 8214 8215 8216 8217 8218 8219 8220 8221
	/*
	 * Consume any pending interrupts, including the possible source of
	 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
	 * An instruction is required after local_irq_enable() to fully unblock
	 * interrupts on processors that implement an interrupt shadow, the
	 * stat.exits increment will do nicely.
	 */
	kvm_before_interrupt(vcpu);
	local_irq_enable();
8222
	++vcpu->stat.exits;
8223 8224
	local_irq_disable();
	kvm_after_interrupt(vcpu);
8225

8226
	guest_exit_irqoff();
8227 8228 8229 8230 8231 8232 8233
	if (lapic_in_kernel(vcpu)) {
		s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta;
		if (delta != S64_MIN) {
			trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta);
			vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN;
		}
	}
8234

8235
	local_irq_enable();
8236 8237
	preempt_enable();

8238
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8239

8240 8241 8242 8243
	/*
	 * Profile KVM exit RIPs:
	 */
	if (unlikely(prof_on == KVM_PROFILING)) {
8244 8245
		unsigned long rip = kvm_rip_read(vcpu);
		profile_hit(KVM_PROFILING, (void *)rip);
8246 8247
	}

8248 8249
	if (unlikely(vcpu->arch.tsc_always_catchup))
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
8250

8251 8252
	if (vcpu->arch.apic_attention)
		kvm_lapic_sync_from_vapic(vcpu);
Avi Kivity's avatar
Avi Kivity committed
8253

8254
	vcpu->arch.gpa_available = false;
8255
	r = kvm_x86_ops->handle_exit(vcpu);
8256 8257 8258 8259
	return r;

cancel_injection:
	kvm_x86_ops->cancel_injection(vcpu);
8260 8261
	if (unlikely(vcpu->arch.apic_attention))
		kvm_lapic_sync_from_vapic(vcpu);
8262 8263 8264
out:
	return r;
}
8265

8266 8267
static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu)
{
8268 8269
	if (!kvm_arch_vcpu_runnable(vcpu) &&
	    (!kvm_x86_ops->pre_block || kvm_x86_ops->pre_block(vcpu) == 0)) {
8270 8271 8272
		srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
		kvm_vcpu_block(vcpu);
		vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8273 8274 8275 8276

		if (kvm_x86_ops->post_block)
			kvm_x86_ops->post_block(vcpu);

8277 8278 8279
		if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
			return 1;
	}
8280 8281 8282 8283 8284 8285 8286

	kvm_apic_accept_events(vcpu);
	switch(vcpu->arch.mp_state) {
	case KVM_MP_STATE_HALTED:
		vcpu->arch.pv.pv_unhalted = false;
		vcpu->arch.mp_state =
			KVM_MP_STATE_RUNNABLE;
8287
		/* fall through */
8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298
	case KVM_MP_STATE_RUNNABLE:
		vcpu->arch.apf.halted = false;
		break;
	case KVM_MP_STATE_INIT_RECEIVED:
		break;
	default:
		return -EINTR;
		break;
	}
	return 1;
}
8299

8300 8301
static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
{
8302 8303 8304
	if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events)
		kvm_x86_ops->check_nested_events(vcpu, false);

8305 8306 8307 8308
	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
		!vcpu->arch.apf.halted);
}

8309
static int vcpu_run(struct kvm_vcpu *vcpu)
8310 8311
{
	int r;
8312
	struct kvm *kvm = vcpu->kvm;
8313

8314
	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8315
	vcpu->arch.l1tf_flush_l1d = true;
8316

8317
	for (;;) {
8318
		if (kvm_vcpu_running(vcpu)) {
8319
			r = vcpu_enter_guest(vcpu);
8320
		} else {
8321
			r = vcpu_block(kvm, vcpu);
8322 8323
		}

8324 8325 8326
		if (r <= 0)
			break;

8327
		kvm_clear_request(KVM_REQ_PENDING_TIMER, vcpu);
8328 8329 8330
		if (kvm_cpu_has_pending_timer(vcpu))
			kvm_inject_pending_timer_irqs(vcpu);

8331 8332
		if (dm_request_for_irq_injection(vcpu) &&
			kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
8333 8334
			r = 0;
			vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
8335
			++vcpu->stat.request_irq_exits;
8336
			break;
8337
		}
8338 8339 8340

		kvm_check_async_pf_completion(vcpu);

8341 8342
		if (signal_pending(current)) {
			r = -EINTR;
8343
			vcpu->run->exit_reason = KVM_EXIT_INTR;
8344
			++vcpu->stat.signal_exits;
8345
			break;
8346 8347
		}
		if (need_resched()) {
8348
			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8349
			cond_resched();
8350
			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
8351
		}
8352 8353
	}

8354
	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
8355 8356 8357 8358

	return r;
}

8359 8360 8361
static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
{
	int r;
8362

8363
	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
8364
	r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
8365
	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
8366
	return r;
8367 8368 8369 8370 8371 8372 8373 8374 8375
}

static int complete_emulated_pio(struct kvm_vcpu *vcpu)
{
	BUG_ON(!vcpu->arch.pio.count);

	return complete_emulated_io(vcpu);
}

Avi Kivity's avatar
Avi Kivity committed
8376 8377 8378 8379 8380
/*
 * Implements the following, as a state machine:
 *
 * read:
 *   for each fragment
8381 8382 8383 8384
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       exit
 *       copy data
Avi Kivity's avatar
Avi Kivity committed
8385 8386 8387 8388
 *   execute insn
 *
 * write:
 *   for each fragment
8389 8390 8391 8392
 *     for each mmio piece in the fragment
 *       write gpa, len
 *       copy data
 *       exit
Avi Kivity's avatar
Avi Kivity committed
8393
 */
8394
static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
8395 8396
{
	struct kvm_run *run = vcpu->run;
Avi Kivity's avatar
Avi Kivity committed
8397
	struct kvm_mmio_fragment *frag;
8398
	unsigned len;
8399

8400
	BUG_ON(!vcpu->mmio_needed);
8401

8402
	/* Complete previous fragment */
8403 8404
	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
	len = min(8u, frag->len);
8405
	if (!vcpu->mmio_is_write)
8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417 8418
		memcpy(frag->data, run->mmio.data, len);

	if (frag->len <= 8) {
		/* Switch to the next fragment. */
		frag++;
		vcpu->mmio_cur_fragment++;
	} else {
		/* Go forward to the next mmio piece. */
		frag->data += len;
		frag->gpa += len;
		frag->len -= len;
	}

8419
	if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
8420
		vcpu->mmio_needed = 0;
8421 8422

		/* FIXME: return into emulator if single-stepping.  */
Avi Kivity's avatar
Avi Kivity committed
8423
		if (vcpu->mmio_is_write)
8424 8425 8426 8427
			return 1;
		vcpu->mmio_read_completed = 1;
		return complete_emulated_io(vcpu);
	}
8428

8429 8430 8431
	run->exit_reason = KVM_EXIT_MMIO;
	run->mmio.phys_addr = frag->gpa;
	if (vcpu->mmio_is_write)
8432 8433
		memcpy(run->mmio.data, frag->data, min(8u, frag->len));
	run->mmio.len = min(8u, frag->len);
8434 8435 8436
	run->mmio.is_write = vcpu->mmio_is_write;
	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
	return 0;
8437 8438
}

8439 8440 8441
/* Swap (qemu) user FPU context for the guest FPU context. */
static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
{
8442 8443
	fpregs_lock();

8444
	copy_fpregs_to_fpstate(vcpu->arch.user_fpu);
8445
	/* PKRU is separately restored in kvm_x86_ops->run.  */
8446
	__copy_kernel_to_fpregs(&vcpu->arch.guest_fpu->state,
8447
				~XFEATURE_MASK_PKRU);
8448 8449 8450 8451

	fpregs_mark_activate();
	fpregs_unlock();

8452 8453 8454 8455 8456 8457
	trace_kvm_fpu(1);
}

/* When vcpu_run ends, restore user space FPU context. */
static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
{
8458 8459
	fpregs_lock();

8460
	copy_fpregs_to_fpstate(vcpu->arch.guest_fpu);
8461
	copy_kernel_to_fpregs(&vcpu->arch.user_fpu->state);
8462 8463 8464 8465

	fpregs_mark_activate();
	fpregs_unlock();

8466 8467 8468 8469
	++vcpu->stat.fpu_reload;
	trace_kvm_fpu(0);
}

8470 8471 8472 8473
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
{
	int r;

8474
	vcpu_load(vcpu);
8475
	kvm_sigset_activate(vcpu);
8476 8477
	kvm_load_guest_fpu(vcpu);

8478
	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
8479 8480 8481 8482
		if (kvm_run->immediate_exit) {
			r = -EINTR;
			goto out;
		}
8483
		kvm_vcpu_block(vcpu);
8484
		kvm_apic_accept_events(vcpu);
8485
		kvm_clear_request(KVM_REQ_UNHALT, vcpu);
8486
		r = -EAGAIN;
8487 8488 8489 8490 8491
		if (signal_pending(current)) {
			r = -EINTR;
			vcpu->run->exit_reason = KVM_EXIT_INTR;
			++vcpu->stat.signal_exits;
		}
8492
		goto out;
8493 8494
	}

Ken Hofsass's avatar
Ken Hofsass committed
8495 8496 8497 8498 8499 8500 8501 8502 8503 8504 8505
	if (vcpu->run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) {
		r = -EINVAL;
		goto out;
	}

	if (vcpu->run->kvm_dirty_regs) {
		r = sync_regs(vcpu);
		if (r != 0)
			goto out;
	}

8506
	/* re-sync apic's tpr */
8507
	if (!lapic_in_kernel(vcpu)) {
Andre Przywara's avatar
Andre Przywara committed
8508 8509 8510 8511 8512
		if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
			r = -EINVAL;
			goto out;
		}
	}
8513

8514 8515 8516 8517 8518
	if (unlikely(vcpu->arch.complete_userspace_io)) {
		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
		vcpu->arch.complete_userspace_io = NULL;
		r = cui(vcpu);
		if (r <= 0)
8519
			goto out;
8520 8521
	} else
		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
8522

8523 8524 8525 8526
	if (kvm_run->immediate_exit)
		r = -EINTR;
	else
		r = vcpu_run(vcpu);
8527 8528

out:
8529
	kvm_put_guest_fpu(vcpu);
Ken Hofsass's avatar
Ken Hofsass committed
8530 8531
	if (vcpu->run->kvm_valid_regs)
		store_regs(vcpu);
8532
	post_kvm_run_save(vcpu);
8533
	kvm_sigset_deactivate(vcpu);
8534

8535
	vcpu_put(vcpu);
8536 8537 8538
	return r;
}

Ken Hofsass's avatar
Ken Hofsass committed
8539
static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8540
{
8541 8542 8543 8544
	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
		/*
		 * We are here if userspace calls get_regs() in the middle of
		 * instruction emulation. Registers state needs to be copied
Guo Chao's avatar
Guo Chao committed
8545
		 * back from emulation context to vcpu. Userspace shouldn't do
8546 8547 8548
		 * that usually, but some bad designed PV devices (vmware
		 * backdoor interface) need this to work
		 */
8549
		emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
8550 8551
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
	}
8552 8553 8554 8555 8556 8557
	regs->rax = kvm_rax_read(vcpu);
	regs->rbx = kvm_rbx_read(vcpu);
	regs->rcx = kvm_rcx_read(vcpu);
	regs->rdx = kvm_rdx_read(vcpu);
	regs->rsi = kvm_rsi_read(vcpu);
	regs->rdi = kvm_rdi_read(vcpu);
8558
	regs->rsp = kvm_rsp_read(vcpu);
8559
	regs->rbp = kvm_rbp_read(vcpu);
8560
#ifdef CONFIG_X86_64
8561 8562 8563 8564 8565 8566 8567 8568
	regs->r8 = kvm_r8_read(vcpu);
	regs->r9 = kvm_r9_read(vcpu);
	regs->r10 = kvm_r10_read(vcpu);
	regs->r11 = kvm_r11_read(vcpu);
	regs->r12 = kvm_r12_read(vcpu);
	regs->r13 = kvm_r13_read(vcpu);
	regs->r14 = kvm_r14_read(vcpu);
	regs->r15 = kvm_r15_read(vcpu);
8569 8570
#endif

8571
	regs->rip = kvm_rip_read(vcpu);
8572
	regs->rflags = kvm_get_rflags(vcpu);
Ken Hofsass's avatar
Ken Hofsass committed
8573
}
8574

Ken Hofsass's avatar
Ken Hofsass committed
8575 8576 8577 8578
int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__get_regs(vcpu, regs);
8579
	vcpu_put(vcpu);
8580 8581 8582
	return 0;
}

Ken Hofsass's avatar
Ken Hofsass committed
8583
static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
8584
{
8585 8586 8587
	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;

8588 8589 8590 8591 8592 8593
	kvm_rax_write(vcpu, regs->rax);
	kvm_rbx_write(vcpu, regs->rbx);
	kvm_rcx_write(vcpu, regs->rcx);
	kvm_rdx_write(vcpu, regs->rdx);
	kvm_rsi_write(vcpu, regs->rsi);
	kvm_rdi_write(vcpu, regs->rdi);
8594
	kvm_rsp_write(vcpu, regs->rsp);
8595
	kvm_rbp_write(vcpu, regs->rbp);
8596
#ifdef CONFIG_X86_64
8597 8598 8599 8600 8601 8602 8603 8604
	kvm_r8_write(vcpu, regs->r8);
	kvm_r9_write(vcpu, regs->r9);
	kvm_r10_write(vcpu, regs->r10);
	kvm_r11_write(vcpu, regs->r11);
	kvm_r12_write(vcpu, regs->r12);
	kvm_r13_write(vcpu, regs->r13);
	kvm_r14_write(vcpu, regs->r14);
	kvm_r15_write(vcpu, regs->r15);
8605 8606
#endif

8607
	kvm_rip_write(vcpu, regs->rip);
8608
	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
8609

8610 8611
	vcpu->arch.exception.pending = false;

8612
	kvm_make_request(KVM_REQ_EVENT, vcpu);
Ken Hofsass's avatar
Ken Hofsass committed
8613
}
8614

Ken Hofsass's avatar
Ken Hofsass committed
8615 8616 8617 8618
int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	vcpu_load(vcpu);
	__set_regs(vcpu, regs);
8619
	vcpu_put(vcpu);
8620 8621 8622 8623 8624 8625 8626
	return 0;
}

void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
{
	struct kvm_segment cs;

8627
	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
8628 8629 8630 8631 8632
	*db = cs.db;
	*l = cs.l;
}
EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);

Ken Hofsass's avatar
Ken Hofsass committed
8633
static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8634
{
8635
	struct desc_ptr dt;
8636

8637 8638 8639 8640 8641 8642
	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8643

8644 8645
	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
8646 8647

	kvm_x86_ops->get_idt(vcpu, &dt);
8648 8649
	sregs->idt.limit = dt.size;
	sregs->idt.base = dt.address;
8650
	kvm_x86_ops->get_gdt(vcpu, &dt);
8651 8652
	sregs->gdt.limit = dt.size;
	sregs->gdt.base = dt.address;
8653

8654
	sregs->cr0 = kvm_read_cr0(vcpu);
8655
	sregs->cr2 = vcpu->arch.cr2;
8656
	sregs->cr3 = kvm_read_cr3(vcpu);
8657
	sregs->cr4 = kvm_read_cr4(vcpu);
8658
	sregs->cr8 = kvm_get_cr8(vcpu);
8659
	sregs->efer = vcpu->arch.efer;
8660 8661
	sregs->apic_base = kvm_get_apic_base(vcpu);

8662
	memset(sregs->interrupt_bitmap, 0, sizeof(sregs->interrupt_bitmap));
8663

8664
	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
8665 8666
		set_bit(vcpu->arch.interrupt.nr,
			(unsigned long *)sregs->interrupt_bitmap);
Ken Hofsass's avatar
Ken Hofsass committed
8667
}
8668

Ken Hofsass's avatar
Ken Hofsass committed
8669 8670 8671 8672 8673
int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	vcpu_load(vcpu);
	__get_sregs(vcpu, sregs);
8674
	vcpu_put(vcpu);
8675 8676 8677
	return 0;
}

8678 8679 8680
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
8681 8682
	vcpu_load(vcpu);

8683
	kvm_apic_accept_events(vcpu);
8684 8685 8686 8687 8688 8689
	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
					vcpu->arch.pv.pv_unhalted)
		mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
	else
		mp_state->mp_state = vcpu->arch.mp_state;

8690
	vcpu_put(vcpu);
8691 8692 8693 8694 8695 8696
	return 0;
}

int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
				    struct kvm_mp_state *mp_state)
{
8697 8698 8699 8700
	int ret = -EINVAL;

	vcpu_load(vcpu);

8701
	if (!lapic_in_kernel(vcpu) &&
8702
	    mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
8703
		goto out;
8704

8705 8706 8707 8708
	/* INITs are latched while in SMM */
	if ((is_smm(vcpu) || vcpu->arch.smi_pending) &&
	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
8709
		goto out;
8710

8711 8712 8713 8714 8715
	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
		set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
	} else
		vcpu->arch.mp_state = mp_state->mp_state;
8716
	kvm_make_request(KVM_REQ_EVENT, vcpu);
8717 8718 8719 8720 8721

	ret = 0;
out:
	vcpu_put(vcpu);
	return ret;
8722 8723
}

8724 8725
int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
		    int reason, bool has_error_code, u32 error_code)
8726
{
8727
	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
8728
	int ret;
8729

8730
	init_emulate_ctxt(vcpu);
8731

8732
	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
8733
				   has_error_code, error_code);
8734 8735 8736 8737
	if (ret) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
8738
		return 0;
8739
	}
8740

8741 8742
	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
8743
	kvm_make_request(KVM_REQ_EVENT, vcpu);
8744
	return 1;
8745 8746 8747
}
EXPORT_SYMBOL_GPL(kvm_task_switch);

Peng Hao's avatar
Peng Hao committed
8748
static int kvm_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8749
{
8750
	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
8751 8752 8753 8754 8755
		/*
		 * When EFER.LME and CR0.PG are set, the processor is in
		 * 64-bit mode (though maybe in a 32-bit code segment).
		 * CR4.PAE and EFER.LMA must be set.
		 */
8756
		if (!(sregs->cr4 & X86_CR4_PAE)
8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767
		    || !(sregs->efer & EFER_LMA))
			return -EINVAL;
	} else {
		/*
		 * Not in 64-bit mode: EFER.LMA is clear and the code
		 * segment cannot be 64-bit.
		 */
		if (sregs->efer & EFER_LMA || sregs->cs.l)
			return -EINVAL;
	}

8768
	return kvm_valid_cr4(vcpu, sregs->cr4);
8769 8770
}

Ken Hofsass's avatar
Ken Hofsass committed
8771
static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8772
{
8773
	struct msr_data apic_base_msr;
8774
	int mmu_reset_needed = 0;
8775
	int cpuid_update_needed = 0;
8776
	int pending_vec, max_bits, idx;
8777
	struct desc_ptr dt;
8778 8779
	int ret = -EINVAL;

8780
	if (kvm_valid_sregs(vcpu, sregs))
8781
		goto out;
8782

8783 8784 8785
	apic_base_msr.data = sregs->apic_base;
	apic_base_msr.host_initiated = true;
	if (kvm_set_apic_base(vcpu, &apic_base_msr))
8786
		goto out;
8787

8788 8789
	dt.size = sregs->idt.limit;
	dt.address = sregs->idt.base;
8790
	kvm_x86_ops->set_idt(vcpu, &dt);
8791 8792
	dt.size = sregs->gdt.limit;
	dt.address = sregs->gdt.base;
8793 8794
	kvm_x86_ops->set_gdt(vcpu, &dt);

8795
	vcpu->arch.cr2 = sregs->cr2;
8796
	mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
8797
	vcpu->arch.cr3 = sregs->cr3;
8798
	__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
8799

8800
	kvm_set_cr8(vcpu, sregs->cr8);
8801

8802
	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
8803 8804
	kvm_x86_ops->set_efer(vcpu, sregs->efer);

8805
	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
8806
	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
8807
	vcpu->arch.cr0 = sregs->cr0;
8808

8809
	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
8810 8811
	cpuid_update_needed |= ((kvm_read_cr4(vcpu) ^ sregs->cr4) &
				(X86_CR4_OSXSAVE | X86_CR4_PKE));
8812
	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
8813
	if (cpuid_update_needed)
8814
		kvm_update_cpuid(vcpu);
8815 8816

	idx = srcu_read_lock(&vcpu->kvm->srcu);
8817
	if (is_pae_paging(vcpu)) {
8818
		load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
8819 8820
		mmu_reset_needed = 1;
	}
8821
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
8822 8823 8824 8825

	if (mmu_reset_needed)
		kvm_mmu_reset_context(vcpu);

8826
	max_bits = KVM_NR_INTERRUPTS;
8827 8828 8829
	pending_vec = find_first_bit(
		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
	if (pending_vec < max_bits) {
8830
		kvm_queue_interrupt(vcpu, pending_vec, false);
8831
		pr_debug("Set back pending irq %d\n", pending_vec);
8832 8833
	}

8834 8835 8836 8837 8838 8839
	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
8840

8841 8842
	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
8843

8844 8845
	update_cr8_intercept(vcpu);

8846
	/* Older userspace won't unhalt the vcpu on reset. */
8847
	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
8848
	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
8849
	    !is_protmode(vcpu))
8850 8851
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;

8852 8853
	kvm_make_request(KVM_REQ_EVENT, vcpu);

8854 8855
	ret = 0;
out:
Ken Hofsass's avatar
Ken Hofsass committed
8856 8857 8858 8859 8860 8861 8862 8863 8864 8865
	return ret;
}

int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
				  struct kvm_sregs *sregs)
{
	int ret;

	vcpu_load(vcpu);
	ret = __set_sregs(vcpu, sregs);
8866 8867
	vcpu_put(vcpu);
	return ret;
8868 8869
}

Jan Kiszka's avatar
Jan Kiszka committed
8870 8871
int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
					struct kvm_guest_debug *dbg)
8872
{
8873
	unsigned long rflags;
8874
	int i, r;
8875

8876 8877
	vcpu_load(vcpu);

8878 8879 8880
	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
		r = -EBUSY;
		if (vcpu->arch.exception.pending)
8881
			goto out;
8882 8883 8884 8885 8886 8887
		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
			kvm_queue_exception(vcpu, DB_VECTOR);
		else
			kvm_queue_exception(vcpu, BP_VECTOR);
	}

8888 8889 8890 8891 8892
	/*
	 * Read rflags as long as potentially injected trace flags are still
	 * filtered out.
	 */
	rflags = kvm_get_rflags(vcpu);
8893 8894 8895 8896 8897 8898

	vcpu->guest_debug = dbg->control;
	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
		vcpu->guest_debug = 0;

	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
8899 8900
		for (i = 0; i < KVM_NR_DB_REGS; ++i)
			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
8901
		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
8902 8903 8904 8905
	} else {
		for (i = 0; i < KVM_NR_DB_REGS; i++)
			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
	}
8906
	kvm_update_dr7(vcpu);
8907

Jan Kiszka's avatar
Jan Kiszka committed
8908 8909 8910
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
			get_segment_base(vcpu, VCPU_SREG_CS);
8911

8912 8913 8914 8915 8916
	/*
	 * Trigger an rflags update that will inject or remove the trace
	 * flags.
	 */
	kvm_set_rflags(vcpu, rflags);
8917

8918
	kvm_x86_ops->update_bp_intercept(vcpu);
8919

8920
	r = 0;
Jan Kiszka's avatar
Jan Kiszka committed
8921

8922
out:
8923
	vcpu_put(vcpu);
8924 8925 8926
	return r;
}

8927 8928 8929 8930 8931 8932 8933 8934
/*
 * Translate a guest virtual address to a guest physical address.
 */
int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
				    struct kvm_translation *tr)
{
	unsigned long vaddr = tr->linear_address;
	gpa_t gpa;
8935
	int idx;
8936

8937 8938
	vcpu_load(vcpu);

8939
	idx = srcu_read_lock(&vcpu->kvm->srcu);
8940
	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
8941
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
8942 8943 8944 8945 8946
	tr->physical_address = gpa;
	tr->valid = gpa != UNMAPPED_GVA;
	tr->writeable = 1;
	tr->usermode = 0;

8947
	vcpu_put(vcpu);
8948 8949 8950
	return 0;
}

8951 8952
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
8953
	struct fxregs_state *fxsave;
8954

8955
	vcpu_load(vcpu);
8956

8957
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
8958 8959 8960 8961 8962 8963 8964
	memcpy(fpu->fpr, fxsave->st_space, 128);
	fpu->fcw = fxsave->cwd;
	fpu->fsw = fxsave->swd;
	fpu->ftwx = fxsave->twd;
	fpu->last_opcode = fxsave->fop;
	fpu->last_ip = fxsave->rip;
	fpu->last_dp = fxsave->rdp;
8965
	memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
8966

8967
	vcpu_put(vcpu);
8968 8969 8970 8971 8972
	return 0;
}

int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
8973 8974 8975 8976
	struct fxregs_state *fxsave;

	vcpu_load(vcpu);

8977
	fxsave = &vcpu->arch.guest_fpu->state.fxsave;
8978 8979 8980 8981 8982 8983 8984 8985

	memcpy(fxsave->st_space, fpu->fpr, 128);
	fxsave->cwd = fpu->fcw;
	fxsave->swd = fpu->fsw;
	fxsave->twd = fpu->ftwx;
	fxsave->fop = fpu->last_opcode;
	fxsave->rip = fpu->last_ip;
	fxsave->rdp = fpu->last_dp;
8986
	memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
8987

8988
	vcpu_put(vcpu);
8989 8990 8991
	return 0;
}

Ken Hofsass's avatar
Ken Hofsass committed
8992 8993 8994 8995 8996 8997 8998 8999 9000 9001 9002 9003 9004 9005 9006 9007 9008 9009 9010 9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021 9022 9023 9024 9025 9026 9027 9028 9029 9030
static void store_regs(struct kvm_vcpu *vcpu)
{
	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
		__get_regs(vcpu, &vcpu->run->s.regs.regs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);

	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
		kvm_vcpu_ioctl_x86_get_vcpu_events(
				vcpu, &vcpu->run->s.regs.events);
}

static int sync_regs(struct kvm_vcpu *vcpu)
{
	if (vcpu->run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)
		return -EINVAL;

	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
		__set_regs(vcpu, &vcpu->run->s.regs.regs);
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
		if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
	}
	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
		if (kvm_vcpu_ioctl_x86_set_vcpu_events(
				vcpu, &vcpu->run->s.regs.events))
			return -EINVAL;
		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
	}

	return 0;
}

9031
static void fx_init(struct kvm_vcpu *vcpu)
9032
{
9033
	fpstate_init(&vcpu->arch.guest_fpu->state);
9034
	if (boot_cpu_has(X86_FEATURE_XSAVES))
9035
		vcpu->arch.guest_fpu->state.xsave.header.xcomp_bv =
9036
			host_xcr0 | XSTATE_COMPACTION_ENABLED;
9037

9038 9039 9040
	/*
	 * Ensure guest xcr0 is valid for loading
	 */
Dave Hansen's avatar
Dave Hansen committed
9041
	vcpu->arch.xcr0 = XFEATURE_MASK_FP;
9042

9043
	vcpu->arch.cr0 |= X86_CR0_ET;
9044 9045
}

9046 9047
void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
{
9048 9049
	void *wbinvd_dirty_mask = vcpu->arch.wbinvd_dirty_mask;

9050
	kvmclock_reset(vcpu);
9051

9052
	kvm_x86_ops->vcpu_free(vcpu);
9053
	free_cpumask_var(wbinvd_dirty_mask);
9054 9055 9056 9057 9058
}

struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
						unsigned int id)
{
9059 9060
	struct kvm_vcpu *vcpu;

9061
	if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
9062 9063 9064
		printk_once(KERN_WARNING
		"kvm: SMP vm created on host with unstable TSC; "
		"guest TSC will not be reliable\n");
9065 9066 9067 9068

	vcpu = kvm_x86_ops->vcpu_create(kvm, id);

	return vcpu;
9069
}
9070

9071 9072
int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
{
9073
	vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
9074
	vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
9075
	kvm_vcpu_mtrr_init(vcpu);
9076
	vcpu_load(vcpu);
9077
	kvm_vcpu_reset(vcpu, false);
9078
	kvm_init_mmu(vcpu, false);
9079
	vcpu_put(vcpu);
9080
	return 0;
9081 9082
}

9083
void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
9084
{
9085
	struct msr_data msr;
9086
	struct kvm *kvm = vcpu->kvm;
9087

9088 9089
	kvm_hv_vcpu_postcreate(vcpu);

9090
	if (mutex_lock_killable(&vcpu->mutex))
9091
		return;
9092
	vcpu_load(vcpu);
9093 9094 9095 9096
	msr.data = 0x0;
	msr.index = MSR_IA32_TSC;
	msr.host_initiated = true;
	kvm_write_tsc(vcpu, &msr);
9097
	vcpu_put(vcpu);
9098 9099 9100 9101

	/* poll control enabled by default */
	vcpu->arch.msr_kvm_poll_control = 1;

9102
	mutex_unlock(&vcpu->mutex);
9103

9104 9105 9106
	if (!kvmclock_periodic_sync)
		return;

9107 9108
	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
					KVMCLOCK_SYNC_PERIOD);
9109 9110
}

9111
void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
9112
{
9113 9114
	vcpu->arch.apf.msr_val = 0;

9115
	vcpu_load(vcpu);
9116 9117 9118 9119 9120 9121
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);

	kvm_x86_ops->vcpu_free(vcpu);
}

9122
void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
9123
{
9124 9125
	kvm_lapic_reset(vcpu, init_event);

9126 9127
	vcpu->arch.hflags = 0;

9128
	vcpu->arch.smi_pending = 0;
9129
	vcpu->arch.smi_count = 0;
Avi Kivity's avatar
Avi Kivity committed
9130 9131
	atomic_set(&vcpu->arch.nmi_queued, 0);
	vcpu->arch.nmi_pending = 0;
9132
	vcpu->arch.nmi_injected = false;
9133 9134
	kvm_clear_interrupt_queue(vcpu);
	kvm_clear_exception_queue(vcpu);
9135
	vcpu->arch.exception.pending = false;
9136

9137
	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
9138
	kvm_update_dr0123(vcpu);
9139
	vcpu->arch.dr6 = DR6_INIT;
Jan Kiszka's avatar
Jan Kiszka committed
9140
	kvm_update_dr6(vcpu);
9141
	vcpu->arch.dr7 = DR7_FIXED_1;
9142
	kvm_update_dr7(vcpu);
9143

9144 9145
	vcpu->arch.cr2 = 0;

9146
	kvm_make_request(KVM_REQ_EVENT, vcpu);
9147
	vcpu->arch.apf.msr_val = 0;
9148
	vcpu->arch.st.msr_val = 0;
9149

9150 9151
	kvmclock_reset(vcpu);

9152 9153 9154
	kvm_clear_async_pf_completion_queue(vcpu);
	kvm_async_pf_hash_reset(vcpu);
	vcpu->arch.apf.halted = false;
9155

9156 9157 9158 9159 9160 9161 9162
	if (kvm_mpx_supported()) {
		void *mpx_state_buffer;

		/*
		 * To avoid have the INIT path from kvm_apic_has_events() that be
		 * called with loaded FPU and does not let userspace fix the state.
		 */
9163 9164
		if (init_event)
			kvm_put_guest_fpu(vcpu);
9165
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9166
					XFEATURE_BNDREGS);
9167 9168
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndreg_state));
9169
		mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu->state.xsave,
9170
					XFEATURE_BNDCSR);
9171 9172
		if (mpx_state_buffer)
			memset(mpx_state_buffer, 0, sizeof(struct mpx_bndcsr));
9173 9174
		if (init_event)
			kvm_load_guest_fpu(vcpu);
9175 9176
	}

9177
	if (!init_event) {
9178
		kvm_pmu_reset(vcpu);
9179
		vcpu->arch.smbase = 0x30000;
9180 9181

		vcpu->arch.msr_misc_features_enables = 0;
9182 9183

		vcpu->arch.xcr0 = XFEATURE_MASK_FP;
9184
	}
9185

9186 9187 9188 9189
	memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
	vcpu->arch.regs_avail = ~0;
	vcpu->arch.regs_dirty = ~0;

9190 9191
	vcpu->arch.ia32_xss = 0;

9192
	kvm_x86_ops->vcpu_reset(vcpu, init_event);
9193 9194
}

9195
void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
9196 9197 9198 9199 9200 9201 9202 9203
{
	struct kvm_segment cs;

	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
	cs.selector = vector << 8;
	cs.base = vector << 12;
	kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
	kvm_rip_write(vcpu, 0);
9204 9205
}

9206
int kvm_arch_hardware_enable(void)
9207
{
9208 9209 9210
	struct kvm *kvm;
	struct kvm_vcpu *vcpu;
	int i;
9211 9212 9213 9214
	int ret;
	u64 local_tsc;
	u64 max_tsc = 0;
	bool stable, backwards_tsc = false;
9215 9216

	kvm_shared_msr_cpu_online();
9217
	ret = kvm_x86_ops->hardware_enable();
9218 9219 9220
	if (ret != 0)
		return ret;

9221
	local_tsc = rdtsc();
9222
	stable = !kvm_check_tsc_unstable();
9223 9224 9225
	list_for_each_entry(kvm, &vm_list, vm_list) {
		kvm_for_each_vcpu(i, vcpu, kvm) {
			if (!stable && vcpu->cpu == smp_processor_id())
9226
				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
9227 9228 9229 9230 9231 9232 9233 9234 9235 9236 9237 9238 9239 9240 9241 9242
			if (stable && vcpu->arch.last_host_tsc > local_tsc) {
				backwards_tsc = true;
				if (vcpu->arch.last_host_tsc > max_tsc)
					max_tsc = vcpu->arch.last_host_tsc;
			}
		}
	}

	/*
	 * Sometimes, even reliable TSCs go backwards.  This happens on
	 * platforms that reset TSC during suspend or hibernate actions, but
	 * maintain synchronization.  We must compensate.  Fortunately, we can
	 * detect that condition here, which happens early in CPU bringup,
	 * before any KVM threads can be running.  Unfortunately, we can't
	 * bring the TSCs fully up to date with real time, as we aren't yet far
	 * enough into CPU bringup that we know how much real time has actually
9243
	 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot
9244 9245 9246 9247 9248 9249 9250 9251 9252 9253 9254 9255 9256 9257 9258 9259 9260 9261 9262 9263 9264 9265 9266 9267
	 * variables that haven't been updated yet.
	 *
	 * So we simply find the maximum observed TSC above, then record the
	 * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
	 * the adjustment will be applied.  Note that we accumulate
	 * adjustments, in case multiple suspend cycles happen before some VCPU
	 * gets a chance to run again.  In the event that no KVM threads get a
	 * chance to run, we will miss the entire elapsed period, as we'll have
	 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
	 * loose cycle time.  This isn't too big a deal, since the loss will be
	 * uniform across all VCPUs (not to mention the scenario is extremely
	 * unlikely). It is possible that a second hibernate recovery happens
	 * much faster than a first, causing the observed TSC here to be
	 * smaller; this would require additional padding adjustment, which is
	 * why we set last_host_tsc to the local tsc observed here.
	 *
	 * N.B. - this code below runs only on platforms with reliable TSC,
	 * as that is the only way backwards_tsc is set above.  Also note
	 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
	 * have the same delta_cyc adjustment applied if backwards_tsc
	 * is detected.  Note further, this adjustment is only done once,
	 * as we reset last_host_tsc on all VCPUs to stop this from being
	 * called multiple times (one for each physical CPU bringup).
	 *
Guo Chao's avatar
Guo Chao committed
9268
	 * Platforms with unreliable TSCs don't have to deal with this, they
9269 9270 9271 9272 9273 9274 9275
	 * will be compensated by the logic in vcpu_load, which sets the TSC to
	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
	 * guarantee that they stay in perfect synchronization.
	 */
	if (backwards_tsc) {
		u64 delta_cyc = max_tsc - local_tsc;
		list_for_each_entry(kvm, &vm_list, vm_list) {
9276
			kvm->arch.backwards_tsc_observed = true;
9277 9278 9279
			kvm_for_each_vcpu(i, vcpu, kvm) {
				vcpu->arch.tsc_offset_adjustment += delta_cyc;
				vcpu->arch.last_host_tsc = local_tsc;
9280
				kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
9281 9282 9283 9284 9285 9286 9287 9288 9289 9290 9291 9292 9293 9294
			}

			/*
			 * We have to disable TSC offset matching.. if you were
			 * booting a VM while issuing an S4 host suspend....
			 * you may have some problem.  Solving this issue is
			 * left as an exercise to the reader.
			 */
			kvm->arch.last_tsc_nsec = 0;
			kvm->arch.last_tsc_write = 0;
		}

	}
	return 0;
9295 9296
}

9297
void kvm_arch_hardware_disable(void)
9298
{
9299 9300
	kvm_x86_ops->hardware_disable();
	drop_user_return_notifiers();
9301 9302 9303 9304
}

int kvm_arch_hardware_setup(void)
{
9305 9306 9307 9308 9309 9310
	int r;

	r = kvm_x86_ops->hardware_setup();
	if (r != 0)
		return r;

9311 9312 9313 9314
	if (kvm_has_tsc_control) {
		/*
		 * Make sure the user can only configure tsc_khz values that
		 * fit into a signed integer.
9315
		 * A min value is not calculated because it will always
9316 9317 9318 9319 9320 9321
		 * be 1 on all machines.
		 */
		u64 max = min(0x7fffffffULL,
			      __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz));
		kvm_max_guest_tsc_khz = max;

9322
		kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits;
9323
	}
9324

9325 9326
	kvm_init_msr_list();
	return 0;
9327 9328 9329 9330 9331 9332 9333
}

void kvm_arch_hardware_unsetup(void)
{
	kvm_x86_ops->hardware_unsetup();
}

9334
int kvm_arch_check_processor_compat(void)
9335
{
9336
	return kvm_x86_ops->check_processor_compatibility();
9337 9338 9339 9340 9341 9342 9343 9344 9345 9346 9347
}

bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
{
	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);

bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
9348 9349
}

9350
struct static_key kvm_no_apic_vcpu __read_mostly;
9351
EXPORT_SYMBOL_GPL(kvm_no_apic_vcpu);
9352

9353 9354 9355 9356 9357
int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
{
	struct page *page;
	int r;

9358
	vcpu->arch.emulate_ctxt.ops = &emulate_ops;
9359
	if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
9360
		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
9361
	else
9362
		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
9363 9364 9365 9366 9367 9368

	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
	if (!page) {
		r = -ENOMEM;
		goto fail;
	}
9369
	vcpu->arch.pio_data = page_address(page);
9370

9371
	kvm_set_tsc_khz(vcpu, max_tsc_khz);
Zachary Amsden's avatar
Zachary Amsden committed
9372

9373 9374 9375 9376
	r = kvm_mmu_create(vcpu);
	if (r < 0)
		goto fail_free_pio_data;

9377
	if (irqchip_in_kernel(vcpu->kvm)) {
9378
		vcpu->arch.apicv_active = kvm_x86_ops->get_enable_apicv(vcpu);
9379
		r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
9380 9381
		if (r < 0)
			goto fail_mmu_destroy;
9382 9383
	} else
		static_key_slow_inc(&kvm_no_apic_vcpu);
9384

Huang Ying's avatar
Huang Ying committed
9385
	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
9386
				       GFP_KERNEL_ACCOUNT);
Huang Ying's avatar
Huang Ying committed
9387 9388
	if (!vcpu->arch.mce_banks) {
		r = -ENOMEM;
9389
		goto fail_free_lapic;
Huang Ying's avatar
Huang Ying committed
9390 9391 9392
	}
	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;

9393 9394
	if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
				GFP_KERNEL_ACCOUNT)) {
9395
		r = -ENOMEM;
9396
		goto fail_free_mce_banks;
9397
	}
9398

9399
	fx_init(vcpu);
9400

9401
	vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
9402

9403 9404
	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);

9405 9406
	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;

9407
	kvm_async_pf_hash_reset(vcpu);
9408
	kvm_pmu_init(vcpu);
9409

9410
	vcpu->arch.pending_external_vector = -1;
9411
	vcpu->arch.preempted_in_kernel = false;
9412

9413 9414
	kvm_hv_vcpu_init(vcpu);

9415
	return 0;
9416

9417 9418
fail_free_mce_banks:
	kfree(vcpu->arch.mce_banks);
9419 9420
fail_free_lapic:
	kvm_free_lapic(vcpu);
9421 9422 9423
fail_mmu_destroy:
	kvm_mmu_destroy(vcpu);
fail_free_pio_data:
9424
	free_page((unsigned long)vcpu->arch.pio_data);
9425 9426 9427 9428 9429 9430
fail:
	return r;
}

void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
{
9431 9432
	int idx;

9433
	kvm_hv_vcpu_uninit(vcpu);
9434
	kvm_pmu_destroy(vcpu);
9435
	kfree(vcpu->arch.mce_banks);
9436
	kvm_free_lapic(vcpu);
9437
	idx = srcu_read_lock(&vcpu->kvm->srcu);
9438
	kvm_mmu_destroy(vcpu);
9439
	srcu_read_unlock(&vcpu->kvm->srcu, idx);
9440
	free_page((unsigned long)vcpu->arch.pio_data);
9441
	if (!lapic_in_kernel(vcpu))
9442
		static_key_slow_dec(&kvm_no_apic_vcpu);
9443
}
9444

Radim Krčmář's avatar
Radim Krčmář committed
9445 9446
void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
{
9447
	vcpu->arch.l1tf_flush_l1d = true;
9448
	kvm_x86_ops->sched_in(vcpu, cpu);
Radim Krčmář's avatar
Radim Krčmář committed
9449 9450
}

9451
int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
9452
{
9453 9454 9455
	if (type)
		return -EINVAL;

9456
	INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
9457
	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
9458
	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
9459
	INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages);
Ben-Ami Yassour's avatar
Ben-Ami Yassour committed
9460
	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
9461
	atomic_set(&kvm->arch.noncoherent_dma_count, 0);
9462

9463 9464
	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
9465 9466 9467
	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
		&kvm->arch.irq_sources_bitmap);
9468

9469
	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
9470
	mutex_init(&kvm->arch.apic_map_lock);
9471 9472
	spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);

9473
	kvm->arch.kvmclock_offset = -ktime_get_boottime_ns();
9474
	pvclock_update_vm_gtod_copy(kvm);
9475

9476 9477
	kvm->arch.guest_can_read_msr_platform_info = true;

9478
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
9479
	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
9480

9481
	kvm_hv_init_vm(kvm);
9482
	kvm_page_track_init(kvm);
9483
	kvm_mmu_init_vm(kvm);
9484

9485
	return kvm_x86_ops->vm_init(kvm);
9486 9487
}

9488 9489 9490 9491 9492
int kvm_arch_post_init_vm(struct kvm *kvm)
{
	return kvm_mmu_post_init_vm(kvm);
}

9493 9494
static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
{
9495
	vcpu_load(vcpu);
9496 9497 9498 9499 9500 9501 9502
	kvm_mmu_unload(vcpu);
	vcpu_put(vcpu);
}

static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned int i;
9503
	struct kvm_vcpu *vcpu;
9504 9505 9506 9507

	/*
	 * Unpin any mmu pages first.
	 */
9508 9509
	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_clear_async_pf_completion_queue(vcpu);
9510
		kvm_unload_vcpu_mmu(vcpu);
9511
	}
9512 9513 9514 9515 9516 9517
	kvm_for_each_vcpu(i, vcpu, kvm)
		kvm_arch_vcpu_free(vcpu);

	mutex_lock(&kvm->lock);
	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
		kvm->vcpus[i] = NULL;
9518

9519 9520
	atomic_set(&kvm->online_vcpus, 0);
	mutex_unlock(&kvm->lock);
9521 9522
}

9523 9524
void kvm_arch_sync_events(struct kvm *kvm)
{
9525
	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
9526
	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
9527
	kvm_free_pit(kvm);
9528 9529
}

9530
int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9531 9532
{
	int i, r;
9533
	unsigned long hva;
9534 9535
	struct kvm_memslots *slots = kvm_memslots(kvm);
	struct kvm_memory_slot *slot, old;
9536 9537

	/* Called with kvm->slots_lock held.  */
9538 9539
	if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
		return -EINVAL;
9540

9541 9542
	slot = id_to_memslot(slots, id);
	if (size) {
9543
		if (slot->npages)
9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556 9557 9558 9559 9560 9561
			return -EEXIST;

		/*
		 * MAP_SHARED to prevent internal slot pages from being moved
		 * by fork()/COW.
		 */
		hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
			      MAP_SHARED | MAP_ANONYMOUS, 0);
		if (IS_ERR((void *)hva))
			return PTR_ERR((void *)hva);
	} else {
		if (!slot->npages)
			return 0;

		hva = 0;
	}

	old = *slot;
9562
	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
9563
		struct kvm_userspace_memory_region m;
9564

9565 9566 9567
		m.slot = id | (i << 16);
		m.flags = 0;
		m.guest_phys_addr = gpa;
9568
		m.userspace_addr = hva;
9569
		m.memory_size = size;
9570 9571 9572 9573 9574
		r = __kvm_set_memory_region(kvm, &m);
		if (r < 0)
			return r;
	}

9575 9576
	if (!size)
		vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE);
9577

9578 9579 9580 9581
	return 0;
}
EXPORT_SYMBOL_GPL(__x86_set_memory_region);

9582
int x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
9583 9584 9585 9586
{
	int r;

	mutex_lock(&kvm->slots_lock);
9587
	r = __x86_set_memory_region(kvm, id, gpa, size);
9588 9589 9590 9591 9592 9593
	mutex_unlock(&kvm->slots_lock);

	return r;
}
EXPORT_SYMBOL_GPL(x86_set_memory_region);

9594 9595 9596 9597 9598
void kvm_arch_pre_destroy_vm(struct kvm *kvm)
{
	kvm_mmu_pre_destroy_vm(kvm);
}

9599 9600
void kvm_arch_destroy_vm(struct kvm *kvm)
{
9601 9602 9603 9604 9605 9606
	if (current->mm == kvm->mm) {
		/*
		 * Free memory regions allocated on behalf of userspace,
		 * unless the the memory map has changed due to process exit
		 * or fd copying.
		 */
9607 9608 9609
		x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0);
		x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 0, 0);
		x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
9610
	}
9611 9612
	if (kvm_x86_ops->vm_destroy)
		kvm_x86_ops->vm_destroy(kvm);
9613 9614
	kvm_pic_destroy(kvm);
	kvm_ioapic_destroy(kvm);
9615
	kvm_free_vcpus(kvm);
9616
	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
Eric Hankland's avatar
Eric Hankland committed
9617
	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
9618
	kvm_mmu_uninit_vm(kvm);
9619
	kvm_page_track_cleanup(kvm);
9620
	kvm_hv_destroy_vm(kvm);
9621
}
9622

9623
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
9624 9625 9626 9627
			   struct kvm_memory_slot *dont)
{
	int i;

9628 9629
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
		if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
Thomas Huth's avatar
Thomas Huth committed
9630
			kvfree(free->arch.rmap[i]);
9631
			free->arch.rmap[i] = NULL;
9632
		}
9633 9634 9635 9636 9637
		if (i == 0)
			continue;

		if (!dont || free->arch.lpage_info[i - 1] !=
			     dont->arch.lpage_info[i - 1]) {
Thomas Huth's avatar
Thomas Huth committed
9638
			kvfree(free->arch.lpage_info[i - 1]);
9639
			free->arch.lpage_info[i - 1] = NULL;
9640 9641
		}
	}
9642 9643

	kvm_page_track_free_memslot(free, dont);
9644 9645
}

9646 9647
int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
			    unsigned long npages)
9648 9649 9650
{
	int i;

9651
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
9652
		struct kvm_lpage_info *linfo;
9653 9654
		unsigned long ugfn;
		int lpages;
9655
		int level = i + 1;
9656 9657 9658 9659

		lpages = gfn_to_index(slot->base_gfn + npages - 1,
				      slot->base_gfn, level) + 1;

9660
		slot->arch.rmap[i] =
9661
			kvcalloc(lpages, sizeof(*slot->arch.rmap[i]),
9662
				 GFP_KERNEL_ACCOUNT);
9663
		if (!slot->arch.rmap[i])
9664
			goto out_free;
9665 9666
		if (i == 0)
			continue;
9667

9668
		linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
9669
		if (!linfo)
9670 9671
			goto out_free;

9672 9673
		slot->arch.lpage_info[i - 1] = linfo;

9674
		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
9675
			linfo[0].disallow_lpage = 1;
9676
		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
9677
			linfo[lpages - 1].disallow_lpage = 1;
9678 9679 9680 9681 9682 9683 9684 9685 9686 9687 9688
		ugfn = slot->userspace_addr >> PAGE_SHIFT;
		/*
		 * If the gfn and userspace address are not aligned wrt each
		 * other, or if explicitly asked to, disable large page
		 * support for this slot
		 */
		if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
		    !kvm_largepages_enabled()) {
			unsigned long j;

			for (j = 0; j < lpages; ++j)
9689
				linfo[j].disallow_lpage = 1;
9690 9691 9692
		}
	}

9693 9694 9695
	if (kvm_page_track_create_memslot(slot, npages))
		goto out_free;

9696 9697 9698
	return 0;

out_free:
9699
	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
Thomas Huth's avatar
Thomas Huth committed
9700
		kvfree(slot->arch.rmap[i]);
9701 9702 9703 9704
		slot->arch.rmap[i] = NULL;
		if (i == 0)
			continue;

Thomas Huth's avatar
Thomas Huth committed
9705
		kvfree(slot->arch.lpage_info[i - 1]);
9706
		slot->arch.lpage_info[i - 1] = NULL;
9707 9708 9709 9710
	}
	return -ENOMEM;
}

9711
void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
9712
{
9713 9714 9715 9716
	/*
	 * memslots->generation has been incremented.
	 * mmio generation may have reached its maximum value.
	 */
9717
	kvm_mmu_invalidate_mmio_sptes(kvm, gen);
9718 9719
}

9720 9721
int kvm_arch_prepare_memory_region(struct kvm *kvm,
				struct kvm_memory_slot *memslot,
9722
				const struct kvm_userspace_memory_region *mem,
9723
				enum kvm_mr_change change)
9724
{
9725 9726 9727
	return 0;
}

9728 9729 9730 9731 9732 9733 9734 9735 9736 9737 9738 9739 9740 9741 9742 9743 9744 9745 9746 9747 9748
static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
				     struct kvm_memory_slot *new)
{
	/* Still write protect RO slot */
	if (new->flags & KVM_MEM_READONLY) {
		kvm_mmu_slot_remove_write_access(kvm, new);
		return;
	}

	/*
	 * Call kvm_x86_ops dirty logging hooks when they are valid.
	 *
	 * kvm_x86_ops->slot_disable_log_dirty is called when:
	 *
	 *  - KVM_MR_CREATE with dirty logging is disabled
	 *  - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag
	 *
	 * The reason is, in case of PML, we need to set D-bit for any slots
	 * with dirty logging disabled in order to eliminate unnecessary GPA
	 * logging in PML buffer (and potential PML buffer full VMEXT). This
	 * guarantees leaving PML enabled during guest's lifetime won't have
Wei Yang's avatar
Wei Yang committed
9749
	 * any additional overhead from PML when guest is running with dirty
9750 9751 9752 9753 9754 9755 9756 9757 9758 9759 9760 9761 9762 9763 9764 9765 9766 9767 9768 9769 9770 9771 9772 9773 9774 9775 9776 9777
	 * logging disabled for memory slots.
	 *
	 * kvm_x86_ops->slot_enable_log_dirty is called when switching new slot
	 * to dirty logging mode.
	 *
	 * If kvm_x86_ops dirty logging hooks are invalid, use write protect.
	 *
	 * In case of write protect:
	 *
	 * Write protect all pages for dirty logging.
	 *
	 * All the sptes including the large sptes which point to this
	 * slot are set to readonly. We can not create any new large
	 * spte on this slot until the end of the logging.
	 *
	 * See the comments in fast_page_fault().
	 */
	if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
		if (kvm_x86_ops->slot_enable_log_dirty)
			kvm_x86_ops->slot_enable_log_dirty(kvm, new);
		else
			kvm_mmu_slot_remove_write_access(kvm, new);
	} else {
		if (kvm_x86_ops->slot_disable_log_dirty)
			kvm_x86_ops->slot_disable_log_dirty(kvm, new);
	}
}

9778
void kvm_arch_commit_memory_region(struct kvm *kvm,
9779
				const struct kvm_userspace_memory_region *mem,
9780
				const struct kvm_memory_slot *old,
9781
				const struct kvm_memory_slot *new,
9782
				enum kvm_mr_change change)
9783
{
9784
	if (!kvm->arch.n_requested_mmu_pages)
9785 9786
		kvm_mmu_change_mmu_pages(kvm,
				kvm_mmu_calculate_default_mmu_pages(kvm));
9787

9788 9789 9790 9791 9792 9793 9794 9795 9796 9797 9798
	/*
	 * Dirty logging tracks sptes in 4k granularity, meaning that large
	 * sptes have to be split.  If live migration is successful, the guest
	 * in the source machine will be destroyed and large sptes will be
	 * created in the destination. However, if the guest continues to run
	 * in the source machine (for example if live migration fails), small
	 * sptes will remain around and cause bad performance.
	 *
	 * Scan sptes if dirty logging has been stopped, dropping those
	 * which can be collapsed into a single large-page spte.  Later
	 * page faults will create the large-page sptes.
9799 9800 9801 9802 9803
	 *
	 * There is no need to do this in any of the following cases:
	 * CREATE:	No dirty mappings will already exist.
	 * MOVE/DELETE:	The old mappings will already have been cleaned up by
	 *		kvm_arch_flush_shadow_memslot()
9804
	 */
9805
	if (change == KVM_MR_FLAGS_ONLY &&
9806 9807 9808 9809
		(old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
		!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
		kvm_mmu_zap_collapsible_sptes(kvm, new);

9810
	/*
9811
	 * Set up write protection and/or dirty logging for the new slot.
9812
	 *
9813 9814 9815 9816
	 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have
	 * been zapped so no dirty logging staff is needed for old slot. For
	 * KVM_MR_FLAGS_ONLY, the old slot is essentially the same one as the
	 * new and it's also covered when dealing with the new slot.
9817 9818
	 *
	 * FIXME: const-ify all uses of struct kvm_memory_slot.
9819
	 */
9820
	if (change != KVM_MR_DELETE)
9821
		kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new);
9822
}
9823

9824
void kvm_arch_flush_shadow_all(struct kvm *kvm)
9825
{
9826
	kvm_mmu_zap_all(kvm);
9827 9828
}

9829 9830 9831
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
				   struct kvm_memory_slot *slot)
{
9832
	kvm_page_track_flush_slot(kvm, slot);
9833 9834
}

9835 9836 9837 9838 9839 9840 9841
static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
{
	return (is_guest_mode(vcpu) &&
			kvm_x86_ops->guest_apic_has_interrupt &&
			kvm_x86_ops->guest_apic_has_interrupt(vcpu));
}

9842 9843 9844 9845 9846 9847 9848 9849 9850 9851 9852
static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
{
	if (!list_empty_careful(&vcpu->async_pf.done))
		return true;

	if (kvm_apic_has_events(vcpu))
		return true;

	if (vcpu->arch.pv.pv_unhalted)
		return true;

9853 9854 9855
	if (vcpu->arch.exception.pending)
		return true;

9856 9857 9858
	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
	    (vcpu->arch.nmi_pending &&
	     kvm_x86_ops->nmi_allowed(vcpu)))
9859 9860
		return true;

9861 9862
	if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
	    (vcpu->arch.smi_pending && !is_smm(vcpu)))
9863 9864
		return true;

9865
	if (kvm_arch_interrupt_allowed(vcpu) &&
9866 9867
	    (kvm_cpu_has_interrupt(vcpu) ||
	    kvm_guest_apic_has_interrupt(vcpu)))
9868 9869
		return true;

9870 9871 9872
	if (kvm_hv_has_stimer_pending(vcpu))
		return true;

9873 9874 9875
	return false;
}

9876 9877
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
9878
	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
9879
}
9880

9881 9882 9883 9884 9885 9886 9887 9888 9889 9890 9891 9892 9893 9894 9895 9896
bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
{
	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
		return true;

	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
		kvm_test_request(KVM_REQ_SMI, vcpu) ||
		 kvm_test_request(KVM_REQ_EVENT, vcpu))
		return true;

	if (vcpu->arch.apicv_active && kvm_x86_ops->dy_apicv_has_pending_interrupt(vcpu))
		return true;

	return false;
}

9897 9898
bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
{
9899
	return vcpu->arch.preempted_in_kernel;
9900 9901
}

9902
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
9903
{
9904
	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
9905
}
9906 9907 9908 9909 9910

int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
{
	return kvm_x86_ops->interrupt_allowed(vcpu);
}
9911

9912
unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
Jan Kiszka's avatar
Jan Kiszka committed
9913
{
9914 9915 9916 9917 9918 9919
	if (is_64_bit_mode(vcpu))
		return kvm_rip_read(vcpu);
	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
		     kvm_rip_read(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
Jan Kiszka's avatar
Jan Kiszka committed
9920

9921 9922 9923
bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
{
	return kvm_get_linear_rip(vcpu) == linear_rip;
Jan Kiszka's avatar
Jan Kiszka committed
9924 9925 9926
}
EXPORT_SYMBOL_GPL(kvm_is_linear_rip);

9927 9928 9929 9930 9931 9932
unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
{
	unsigned long rflags;

	rflags = kvm_x86_ops->get_rflags(vcpu);
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
9933
		rflags &= ~X86_EFLAGS_TF;
9934 9935 9936 9937
	return rflags;
}
EXPORT_SYMBOL_GPL(kvm_get_rflags);

9938
static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
9939 9940
{
	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
Jan Kiszka's avatar
Jan Kiszka committed
9941
	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
9942
		rflags |= X86_EFLAGS_TF;
9943
	kvm_x86_ops->set_rflags(vcpu, rflags);
9944 9945 9946 9947 9948
}

void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
{
	__kvm_set_rflags(vcpu, rflags);
9949
	kvm_make_request(KVM_REQ_EVENT, vcpu);
9950 9951 9952
}
EXPORT_SYMBOL_GPL(kvm_set_rflags);

9953 9954 9955 9956
void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
{
	int r;

9957
	if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) ||
9958
	      work->wakeup_all)
9959 9960 9961 9962 9963 9964
		return;

	r = kvm_mmu_reload(vcpu);
	if (unlikely(r))
		return;

9965 9966
	if (!vcpu->arch.mmu->direct_map &&
	      work->arch.cr3 != vcpu->arch.mmu->get_cr3(vcpu))
9967 9968
		return;

9969
	vcpu->arch.mmu->page_fault(vcpu, work->gva, 0, true);
9970 9971
}

9972 9973 9974 9975 9976 9977 9978 9979 9980 9981 9982 9983 9984 9985 9986 9987 9988 9989 9990 9991 9992 9993 9994 9995 9996 9997
static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
{
	return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
}

static inline u32 kvm_async_pf_next_probe(u32 key)
{
	return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
}

static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 key = kvm_async_pf_hash_fn(gfn);

	while (vcpu->arch.apf.gfns[key] != ~0)
		key = kvm_async_pf_next_probe(key);

	vcpu->arch.apf.gfns[key] = gfn;
}

static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	int i;
	u32 key = kvm_async_pf_hash_fn(gfn);

	for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
9998 9999
		     (vcpu->arch.apf.gfns[key] != gfn &&
		      vcpu->arch.apf.gfns[key] != ~0); i++)
10000 10001 10002 10003 10004 10005 10006 10007 10008 10009 10010 10011 10012 10013 10014 10015 10016 10017 10018 10019 10020 10021 10022 10023 10024 10025 10026 10027 10028 10029 10030 10031 10032
		key = kvm_async_pf_next_probe(key);

	return key;
}

bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
}

static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
	u32 i, j, k;

	i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
	while (true) {
		vcpu->arch.apf.gfns[i] = ~0;
		do {
			j = kvm_async_pf_next_probe(j);
			if (vcpu->arch.apf.gfns[j] == ~0)
				return;
			k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
			/*
			 * k lies cyclically in ]i,j]
			 * |    i.k.j |
			 * |....j i.k.| or  |.k..j i...|
			 */
		} while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
		vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
		i = j;
	}
}

10033 10034
static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
{
10035 10036 10037

	return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
				      sizeof(val));
10038 10039
}

10040 10041 10042 10043 10044 10045 10046
static int apf_get_user(struct kvm_vcpu *vcpu, u32 *val)
{

	return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, val,
				      sizeof(u32));
}

10047 10048 10049 10050 10051 10052 10053 10054 10055 10056 10057 10058 10059 10060 10061 10062 10063 10064 10065 10066 10067 10068 10069 10070 10071 10072 10073 10074 10075 10076
static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
{
	if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu))
		return false;

	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
	    (vcpu->arch.apf.send_user_only &&
	     kvm_x86_ops->get_cpl(vcpu) == 0))
		return false;

	return true;
}

bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
{
	if (unlikely(!lapic_in_kernel(vcpu) ||
		     kvm_event_needs_reinjection(vcpu) ||
		     vcpu->arch.exception.pending))
		return false;

	if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
		return false;

	/*
	 * If interrupts are off we cannot even use an artificial
	 * halt state.
	 */
	return kvm_x86_ops->interrupt_allowed(vcpu);
}

10077 10078 10079
void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
				     struct kvm_async_pf *work)
{
10080 10081
	struct x86_exception fault;

10082
	trace_kvm_async_pf_not_present(work->arch.token, work->gva);
10083
	kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
10084

10085 10086
	if (kvm_can_deliver_async_pf(vcpu) &&
	    !apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
10087 10088 10089 10090 10091
		fault.vector = PF_VECTOR;
		fault.error_code_valid = true;
		fault.error_code = 0;
		fault.nested_page_fault = false;
		fault.address = work->arch.token;
10092
		fault.async_page_fault = true;
10093
		kvm_inject_page_fault(vcpu, &fault);
10094 10095 10096 10097 10098 10099 10100 10101 10102 10103
	} else {
		/*
		 * It is not possible to deliver a paravirtualized asynchronous
		 * page fault, but putting the guest in an artificial halt state
		 * can be beneficial nevertheless: if an interrupt arrives, we
		 * can deliver it timely and perhaps the guest will schedule
		 * another process.  When the instruction that triggered a page
		 * fault is retried, hopefully the page will be ready in the host.
		 */
		kvm_make_request(KVM_REQ_APF_HALT, vcpu);
10104
	}
10105 10106 10107 10108 10109
}

void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
				 struct kvm_async_pf *work)
{
10110
	struct x86_exception fault;
10111
	u32 val;
10112

10113
	if (work->wakeup_all)
10114 10115 10116
		work->arch.token = ~0; /* broadcast wakeup */
	else
		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
10117
	trace_kvm_async_pf_ready(work->arch.token, work->gva);
10118

10119 10120 10121 10122 10123 10124 10125 10126 10127 10128 10129
	if (vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED &&
	    !apf_get_user(vcpu, &val)) {
		if (val == KVM_PV_REASON_PAGE_NOT_PRESENT &&
		    vcpu->arch.exception.pending &&
		    vcpu->arch.exception.nr == PF_VECTOR &&
		    !apf_put_user(vcpu, 0)) {
			vcpu->arch.exception.injected = false;
			vcpu->arch.exception.pending = false;
			vcpu->arch.exception.nr = 0;
			vcpu->arch.exception.has_error_code = false;
			vcpu->arch.exception.error_code = 0;
10130 10131
			vcpu->arch.exception.has_payload = false;
			vcpu->arch.exception.payload = 0;
10132 10133 10134 10135 10136 10137 10138 10139 10140
		} else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
			fault.vector = PF_VECTOR;
			fault.error_code_valid = true;
			fault.error_code = 0;
			fault.nested_page_fault = false;
			fault.address = work->arch.token;
			fault.async_page_fault = true;
			kvm_inject_page_fault(vcpu, &fault);
		}
10141
	}
10142
	vcpu->arch.apf.halted = false;
10143
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
10144 10145 10146 10147 10148 10149 10150
}

bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
{
	if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
		return true;
	else
10151
		return kvm_can_do_async_pf(vcpu);
10152 10153
}

10154 10155 10156 10157 10158 10159 10160 10161 10162 10163 10164 10165 10166 10167 10168 10169 10170 10171
void kvm_arch_start_assignment(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);

void kvm_arch_end_assignment(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);

bool kvm_arch_has_assigned_device(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.assigned_device_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);

10172 10173 10174 10175 10176 10177 10178 10179 10180 10181 10182 10183 10184 10185 10186 10187 10188 10189
void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
{
	atomic_inc(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);

void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
{
	atomic_dec(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);

bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
{
	return atomic_read(&kvm->arch.noncoherent_dma_count);
}
EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);

10190 10191
bool kvm_arch_has_irq_bypass(void)
{
10192
	return true;
10193 10194
}

10195 10196 10197 10198 10199 10200
int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

10201
	irqfd->producer = prod;
10202

10203 10204
	return kvm_x86_ops->update_pi_irte(irqfd->kvm,
					   prod->irq, irqfd->gsi, 1);
10205 10206 10207 10208 10209 10210 10211 10212 10213 10214 10215 10216 10217 10218 10219
}

void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
				      struct irq_bypass_producer *prod)
{
	int ret;
	struct kvm_kernel_irqfd *irqfd =
		container_of(cons, struct kvm_kernel_irqfd, consumer);

	WARN_ON(irqfd->producer != prod);
	irqfd->producer = NULL;

	/*
	 * When producer of consumer is unregistered, we change back to
	 * remapped mode, so we can re-use the current implementation
Andrea Gelmini's avatar
Andrea Gelmini committed
10220
	 * when the irq is masked/disabled or the consumer side (KVM
10221 10222 10223 10224 10225 10226 10227 10228 10229 10230 10231 10232 10233 10234
	 * int this case doesn't want to receive the interrupts.
	*/
	ret = kvm_x86_ops->update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0);
	if (ret)
		printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
		       " fails: %d\n", irqfd->consumer.token, ret);
}

int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
				   uint32_t guest_irq, bool set)
{
	return kvm_x86_ops->update_pi_irte(kvm, host_irq, guest_irq, set);
}

10235 10236 10237 10238 10239 10240
bool kvm_vector_hashing_enabled(void)
{
	return vector_hashing;
}
EXPORT_SYMBOL_GPL(kvm_vector_hashing_enabled);

10241 10242 10243 10244 10245 10246 10247
bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
{
	return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
}
EXPORT_SYMBOL_GPL(kvm_arch_no_poll);


10248
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
10249
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
10250 10251 10252 10253
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
10254
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
10255
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
10256
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
10257
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
10258
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
10259
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
10260
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
10261
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
10262
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
10263
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
10264
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
10265
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
10266 10267
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);