Commit b01c8b54 authored by Paul Mackerras's avatar Paul Mackerras Committed by Avi Kivity

powerpc, KVM: Rework KVM checks in first-level interrupt handlers

Instead of branching out-of-line with the DO_KVM macro to check if we
are in a KVM guest at the time of an interrupt, this moves the KVM
check inline in the first-level interrupt handlers.  This speeds up
the non-KVM case and makes sure that none of the interrupt handlers
are missing the check.

Because the first-level interrupt handlers are now larger, some things
had to be move out of line in exceptions-64s.S.

This all necessitated some minor changes to the interrupt entry code
in KVM.  This also streamlines the book3s_32 KVM test.
Signed-off-by: default avatarPaul Mackerras <paulus@samba.org>
Signed-off-by: default avatarAlexander Graf <agraf@suse.de>
parent f05ed4d5
...@@ -61,19 +61,22 @@ ...@@ -61,19 +61,22 @@
#define EXC_HV H #define EXC_HV H
#define EXC_STD #define EXC_STD
#define EXCEPTION_PROLOG_1(area) \ #define __EXCEPTION_PROLOG_1(area, extra, vec) \
GET_PACA(r13); \ GET_PACA(r13); \
std r9,area+EX_R9(r13); /* save r9 - r12 */ \ std r9,area+EX_R9(r13); /* save r9 - r12 */ \
std r10,area+EX_R10(r13); \ std r10,area+EX_R10(r13); \
std r11,area+EX_R11(r13); \
std r12,area+EX_R12(r13); \
BEGIN_FTR_SECTION_NESTED(66); \ BEGIN_FTR_SECTION_NESTED(66); \
mfspr r10,SPRN_CFAR; \ mfspr r10,SPRN_CFAR; \
std r10,area+EX_CFAR(r13); \ std r10,area+EX_CFAR(r13); \
END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \
GET_SCRATCH0(r9); \ mfcr r9; \
std r9,area+EX_R13(r13); \ extra(vec); \
mfcr r9 std r11,area+EX_R11(r13); \
std r12,area+EX_R12(r13); \
GET_SCRATCH0(r10); \
std r10,area+EX_R13(r13)
#define EXCEPTION_PROLOG_1(area, extra, vec) \
__EXCEPTION_PROLOG_1(area, extra, vec)
#define __EXCEPTION_PROLOG_PSERIES_1(label, h) \ #define __EXCEPTION_PROLOG_PSERIES_1(label, h) \
ld r12,PACAKBASE(r13); /* get high part of &label */ \ ld r12,PACAKBASE(r13); /* get high part of &label */ \
...@@ -85,13 +88,54 @@ ...@@ -85,13 +88,54 @@
mtspr SPRN_##h##SRR1,r10; \ mtspr SPRN_##h##SRR1,r10; \
h##rfid; \ h##rfid; \
b . /* prevent speculative execution */ b . /* prevent speculative execution */
#define EXCEPTION_PROLOG_PSERIES_1(label, h) \ #define EXCEPTION_PROLOG_PSERIES_1(label, h) \
__EXCEPTION_PROLOG_PSERIES_1(label, h) __EXCEPTION_PROLOG_PSERIES_1(label, h)
#define EXCEPTION_PROLOG_PSERIES(area, label, h) \ #define EXCEPTION_PROLOG_PSERIES(area, label, h, extra, vec) \
EXCEPTION_PROLOG_1(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \
EXCEPTION_PROLOG_PSERIES_1(label, h); EXCEPTION_PROLOG_PSERIES_1(label, h);
#define __KVMTEST(n) \
lbz r10,PACA_KVM_SVCPU+SVCPU_IN_GUEST(r13); \
cmpwi r10,0; \
bne do_kvm_##n
#define __KVM_HANDLER(area, h, n) \
do_kvm_##n: \
ld r10,area+EX_R10(r13); \
stw r9,PACA_KVM_SVCPU+SVCPU_SCRATCH1(r13); \
ld r9,area+EX_R9(r13); \
std r12,PACA_KVM_SVCPU+SVCPU_SCRATCH0(r13); \
li r12,n; \
b kvmppc_interrupt
#define __KVM_HANDLER_SKIP(area, h, n) \
do_kvm_##n: \
cmpwi r10,KVM_GUEST_MODE_SKIP; \
ld r10,area+EX_R10(r13); \
beq 89f; \
stw r9,PACA_KVM_SVCPU+SVCPU_SCRATCH1(r13); \
ld r9,area+EX_R9(r13); \
std r12,PACA_KVM_SVCPU+SVCPU_SCRATCH0(r13); \
li r12,n; \
b kvmppc_interrupt; \
89: mtocrf 0x80,r9; \
ld r9,area+EX_R9(r13); \
b kvmppc_skip_##h##interrupt
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
#define KVMTEST(n) __KVMTEST(n)
#define KVM_HANDLER(area, h, n) __KVM_HANDLER(area, h, n)
#define KVM_HANDLER_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n)
#else
#define KVMTEST(n)
#define KVM_HANDLER(area, h, n)
#define KVM_HANDLER_SKIP(area, h, n)
#endif
#define NOTEST(n)
/* /*
* The common exception prolog is used for all except a few exceptions * The common exception prolog is used for all except a few exceptions
* such as a segment miss on a kernel address. We have to be prepared * such as a segment miss on a kernel address. We have to be prepared
...@@ -164,57 +208,54 @@ ...@@ -164,57 +208,54 @@
.globl label##_pSeries; \ .globl label##_pSeries; \
label##_pSeries: \ label##_pSeries: \
HMT_MEDIUM; \ HMT_MEDIUM; \
DO_KVM vec; \
SET_SCRATCH0(r13); /* save r13 */ \ SET_SCRATCH0(r13); /* save r13 */ \
EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, EXC_STD) EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \
EXC_STD, KVMTEST, vec)
#define STD_EXCEPTION_HV(loc, vec, label) \ #define STD_EXCEPTION_HV(loc, vec, label) \
. = loc; \ . = loc; \
.globl label##_hv; \ .globl label##_hv; \
label##_hv: \ label##_hv: \
HMT_MEDIUM; \ HMT_MEDIUM; \
DO_KVM vec; \ SET_SCRATCH0(r13); /* save r13 */ \
SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \
EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, EXC_HV) EXC_HV, KVMTEST, vec)
#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h) \ #define __SOFTEN_TEST(h) \
HMT_MEDIUM; \
DO_KVM vec; \
SET_SCRATCH0(r13); /* save r13 */ \
GET_PACA(r13); \
std r9,PACA_EXGEN+EX_R9(r13); /* save r9, r10 */ \
std r10,PACA_EXGEN+EX_R10(r13); \
lbz r10,PACASOFTIRQEN(r13); \ lbz r10,PACASOFTIRQEN(r13); \
mfcr r9; \
cmpwi r10,0; \ cmpwi r10,0; \
beq masked_##h##interrupt; \ beq masked_##h##interrupt
GET_SCRATCH0(r10); \ #define _SOFTEN_TEST(h) __SOFTEN_TEST(h)
std r10,PACA_EXGEN+EX_R13(r13); \
std r11,PACA_EXGEN+EX_R11(r13); \ #define SOFTEN_TEST(vec) \
std r12,PACA_EXGEN+EX_R12(r13); \ KVMTEST(vec); \
ld r12,PACAKBASE(r13); /* get high part of &label */ \ _SOFTEN_TEST(EXC_STD)
ld r10,PACAKMSR(r13); /* get MSR value for kernel */ \
mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ #define SOFTEN_TEST_HV(vec) \
LOAD_HANDLER(r12,label##_common) \ KVMTEST(vec); \
mtspr SPRN_##h##SRR0,r12; \ _SOFTEN_TEST(EXC_HV)
mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \
mtspr SPRN_##h##SRR1,r10; \ #define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \
h##rfid; \ HMT_MEDIUM; \
b . /* prevent speculative execution */ SET_SCRATCH0(r13); /* save r13 */ \
#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h) \ __EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec); \
__MASKABLE_EXCEPTION_PSERIES(vec, label, h) EXCEPTION_PROLOG_PSERIES_1(label##_common, h);
#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \
__MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)
#define MASKABLE_EXCEPTION_PSERIES(loc, vec, label) \ #define MASKABLE_EXCEPTION_PSERIES(loc, vec, label) \
. = loc; \ . = loc; \
.globl label##_pSeries; \ .globl label##_pSeries; \
label##_pSeries: \ label##_pSeries: \
_MASKABLE_EXCEPTION_PSERIES(vec, label, EXC_STD) _MASKABLE_EXCEPTION_PSERIES(vec, label, \
EXC_STD, SOFTEN_TEST)
#define MASKABLE_EXCEPTION_HV(loc, vec, label) \ #define MASKABLE_EXCEPTION_HV(loc, vec, label) \
. = loc; \ . = loc; \
.globl label##_hv; \ .globl label##_hv; \
label##_hv: \ label##_hv: \
_MASKABLE_EXCEPTION_PSERIES(vec, label, EXC_HV) _MASKABLE_EXCEPTION_PSERIES(vec, label, \
EXC_HV, SOFTEN_TEST_HV)
#ifdef CONFIG_PPC_ISERIES #ifdef CONFIG_PPC_ISERIES
#define DISABLE_INTS \ #define DISABLE_INTS \
......
...@@ -40,7 +40,6 @@ __start_interrupts: ...@@ -40,7 +40,6 @@ __start_interrupts:
.globl system_reset_pSeries; .globl system_reset_pSeries;
system_reset_pSeries: system_reset_pSeries:
HMT_MEDIUM; HMT_MEDIUM;
DO_KVM 0x100;
SET_SCRATCH0(r13) SET_SCRATCH0(r13)
#ifdef CONFIG_PPC_P7_NAP #ifdef CONFIG_PPC_P7_NAP
BEGIN_FTR_SECTION BEGIN_FTR_SECTION
...@@ -65,67 +64,45 @@ BEGIN_FTR_SECTION ...@@ -65,67 +64,45 @@ BEGIN_FTR_SECTION
beq . beq .
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE_206) END_FTR_SECTION_IFSET(CPU_FTR_HVMODE_206)
#endif /* CONFIG_PPC_P7_NAP */ #endif /* CONFIG_PPC_P7_NAP */
EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD) EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
NOTEST, 0x100)
. = 0x200 . = 0x200
_machine_check_pSeries: machine_check_pSeries_1:
HMT_MEDIUM /* This is moved out of line as it can be patched by FW, but
DO_KVM 0x200 * some code path might still want to branch into the original
SET_SCRATCH0(r13) * vector
EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common, EXC_STD) */
b machine_check_pSeries
. = 0x300 . = 0x300
.globl data_access_pSeries .globl data_access_pSeries
data_access_pSeries: data_access_pSeries:
HMT_MEDIUM HMT_MEDIUM
DO_KVM 0x300
SET_SCRATCH0(r13) SET_SCRATCH0(r13)
#ifndef CONFIG_POWER4_ONLY
BEGIN_FTR_SECTION BEGIN_FTR_SECTION
GET_PACA(r13) b data_access_check_stab
std r9,PACA_EXSLB+EX_R9(r13) data_access_not_stab:
std r10,PACA_EXSLB+EX_R10(r13) END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
mfspr r10,SPRN_DAR #endif
mfspr r9,SPRN_DSISR EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD,
srdi r10,r10,60 KVMTEST, 0x300)
rlwimi r10,r9,16,0x20
mfcr r9
cmpwi r10,0x2c
beq do_stab_bolted_pSeries
ld r10,PACA_EXSLB+EX_R10(r13)
std r11,PACA_EXGEN+EX_R11(r13)
ld r11,PACA_EXSLB+EX_R9(r13)
std r12,PACA_EXGEN+EX_R12(r13)
GET_SCRATCH0(r12)
std r10,PACA_EXGEN+EX_R10(r13)
std r11,PACA_EXGEN+EX_R9(r13)
std r12,PACA_EXGEN+EX_R13(r13)
EXCEPTION_PROLOG_PSERIES_1(data_access_common, EXC_STD)
FTR_SECTION_ELSE
EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD)
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_SLB)
. = 0x380 . = 0x380
.globl data_access_slb_pSeries .globl data_access_slb_pSeries
data_access_slb_pSeries: data_access_slb_pSeries:
HMT_MEDIUM HMT_MEDIUM
DO_KVM 0x380
SET_SCRATCH0(r13) SET_SCRATCH0(r13)
GET_PACA(r13) EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x380)
std r3,PACA_EXSLB+EX_R3(r13) std r3,PACA_EXSLB+EX_R3(r13)
mfspr r3,SPRN_DAR mfspr r3,SPRN_DAR
std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
mfcr r9
#ifdef __DISABLED__ #ifdef __DISABLED__
/* Keep that around for when we re-implement dynamic VSIDs */ /* Keep that around for when we re-implement dynamic VSIDs */
cmpdi r3,0 cmpdi r3,0
bge slb_miss_user_pseries bge slb_miss_user_pseries
#endif /* __DISABLED__ */ #endif /* __DISABLED__ */
std r10,PACA_EXSLB+EX_R10(r13) mfspr r12,SPRN_SRR1
std r11,PACA_EXSLB+EX_R11(r13)
std r12,PACA_EXSLB+EX_R12(r13)
GET_SCRATCH0(r10)
std r10,PACA_EXSLB+EX_R13(r13)
mfspr r12,SPRN_SRR1 /* and SRR1 */
#ifndef CONFIG_RELOCATABLE #ifndef CONFIG_RELOCATABLE
b .slb_miss_realmode b .slb_miss_realmode
#else #else
...@@ -147,24 +124,16 @@ data_access_slb_pSeries: ...@@ -147,24 +124,16 @@ data_access_slb_pSeries:
.globl instruction_access_slb_pSeries .globl instruction_access_slb_pSeries
instruction_access_slb_pSeries: instruction_access_slb_pSeries:
HMT_MEDIUM HMT_MEDIUM
DO_KVM 0x480
SET_SCRATCH0(r13) SET_SCRATCH0(r13)
GET_PACA(r13) EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x480)
std r3,PACA_EXSLB+EX_R3(r13) std r3,PACA_EXSLB+EX_R3(r13)
mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */ mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
mfcr r9
#ifdef __DISABLED__ #ifdef __DISABLED__
/* Keep that around for when we re-implement dynamic VSIDs */ /* Keep that around for when we re-implement dynamic VSIDs */
cmpdi r3,0 cmpdi r3,0
bge slb_miss_user_pseries bge slb_miss_user_pseries
#endif /* __DISABLED__ */ #endif /* __DISABLED__ */
std r10,PACA_EXSLB+EX_R10(r13) mfspr r12,SPRN_SRR1
std r11,PACA_EXSLB+EX_R11(r13)
std r12,PACA_EXSLB+EX_R12(r13)
GET_SCRATCH0(r10)
std r10,PACA_EXSLB+EX_R13(r13)
mfspr r12,SPRN_SRR1 /* and SRR1 */
#ifndef CONFIG_RELOCATABLE #ifndef CONFIG_RELOCATABLE
b .slb_miss_realmode b .slb_miss_realmode
#else #else
...@@ -184,26 +153,46 @@ instruction_access_slb_pSeries: ...@@ -184,26 +153,46 @@ instruction_access_slb_pSeries:
hardware_interrupt_pSeries: hardware_interrupt_pSeries:
hardware_interrupt_hv: hardware_interrupt_hv:
BEGIN_FTR_SECTION BEGIN_FTR_SECTION
_MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt, EXC_STD) _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt,
EXC_STD, SOFTEN_TEST)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
FTR_SECTION_ELSE FTR_SECTION_ELSE
_MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt, EXC_HV) _MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt,
EXC_HV, SOFTEN_TEST_HV)
KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_HVMODE_206) ALT_FTR_SECTION_END_IFCLR(CPU_FTR_HVMODE_206)
STD_EXCEPTION_PSERIES(0x600, 0x600, alignment) STD_EXCEPTION_PSERIES(0x600, 0x600, alignment)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x600)
STD_EXCEPTION_PSERIES(0x700, 0x700, program_check) STD_EXCEPTION_PSERIES(0x700, 0x700, program_check)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x700)
STD_EXCEPTION_PSERIES(0x800, 0x800, fp_unavailable) STD_EXCEPTION_PSERIES(0x800, 0x800, fp_unavailable)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x800)
MASKABLE_EXCEPTION_PSERIES(0x900, 0x900, decrementer) MASKABLE_EXCEPTION_PSERIES(0x900, 0x900, decrementer)
MASKABLE_EXCEPTION_HV(0x980, 0x980, decrementer) MASKABLE_EXCEPTION_HV(0x980, 0x982, decrementer)
STD_EXCEPTION_PSERIES(0xa00, 0xa00, trap_0a) STD_EXCEPTION_PSERIES(0xa00, 0xa00, trap_0a)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xa00)
STD_EXCEPTION_PSERIES(0xb00, 0xb00, trap_0b) STD_EXCEPTION_PSERIES(0xb00, 0xb00, trap_0b)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xb00)
. = 0xc00 . = 0xc00
.globl system_call_pSeries .globl system_call_pSeries
system_call_pSeries: system_call_pSeries:
HMT_MEDIUM HMT_MEDIUM
DO_KVM 0xc00 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
SET_SCRATCH0(r13)
GET_PACA(r13)
std r9,PACA_EXGEN+EX_R9(r13)
std r10,PACA_EXGEN+EX_R10(r13)
mfcr r9
KVMTEST(0xc00)
GET_SCRATCH0(r13)
#endif
BEGIN_FTR_SECTION BEGIN_FTR_SECTION
cmpdi r0,0x1ebe cmpdi r0,0x1ebe
beq- 1f beq- 1f
...@@ -220,6 +209,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) ...@@ -220,6 +209,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
rfid rfid
b . /* prevent speculative execution */ b . /* prevent speculative execution */
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00)
/* Fast LE/BE switch system call */ /* Fast LE/BE switch system call */
1: mfspr r12,SPRN_SRR1 1: mfspr r12,SPRN_SRR1
xori r12,r12,MSR_LE xori r12,r12,MSR_LE
...@@ -228,6 +219,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) ...@@ -228,6 +219,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
b . b .
STD_EXCEPTION_PSERIES(0xd00, 0xd00, single_step) STD_EXCEPTION_PSERIES(0xd00, 0xd00, single_step)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xd00)
/* At 0xe??? we have a bunch of hypervisor exceptions, we branch /* At 0xe??? we have a bunch of hypervisor exceptions, we branch
* out of line to handle them * out of line to handle them
...@@ -262,30 +254,93 @@ vsx_unavailable_pSeries_1: ...@@ -262,30 +254,93 @@ vsx_unavailable_pSeries_1:
#ifdef CONFIG_CBE_RAS #ifdef CONFIG_CBE_RAS
STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error) STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error)
KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1202)
#endif /* CONFIG_CBE_RAS */ #endif /* CONFIG_CBE_RAS */
STD_EXCEPTION_PSERIES(0x1300, 0x1300, instruction_breakpoint) STD_EXCEPTION_PSERIES(0x1300, 0x1300, instruction_breakpoint)
KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x1300)
#ifdef CONFIG_CBE_RAS #ifdef CONFIG_CBE_RAS
STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance) STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance)
KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1602)
#endif /* CONFIG_CBE_RAS */ #endif /* CONFIG_CBE_RAS */
STD_EXCEPTION_PSERIES(0x1700, 0x1700, altivec_assist) STD_EXCEPTION_PSERIES(0x1700, 0x1700, altivec_assist)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x1700)
#ifdef CONFIG_CBE_RAS #ifdef CONFIG_CBE_RAS
STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal) STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal)
KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1802)
#endif /* CONFIG_CBE_RAS */ #endif /* CONFIG_CBE_RAS */
. = 0x3000 . = 0x3000
/*** Out of line interrupts support ***/ /*** Out of line interrupts support ***/
/* moved from 0x200 */
machine_check_pSeries:
.globl machine_check_fwnmi
machine_check_fwnmi:
HMT_MEDIUM
SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common,
EXC_STD, KVMTEST, 0x200)
KVM_HANDLER_SKIP(PACA_EXMC, EXC_STD, 0x200)
#ifndef CONFIG_POWER4_ONLY
/* moved from 0x300 */
data_access_check_stab:
GET_PACA(r13)
std r9,PACA_EXSLB+EX_R9(r13)
std r10,PACA_EXSLB+EX_R10(r13)
mfspr r10,SPRN_DAR
mfspr r9,SPRN_DSISR
srdi r10,r10,60
rlwimi r10,r9,16,0x20
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
lbz r9,PACA_KVM_SVCPU+SVCPU_IN_GUEST(r13)
rlwimi r10,r9,8,0x300
#endif
mfcr r9
cmpwi r10,0x2c
beq do_stab_bolted_pSeries
mtcrf 0x80,r9
ld r9,PACA_EXSLB+EX_R9(r13)
ld r10,PACA_EXSLB+EX_R10(r13)
b data_access_not_stab
do_stab_bolted_pSeries:
std r11,PACA_EXSLB+EX_R11(r13)
std r12,PACA_EXSLB+EX_R12(r13)
GET_SCRATCH0(r10)
std r10,PACA_EXSLB+EX_R13(r13)
EXCEPTION_PROLOG_PSERIES_1(.do_stab_bolted, EXC_STD)
#endif /* CONFIG_POWER4_ONLY */
KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x300)
KVM_HANDLER_SKIP(PACA_EXSLB, EXC_STD, 0x380)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x400)
KVM_HANDLER(PACA_EXSLB, EXC_STD, 0x480)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x900)
KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x982)
.align 7
/* moved from 0xe00 */ /* moved from 0xe00 */
STD_EXCEPTION_HV(., 0xe00, h_data_storage) STD_EXCEPTION_HV(., 0xe02, h_data_storage)
STD_EXCEPTION_HV(., 0xe20, h_instr_storage) KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0xe02)
STD_EXCEPTION_HV(., 0xe40, emulation_assist) STD_EXCEPTION_HV(., 0xe22, h_instr_storage)
STD_EXCEPTION_HV(., 0xe60, hmi_exception) /* need to flush cache ? */ KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe22)
STD_EXCEPTION_HV(., 0xe42, emulation_assist)
KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe42)
STD_EXCEPTION_HV(., 0xe62, hmi_exception) /* need to flush cache ? */
KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe62)
/* moved from 0xf00 */ /* moved from 0xf00 */
STD_EXCEPTION_PSERIES(., 0xf00, performance_monitor) STD_EXCEPTION_PSERIES(., 0xf00, performance_monitor)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf00)
STD_EXCEPTION_PSERIES(., 0xf20, altivec_unavailable) STD_EXCEPTION_PSERIES(., 0xf20, altivec_unavailable)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf20)
STD_EXCEPTION_PSERIES(., 0xf40, vsx_unavailable) STD_EXCEPTION_PSERIES(., 0xf40, vsx_unavailable)
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf40)
/* /*
* An interrupt came in while soft-disabled; clear EE in SRR1, * An interrupt came in while soft-disabled; clear EE in SRR1,
...@@ -317,14 +372,6 @@ masked_Hinterrupt: ...@@ -317,14 +372,6 @@ masked_Hinterrupt:
hrfid hrfid
b . b .
.align 7
do_stab_bolted_pSeries:
std r11,PACA_EXSLB+EX_R11(r13)
std r12,PACA_EXSLB+EX_R12(r13)
GET_SCRATCH0(r10)
std r10,PACA_EXSLB+EX_R13(r13)
EXCEPTION_PROLOG_PSERIES_1(.do_stab_bolted, EXC_STD)
#ifdef CONFIG_PPC_PSERIES #ifdef CONFIG_PPC_PSERIES
/* /*
* Vectors for the FWNMI option. Share common code. * Vectors for the FWNMI option. Share common code.
...@@ -334,14 +381,8 @@ do_stab_bolted_pSeries: ...@@ -334,14 +381,8 @@ do_stab_bolted_pSeries:
system_reset_fwnmi: system_reset_fwnmi:
HMT_MEDIUM HMT_MEDIUM
SET_SCRATCH0(r13) /* save r13 */ SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD) EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
NOTEST, 0x100)
.globl machine_check_fwnmi
.align 7
machine_check_fwnmi:
HMT_MEDIUM
SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common, EXC_STD)
#endif /* CONFIG_PPC_PSERIES */ #endif /* CONFIG_PPC_PSERIES */
......
...@@ -40,37 +40,42 @@ ...@@ -40,37 +40,42 @@
#define MSR_NOIRQ MSR_KERNEL & ~(MSR_IR | MSR_DR) #define MSR_NOIRQ MSR_KERNEL & ~(MSR_IR | MSR_DR)
#define FUNC(name) GLUE(.,name) #define FUNC(name) GLUE(.,name)
#elif defined(CONFIG_PPC_BOOK3S_32) kvmppc_skip_interrupt:
/*
* Here all GPRs are unchanged from when the interrupt happened
* except for r13, which is saved in SPRG_SCRATCH0.
*/
mfspr r13, SPRN_SRR0
addi r13, r13, 4
mtspr SPRN_SRR0, r13
GET_SCRATCH0(r13)
rfid
b .
kvmppc_skip_Hinterrupt:
/*
* Here all GPRs are unchanged from when the interrupt happened
* except for r13, which is saved in SPRG_SCRATCH0.
*/
mfspr r13, SPRN_HSRR0
addi r13, r13, 4
mtspr SPRN_HSRR0, r13
GET_SCRATCH0(r13)
hrfid
b .
#define LOAD_SHADOW_VCPU(reg) \ #elif defined(CONFIG_PPC_BOOK3S_32)
mfspr reg, SPRN_SPRG_THREAD; \
lwz reg, THREAD_KVM_SVCPU(reg); \
/* PPC32 can have a NULL pointer - let's check for that */ \
mtspr SPRN_SPRG_SCRATCH1, r12; /* Save r12 */ \
mfcr r12; \
cmpwi reg, 0; \
bne 1f; \
mfspr reg, SPRN_SPRG_SCRATCH0; \
mtcr r12; \
mfspr r12, SPRN_SPRG_SCRATCH1; \
b kvmppc_resume_\intno; \
1:; \
mtcr r12; \
mfspr r12, SPRN_SPRG_SCRATCH1; \
tophys(reg, reg)
#define SHADOW_VCPU_OFF 0 #define SHADOW_VCPU_OFF 0
#define MSR_NOIRQ MSR_KERNEL #define MSR_NOIRQ MSR_KERNEL
#define FUNC(name) name #define FUNC(name) name
#endif
.macro INTERRUPT_TRAMPOLINE intno .macro INTERRUPT_TRAMPOLINE intno
.global kvmppc_trampoline_\intno .global kvmppc_trampoline_\intno
kvmppc_trampoline_\intno: kvmppc_trampoline_\intno:
SET_SCRATCH0(r13) /* Save r13 */ mtspr SPRN_SPRG_SCRATCH0, r13 /* Save r13 */
/* /*
* First thing to do is to find out if we're coming * First thing to do is to find out if we're coming
...@@ -78,19 +83,28 @@ kvmppc_trampoline_\intno: ...@@ -78,19 +83,28 @@ kvmppc_trampoline_\intno:
* *
* To distinguish, we check a magic byte in the PACA/current * To distinguish, we check a magic byte in the PACA/current
*/ */
LOAD_SHADOW_VCPU(r13) mfspr r13, SPRN_SPRG_THREAD
PPC_STL r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH0)(r13) lwz r13, THREAD_KVM_SVCPU(r13)
/* PPC32 can have a NULL pointer - let's check for that */
mtspr SPRN_SPRG_SCRATCH1, r12 /* Save r12 */
mfcr r12 mfcr r12
cmpwi r13, 0
bne 1f
2: mtcr r12
mfspr r12, SPRN_SPRG_SCRATCH1
mfspr r13, SPRN_SPRG_SCRATCH0 /* r13 = original r13 */
b kvmppc_resume_\intno /* Get back original handler */
1: tophys(r13, r13)
stw r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH1)(r13) stw r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH1)(r13)
mfspr r12, SPRN_SPRG_SCRATCH1
stw r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH0)(r13)
lbz r12, (SHADOW_VCPU_OFF + SVCPU_IN_GUEST)(r13) lbz r12, (SHADOW_VCPU_OFF + SVCPU_IN_GUEST)(r13)
cmpwi r12, KVM_GUEST_MODE_NONE cmpwi r12, KVM_GUEST_MODE_NONE
bne ..kvmppc_handler_hasmagic_\intno bne ..kvmppc_handler_hasmagic_\intno
/* No KVM guest? Then jump back to the Linux handler! */ /* No KVM guest? Then jump back to the Linux handler! */
lwz r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH1)(r13) lwz r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH1)(r13)
mtcr r12 b 2b
PPC_LL r12, (SHADOW_VCPU_OFF + SVCPU_SCRATCH0)(r13)
GET_SCRATCH0(r13) /* r13 = original r13 */
b kvmppc_resume_\intno /* Get back original handler */
/* Now we know we're handling a KVM guest */ /* Now we know we're handling a KVM guest */
..kvmppc_handler_hasmagic_\intno: ..kvmppc_handler_hasmagic_\intno:
...@@ -112,9 +126,6 @@ INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_MACHINE_CHECK ...@@ -112,9 +126,6 @@ INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_MACHINE_CHECK
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_DATA_STORAGE INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_DATA_STORAGE
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_INST_STORAGE INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_INST_STORAGE
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_EXTERNAL INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_EXTERNAL
#ifdef CONFIG_PPC_BOOK3S_64
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_EXTERNAL_HV
#endif
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_ALIGNMENT INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_ALIGNMENT
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_PROGRAM INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_PROGRAM
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_FP_UNAVAIL INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_FP_UNAVAIL
...@@ -124,14 +135,6 @@ INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_TRACE ...@@ -124,14 +135,6 @@ INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_TRACE
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_PERFMON INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_PERFMON
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_ALTIVEC INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_ALTIVEC
/* Those are only available on 64 bit machines */
#ifdef CONFIG_PPC_BOOK3S_64
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_DATA_SEGMENT
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_INST_SEGMENT
INTERRUPT_TRAMPOLINE BOOK3S_INTERRUPT_VSX
#endif
/* /*
* Bring us back to the faulting code, but skip the * Bring us back to the faulting code, but skip the
* faulting instruction. * faulting instruction.
...@@ -163,6 +166,7 @@ kvmppc_handler_skip_ins: ...@@ -163,6 +166,7 @@ kvmppc_handler_skip_ins:
/* And get back into the code */ /* And get back into the code */
RFI RFI
#endif
/* /*
* This trampoline brings us back to a real mode handler * This trampoline brings us back to a real mode handler
......
...@@ -125,6 +125,9 @@ kvmppc_handler_trampoline_enter_end: ...@@ -125,6 +125,9 @@ kvmppc_handler_trampoline_enter_end:
.global kvmppc_handler_trampoline_exit .global kvmppc_handler_trampoline_exit
kvmppc_handler_trampoline_exit: kvmppc_handler_trampoline_exit:
.global kvmppc_interrupt
kvmppc_interrupt:
/* Register usage at this point: /* Register usage at this point:
* *
* SPRG_SCRATCH0 = guest R13 * SPRG_SCRATCH0 = guest R13
...@@ -155,12 +158,16 @@ kvmppc_handler_trampoline_exit: ...@@ -155,12 +158,16 @@ kvmppc_handler_trampoline_exit:
PPC_LL r2, (SHADOW_VCPU_OFF + SVCPU_HOST_R2)(r13) PPC_LL r2, (SHADOW_VCPU_OFF + SVCPU_HOST_R2)(r13)
/* Save guest PC and MSR */ /* Save guest PC and MSR */
#ifdef CONFIG_PPC64
BEGIN_FTR_SECTION
andi. r0,r12,0x2 andi. r0,r12,0x2
beq 1f beq 1f
mfspr r3,SPRN_HSRR0 mfspr r3,SPRN_HSRR0
mfspr r4,SPRN_HSRR1 mfspr r4,SPRN_HSRR1
andi. r12,r12,0x3ffd andi. r12,r12,0x3ffd
b 2f b 2f
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE_206)
#endif
1: mfsrr0 r3 1: mfsrr0 r3
mfsrr1 r4 mfsrr1 r4
2: 2:
......
...@@ -167,7 +167,7 @@ BEGIN_FTR_SECTION ...@@ -167,7 +167,7 @@ BEGIN_FTR_SECTION
std r12,PACA_EXGEN+EX_R13(r13) std r12,PACA_EXGEN+EX_R13(r13)
EXCEPTION_PROLOG_ISERIES_1 EXCEPTION_PROLOG_ISERIES_1
FTR_SECTION_ELSE FTR_SECTION_ELSE
EXCEPTION_PROLOG_1(PACA_EXGEN) EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0)
EXCEPTION_PROLOG_ISERIES_1 EXCEPTION_PROLOG_ISERIES_1
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_SLB) ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_SLB)
b data_access_common b data_access_common
......
...@@ -39,7 +39,7 @@ ...@@ -39,7 +39,7 @@
label##_iSeries: \ label##_iSeries: \
HMT_MEDIUM; \ HMT_MEDIUM; \
mtspr SPRN_SPRG_SCRATCH0,r13; /* save r13 */ \ mtspr SPRN_SPRG_SCRATCH0,r13; /* save r13 */ \
EXCEPTION_PROLOG_1(area); \ EXCEPTION_PROLOG_1(area, NOTEST, 0); \
EXCEPTION_PROLOG_ISERIES_1; \ EXCEPTION_PROLOG_ISERIES_1; \
b label##_common b label##_common
...@@ -48,7 +48,7 @@ label##_iSeries: \ ...@@ -48,7 +48,7 @@ label##_iSeries: \
label##_iSeries: \ label##_iSeries: \
HMT_MEDIUM; \ HMT_MEDIUM; \
mtspr SPRN_SPRG_SCRATCH0,r13; /* save r13 */ \ mtspr SPRN_SPRG_SCRATCH0,r13; /* save r13 */ \
EXCEPTION_PROLOG_1(PACA_EXGEN); \ EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0); \
lbz r10,PACASOFTIRQEN(r13); \ lbz r10,PACASOFTIRQEN(r13); \
cmpwi 0,r10,0; \ cmpwi 0,r10,0; \
beq- label##_iSeries_masked; \ beq- label##_iSeries_masked; \
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment