Commit 6f22bd32 authored by Alexander Graf's avatar Alexander Graf

KVM: PPC: Book3S HV: Make HTAB code LE host aware

When running on an LE host all data structures are kept in little endian
byte order. However, the HTAB still needs to be maintained in big endian.

So every time we access any HTAB we need to make sure we do so in the right
byte order. Fix up all accesses to manually byte swap.
Signed-off-by: default avatarAlexander Graf <agraf@suse.de>
parent 8f6822c4
...@@ -162,9 +162,9 @@ extern pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, bool writing, ...@@ -162,9 +162,9 @@ extern pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, bool writing,
bool *writable); bool *writable);
extern void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev, extern void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev,
unsigned long *rmap, long pte_index, int realmode); unsigned long *rmap, long pte_index, int realmode);
extern void kvmppc_invalidate_hpte(struct kvm *kvm, unsigned long *hptep, extern void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep,
unsigned long pte_index); unsigned long pte_index);
void kvmppc_clear_ref_hpte(struct kvm *kvm, unsigned long *hptep, void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep,
unsigned long pte_index); unsigned long pte_index);
extern void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long addr, extern void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long addr,
unsigned long *nb_ret); unsigned long *nb_ret);
......
...@@ -59,20 +59,29 @@ extern unsigned long kvm_rma_pages; ...@@ -59,20 +59,29 @@ extern unsigned long kvm_rma_pages;
/* These bits are reserved in the guest view of the HPTE */ /* These bits are reserved in the guest view of the HPTE */
#define HPTE_GR_RESERVED HPTE_GR_MODIFIED #define HPTE_GR_RESERVED HPTE_GR_MODIFIED
static inline long try_lock_hpte(unsigned long *hpte, unsigned long bits) static inline long try_lock_hpte(__be64 *hpte, unsigned long bits)
{ {
unsigned long tmp, old; unsigned long tmp, old;
__be64 be_lockbit, be_bits;
/*
* We load/store in native endian, but the HTAB is in big endian. If
* we byte swap all data we apply on the PTE we're implicitly correct
* again.
*/
be_lockbit = cpu_to_be64(HPTE_V_HVLOCK);
be_bits = cpu_to_be64(bits);
asm volatile(" ldarx %0,0,%2\n" asm volatile(" ldarx %0,0,%2\n"
" and. %1,%0,%3\n" " and. %1,%0,%3\n"
" bne 2f\n" " bne 2f\n"
" ori %0,%0,%4\n" " or %0,%0,%4\n"
" stdcx. %0,0,%2\n" " stdcx. %0,0,%2\n"
" beq+ 2f\n" " beq+ 2f\n"
" mr %1,%3\n" " mr %1,%3\n"
"2: isync" "2: isync"
: "=&r" (tmp), "=&r" (old) : "=&r" (tmp), "=&r" (old)
: "r" (hpte), "r" (bits), "i" (HPTE_V_HVLOCK) : "r" (hpte), "r" (be_bits), "r" (be_lockbit)
: "cc", "memory"); : "cc", "memory");
return old == 0; return old == 0;
} }
......
...@@ -450,7 +450,7 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, ...@@ -450,7 +450,7 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
unsigned long slb_v; unsigned long slb_v;
unsigned long pp, key; unsigned long pp, key;
unsigned long v, gr; unsigned long v, gr;
unsigned long *hptep; __be64 *hptep;
int index; int index;
int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR); int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR);
...@@ -473,13 +473,13 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, ...@@ -473,13 +473,13 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
preempt_enable(); preempt_enable();
return -ENOENT; return -ENOENT;
} }
hptep = (unsigned long *)(kvm->arch.hpt_virt + (index << 4)); hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
v = hptep[0] & ~HPTE_V_HVLOCK; v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
gr = kvm->arch.revmap[index].guest_rpte; gr = kvm->arch.revmap[index].guest_rpte;
/* Unlock the HPTE */ /* Unlock the HPTE */
asm volatile("lwsync" : : : "memory"); asm volatile("lwsync" : : : "memory");
hptep[0] = v; hptep[0] = cpu_to_be64(v);
preempt_enable(); preempt_enable();
gpte->eaddr = eaddr; gpte->eaddr = eaddr;
...@@ -583,7 +583,8 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, ...@@ -583,7 +583,8 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
unsigned long ea, unsigned long dsisr) unsigned long ea, unsigned long dsisr)
{ {
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
unsigned long *hptep, hpte[3], r; unsigned long hpte[3], r;
__be64 *hptep;
unsigned long mmu_seq, psize, pte_size; unsigned long mmu_seq, psize, pte_size;
unsigned long gpa_base, gfn_base; unsigned long gpa_base, gfn_base;
unsigned long gpa, gfn, hva, pfn; unsigned long gpa, gfn, hva, pfn;
...@@ -606,16 +607,16 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, ...@@ -606,16 +607,16 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
if (ea != vcpu->arch.pgfault_addr) if (ea != vcpu->arch.pgfault_addr)
return RESUME_GUEST; return RESUME_GUEST;
index = vcpu->arch.pgfault_index; index = vcpu->arch.pgfault_index;
hptep = (unsigned long *)(kvm->arch.hpt_virt + (index << 4)); hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
rev = &kvm->arch.revmap[index]; rev = &kvm->arch.revmap[index];
preempt_disable(); preempt_disable();
while (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
cpu_relax(); cpu_relax();
hpte[0] = hptep[0] & ~HPTE_V_HVLOCK; hpte[0] = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
hpte[1] = hptep[1]; hpte[1] = be64_to_cpu(hptep[1]);
hpte[2] = r = rev->guest_rpte; hpte[2] = r = rev->guest_rpte;
asm volatile("lwsync" : : : "memory"); asm volatile("lwsync" : : : "memory");
hptep[0] = hpte[0]; hptep[0] = cpu_to_be64(hpte[0]);
preempt_enable(); preempt_enable();
if (hpte[0] != vcpu->arch.pgfault_hpte[0] || if (hpte[0] != vcpu->arch.pgfault_hpte[0] ||
...@@ -731,7 +732,8 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, ...@@ -731,7 +732,8 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
preempt_disable(); preempt_disable();
while (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
cpu_relax(); cpu_relax();
if ((hptep[0] & ~HPTE_V_HVLOCK) != hpte[0] || hptep[1] != hpte[1] || if ((be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK) != hpte[0] ||
be64_to_cpu(hptep[1]) != hpte[1] ||
rev->guest_rpte != hpte[2]) rev->guest_rpte != hpte[2])
/* HPTE has been changed under us; let the guest retry */ /* HPTE has been changed under us; let the guest retry */
goto out_unlock; goto out_unlock;
...@@ -752,20 +754,20 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, ...@@ -752,20 +754,20 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT; rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT;
r &= rcbits | ~(HPTE_R_R | HPTE_R_C); r &= rcbits | ~(HPTE_R_R | HPTE_R_C);
if (hptep[0] & HPTE_V_VALID) { if (be64_to_cpu(hptep[0]) & HPTE_V_VALID) {
/* HPTE was previously valid, so we need to invalidate it */ /* HPTE was previously valid, so we need to invalidate it */
unlock_rmap(rmap); unlock_rmap(rmap);
hptep[0] |= HPTE_V_ABSENT; hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
kvmppc_invalidate_hpte(kvm, hptep, index); kvmppc_invalidate_hpte(kvm, hptep, index);
/* don't lose previous R and C bits */ /* don't lose previous R and C bits */
r |= hptep[1] & (HPTE_R_R | HPTE_R_C); r |= be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
} else { } else {
kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0); kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);
} }
hptep[1] = r; hptep[1] = cpu_to_be64(r);
eieio(); eieio();
hptep[0] = hpte[0]; hptep[0] = cpu_to_be64(hpte[0]);
asm volatile("ptesync" : : : "memory"); asm volatile("ptesync" : : : "memory");
preempt_enable(); preempt_enable();
if (page && hpte_is_writable(r)) if (page && hpte_is_writable(r))
...@@ -784,7 +786,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, ...@@ -784,7 +786,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
return ret; return ret;
out_unlock: out_unlock:
hptep[0] &= ~HPTE_V_HVLOCK; hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
preempt_enable(); preempt_enable();
goto out_put; goto out_put;
} }
...@@ -860,7 +862,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, ...@@ -860,7 +862,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
{ {
struct revmap_entry *rev = kvm->arch.revmap; struct revmap_entry *rev = kvm->arch.revmap;
unsigned long h, i, j; unsigned long h, i, j;
unsigned long *hptep; __be64 *hptep;
unsigned long ptel, psize, rcbits; unsigned long ptel, psize, rcbits;
for (;;) { for (;;) {
...@@ -876,11 +878,11 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, ...@@ -876,11 +878,11 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
* rmap chain lock. * rmap chain lock.
*/ */
i = *rmapp & KVMPPC_RMAP_INDEX; i = *rmapp & KVMPPC_RMAP_INDEX;
hptep = (unsigned long *) (kvm->arch.hpt_virt + (i << 4)); hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));
if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) { if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
/* unlock rmap before spinning on the HPTE lock */ /* unlock rmap before spinning on the HPTE lock */
unlock_rmap(rmapp); unlock_rmap(rmapp);
while (hptep[0] & HPTE_V_HVLOCK) while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
cpu_relax(); cpu_relax();
continue; continue;
} }
...@@ -899,14 +901,14 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, ...@@ -899,14 +901,14 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
/* Now check and modify the HPTE */ /* Now check and modify the HPTE */
ptel = rev[i].guest_rpte; ptel = rev[i].guest_rpte;
psize = hpte_page_size(hptep[0], ptel); psize = hpte_page_size(be64_to_cpu(hptep[0]), ptel);
if ((hptep[0] & HPTE_V_VALID) && if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
hpte_rpn(ptel, psize) == gfn) { hpte_rpn(ptel, psize) == gfn) {
if (kvm->arch.using_mmu_notifiers) if (kvm->arch.using_mmu_notifiers)
hptep[0] |= HPTE_V_ABSENT; hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
kvmppc_invalidate_hpte(kvm, hptep, i); kvmppc_invalidate_hpte(kvm, hptep, i);
/* Harvest R and C */ /* Harvest R and C */
rcbits = hptep[1] & (HPTE_R_R | HPTE_R_C); rcbits = be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
*rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT; *rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT;
if (rcbits & ~rev[i].guest_rpte) { if (rcbits & ~rev[i].guest_rpte) {
rev[i].guest_rpte = ptel | rcbits; rev[i].guest_rpte = ptel | rcbits;
...@@ -914,7 +916,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, ...@@ -914,7 +916,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
} }
} }
unlock_rmap(rmapp); unlock_rmap(rmapp);
hptep[0] &= ~HPTE_V_HVLOCK; hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
} }
return 0; return 0;
} }
...@@ -961,7 +963,7 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, ...@@ -961,7 +963,7 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
{ {
struct revmap_entry *rev = kvm->arch.revmap; struct revmap_entry *rev = kvm->arch.revmap;
unsigned long head, i, j; unsigned long head, i, j;
unsigned long *hptep; __be64 *hptep;
int ret = 0; int ret = 0;
retry: retry:
...@@ -977,23 +979,24 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, ...@@ -977,23 +979,24 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
i = head = *rmapp & KVMPPC_RMAP_INDEX; i = head = *rmapp & KVMPPC_RMAP_INDEX;
do { do {
hptep = (unsigned long *) (kvm->arch.hpt_virt + (i << 4)); hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));
j = rev[i].forw; j = rev[i].forw;
/* If this HPTE isn't referenced, ignore it */ /* If this HPTE isn't referenced, ignore it */
if (!(hptep[1] & HPTE_R_R)) if (!(be64_to_cpu(hptep[1]) & HPTE_R_R))
continue; continue;
if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) { if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
/* unlock rmap before spinning on the HPTE lock */ /* unlock rmap before spinning on the HPTE lock */
unlock_rmap(rmapp); unlock_rmap(rmapp);
while (hptep[0] & HPTE_V_HVLOCK) while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
cpu_relax(); cpu_relax();
goto retry; goto retry;
} }
/* Now check and modify the HPTE */ /* Now check and modify the HPTE */
if ((hptep[0] & HPTE_V_VALID) && (hptep[1] & HPTE_R_R)) { if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
(be64_to_cpu(hptep[1]) & HPTE_R_R)) {
kvmppc_clear_ref_hpte(kvm, hptep, i); kvmppc_clear_ref_hpte(kvm, hptep, i);
if (!(rev[i].guest_rpte & HPTE_R_R)) { if (!(rev[i].guest_rpte & HPTE_R_R)) {
rev[i].guest_rpte |= HPTE_R_R; rev[i].guest_rpte |= HPTE_R_R;
...@@ -1001,7 +1004,7 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, ...@@ -1001,7 +1004,7 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
} }
ret = 1; ret = 1;
} }
hptep[0] &= ~HPTE_V_HVLOCK; hptep[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
} while ((i = j) != head); } while ((i = j) != head);
unlock_rmap(rmapp); unlock_rmap(rmapp);
...@@ -1035,7 +1038,7 @@ static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp, ...@@ -1035,7 +1038,7 @@ static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
do { do {
hp = (unsigned long *)(kvm->arch.hpt_virt + (i << 4)); hp = (unsigned long *)(kvm->arch.hpt_virt + (i << 4));
j = rev[i].forw; j = rev[i].forw;
if (hp[1] & HPTE_R_R) if (be64_to_cpu(hp[1]) & HPTE_R_R)
goto out; goto out;
} while ((i = j) != head); } while ((i = j) != head);
} }
...@@ -1075,7 +1078,7 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp) ...@@ -1075,7 +1078,7 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
unsigned long head, i, j; unsigned long head, i, j;
unsigned long n; unsigned long n;
unsigned long v, r; unsigned long v, r;
unsigned long *hptep; __be64 *hptep;
int npages_dirty = 0; int npages_dirty = 0;
retry: retry:
...@@ -1091,7 +1094,8 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp) ...@@ -1091,7 +1094,8 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
i = head = *rmapp & KVMPPC_RMAP_INDEX; i = head = *rmapp & KVMPPC_RMAP_INDEX;
do { do {
hptep = (unsigned long *) (kvm->arch.hpt_virt + (i << 4)); unsigned long hptep1;
hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));
j = rev[i].forw; j = rev[i].forw;
/* /*
...@@ -1108,29 +1112,30 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp) ...@@ -1108,29 +1112,30 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
* Otherwise we need to do the tlbie even if C==0 in * Otherwise we need to do the tlbie even if C==0 in
* order to pick up any delayed writeback of C. * order to pick up any delayed writeback of C.
*/ */
if (!(hptep[1] & HPTE_R_C) && hptep1 = be64_to_cpu(hptep[1]);
(!hpte_is_writable(hptep[1]) || vcpus_running(kvm))) if (!(hptep1 & HPTE_R_C) &&
(!hpte_is_writable(hptep1) || vcpus_running(kvm)))
continue; continue;
if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) { if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
/* unlock rmap before spinning on the HPTE lock */ /* unlock rmap before spinning on the HPTE lock */
unlock_rmap(rmapp); unlock_rmap(rmapp);
while (hptep[0] & HPTE_V_HVLOCK) while (hptep[0] & cpu_to_be64(HPTE_V_HVLOCK))
cpu_relax(); cpu_relax();
goto retry; goto retry;
} }
/* Now check and modify the HPTE */ /* Now check and modify the HPTE */
if (!(hptep[0] & HPTE_V_VALID)) if (!(hptep[0] & cpu_to_be64(HPTE_V_VALID)))
continue; continue;
/* need to make it temporarily absent so C is stable */ /* need to make it temporarily absent so C is stable */
hptep[0] |= HPTE_V_ABSENT; hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
kvmppc_invalidate_hpte(kvm, hptep, i); kvmppc_invalidate_hpte(kvm, hptep, i);
v = hptep[0]; v = be64_to_cpu(hptep[0]);
r = hptep[1]; r = be64_to_cpu(hptep[1]);
if (r & HPTE_R_C) { if (r & HPTE_R_C) {
hptep[1] = r & ~HPTE_R_C; hptep[1] = cpu_to_be64(r & ~HPTE_R_C);
if (!(rev[i].guest_rpte & HPTE_R_C)) { if (!(rev[i].guest_rpte & HPTE_R_C)) {
rev[i].guest_rpte |= HPTE_R_C; rev[i].guest_rpte |= HPTE_R_C;
note_hpte_modification(kvm, &rev[i]); note_hpte_modification(kvm, &rev[i]);
...@@ -1143,7 +1148,7 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp) ...@@ -1143,7 +1148,7 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
} }
v &= ~(HPTE_V_ABSENT | HPTE_V_HVLOCK); v &= ~(HPTE_V_ABSENT | HPTE_V_HVLOCK);
v |= HPTE_V_VALID; v |= HPTE_V_VALID;
hptep[0] = v; hptep[0] = cpu_to_be64(v);
} while ((i = j) != head); } while ((i = j) != head);
unlock_rmap(rmapp); unlock_rmap(rmapp);
...@@ -1307,7 +1312,7 @@ struct kvm_htab_ctx { ...@@ -1307,7 +1312,7 @@ struct kvm_htab_ctx {
* Returns 1 if this HPT entry has been modified or has pending * Returns 1 if this HPT entry has been modified or has pending
* R/C bit changes. * R/C bit changes.
*/ */
static int hpte_dirty(struct revmap_entry *revp, unsigned long *hptp) static int hpte_dirty(struct revmap_entry *revp, __be64 *hptp)
{ {
unsigned long rcbits_unset; unsigned long rcbits_unset;
...@@ -1316,13 +1321,14 @@ static int hpte_dirty(struct revmap_entry *revp, unsigned long *hptp) ...@@ -1316,13 +1321,14 @@ static int hpte_dirty(struct revmap_entry *revp, unsigned long *hptp)
/* Also need to consider changes in reference and changed bits */ /* Also need to consider changes in reference and changed bits */
rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C); rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
if ((hptp[0] & HPTE_V_VALID) && (hptp[1] & rcbits_unset)) if ((be64_to_cpu(hptp[0]) & HPTE_V_VALID) &&
(be64_to_cpu(hptp[1]) & rcbits_unset))
return 1; return 1;
return 0; return 0;
} }
static long record_hpte(unsigned long flags, unsigned long *hptp, static long record_hpte(unsigned long flags, __be64 *hptp,
unsigned long *hpte, struct revmap_entry *revp, unsigned long *hpte, struct revmap_entry *revp,
int want_valid, int first_pass) int want_valid, int first_pass)
{ {
...@@ -1337,10 +1343,10 @@ static long record_hpte(unsigned long flags, unsigned long *hptp, ...@@ -1337,10 +1343,10 @@ static long record_hpte(unsigned long flags, unsigned long *hptp,
return 0; return 0;
valid = 0; valid = 0;
if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT)) { if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)) {
valid = 1; valid = 1;
if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && if ((flags & KVM_GET_HTAB_BOLTED_ONLY) &&
!(hptp[0] & HPTE_V_BOLTED)) !(be64_to_cpu(hptp[0]) & HPTE_V_BOLTED))
valid = 0; valid = 0;
} }
if (valid != want_valid) if (valid != want_valid)
...@@ -1352,7 +1358,7 @@ static long record_hpte(unsigned long flags, unsigned long *hptp, ...@@ -1352,7 +1358,7 @@ static long record_hpte(unsigned long flags, unsigned long *hptp,
preempt_disable(); preempt_disable();
while (!try_lock_hpte(hptp, HPTE_V_HVLOCK)) while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
cpu_relax(); cpu_relax();
v = hptp[0]; v = be64_to_cpu(hptp[0]);
/* re-evaluate valid and dirty from synchronized HPTE value */ /* re-evaluate valid and dirty from synchronized HPTE value */
valid = !!(v & HPTE_V_VALID); valid = !!(v & HPTE_V_VALID);
...@@ -1360,9 +1366,9 @@ static long record_hpte(unsigned long flags, unsigned long *hptp, ...@@ -1360,9 +1366,9 @@ static long record_hpte(unsigned long flags, unsigned long *hptp,
/* Harvest R and C into guest view if necessary */ /* Harvest R and C into guest view if necessary */
rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C); rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
if (valid && (rcbits_unset & hptp[1])) { if (valid && (rcbits_unset & be64_to_cpu(hptp[1]))) {
revp->guest_rpte |= (hptp[1] & (HPTE_R_R | HPTE_R_C)) | revp->guest_rpte |= (be64_to_cpu(hptp[1]) &
HPTE_GR_MODIFIED; (HPTE_R_R | HPTE_R_C)) | HPTE_GR_MODIFIED;
dirty = 1; dirty = 1;
} }
...@@ -1381,13 +1387,13 @@ static long record_hpte(unsigned long flags, unsigned long *hptp, ...@@ -1381,13 +1387,13 @@ static long record_hpte(unsigned long flags, unsigned long *hptp,
revp->guest_rpte = r; revp->guest_rpte = r;
} }
asm volatile(PPC_RELEASE_BARRIER "" : : : "memory"); asm volatile(PPC_RELEASE_BARRIER "" : : : "memory");
hptp[0] &= ~HPTE_V_HVLOCK; hptp[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
preempt_enable(); preempt_enable();
if (!(valid == want_valid && (first_pass || dirty))) if (!(valid == want_valid && (first_pass || dirty)))
ok = 0; ok = 0;
} }
hpte[0] = v; hpte[0] = cpu_to_be64(v);
hpte[1] = r; hpte[1] = cpu_to_be64(r);
return ok; return ok;
} }
...@@ -1397,7 +1403,7 @@ static ssize_t kvm_htab_read(struct file *file, char __user *buf, ...@@ -1397,7 +1403,7 @@ static ssize_t kvm_htab_read(struct file *file, char __user *buf,
struct kvm_htab_ctx *ctx = file->private_data; struct kvm_htab_ctx *ctx = file->private_data;
struct kvm *kvm = ctx->kvm; struct kvm *kvm = ctx->kvm;
struct kvm_get_htab_header hdr; struct kvm_get_htab_header hdr;
unsigned long *hptp; __be64 *hptp;
struct revmap_entry *revp; struct revmap_entry *revp;
unsigned long i, nb, nw; unsigned long i, nb, nw;
unsigned long __user *lbuf; unsigned long __user *lbuf;
...@@ -1413,7 +1419,7 @@ static ssize_t kvm_htab_read(struct file *file, char __user *buf, ...@@ -1413,7 +1419,7 @@ static ssize_t kvm_htab_read(struct file *file, char __user *buf,
flags = ctx->flags; flags = ctx->flags;
i = ctx->index; i = ctx->index;
hptp = (unsigned long *)(kvm->arch.hpt_virt + (i * HPTE_SIZE)); hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));
revp = kvm->arch.revmap + i; revp = kvm->arch.revmap + i;
lbuf = (unsigned long __user *)buf; lbuf = (unsigned long __user *)buf;
...@@ -1497,7 +1503,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf, ...@@ -1497,7 +1503,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
unsigned long i, j; unsigned long i, j;
unsigned long v, r; unsigned long v, r;
unsigned long __user *lbuf; unsigned long __user *lbuf;
unsigned long *hptp; __be64 *hptp;
unsigned long tmp[2]; unsigned long tmp[2];
ssize_t nb; ssize_t nb;
long int err, ret; long int err, ret;
...@@ -1539,7 +1545,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf, ...@@ -1539,7 +1545,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
i + hdr.n_valid + hdr.n_invalid > kvm->arch.hpt_npte) i + hdr.n_valid + hdr.n_invalid > kvm->arch.hpt_npte)
break; break;
hptp = (unsigned long *)(kvm->arch.hpt_virt + (i * HPTE_SIZE)); hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));
lbuf = (unsigned long __user *)buf; lbuf = (unsigned long __user *)buf;
for (j = 0; j < hdr.n_valid; ++j) { for (j = 0; j < hdr.n_valid; ++j) {
err = -EFAULT; err = -EFAULT;
...@@ -1551,7 +1557,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf, ...@@ -1551,7 +1557,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
lbuf += 2; lbuf += 2;
nb += HPTE_SIZE; nb += HPTE_SIZE;
if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT)) if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
kvmppc_do_h_remove(kvm, 0, i, 0, tmp); kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
err = -EIO; err = -EIO;
ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r, ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,
...@@ -1577,7 +1583,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf, ...@@ -1577,7 +1583,7 @@ static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
} }
for (j = 0; j < hdr.n_invalid; ++j) { for (j = 0; j < hdr.n_invalid; ++j) {
if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT)) if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
kvmppc_do_h_remove(kvm, 0, i, 0, tmp); kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
++i; ++i;
hptp += 2; hptp += 2;
......
...@@ -154,10 +154,10 @@ static pte_t lookup_linux_pte_and_update(pgd_t *pgdir, unsigned long hva, ...@@ -154,10 +154,10 @@ static pte_t lookup_linux_pte_and_update(pgd_t *pgdir, unsigned long hva,
return kvmppc_read_update_linux_pte(ptep, writing, hugepage_shift); return kvmppc_read_update_linux_pte(ptep, writing, hugepage_shift);
} }
static inline void unlock_hpte(unsigned long *hpte, unsigned long hpte_v) static inline void unlock_hpte(__be64 *hpte, unsigned long hpte_v)
{ {
asm volatile(PPC_RELEASE_BARRIER "" : : : "memory"); asm volatile(PPC_RELEASE_BARRIER "" : : : "memory");
hpte[0] = hpte_v; hpte[0] = cpu_to_be64(hpte_v);
} }
long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
...@@ -166,7 +166,7 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, ...@@ -166,7 +166,7 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
{ {
unsigned long i, pa, gpa, gfn, psize; unsigned long i, pa, gpa, gfn, psize;
unsigned long slot_fn, hva; unsigned long slot_fn, hva;
unsigned long *hpte; __be64 *hpte;
struct revmap_entry *rev; struct revmap_entry *rev;
unsigned long g_ptel; unsigned long g_ptel;
struct kvm_memory_slot *memslot; struct kvm_memory_slot *memslot;
...@@ -275,9 +275,9 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, ...@@ -275,9 +275,9 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
return H_PARAMETER; return H_PARAMETER;
if (likely((flags & H_EXACT) == 0)) { if (likely((flags & H_EXACT) == 0)) {
pte_index &= ~7UL; pte_index &= ~7UL;
hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4)); hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));
for (i = 0; i < 8; ++i) { for (i = 0; i < 8; ++i) {
if ((*hpte & HPTE_V_VALID) == 0 && if ((be64_to_cpu(*hpte) & HPTE_V_VALID) == 0 &&
try_lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID | try_lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID |
HPTE_V_ABSENT)) HPTE_V_ABSENT))
break; break;
...@@ -292,11 +292,13 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, ...@@ -292,11 +292,13 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
*/ */
hpte -= 16; hpte -= 16;
for (i = 0; i < 8; ++i) { for (i = 0; i < 8; ++i) {
u64 pte;
while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
cpu_relax(); cpu_relax();
if (!(*hpte & (HPTE_V_VALID | HPTE_V_ABSENT))) pte = be64_to_cpu(*hpte);
if (!(pte & (HPTE_V_VALID | HPTE_V_ABSENT)))
break; break;
*hpte &= ~HPTE_V_HVLOCK; *hpte &= ~cpu_to_be64(HPTE_V_HVLOCK);
hpte += 2; hpte += 2;
} }
if (i == 8) if (i == 8)
...@@ -304,14 +306,17 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, ...@@ -304,14 +306,17 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
} }
pte_index += i; pte_index += i;
} else { } else {
hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4)); hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));
if (!try_lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID | if (!try_lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID |
HPTE_V_ABSENT)) { HPTE_V_ABSENT)) {
/* Lock the slot and check again */ /* Lock the slot and check again */
u64 pte;
while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
cpu_relax(); cpu_relax();
if (*hpte & (HPTE_V_VALID | HPTE_V_ABSENT)) { pte = be64_to_cpu(*hpte);
*hpte &= ~HPTE_V_HVLOCK; if (pte & (HPTE_V_VALID | HPTE_V_ABSENT)) {
*hpte &= ~cpu_to_be64(HPTE_V_HVLOCK);
return H_PTEG_FULL; return H_PTEG_FULL;
} }
} }
...@@ -347,11 +352,11 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, ...@@ -347,11 +352,11 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
} }
} }
hpte[1] = ptel; hpte[1] = cpu_to_be64(ptel);
/* Write the first HPTE dword, unlocking the HPTE and making it valid */ /* Write the first HPTE dword, unlocking the HPTE and making it valid */
eieio(); eieio();
hpte[0] = pteh; hpte[0] = cpu_to_be64(pteh);
asm volatile("ptesync" : : : "memory"); asm volatile("ptesync" : : : "memory");
*pte_idx_ret = pte_index; *pte_idx_ret = pte_index;
...@@ -468,30 +473,35 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags, ...@@ -468,30 +473,35 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,
unsigned long pte_index, unsigned long avpn, unsigned long pte_index, unsigned long avpn,
unsigned long *hpret) unsigned long *hpret)
{ {
unsigned long *hpte; __be64 *hpte;
unsigned long v, r, rb; unsigned long v, r, rb;
struct revmap_entry *rev; struct revmap_entry *rev;
u64 pte;
if (pte_index >= kvm->arch.hpt_npte) if (pte_index >= kvm->arch.hpt_npte)
return H_PARAMETER; return H_PARAMETER;
hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4)); hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));
while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
cpu_relax(); cpu_relax();
if ((hpte[0] & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 || pte = be64_to_cpu(hpte[0]);
((flags & H_AVPN) && (hpte[0] & ~0x7fUL) != avpn) || if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 ||
((flags & H_ANDCOND) && (hpte[0] & avpn) != 0)) { ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn) ||
hpte[0] &= ~HPTE_V_HVLOCK; ((flags & H_ANDCOND) && (pte & avpn) != 0)) {
hpte[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
return H_NOT_FOUND; return H_NOT_FOUND;
} }
rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]); rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);
v = hpte[0] & ~HPTE_V_HVLOCK; v = pte & ~HPTE_V_HVLOCK;
if (v & HPTE_V_VALID) { if (v & HPTE_V_VALID) {
hpte[0] &= ~HPTE_V_VALID; u64 pte1;
rb = compute_tlbie_rb(v, hpte[1], pte_index);
pte1 = be64_to_cpu(hpte[1]);
hpte[0] &= ~cpu_to_be64(HPTE_V_VALID);
rb = compute_tlbie_rb(v, pte1, pte_index);
do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true); do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true);
/* Read PTE low word after tlbie to get final R/C values */ /* Read PTE low word after tlbie to get final R/C values */
remove_revmap_chain(kvm, pte_index, rev, v, hpte[1]); remove_revmap_chain(kvm, pte_index, rev, v, pte1);
} }
r = rev->guest_rpte & ~HPTE_GR_RESERVED; r = rev->guest_rpte & ~HPTE_GR_RESERVED;
note_hpte_modification(kvm, rev); note_hpte_modification(kvm, rev);
...@@ -514,12 +524,14 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) ...@@ -514,12 +524,14 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
{ {
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
unsigned long *args = &vcpu->arch.gpr[4]; unsigned long *args = &vcpu->arch.gpr[4];
unsigned long *hp, *hptes[4], tlbrb[4]; __be64 *hp, *hptes[4];
unsigned long tlbrb[4];
long int i, j, k, n, found, indexes[4]; long int i, j, k, n, found, indexes[4];
unsigned long flags, req, pte_index, rcbits; unsigned long flags, req, pte_index, rcbits;
int global; int global;
long int ret = H_SUCCESS; long int ret = H_SUCCESS;
struct revmap_entry *rev, *revs[4]; struct revmap_entry *rev, *revs[4];
u64 hp0;
global = global_invalidates(kvm, 0); global = global_invalidates(kvm, 0);
for (i = 0; i < 4 && ret == H_SUCCESS; ) { for (i = 0; i < 4 && ret == H_SUCCESS; ) {
...@@ -542,8 +554,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) ...@@ -542,8 +554,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
ret = H_PARAMETER; ret = H_PARAMETER;
break; break;
} }
hp = (unsigned long *) hp = (__be64 *) (kvm->arch.hpt_virt + (pte_index << 4));
(kvm->arch.hpt_virt + (pte_index << 4));
/* to avoid deadlock, don't spin except for first */ /* to avoid deadlock, don't spin except for first */
if (!try_lock_hpte(hp, HPTE_V_HVLOCK)) { if (!try_lock_hpte(hp, HPTE_V_HVLOCK)) {
if (n) if (n)
...@@ -552,23 +563,24 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) ...@@ -552,23 +563,24 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
cpu_relax(); cpu_relax();
} }
found = 0; found = 0;
if (hp[0] & (HPTE_V_ABSENT | HPTE_V_VALID)) { hp0 = be64_to_cpu(hp[0]);
if (hp0 & (HPTE_V_ABSENT | HPTE_V_VALID)) {
switch (flags & 3) { switch (flags & 3) {
case 0: /* absolute */ case 0: /* absolute */
found = 1; found = 1;
break; break;
case 1: /* andcond */ case 1: /* andcond */
if (!(hp[0] & args[j + 1])) if (!(hp0 & args[j + 1]))
found = 1; found = 1;
break; break;
case 2: /* AVPN */ case 2: /* AVPN */
if ((hp[0] & ~0x7fUL) == args[j + 1]) if ((hp0 & ~0x7fUL) == args[j + 1])
found = 1; found = 1;
break; break;
} }
} }
if (!found) { if (!found) {
hp[0] &= ~HPTE_V_HVLOCK; hp[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
args[j] = ((0x90 | flags) << 56) + pte_index; args[j] = ((0x90 | flags) << 56) + pte_index;
continue; continue;
} }
...@@ -577,7 +589,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) ...@@ -577,7 +589,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]); rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);
note_hpte_modification(kvm, rev); note_hpte_modification(kvm, rev);
if (!(hp[0] & HPTE_V_VALID)) { if (!(hp0 & HPTE_V_VALID)) {
/* insert R and C bits from PTE */ /* insert R and C bits from PTE */
rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C); rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C);
args[j] |= rcbits << (56 - 5); args[j] |= rcbits << (56 - 5);
...@@ -585,8 +597,10 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) ...@@ -585,8 +597,10 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
continue; continue;
} }
hp[0] &= ~HPTE_V_VALID; /* leave it locked */ /* leave it locked */
tlbrb[n] = compute_tlbie_rb(hp[0], hp[1], pte_index); hp[0] &= ~cpu_to_be64(HPTE_V_VALID);
tlbrb[n] = compute_tlbie_rb(be64_to_cpu(hp[0]),
be64_to_cpu(hp[1]), pte_index);
indexes[n] = j; indexes[n] = j;
hptes[n] = hp; hptes[n] = hp;
revs[n] = rev; revs[n] = rev;
...@@ -605,7 +619,8 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) ...@@ -605,7 +619,8 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
pte_index = args[j] & ((1ul << 56) - 1); pte_index = args[j] & ((1ul << 56) - 1);
hp = hptes[k]; hp = hptes[k];
rev = revs[k]; rev = revs[k];
remove_revmap_chain(kvm, pte_index, rev, hp[0], hp[1]); remove_revmap_chain(kvm, pte_index, rev,
be64_to_cpu(hp[0]), be64_to_cpu(hp[1]));
rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C); rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C);
args[j] |= rcbits << (56 - 5); args[j] |= rcbits << (56 - 5);
hp[0] = 0; hp[0] = 0;
...@@ -620,23 +635,25 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, ...@@ -620,23 +635,25 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
unsigned long va) unsigned long va)
{ {
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
unsigned long *hpte; __be64 *hpte;
struct revmap_entry *rev; struct revmap_entry *rev;
unsigned long v, r, rb, mask, bits; unsigned long v, r, rb, mask, bits;
u64 pte;
if (pte_index >= kvm->arch.hpt_npte) if (pte_index >= kvm->arch.hpt_npte)
return H_PARAMETER; return H_PARAMETER;
hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4)); hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));
while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
cpu_relax(); cpu_relax();
if ((hpte[0] & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 || pte = be64_to_cpu(hpte[0]);
((flags & H_AVPN) && (hpte[0] & ~0x7fUL) != avpn)) { if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 ||
hpte[0] &= ~HPTE_V_HVLOCK; ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn)) {
hpte[0] &= ~cpu_to_be64(HPTE_V_HVLOCK);
return H_NOT_FOUND; return H_NOT_FOUND;
} }
v = hpte[0]; v = pte;
bits = (flags << 55) & HPTE_R_PP0; bits = (flags << 55) & HPTE_R_PP0;
bits |= (flags << 48) & HPTE_R_KEY_HI; bits |= (flags << 48) & HPTE_R_KEY_HI;
bits |= flags & (HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_LO); bits |= flags & (HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_LO);
...@@ -650,12 +667,12 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, ...@@ -650,12 +667,12 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
rev->guest_rpte = r; rev->guest_rpte = r;
note_hpte_modification(kvm, rev); note_hpte_modification(kvm, rev);
} }
r = (hpte[1] & ~mask) | bits; r = (be64_to_cpu(hpte[1]) & ~mask) | bits;
/* Update HPTE */ /* Update HPTE */
if (v & HPTE_V_VALID) { if (v & HPTE_V_VALID) {
rb = compute_tlbie_rb(v, r, pte_index); rb = compute_tlbie_rb(v, r, pte_index);
hpte[0] = v & ~HPTE_V_VALID; hpte[0] = cpu_to_be64(v & ~HPTE_V_VALID);
do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true); do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true);
/* /*
* If the host has this page as readonly but the guest * If the host has this page as readonly but the guest
...@@ -681,9 +698,9 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, ...@@ -681,9 +698,9 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
} }
} }
} }
hpte[1] = r; hpte[1] = cpu_to_be64(r);
eieio(); eieio();
hpte[0] = v & ~HPTE_V_HVLOCK; hpte[0] = cpu_to_be64(v & ~HPTE_V_HVLOCK);
asm volatile("ptesync" : : : "memory"); asm volatile("ptesync" : : : "memory");
return H_SUCCESS; return H_SUCCESS;
} }
...@@ -692,7 +709,8 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, ...@@ -692,7 +709,8 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags,
unsigned long pte_index) unsigned long pte_index)
{ {
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
unsigned long *hpte, v, r; __be64 *hpte;
unsigned long v, r;
int i, n = 1; int i, n = 1;
struct revmap_entry *rev = NULL; struct revmap_entry *rev = NULL;
...@@ -704,9 +722,9 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, ...@@ -704,9 +722,9 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags,
} }
rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]); rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);
for (i = 0; i < n; ++i, ++pte_index) { for (i = 0; i < n; ++i, ++pte_index) {
hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4)); hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));
v = hpte[0] & ~HPTE_V_HVLOCK; v = be64_to_cpu(hpte[0]) & ~HPTE_V_HVLOCK;
r = hpte[1]; r = be64_to_cpu(hpte[1]);
if (v & HPTE_V_ABSENT) { if (v & HPTE_V_ABSENT) {
v &= ~HPTE_V_ABSENT; v &= ~HPTE_V_ABSENT;
v |= HPTE_V_VALID; v |= HPTE_V_VALID;
...@@ -721,25 +739,27 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, ...@@ -721,25 +739,27 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags,
return H_SUCCESS; return H_SUCCESS;
} }
void kvmppc_invalidate_hpte(struct kvm *kvm, unsigned long *hptep, void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep,
unsigned long pte_index) unsigned long pte_index)
{ {
unsigned long rb; unsigned long rb;
hptep[0] &= ~HPTE_V_VALID; hptep[0] &= ~cpu_to_be64(HPTE_V_VALID);
rb = compute_tlbie_rb(hptep[0], hptep[1], pte_index); rb = compute_tlbie_rb(be64_to_cpu(hptep[0]), be64_to_cpu(hptep[1]),
pte_index);
do_tlbies(kvm, &rb, 1, 1, true); do_tlbies(kvm, &rb, 1, 1, true);
} }
EXPORT_SYMBOL_GPL(kvmppc_invalidate_hpte); EXPORT_SYMBOL_GPL(kvmppc_invalidate_hpte);
void kvmppc_clear_ref_hpte(struct kvm *kvm, unsigned long *hptep, void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep,
unsigned long pte_index) unsigned long pte_index)
{ {
unsigned long rb; unsigned long rb;
unsigned char rbyte; unsigned char rbyte;
rb = compute_tlbie_rb(hptep[0], hptep[1], pte_index); rb = compute_tlbie_rb(be64_to_cpu(hptep[0]), be64_to_cpu(hptep[1]),
rbyte = (hptep[1] & ~HPTE_R_R) >> 8; pte_index);
rbyte = (be64_to_cpu(hptep[1]) & ~HPTE_R_R) >> 8;
/* modify only the second-last byte, which contains the ref bit */ /* modify only the second-last byte, which contains the ref bit */
*((char *)hptep + 14) = rbyte; *((char *)hptep + 14) = rbyte;
do_tlbies(kvm, &rb, 1, 1, false); do_tlbies(kvm, &rb, 1, 1, false);
...@@ -765,7 +785,7 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, ...@@ -765,7 +785,7 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
unsigned long somask; unsigned long somask;
unsigned long vsid, hash; unsigned long vsid, hash;
unsigned long avpn; unsigned long avpn;
unsigned long *hpte; __be64 *hpte;
unsigned long mask, val; unsigned long mask, val;
unsigned long v, r; unsigned long v, r;
...@@ -797,11 +817,11 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, ...@@ -797,11 +817,11 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
val |= avpn; val |= avpn;
for (;;) { for (;;) {
hpte = (unsigned long *)(kvm->arch.hpt_virt + (hash << 7)); hpte = (__be64 *)(kvm->arch.hpt_virt + (hash << 7));
for (i = 0; i < 16; i += 2) { for (i = 0; i < 16; i += 2) {
/* Read the PTE racily */ /* Read the PTE racily */
v = hpte[i] & ~HPTE_V_HVLOCK; v = be64_to_cpu(hpte[i]) & ~HPTE_V_HVLOCK;
/* Check valid/absent, hash, segment size and AVPN */ /* Check valid/absent, hash, segment size and AVPN */
if (!(v & valid) || (v & mask) != val) if (!(v & valid) || (v & mask) != val)
...@@ -810,8 +830,8 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, ...@@ -810,8 +830,8 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
/* Lock the PTE and read it under the lock */ /* Lock the PTE and read it under the lock */
while (!try_lock_hpte(&hpte[i], HPTE_V_HVLOCK)) while (!try_lock_hpte(&hpte[i], HPTE_V_HVLOCK))
cpu_relax(); cpu_relax();
v = hpte[i] & ~HPTE_V_HVLOCK; v = be64_to_cpu(hpte[i]) & ~HPTE_V_HVLOCK;
r = hpte[i+1]; r = be64_to_cpu(hpte[i+1]);
/* /*
* Check the HPTE again, including large page size * Check the HPTE again, including large page size
...@@ -825,7 +845,7 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, ...@@ -825,7 +845,7 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
return (hash << 3) + (i >> 1); return (hash << 3) + (i >> 1);
/* Unlock and move on */ /* Unlock and move on */
hpte[i] = v; hpte[i] = cpu_to_be64(v);
} }
if (val & HPTE_V_SECONDARY) if (val & HPTE_V_SECONDARY)
...@@ -854,7 +874,7 @@ long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr, ...@@ -854,7 +874,7 @@ long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr,
struct kvm *kvm = vcpu->kvm; struct kvm *kvm = vcpu->kvm;
long int index; long int index;
unsigned long v, r, gr; unsigned long v, r, gr;
unsigned long *hpte; __be64 *hpte;
unsigned long valid; unsigned long valid;
struct revmap_entry *rev; struct revmap_entry *rev;
unsigned long pp, key; unsigned long pp, key;
...@@ -870,9 +890,9 @@ long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr, ...@@ -870,9 +890,9 @@ long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr,
return status; /* there really was no HPTE */ return status; /* there really was no HPTE */
return 0; /* for prot fault, HPTE disappeared */ return 0; /* for prot fault, HPTE disappeared */
} }
hpte = (unsigned long *)(kvm->arch.hpt_virt + (index << 4)); hpte = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
v = hpte[0] & ~HPTE_V_HVLOCK; v = be64_to_cpu(hpte[0]) & ~HPTE_V_HVLOCK;
r = hpte[1]; r = be64_to_cpu(hpte[1]);
rev = real_vmalloc_addr(&kvm->arch.revmap[index]); rev = real_vmalloc_addr(&kvm->arch.revmap[index]);
gr = rev->guest_rpte; gr = rev->guest_rpte;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment