Commit daf3dc0f authored by Chris Wilson's avatar Chris Wilson

drm/i915/gtt: Record the scratch pte

Record the scratch PTE encoding upon creation rather than recomputing
the bits everytime. This is important for the next patch where we forgo
having a valid scratch page with which we may compute the bits and so
require keeping the PTE value instead.

v2: Fix up scrub_64K to use scratch_pte as well.
Signed-off-by: default avatarChris Wilson <chris@chris-wilson.co.uk>
Cc: Matthew Auld <matthew.auld@intel.com>
Reviewed-by: default avatarMatthew Auld <matthew.auld@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20181029182721.29568-1-chris@chris-wilson.co.uk
parent 4c8d351d
...@@ -186,9 +186,9 @@ static void clear_pages(struct i915_vma *vma) ...@@ -186,9 +186,9 @@ static void clear_pages(struct i915_vma *vma)
memset(&vma->page_sizes, 0, sizeof(vma->page_sizes)); memset(&vma->page_sizes, 0, sizeof(vma->page_sizes));
} }
static gen8_pte_t gen8_pte_encode(dma_addr_t addr, static u64 gen8_pte_encode(dma_addr_t addr,
enum i915_cache_level level, enum i915_cache_level level,
u32 flags) u32 flags)
{ {
gen8_pte_t pte = addr | _PAGE_PRESENT | _PAGE_RW; gen8_pte_t pte = addr | _PAGE_PRESENT | _PAGE_RW;
...@@ -225,9 +225,9 @@ static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, ...@@ -225,9 +225,9 @@ static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
#define gen8_pdpe_encode gen8_pde_encode #define gen8_pdpe_encode gen8_pde_encode
#define gen8_pml4e_encode gen8_pde_encode #define gen8_pml4e_encode gen8_pde_encode
static gen6_pte_t snb_pte_encode(dma_addr_t addr, static u64 snb_pte_encode(dma_addr_t addr,
enum i915_cache_level level, enum i915_cache_level level,
u32 unused) u32 flags)
{ {
gen6_pte_t pte = GEN6_PTE_VALID; gen6_pte_t pte = GEN6_PTE_VALID;
pte |= GEN6_PTE_ADDR_ENCODE(addr); pte |= GEN6_PTE_ADDR_ENCODE(addr);
...@@ -247,9 +247,9 @@ static gen6_pte_t snb_pte_encode(dma_addr_t addr, ...@@ -247,9 +247,9 @@ static gen6_pte_t snb_pte_encode(dma_addr_t addr,
return pte; return pte;
} }
static gen6_pte_t ivb_pte_encode(dma_addr_t addr, static u64 ivb_pte_encode(dma_addr_t addr,
enum i915_cache_level level, enum i915_cache_level level,
u32 unused) u32 flags)
{ {
gen6_pte_t pte = GEN6_PTE_VALID; gen6_pte_t pte = GEN6_PTE_VALID;
pte |= GEN6_PTE_ADDR_ENCODE(addr); pte |= GEN6_PTE_ADDR_ENCODE(addr);
...@@ -271,9 +271,9 @@ static gen6_pte_t ivb_pte_encode(dma_addr_t addr, ...@@ -271,9 +271,9 @@ static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
return pte; return pte;
} }
static gen6_pte_t byt_pte_encode(dma_addr_t addr, static u64 byt_pte_encode(dma_addr_t addr,
enum i915_cache_level level, enum i915_cache_level level,
u32 flags) u32 flags)
{ {
gen6_pte_t pte = GEN6_PTE_VALID; gen6_pte_t pte = GEN6_PTE_VALID;
pte |= GEN6_PTE_ADDR_ENCODE(addr); pte |= GEN6_PTE_ADDR_ENCODE(addr);
...@@ -287,9 +287,9 @@ static gen6_pte_t byt_pte_encode(dma_addr_t addr, ...@@ -287,9 +287,9 @@ static gen6_pte_t byt_pte_encode(dma_addr_t addr,
return pte; return pte;
} }
static gen6_pte_t hsw_pte_encode(dma_addr_t addr, static u64 hsw_pte_encode(dma_addr_t addr,
enum i915_cache_level level, enum i915_cache_level level,
u32 unused) u32 flags)
{ {
gen6_pte_t pte = GEN6_PTE_VALID; gen6_pte_t pte = GEN6_PTE_VALID;
pte |= HSW_PTE_ADDR_ENCODE(addr); pte |= HSW_PTE_ADDR_ENCODE(addr);
...@@ -300,9 +300,9 @@ static gen6_pte_t hsw_pte_encode(dma_addr_t addr, ...@@ -300,9 +300,9 @@ static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
return pte; return pte;
} }
static gen6_pte_t iris_pte_encode(dma_addr_t addr, static u64 iris_pte_encode(dma_addr_t addr,
enum i915_cache_level level, enum i915_cache_level level,
u32 unused) u32 flags)
{ {
gen6_pte_t pte = GEN6_PTE_VALID; gen6_pte_t pte = GEN6_PTE_VALID;
pte |= HSW_PTE_ADDR_ENCODE(addr); pte |= HSW_PTE_ADDR_ENCODE(addr);
...@@ -666,14 +666,13 @@ static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt) ...@@ -666,14 +666,13 @@ static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt)
static void gen8_initialize_pt(struct i915_address_space *vm, static void gen8_initialize_pt(struct i915_address_space *vm,
struct i915_page_table *pt) struct i915_page_table *pt)
{ {
fill_px(vm, pt, fill_px(vm, pt, vm->scratch_pte);
gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0));
} }
static void gen6_initialize_pt(struct gen6_hw_ppgtt *ppgtt, static void gen6_initialize_pt(struct i915_address_space *vm,
struct i915_page_table *pt) struct i915_page_table *pt)
{ {
fill32_px(&ppgtt->base.vm, pt, ppgtt->scratch_pte); fill32_px(vm, pt, vm->scratch_pte);
} }
static struct i915_page_directory *alloc_pd(struct i915_address_space *vm) static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
...@@ -807,15 +806,13 @@ static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) ...@@ -807,15 +806,13 @@ static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
/* Removes entries from a single page table, releasing it if it's empty. /* Removes entries from a single page table, releasing it if it's empty.
* Caller can use the return value to update higher-level entries. * Caller can use the return value to update higher-level entries.
*/ */
static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
struct i915_page_table *pt, struct i915_page_table *pt,
u64 start, u64 length) u64 start, u64 length)
{ {
unsigned int num_entries = gen8_pte_count(start, length); unsigned int num_entries = gen8_pte_count(start, length);
unsigned int pte = gen8_pte_index(start); unsigned int pte = gen8_pte_index(start);
unsigned int pte_end = pte + num_entries; unsigned int pte_end = pte + num_entries;
const gen8_pte_t scratch_pte =
gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0);
gen8_pte_t *vaddr; gen8_pte_t *vaddr;
GEM_BUG_ON(num_entries > pt->used_ptes); GEM_BUG_ON(num_entries > pt->used_ptes);
...@@ -826,7 +823,7 @@ static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, ...@@ -826,7 +823,7 @@ static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
vaddr = kmap_atomic_px(pt); vaddr = kmap_atomic_px(pt);
while (pte < pte_end) while (pte < pte_end)
vaddr[pte++] = scratch_pte; vaddr[pte++] = vm->scratch_pte;
kunmap_atomic(vaddr); kunmap_atomic(vaddr);
return false; return false;
...@@ -1159,7 +1156,7 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma, ...@@ -1159,7 +1156,7 @@ static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) { if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) {
u16 i; u16 i;
encode = pte_encode | vma->vm->scratch_page.daddr; encode = vma->vm->scratch_pte;
vaddr = kmap_atomic_px(pd->page_table[idx.pde]); vaddr = kmap_atomic_px(pd->page_table[idx.pde]);
for (i = 1; i < index; i += 16) for (i = 1; i < index; i += 16)
...@@ -1216,6 +1213,11 @@ static int gen8_init_scratch(struct i915_address_space *vm) ...@@ -1216,6 +1213,11 @@ static int gen8_init_scratch(struct i915_address_space *vm)
if (ret) if (ret)
return ret; return ret;
vm->scratch_pte =
gen8_pte_encode(vm->scratch_page.daddr,
I915_CACHE_LLC,
PTE_READ_ONLY);
vm->scratch_pt = alloc_pt(vm); vm->scratch_pt = alloc_pt(vm);
if (IS_ERR(vm->scratch_pt)) { if (IS_ERR(vm->scratch_pt)) {
ret = PTR_ERR(vm->scratch_pt); ret = PTR_ERR(vm->scratch_pt);
...@@ -1524,8 +1526,7 @@ static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt, ...@@ -1524,8 +1526,7 @@ static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt,
static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
{ {
struct i915_address_space *vm = &ppgtt->vm; struct i915_address_space *vm = &ppgtt->vm;
const gen8_pte_t scratch_pte = const gen8_pte_t scratch_pte = vm->scratch_pte;
gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0);
u64 start = 0, length = ppgtt->vm.total; u64 start = 0, length = ppgtt->vm.total;
if (use_4lvl(vm)) { if (use_4lvl(vm)) {
...@@ -1672,7 +1673,7 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915) ...@@ -1672,7 +1673,7 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
static void gen6_dump_ppgtt(struct i915_hw_ppgtt *base, struct seq_file *m) static void gen6_dump_ppgtt(struct i915_hw_ppgtt *base, struct seq_file *m)
{ {
struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base); struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base);
const gen6_pte_t scratch_pte = ppgtt->scratch_pte; const gen6_pte_t scratch_pte = base->vm.scratch_pte;
struct i915_page_table *pt; struct i915_page_table *pt;
u32 pte, pde; u32 pte, pde;
...@@ -1785,7 +1786,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, ...@@ -1785,7 +1786,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
unsigned int pde = first_entry / GEN6_PTES; unsigned int pde = first_entry / GEN6_PTES;
unsigned int pte = first_entry % GEN6_PTES; unsigned int pte = first_entry % GEN6_PTES;
unsigned int num_entries = length / I915_GTT_PAGE_SIZE; unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
const gen6_pte_t scratch_pte = ppgtt->scratch_pte; const gen6_pte_t scratch_pte = vm->scratch_pte;
while (num_entries) { while (num_entries) {
struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++]; struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++];
...@@ -1876,7 +1877,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, ...@@ -1876,7 +1877,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
if (IS_ERR(pt)) if (IS_ERR(pt))
goto unwind_out; goto unwind_out;
gen6_initialize_pt(ppgtt, pt); gen6_initialize_pt(vm, pt);
ppgtt->base.pd.page_table[pde] = pt; ppgtt->base.pd.page_table[pde] = pt;
if (i915_vma_is_bound(ppgtt->vma, if (i915_vma_is_bound(ppgtt->vma,
...@@ -1914,9 +1915,9 @@ static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt) ...@@ -1914,9 +1915,9 @@ static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt)
if (ret) if (ret)
return ret; return ret;
ppgtt->scratch_pte = vm->scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
vm->pte_encode(vm->scratch_page.daddr, I915_CACHE_NONE,
I915_CACHE_NONE, PTE_READ_ONLY); PTE_READ_ONLY);
vm->scratch_pt = alloc_pt(vm); vm->scratch_pt = alloc_pt(vm);
if (IS_ERR(vm->scratch_pt)) { if (IS_ERR(vm->scratch_pt)) {
...@@ -1924,7 +1925,7 @@ static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt) ...@@ -1924,7 +1925,7 @@ static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt)
return PTR_ERR(vm->scratch_pt); return PTR_ERR(vm->scratch_pt);
} }
gen6_initialize_pt(ppgtt, vm->scratch_pt); gen6_initialize_pt(vm, vm->scratch_pt);
gen6_for_all_pdes(unused, &ppgtt->base.pd, pde) gen6_for_all_pdes(unused, &ppgtt->base.pd, pde)
ppgtt->base.pd.page_table[pde] = vm->scratch_pt; ppgtt->base.pd.page_table[pde] = vm->scratch_pt;
...@@ -2469,8 +2470,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, ...@@ -2469,8 +2470,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
unsigned first_entry = start / I915_GTT_PAGE_SIZE; unsigned first_entry = start / I915_GTT_PAGE_SIZE;
unsigned num_entries = length / I915_GTT_PAGE_SIZE; unsigned num_entries = length / I915_GTT_PAGE_SIZE;
const gen8_pte_t scratch_pte = const gen8_pte_t scratch_pte = vm->scratch_pte;
gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC, 0);
gen8_pte_t __iomem *gtt_base = gen8_pte_t __iomem *gtt_base =
(gen8_pte_t __iomem *)ggtt->gsm + first_entry; (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
const int max_entries = ggtt_total_entries(ggtt) - first_entry; const int max_entries = ggtt_total_entries(ggtt) - first_entry;
...@@ -2595,8 +2595,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, ...@@ -2595,8 +2595,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
first_entry, num_entries, max_entries)) first_entry, num_entries, max_entries))
num_entries = max_entries; num_entries = max_entries;
scratch_pte = vm->pte_encode(vm->scratch_page.daddr, scratch_pte = vm->scratch_pte;
I915_CACHE_LLC, 0);
for (i = 0; i < num_entries; i++) for (i = 0; i < num_entries; i++)
iowrite32(scratch_pte, &gtt_base[i]); iowrite32(scratch_pte, &gtt_base[i]);
...@@ -3002,6 +3001,10 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) ...@@ -3002,6 +3001,10 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
return ret; return ret;
} }
ggtt->vm.scratch_pte =
ggtt->vm.pte_encode(ggtt->vm.scratch_page.daddr,
I915_CACHE_NONE, 0);
return 0; return 0;
} }
...@@ -3348,6 +3351,8 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) ...@@ -3348,6 +3351,8 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
ggtt->vm.vma_ops.set_pages = ggtt_set_pages; ggtt->vm.vma_ops.set_pages = ggtt_set_pages;
ggtt->vm.vma_ops.clear_pages = clear_pages; ggtt->vm.vma_ops.clear_pages = clear_pages;
ggtt->vm.pte_encode = gen8_pte_encode;
setup_private_pat(dev_priv); setup_private_pat(dev_priv);
return ggtt_probe_common(ggtt, size); return ggtt_probe_common(ggtt, size);
......
...@@ -289,6 +289,7 @@ struct i915_address_space { ...@@ -289,6 +289,7 @@ struct i915_address_space {
struct mutex mutex; /* protects vma and our lists */ struct mutex mutex; /* protects vma and our lists */
u64 scratch_pte;
struct i915_page_dma scratch_page; struct i915_page_dma scratch_page;
struct i915_page_table *scratch_pt; struct i915_page_table *scratch_pt;
struct i915_page_directory *scratch_pd; struct i915_page_directory *scratch_pd;
...@@ -335,12 +336,11 @@ struct i915_address_space { ...@@ -335,12 +336,11 @@ struct i915_address_space {
/* Some systems support read-only mappings for GGTT and/or PPGTT */ /* Some systems support read-only mappings for GGTT and/or PPGTT */
bool has_read_only:1; bool has_read_only:1;
/* FIXME: Need a more generic return type */ u64 (*pte_encode)(dma_addr_t addr,
gen6_pte_t (*pte_encode)(dma_addr_t addr, enum i915_cache_level level,
enum i915_cache_level level, u32 flags); /* Create a valid PTE */
u32 flags); /* Create a valid PTE */
/* flags for pte_encode */
#define PTE_READ_ONLY (1<<0) #define PTE_READ_ONLY (1<<0)
int (*allocate_va_range)(struct i915_address_space *vm, int (*allocate_va_range)(struct i915_address_space *vm,
u64 start, u64 length); u64 start, u64 length);
void (*clear_range)(struct i915_address_space *vm, void (*clear_range)(struct i915_address_space *vm,
...@@ -422,7 +422,6 @@ struct gen6_hw_ppgtt { ...@@ -422,7 +422,6 @@ struct gen6_hw_ppgtt {
struct i915_vma *vma; struct i915_vma *vma;
gen6_pte_t __iomem *pd_addr; gen6_pte_t __iomem *pd_addr;
gen6_pte_t scratch_pte;
unsigned int pin_count; unsigned int pin_count;
bool scan_for_unused_pt; bool scan_for_unused_pt;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment