drm/i915/gtt: Replace struct_mutex serialisation for allocation

Instead of relying on the caller holding struct_mutex across the
allocation, push the allocation under a tree of spinlocks stored inside
the page tables. Not only should this allow us to avoid struct_mutex
here, but it will allow multiple users to lock independent ranges for
concurrent allocations, and operate independently. This is vital for
pushing the GTT manipulation into a background thread where dependency
on struct_mutex is verboten, and for allowing other callers to avoid
struct_mutex altogether.

v2: Restore lost GEM_BUG_ON for removing too many PTE from
gen6_ppgtt_clear_range.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Matthew Auld <matthew.auld@intel.com>
Cc: Mika Kuoppala <mika.kuoppala@intel.com>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Acked-by: Mika Kuoppala <mika.kuoppala@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20190604153830.19096-1-chris@chris-wilson.co.uk

drivers/gpu/drm/i915/i915_gem_gtt.c

@@ -659,7 +659,7 @@ static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
 		return ERR_PTR(-ENOMEM);
 	}
 
-	pt->used_ptes = 0;
+	atomic_set(&pt->used_ptes, 0);
 	return pt;
 }
 
@@ -694,7 +694,8 @@ static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
 		return ERR_PTR(-ENOMEM);
 	}
 
-	pd->used_pdes = 0;
+	atomic_set(&pd->used_pdes, 0);
+	spin_lock_init(&pd->lock);
 	return pd;
 }
 
@@ -725,6 +726,8 @@ static int __pdp_init(struct i915_address_space *vm,
 
 	memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes);
 
+	atomic_set(&pdp->used_pdpes, 0);
+	spin_lock_init(&pdp->lock);
 	return 0;
 }
 
@@ -779,11 +782,8 @@ static void free_pdp(struct i915_address_space *vm,
 static void gen8_initialize_pdp(struct i915_address_space *vm,
 				struct i915_page_directory_pointer *pdp)
 {
-	gen8_ppgtt_pdpe_t scratch_pdpe;
-
-	scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
-
-	fill_px(vm, pdp, scratch_pdpe);
+	fill_px(vm, pdp,
+		gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC));
 }
 
 static void gen8_initialize_pml4(struct i915_address_space *vm,
@@ -792,6 +792,7 @@ static void gen8_initialize_pml4(struct i915_address_space *vm,
 	fill_px(vm, pml4,
 		gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC));
 	memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
+	spin_lock_init(&pml4->lock);
 }
 
 /*
@@ -815,17 +816,12 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
 	unsigned int num_entries = gen8_pte_count(start, length);
 	gen8_pte_t *vaddr;
 
-	GEM_BUG_ON(num_entries > pt->used_ptes);
-
-	pt->used_ptes -= num_entries;
-	if (!pt->used_ptes)
-		return true;
-
 	vaddr = kmap_atomic_px(pt);
 	memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries);
 	kunmap_atomic(vaddr);
 
-	return false;
+	GEM_BUG_ON(num_entries > atomic_read(&pt->used_ptes));
+	return !atomic_sub_return(num_entries, &pt->used_ptes);
 }
 
 static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
@@ -835,8 +831,6 @@ static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
 {
 	gen8_pde_t *vaddr;
 
-	pd->page_table[pde] = pt;
-
 	vaddr = kmap_atomic_px(pd);
 	vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
 	kunmap_atomic(vaddr);
@@ -850,19 +844,28 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
 	u32 pde;
 
 	gen8_for_each_pde(pt, pd, start, length, pde) {
+		bool free = false;
+
 		GEM_BUG_ON(pt == vm->scratch_pt);
 
 		if (!gen8_ppgtt_clear_pt(vm, pt, start, length))
 			continue;
 
-		gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
-		GEM_BUG_ON(!pd->used_pdes);
-		pd->used_pdes--;
+		spin_lock(&pd->lock);
+		if (!atomic_read(&pt->used_ptes)) {
+			gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
+			pd->page_table[pde] = vm->scratch_pt;
 
-		free_pt(vm, pt);
+			GEM_BUG_ON(!atomic_read(&pd->used_pdes));
+			atomic_dec(&pd->used_pdes);
+			free = true;
+		}
+		spin_unlock(&pd->lock);
+		if (free)
+			free_pt(vm, pt);
 	}
 
-	return !pd->used_pdes;
+	return !atomic_read(&pd->used_pdes);
 }
 
 static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
@@ -872,7 +875,6 @@ static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
 {
 	gen8_ppgtt_pdpe_t *vaddr;
 
-	pdp->page_directory[pdpe] = pd;
 	if (!i915_vm_is_4lvl(vm))
 		return;
 
@@ -892,19 +894,28 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
 	unsigned int pdpe;
 
 	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
+		bool free = false;
+
 		GEM_BUG_ON(pd == vm->scratch_pd);
 
 		if (!gen8_ppgtt_clear_pd(vm, pd, start, length))
 			continue;
 
-		gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
-		GEM_BUG_ON(!pdp->used_pdpes);
-		pdp->used_pdpes--;
+		spin_lock(&pdp->lock);
+		if (!atomic_read(&pd->used_pdes)) {
+			gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
+			pdp->page_directory[pdpe] = vm->scratch_pd;
 
-		free_pd(vm, pd);
+			GEM_BUG_ON(!atomic_read(&pdp->used_pdpes));
+			atomic_dec(&pdp->used_pdpes);
+			free = true;
+		}
+		spin_unlock(&pdp->lock);
+		if (free)
+			free_pd(vm, pd);
 	}
 
-	return !pdp->used_pdpes;
+	return !atomic_read(&pdp->used_pdpes);
 }
 
 static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
@@ -919,8 +930,6 @@ static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,
 {
 	gen8_ppgtt_pml4e_t *vaddr;
 
-	pml4->pdps[pml4e] = pdp;
-
 	vaddr = kmap_atomic_px(pml4);
 	vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
 	kunmap_atomic(vaddr);
@@ -941,14 +950,21 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
 	GEM_BUG_ON(!i915_vm_is_4lvl(vm));
 
 	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
+		bool free = false;
 		GEM_BUG_ON(pdp == vm->scratch_pdp);
 
 		if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length))
 			continue;
 
-		gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
-
-		free_pdp(vm, pdp);
+		spin_lock(&pml4->lock);
+		if (!atomic_read(&pdp->used_pdpes)) {
+			gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
+			pml4->pdps[pml4e] = vm->scratch_pdp;
+			free = true;
+		}
+		spin_unlock(&pml4->lock);
+		if (free)
+			free_pdp(vm, pdp);
 	}
 }
 
@@ -1373,27 +1389,38 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
 	u64 from = start;
 	unsigned int pde;
 
+	spin_lock(&pd->lock);
 	gen8_for_each_pde(pt, pd, start, length, pde) {
-		int count = gen8_pte_count(start, length);
+		const int count = gen8_pte_count(start, length);
 
 		if (pt == vm->scratch_pt) {
-			pd->used_pdes++;
+			struct i915_page_table *old;
+
+			spin_unlock(&pd->lock);
 
 			pt = alloc_pt(vm);
-			if (IS_ERR(pt)) {
-				pd->used_pdes--;
+			if (IS_ERR(pt))
 				goto unwind;
-			}
 
 			if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
 				gen8_initialize_pt(vm, pt);
 
-			gen8_ppgtt_set_pde(vm, pd, pt, pde);
-			GEM_BUG_ON(pd->used_pdes > I915_PDES);
+			old = cmpxchg(&pd->page_table[pde], vm->scratch_pt, pt);
+			if (old == vm->scratch_pt) {
+				gen8_ppgtt_set_pde(vm, pd, pt, pde);
+				atomic_inc(&pd->used_pdes);
+			} else {
+				free_pt(vm, pt);
+				pt = old;
+			}
+
+			spin_lock(&pd->lock);
 		}
 
-		pt->used_ptes += count;
+		atomic_add(count, &pt->used_ptes);
 	}
+	spin_unlock(&pd->lock);
+
 	return 0;
 
 unwind:
@@ -1410,35 +1437,54 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
 	unsigned int pdpe;
 	int ret;
 
+	spin_lock(&pdp->lock);
 	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 		if (pd == vm->scratch_pd) {
-			pdp->used_pdpes++;
+			struct i915_page_directory *old;
+
+			spin_unlock(&pdp->lock);
 
 			pd = alloc_pd(vm);
-			if (IS_ERR(pd)) {
-				pdp->used_pdpes--;
+			if (IS_ERR(pd))
 				goto unwind;
-			}
 
 			gen8_initialize_pd(vm, pd);
-			gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
-			GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
+
+			old = cmpxchg(&pdp->page_directory[pdpe],
+				      vm->scratch_pd, pd);
+			if (old == vm->scratch_pd) {
+				gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
+				atomic_inc(&pdp->used_pdpes);
+			} else {
+				free_pd(vm, pd);
+				pd = old;
+			}
+
+			spin_lock(&pdp->lock);
 		}
+		atomic_inc(&pd->used_pdes);
+		spin_unlock(&pdp->lock);
 
 		ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
 		if (unlikely(ret))
 			goto unwind_pd;
+
+		spin_lock(&pdp->lock);
+		atomic_dec(&pd->used_pdes);
 	}
+	spin_unlock(&pdp->lock);
 
 	return 0;
 
 unwind_pd:
-	if (!pd->used_pdes) {
+	spin_lock(&pdp->lock);
+	if (atomic_dec_and_test(&pd->used_pdes)) {
 		gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
-		GEM_BUG_ON(!pdp->used_pdpes);
-		pdp->used_pdpes--;
+		GEM_BUG_ON(!atomic_read(&pdp->used_pdpes));
+		atomic_dec(&pdp->used_pdpes);
 		free_pd(vm, pd);
 	}
+	spin_unlock(&pdp->lock);
 unwind:
 	gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
 	return -ENOMEM;
@@ -1461,28 +1507,50 @@ static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
 	u32 pml4e;
 	int ret;
 
+	spin_lock(&pml4->lock);
 	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
-		if (pml4->pdps[pml4e] == vm->scratch_pdp) {
+		if (pdp == vm->scratch_pdp) {
+			struct i915_page_directory_pointer *old;
+
+			spin_unlock(&pml4->lock);
+
 			pdp = alloc_pdp(vm);
 			if (IS_ERR(pdp))
 				goto unwind;
 
 			gen8_initialize_pdp(vm, pdp);
-			gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
+
+			old = cmpxchg(&pml4->pdps[pml4e], vm->scratch_pdp, pdp);
+			if (old == vm->scratch_pdp) {
+				gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
+			} else {
+				free_pdp(vm, pdp);
+				pdp = old;
+			}
+
+			spin_lock(&pml4->lock);
 		}
+		atomic_inc(&pdp->used_pdpes);
+		spin_unlock(&pml4->lock);
 
 		ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
 		if (unlikely(ret))
 			goto unwind_pdp;
+
+		spin_lock(&pml4->lock);
+		atomic_dec(&pdp->used_pdpes);
 	}
+	spin_unlock(&pml4->lock);
 
 	return 0;
 
 unwind_pdp:
-	if (!pdp->used_pdpes) {
+	spin_lock(&pml4->lock);
+	if (atomic_dec_and_test(&pdp->used_pdpes)) {
 		gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
 		free_pdp(vm, pdp);
 	}
+	spin_unlock(&pml4->lock);
 unwind:
 	gen8_ppgtt_clear_4lvl(vm, from, start - from);
 	return -ENOMEM;
@@ -1504,10 +1572,10 @@ static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)
 
 		gen8_initialize_pd(vm, pd);
 		gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
-		pdp->used_pdpes++;
+		atomic_inc(&pdp->used_pdpes);
 	}
 
-	pdp->used_pdpes++; /* never remove */
+	atomic_inc(&pdp->used_pdpes); /* never remove */
 	return 0;
 
 unwind:
@@ -1516,7 +1584,7 @@ static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)
 		gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
 		free_pd(vm, pd);
 	}
-	pdp->used_pdpes = 0;
+	atomic_set(&pdp->used_pdpes, 0);
 	return -ENOMEM;
 }
 
@@ -1688,9 +1756,8 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 
 		num_entries -= count;
 
-		GEM_BUG_ON(count > pt->used_ptes);
-		pt->used_ptes -= count;
-		if (!pt->used_ptes)
+		GEM_BUG_ON(count > atomic_read(&pt->used_ptes));
+		if (!atomic_sub_return(count, &pt->used_ptes))
 			ppgtt->scan_for_unused_pt = true;
 
 		/*
@@ -1760,28 +1827,41 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
 
 	wakeref = intel_runtime_pm_get(vm->i915);
 
+	spin_lock(&ppgtt->base.pd.lock);
 	gen6_for_each_pde(pt, &ppgtt->base.pd, start, length, pde) {
 		const unsigned int count = gen6_pte_count(start, length);
 
 		if (pt == vm->scratch_pt) {
+			struct i915_page_table *old;
+
+			spin_unlock(&ppgtt->base.pd.lock);
+
 			pt = alloc_pt(vm);
 			if (IS_ERR(pt))
 				goto unwind_out;
 
 			gen6_initialize_pt(vm, pt);
-			ppgtt->base.pd.page_table[pde] = pt;
 
-			if (i915_vma_is_bound(ppgtt->vma,
-					      I915_VMA_GLOBAL_BIND)) {
-				gen6_write_pde(ppgtt, pde, pt);
-				flush = true;
+			old = cmpxchg(&ppgtt->base.pd.page_table[pde],
+				      vm->scratch_pt, pt);
+			if (old == vm->scratch_pt) {
+				ppgtt->base.pd.page_table[pde] = pt;
+				if (i915_vma_is_bound(ppgtt->vma,
+						      I915_VMA_GLOBAL_BIND)) {
+					gen6_write_pde(ppgtt, pde, pt);
+					flush = true;
+				}
+			} else {
+				free_pt(vm, pt);
+				pt = old;
 			}
 
-			GEM_BUG_ON(pt->used_ptes);
+			spin_lock(&ppgtt->base.pd.lock);
 		}
 
-		pt->used_ptes += count;
+		atomic_add(count, &pt->used_ptes);
 	}
+	spin_unlock(&ppgtt->base.pd.lock);
 
 	if (flush) {
 		mark_tlbs_dirty(&ppgtt->base);
@@ -1822,6 +1902,7 @@ static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt)
 	gen6_initialize_pt(vm, vm->scratch_pt);
 	gen6_for_all_pdes(unused, &ppgtt->base.pd, pde)
 		ppgtt->base.pd.page_table[pde] = vm->scratch_pt;
+	spin_lock_init(&ppgtt->base.pd.lock);
 
 	return 0;
 }
@@ -1950,7 +2031,7 @@ static void pd_vma_unbind(struct i915_vma *vma)
 
 	/* Free all no longer used page tables */
 	gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) {
-		if (pt->used_ptes || pt == scratch_pt)
+		if (atomic_read(&pt->used_ptes) || pt == scratch_pt)
 			continue;
 
 		free_pt(&ppgtt->base.vm, pt);

drivers/gpu/drm/i915/i915_gem_gtt.h

@@ -248,25 +248,28 @@ struct i915_page_dma {
 
 struct i915_page_table {
 	struct i915_page_dma base;
-	unsigned int used_ptes;
+	atomic_t used_ptes;
 };
 
 struct i915_page_directory {
 	struct i915_page_dma base;
 
 	struct i915_page_table *page_table[I915_PDES]; /* PDEs */
-	unsigned int used_pdes;
+	atomic_t used_pdes;
+	spinlock_t lock;
 };
 
 struct i915_page_directory_pointer {
 	struct i915_page_dma base;
 	struct i915_page_directory **page_directory;
-	unsigned int used_pdpes;
+	atomic_t used_pdpes;
+	spinlock_t lock;
 };
 
 struct i915_pml4 {
 	struct i915_page_dma base;
 	struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4];
+	spinlock_t lock;
 };
 
 struct i915_vma_ops {