Commit c12a2ee5 authored by Christian König's avatar Christian König Committed by Alex Deucher

drm/amdgpu: separate per VM BOs from normal in the moved state

Allows us to avoid taking the spinlock in more places.
Signed-off-by: default avatarChristian König <christian.koenig@amd.com>
Reviewed-by: default avatarJunwei Zhang <Jerry.Zhang@amd.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
parent c460f8a6
...@@ -342,9 +342,7 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, ...@@ -342,9 +342,7 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
break; break;
if (bo->tbo.type != ttm_bo_type_kernel) { if (bo->tbo.type != ttm_bo_type_kernel) {
spin_lock(&vm->moved_lock);
list_move(&bo_base->vm_status, &vm->moved); list_move(&bo_base->vm_status, &vm->moved);
spin_unlock(&vm->moved_lock);
} else { } else {
if (vm->use_cpu_for_update) if (vm->use_cpu_for_update)
r = amdgpu_bo_kmap(bo, NULL); r = amdgpu_bo_kmap(bo, NULL);
...@@ -1734,10 +1732,6 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, ...@@ -1734,10 +1732,6 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
amdgpu_asic_flush_hdp(adev, NULL); amdgpu_asic_flush_hdp(adev, NULL);
} }
spin_lock(&vm->moved_lock);
list_del_init(&bo_va->base.vm_status);
spin_unlock(&vm->moved_lock);
/* If the BO is not in its preferred location add it back to /* If the BO is not in its preferred location add it back to
* the evicted list so that it gets validated again on the * the evicted list so that it gets validated again on the
* next command submission. * next command submission.
...@@ -1746,9 +1740,13 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, ...@@ -1746,9 +1740,13 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
uint32_t mem_type = bo->tbo.mem.mem_type; uint32_t mem_type = bo->tbo.mem.mem_type;
if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type))) if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type)))
list_add_tail(&bo_va->base.vm_status, &vm->evicted); list_move_tail(&bo_va->base.vm_status, &vm->evicted);
else else
list_add(&bo_va->base.vm_status, &vm->idle); list_move(&bo_va->base.vm_status, &vm->idle);
} else {
spin_lock(&vm->invalidated_lock);
list_del_init(&bo_va->base.vm_status);
spin_unlock(&vm->invalidated_lock);
} }
list_splice_init(&bo_va->invalids, &bo_va->valids); list_splice_init(&bo_va->invalids, &bo_va->valids);
...@@ -1974,40 +1972,40 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev, ...@@ -1974,40 +1972,40 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
struct amdgpu_vm *vm) struct amdgpu_vm *vm)
{ {
struct amdgpu_bo_va *bo_va, *tmp; struct amdgpu_bo_va *bo_va, *tmp;
struct list_head moved; struct reservation_object *resv;
bool clear; bool clear;
int r; int r;
INIT_LIST_HEAD(&moved); list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
spin_lock(&vm->moved_lock); /* Per VM BOs never need to bo cleared in the page tables */
list_splice_init(&vm->moved, &moved); r = amdgpu_vm_bo_update(adev, bo_va, false);
spin_unlock(&vm->moved_lock); if (r)
return r;
}
list_for_each_entry_safe(bo_va, tmp, &moved, base.vm_status) { spin_lock(&vm->invalidated_lock);
struct reservation_object *resv = bo_va->base.bo->tbo.resv; while (!list_empty(&vm->invalidated)) {
bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
base.vm_status);
resv = bo_va->base.bo->tbo.resv;
spin_unlock(&vm->invalidated_lock);
/* Per VM BOs never need to bo cleared in the page tables */
if (resv == vm->root.base.bo->tbo.resv)
clear = false;
/* Try to reserve the BO to avoid clearing its ptes */ /* Try to reserve the BO to avoid clearing its ptes */
else if (!amdgpu_vm_debug && reservation_object_trylock(resv)) if (!amdgpu_vm_debug && reservation_object_trylock(resv))
clear = false; clear = false;
/* Somebody else is using the BO right now */ /* Somebody else is using the BO right now */
else else
clear = true; clear = true;
r = amdgpu_vm_bo_update(adev, bo_va, clear); r = amdgpu_vm_bo_update(adev, bo_va, clear);
if (r) { if (r)
spin_lock(&vm->moved_lock);
list_splice(&moved, &vm->moved);
spin_unlock(&vm->moved_lock);
return r; return r;
}
if (!clear && resv != vm->root.base.bo->tbo.resv) if (!clear)
reservation_object_unlock(resv); reservation_object_unlock(resv);
spin_lock(&vm->invalidated_lock);
} }
spin_unlock(&vm->invalidated_lock);
return 0; return 0;
} }
...@@ -2072,9 +2070,7 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, ...@@ -2072,9 +2070,7 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv && if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
!bo_va->base.moved) { !bo_va->base.moved) {
spin_lock(&vm->moved_lock);
list_move(&bo_va->base.vm_status, &vm->moved); list_move(&bo_va->base.vm_status, &vm->moved);
spin_unlock(&vm->moved_lock);
} }
trace_amdgpu_vm_bo_map(bo_va, mapping); trace_amdgpu_vm_bo_map(bo_va, mapping);
} }
...@@ -2430,9 +2426,9 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, ...@@ -2430,9 +2426,9 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
list_del(&bo_va->base.bo_list); list_del(&bo_va->base.bo_list);
spin_lock(&vm->moved_lock); spin_lock(&vm->invalidated_lock);
list_del(&bo_va->base.vm_status); list_del(&bo_va->base.vm_status);
spin_unlock(&vm->moved_lock); spin_unlock(&vm->invalidated_lock);
list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
list_del(&mapping->list); list_del(&mapping->list);
...@@ -2489,10 +2485,12 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, ...@@ -2489,10 +2485,12 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
if (bo->tbo.type == ttm_bo_type_kernel) { if (bo->tbo.type == ttm_bo_type_kernel) {
list_move(&bo_base->vm_status, &vm->relocated); list_move(&bo_base->vm_status, &vm->relocated);
} else { } else if (bo->tbo.resv == vm->root.base.bo->tbo.resv) {
spin_lock(&bo_base->vm->moved_lock);
list_move(&bo_base->vm_status, &vm->moved); list_move(&bo_base->vm_status, &vm->moved);
spin_unlock(&bo_base->vm->moved_lock); } else {
spin_lock(&vm->invalidated_lock);
list_move(&bo_base->vm_status, &vm->invalidated);
spin_unlock(&vm->invalidated_lock);
} }
} }
} }
...@@ -2637,9 +2635,10 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, ...@@ -2637,9 +2635,10 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
vm->reserved_vmid[i] = NULL; vm->reserved_vmid[i] = NULL;
INIT_LIST_HEAD(&vm->evicted); INIT_LIST_HEAD(&vm->evicted);
INIT_LIST_HEAD(&vm->relocated); INIT_LIST_HEAD(&vm->relocated);
spin_lock_init(&vm->moved_lock);
INIT_LIST_HEAD(&vm->moved); INIT_LIST_HEAD(&vm->moved);
INIT_LIST_HEAD(&vm->idle); INIT_LIST_HEAD(&vm->idle);
INIT_LIST_HEAD(&vm->invalidated);
spin_lock_init(&vm->invalidated_lock);
INIT_LIST_HEAD(&vm->freed); INIT_LIST_HEAD(&vm->freed);
/* create scheduler entity for page table updates */ /* create scheduler entity for page table updates */
......
...@@ -201,13 +201,16 @@ struct amdgpu_vm { ...@@ -201,13 +201,16 @@ struct amdgpu_vm {
/* PT BOs which relocated and their parent need an update */ /* PT BOs which relocated and their parent need an update */
struct list_head relocated; struct list_head relocated;
/* BOs moved, but not yet updated in the PT */ /* per VM BOs moved, but not yet updated in the PT */
struct list_head moved; struct list_head moved;
spinlock_t moved_lock;
/* All BOs of this VM not currently in the state machine */ /* All BOs of this VM not currently in the state machine */
struct list_head idle; struct list_head idle;
/* regular invalidated BOs, but not yet updated in the PT */
struct list_head invalidated;
spinlock_t invalidated_lock;
/* BO mappings freed, but not yet updated in the PT */ /* BO mappings freed, but not yet updated in the PT */
struct list_head freed; struct list_head freed;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment