Commit 46ae2af9 authored by Philip Yang's avatar Philip Yang Committed by Alex Deucher

drm/amdkfd: Add unmap from GPU SMI event

SVM range unmapped from GPUs when range is unmapped from CPU, or with
xnack on from MMU notifier when range is evicted or migrated.
Signed-off-by: default avatarPhilip Yang <Philip.Yang@amd.com>
Reviewed-by: default avatarFelix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
parent c7f21978
...@@ -318,6 +318,15 @@ void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm) ...@@ -318,6 +318,15 @@ void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm)
kfd_unref_process(p); kfd_unref_process(p);
} }
void kfd_smi_event_unmap_from_gpu(struct kfd_dev *dev, pid_t pid,
unsigned long address, unsigned long last,
uint32_t trigger)
{
kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_UNMAP_FROM_GPU,
"%lld -%d @%lx(%lx) %x %d\n", ktime_get_boottime_ns(),
pid, address, last - address + 1, dev->id, trigger);
}
int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd) int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd)
{ {
struct kfd_smi_client *client; struct kfd_smi_client *client;
......
...@@ -46,4 +46,7 @@ void kfd_smi_event_queue_eviction(struct kfd_dev *dev, pid_t pid, ...@@ -46,4 +46,7 @@ void kfd_smi_event_queue_eviction(struct kfd_dev *dev, pid_t pid,
uint32_t trigger); uint32_t trigger);
void kfd_smi_event_queue_restore(struct kfd_dev *dev, pid_t pid); void kfd_smi_event_queue_restore(struct kfd_dev *dev, pid_t pid);
void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm); void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm);
void kfd_smi_event_unmap_from_gpu(struct kfd_dev *dev, pid_t pid,
unsigned long address, unsigned long last,
uint32_t trigger);
#endif #endif
...@@ -1200,7 +1200,7 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, ...@@ -1200,7 +1200,7 @@ svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
static int static int
svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start, svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
unsigned long last) unsigned long last, uint32_t trigger)
{ {
DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE); DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
struct kfd_process_device *pdd; struct kfd_process_device *pdd;
...@@ -1232,6 +1232,9 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start, ...@@ -1232,6 +1232,9 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
return -EINVAL; return -EINVAL;
} }
kfd_smi_event_unmap_from_gpu(pdd->dev, p->lead_thread->pid,
start, last, trigger);
r = svm_range_unmap_from_gpu(pdd->dev->adev, r = svm_range_unmap_from_gpu(pdd->dev->adev,
drm_priv_to_vm(pdd->drm_priv), drm_priv_to_vm(pdd->drm_priv),
start, last, &fence); start, last, &fence);
...@@ -1759,7 +1762,8 @@ static void svm_range_restore_work(struct work_struct *work) ...@@ -1759,7 +1762,8 @@ static void svm_range_restore_work(struct work_struct *work)
*/ */
static int static int
svm_range_evict(struct svm_range *prange, struct mm_struct *mm, svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
unsigned long start, unsigned long last) unsigned long start, unsigned long last,
enum mmu_notifier_event event)
{ {
struct svm_range_list *svms = prange->svms; struct svm_range_list *svms = prange->svms;
struct svm_range *pchild; struct svm_range *pchild;
...@@ -1804,6 +1808,12 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm, ...@@ -1804,6 +1808,12 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS)); msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
} else { } else {
unsigned long s, l; unsigned long s, l;
uint32_t trigger;
if (event == MMU_NOTIFY_MIGRATE)
trigger = KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY_MIGRATE;
else
trigger = KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY;
pr_debug("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n", pr_debug("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n",
prange->svms, start, last); prange->svms, start, last);
...@@ -1812,13 +1822,13 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm, ...@@ -1812,13 +1822,13 @@ svm_range_evict(struct svm_range *prange, struct mm_struct *mm,
s = max(start, pchild->start); s = max(start, pchild->start);
l = min(last, pchild->last); l = min(last, pchild->last);
if (l >= s) if (l >= s)
svm_range_unmap_from_gpus(pchild, s, l); svm_range_unmap_from_gpus(pchild, s, l, trigger);
mutex_unlock(&pchild->lock); mutex_unlock(&pchild->lock);
} }
s = max(start, prange->start); s = max(start, prange->start);
l = min(last, prange->last); l = min(last, prange->last);
if (l >= s) if (l >= s)
svm_range_unmap_from_gpus(prange, s, l); svm_range_unmap_from_gpus(prange, s, l, trigger);
} }
return r; return r;
...@@ -2232,6 +2242,7 @@ static void ...@@ -2232,6 +2242,7 @@ static void
svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,
unsigned long start, unsigned long last) unsigned long start, unsigned long last)
{ {
uint32_t trigger = KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU;
struct svm_range_list *svms; struct svm_range_list *svms;
struct svm_range *pchild; struct svm_range *pchild;
struct kfd_process *p; struct kfd_process *p;
...@@ -2259,14 +2270,14 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, ...@@ -2259,14 +2270,14 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange,
s = max(start, pchild->start); s = max(start, pchild->start);
l = min(last, pchild->last); l = min(last, pchild->last);
if (l >= s) if (l >= s)
svm_range_unmap_from_gpus(pchild, s, l); svm_range_unmap_from_gpus(pchild, s, l, trigger);
svm_range_unmap_split(mm, prange, pchild, start, last); svm_range_unmap_split(mm, prange, pchild, start, last);
mutex_unlock(&pchild->lock); mutex_unlock(&pchild->lock);
} }
s = max(start, prange->start); s = max(start, prange->start);
l = min(last, prange->last); l = min(last, prange->last);
if (l >= s) if (l >= s)
svm_range_unmap_from_gpus(prange, s, l); svm_range_unmap_from_gpus(prange, s, l, trigger);
svm_range_unmap_split(mm, prange, prange, start, last); svm_range_unmap_split(mm, prange, prange, start, last);
if (unmap_parent) if (unmap_parent)
...@@ -2333,7 +2344,7 @@ svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, ...@@ -2333,7 +2344,7 @@ svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
svm_range_unmap_from_cpu(mni->mm, prange, start, last); svm_range_unmap_from_cpu(mni->mm, prange, start, last);
break; break;
default: default:
svm_range_evict(prange, mni->mm, start, last); svm_range_evict(prange, mni->mm, start, last, range->event);
break; break;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment