[PATCH] drm/amdkfd: don't repeat a SVM migration that collected no pages

svm_range_restore_pages() migrates the faulting granule to best_loc
whenever prange->actual_loc or best_loc is set, without checking whether
that window already lives there.

When it does, MIGRATE_VMA_SELECT_SYSTEM has nothing to collect and
migrate_vma_setup() returns cpages == 0. By then it has already issued
MMU_NOTIFY_MIGRATE, which svm_range_cpu_invalidate_pagetables() turns
into svm_range_unmap_from_gpus() for that window, so the mapping the
fault needed is torn down to service a migration that moves nothing and
the fault is re-armed rather than resolved. Under XNACK on an
oversubscribed range this becomes a loop: svm_range_restore_pages() and
svm_migrate_to_vram() are entered thousands of times per second,
svm_migrate_copy_to_vram() is never reached, and the per process
migration counters stop advancing while the GPU stays busy.

The notification cannot be filtered by pgmap owner, as it is what
invalidates the PTEs for pages that really do move. Avoid starting a
migration that cannot move anything instead. Per window residency is not
cheaply available beforehand, so record the window of a migration that
collected nothing and do not repeat it there until a short backoff
expires.

Fixes: a546a2768440 ("drm/amdkfd: Use partial migrations/mapping for GPU/CPU page faults in SVM")
Signed-off-by: William Palacek <[email protected]>
---
 drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 23 +++++++++++++++++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_svm.h |  4 ++++
 2 files changed, 27 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index fa4054d51f60..b946ae95ff12 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -50,6 +50,11 @@
  * page table is updated.
  */
 #define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING	(2UL * NSEC_PER_MSEC)
+
+/* Long enough that a retry fault storm cannot re-arm a migration which
+ * has already been found to have nothing to collect.
+ */
+#define AMDGPU_SVM_RANGE_NOOP_MIGRATE_BACKOFF	(100UL * NSEC_PER_MSEC)
 #if IS_ENABLED(CONFIG_DYNAMIC_DEBUG)
 #define dynamic_svm_range_dump(svms) \
 	_dynamic_func_call_no_desc("svm_range_dump", svm_range_debug_dump, svms)
@@ -3221,8 +3226,25 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 	last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last);
 	if (prange->actual_loc != 0 || best_loc != 0) {
 		if (best_loc) {
+			uint64_t vram_pages = prange->vram_pages;
+
+			/* migrate_vma_setup() unmaps the window before it knows
+			 * whether it can collect anything, so a migration that
+			 * collects nothing re-arms this fault instead of
+			 * resolving it.
+			 */
+			if (start == prange->noop_migrate_start &&
+			    ktime_before(timestamp,
+					 ktime_add_ns(prange->noop_migrate_timestamp,
+						      AMDGPU_SVM_RANGE_NOOP_MIGRATE_BACKOFF)))
+				goto skip_migrate;
+
 			r = svm_migrate_to_vram(prange, best_loc, start, last,
 					mm, KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU);
+			if (!r && prange->vram_pages == vram_pages) {
+				prange->noop_migrate_start = start;
+				prange->noop_migrate_timestamp = ktime_get_boottime();
+			}
 			if (r) {
 				pr_debug("svm_migrate_to_vram failed (%d) at %llx, falling back to system memory\n",
 					 r, addr);
@@ -3248,6 +3270,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
 		}
 	}
 
+skip_migrate:
 	r = svm_range_validate_and_map(mm, start, last, prange, gpuidx, false,
 				       false, false);
 	if (r)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index c7d7adae4476..508a086195ac 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -93,6 +93,8 @@ struct svm_work_list_item {
  * @granularity:migration granularity, log2 num pages
  * @invalid:    not 0 means cpu page table is invalidated
  * @validate_timestamp: system timestamp when range is validated
+ * @noop_migrate_start: start of the last migration that collected no pages
+ * @noop_migrate_timestamp: system timestamp of that migration
  * @notifier:   register mmu interval notifier
  * @work_item:  deferred work item information
  * @deferred_list: list header used to add range to deferred list
@@ -131,6 +133,8 @@ struct svm_range {
 	uint8_t				granularity;
 	atomic_t			invalid;
 	ktime_t				validate_timestamp;
+	unsigned long			noop_migrate_start;
+	ktime_t				noop_migrate_timestamp;
 	struct mmu_interval_notifier	notifier;
 	struct svm_work_list_item	work_item;
 	struct list_head		deferred_list;
-- 
2.34.1