[PATCH v9 08/18] drm/amdgpu: implement SVM range notifier and GC helpers

From: Honglei Huang <[email protected]>

Add MMU notifier event handling and garbage collection infrastructure:
- amdgpu_svm_range_remove: unmap pages and remove from gpusvm
- amdgpu_svm_range_notifier_event_begin: compute the [start_page,
  last_page] window from the notifier range and the gpusvm range,
  then zap PTEs via amdgpu_svm_range_zap_ptes and invalidate the
  GPU mapping
- amdgpu_svm_gc_enqueue: queue range for GC worker and set queue state
- amdgpu_svm_gc_add_range: mark range unmapped and enqueue for GC
- amdgpu_svm_range_notifier_event_end: DMA unmap and GC on MMU_UNMAP
- amdgpu_svm_range_invalidate_interval: cursor based PTE clearing
  across notifiers/ranges; clears PTEs through
  amdgpu_svm_range_zap_ptes
- amdgpu_svm_range_dequeue_locked: dequeue work item with atomic
  pending state transfer
- amdgpu_svm_range_put_if_dequeued: release range ref after dequeue,
  re-dispatch if new work was enqueued during processing
- amdgpu_svm_range_unqueue: locked context helper that pulls a range
  off the GC list

Signed-off-by: Honglei Huang <[email protected]>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 311 ++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h |  16 +
 2 files changed, 327 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
index 335fe5a551f0f..9e348390795a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
@@ -417,3 +417,314 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm,
 	return 0;
 }
 
+void amdgpu_svm_range_remove(struct amdgpu_svm *svm,
+			     struct amdgpu_svm_range *range,
+			     struct drm_gpusvm_ctx *ctx)
+{
+	struct drm_gpusvm_range *base = &range->base;
+
+	amdgpu_svm_assert_locked(svm);
+
+	if (!base->pages.flags.unmapped && !base->pages.flags.partial_unmap)
+		drm_gpusvm_range_unmap_pages(&svm->gpusvm, base, ctx);
+
+	amdgpu_svm_range_invalidate_gpu_mapping(range);
+	drm_gpusvm_range_remove(&svm->gpusvm, base);
+}
+
+/**
+ * amdgpu_svm_range_notifier_event_begin() - First MMU notifier phase
+ * @svm: The SVM context.
+ * @range: The GPU SVM range covered by the notifier event.
+ * @mmu_range: The MMU notifier range describing the invalidation window.
+ *
+ * Invalidate the GPU view of @range over the notifier window: zap the
+ * affected PTEs and clear the gpu_mapped flag. The range is not removed
+ * here; structural removal if needed happens in the event-end phase.
+ *
+ * Return: true if PTEs were zapped, false otherwise.
+ */
+bool
+amdgpu_svm_range_notifier_event_begin(struct amdgpu_svm *svm,
+				      struct drm_gpusvm_range *range,
+				      const struct mmu_notifier_range *mmu_range)
+{
+	struct amdgpu_svm_range *svm_range = to_amdgpu_svm_range(range);
+	unsigned long start_page, last_page;
+
+	amdgpu_svm_assert_in_notifier(svm);
+
+	AMDGPU_SVM_RANGE_TRACE(svm_range, "NOTIFIER");
+
+	if (range->pages.flags.unmapped || !svm_range->gpu_mapped)
+		return false;
+
+	AMDGPU_SVM_RANGE_TRACE(svm_range, "NOTIFIER - EXECUTE");
+
+	start_page = max(drm_gpusvm_range_start(range),
+			 mmu_range->start) >> PAGE_SHIFT;
+	last_page = (min(drm_gpusvm_range_end(range),
+			 mmu_range->end) >> PAGE_SHIFT) - 1;
+
+	amdgpu_svm_range_zap_ptes(svm, svm_range, start_page, last_page);
+	amdgpu_svm_range_invalidate_gpu_mapping(svm_range);
+
+	return true;
+}
+
+/**
+ * amdgpu_svm_gc_enqueue() - Queue a range for work queue processing
+ * @svm: The SVM context.
+ * @range: The SVM range to queue.
+ * @start_page: First page of the pending invalidation interval.
+ * @last_page: Last page of the pending invalidation interval.
+ *
+ * Queue @range on the GC list and kick the GC worker. A range reference
+ * is taken when it first enters IN_GC and dropped only when it returns
+ * to NOT_QUEUED, so the range stays alive while on the GC list or being
+ * processed. All transitions are serialized by svm->work_lock.
+ */
+static void
+amdgpu_svm_gc_enqueue(struct amdgpu_svm *svm,
+		      struct amdgpu_svm_range *range,
+		      unsigned long start_page, unsigned long last_page)
+{
+	if (atomic_read(&svm->exiting))
+		return;
+
+	spin_lock(&svm->work_lock);
+	if (range->queue_state == AMDGPU_SVM_RANGE_NOT_QUEUED) {
+		drm_gpusvm_range_get(&range->base);
+		range->queue_state = AMDGPU_SVM_RANGE_IN_GC;
+	}
+
+	range->pending_start_page = min(range->pending_start_page, start_page);
+	range->pending_last_page = max(range->pending_last_page, last_page);
+	if (range->pending_ops == AMDGPU_SVM_RANGE_OP_NONE)
+		list_add_tail(&range->work_node, &svm->gc.list);
+	range->pending_ops |= AMDGPU_SVM_RANGE_OP_UNMAP;
+	spin_unlock(&svm->work_lock);
+
+	queue_work(svm->gc.wq, &svm->gc.work);
+}
+
+static void
+amdgpu_svm_gc_add_range(struct amdgpu_svm *svm,
+			struct amdgpu_svm_range *svm_range,
+			const struct mmu_notifier_range *mmu_range)
+{
+	unsigned long start_page = max(drm_gpusvm_range_start(&svm_range->base),
+				       mmu_range->start) >> PAGE_SHIFT;
+	unsigned long last_page = (min(drm_gpusvm_range_end(&svm_range->base),
+				       mmu_range->end) >> PAGE_SHIFT) - 1;
+
+	AMDGPU_SVM_RANGE_TRACE(svm_range, "GARBAGE COLLECTOR ADD");
+
+	drm_gpusvm_range_set_unmapped(&svm_range->base, mmu_range);
+	amdgpu_svm_gc_enqueue(svm, svm_range, start_page, last_page);
+}
+
+/**
+ * amdgpu_svm_range_notifier_event_end() - Second MMU notifier phase
+ * @svm: The SVM context.
+ * @range: The GPU SVM range covered by the notifier event.
+ * @mmu_range: The MMU notifier range describing the event.
+ *
+ * DMA-unmap the pages for @range. On an actual unmap event, also queue the
+ * range to the GC so it gets structurally removed from the gpusvm trees.
+ */
+static void
+amdgpu_svm_range_notifier_event_end(struct amdgpu_svm *svm,
+				    struct drm_gpusvm_range *range,
+				    const struct mmu_notifier_range *mmu_range)
+{
+	struct drm_gpusvm_ctx ctx = { .in_notifier = true, };
+
+	amdgpu_svm_assert_in_notifier(svm);
+
+	drm_gpusvm_range_unmap_pages(&svm->gpusvm, range, &ctx);
+	if (mmu_range->event == MMU_NOTIFY_UNMAP)
+		amdgpu_svm_gc_add_range(svm, to_amdgpu_svm_range(range),
+					mmu_range);
+}
+
+int
+amdgpu_svm_range_invalidate_interval(struct amdgpu_svm *svm,
+				     unsigned long start_page,
+				     unsigned long last_page)
+{
+	unsigned long start = start_page << PAGE_SHIFT;
+	unsigned long end = (last_page + 1) << PAGE_SHIFT;
+	struct drm_gpusvm_notifier *notifier, *next_notifier;
+	struct drm_gpusvm_ctx ctx = { .in_notifier = false };
+	struct drm_exec exec;
+	bool needs_flush = false;
+	int ret;
+
+	amdgpu_svm_assert_locked(svm);
+
+	ret = amdgpu_svm_range_lock_vm_pd(svm, &exec, true);
+	if (ret)
+		return ret;
+
+	drm_gpusvm_for_each_notifier_safe(notifier, next_notifier, &svm->gpusvm,
+					  start, end) {
+		struct drm_gpusvm_range *range, *next_range;
+
+		drm_gpusvm_for_each_range_safe(range, next_range, notifier,
+					       start, end) {
+			struct amdgpu_svm_range *svm_range = to_amdgpu_svm_range(range);
+			bool crosses_boundary = start > drm_gpusvm_range_start(range) ||
+						end < drm_gpusvm_range_end(range);
+
+			if (svm_range->gpu_mapped) {
+				AMDGPU_SVM_RANGE_TRACE(svm_range,
+					crosses_boundary ? "ATTR DESTROY" :
+							   "ATTR ZAP PTE");
+
+				ret = amdgpu_svm_range_zap_ptes(svm, svm_range,
+						drm_gpusvm_range_start(range) >> PAGE_SHIFT,
+						(drm_gpusvm_range_end(range) >> PAGE_SHIFT) - 1);
+				if (ret < 0) {
+					drm_exec_fini(&exec);
+					return ret;
+				}
+
+				amdgpu_svm_range_invalidate_gpu_mapping(svm_range);
+				needs_flush = true;
+			}
+
+			if (crosses_boundary) {
+				/* remove ranges crossing the boundary so GPU fault
+				 * creates new ranges bounded by the updated
+				 * attr_range boundaries.
+				 * Evict devmem-backed pages back to sysmem first
+				 * so VRAM-resident data is not lost when the range
+				 * is destroyed. No-op for sysmem-only ranges.
+				 */
+				drm_gpusvm_range_get(range);
+
+				amdgpu_svm_range_evict(svm, range);
+				amdgpu_svm_range_remove(svm, svm_range, &ctx);
+
+				amdgpu_svm_range_unqueue(svm, svm_range);
+
+				drm_gpusvm_range_put(range);
+			}
+		}
+	}
+
+	drm_exec_fini(&exec);
+
+	if (needs_flush)
+		amdgpu_svm_flush_tlb(svm);
+
+	AMDGPU_SVM_DBG("attr invalidate done [0x%lx-0x%lx]-0x%lx needs_flush=%d\n",
+			 start_page, last_page, last_page - start_page + 1,
+			 needs_flush ? 1 : 0);
+
+	return 0;
+}
+
+/**
+ * amdgpu_svm_range_dequeue_locked() - Pull the next range off a work list
+ * @svm: The SVM context.
+ * @work_list: The work list to dequeue from.
+ * @op_ctx: Output context filled with the dequeued range's pending state.
+ *
+ * Move the first range on @work_list to PROCESSING and atomically snapshot
+ * its pending op/interval into @op_ctx while resetting them on the range, so
+ * invalidations that arrive during processing accumulate afresh. Caller must
+ * hold svm->work_lock.
+ *
+ * Return: true if a range was dequeued, false if @work_list was empty.
+ */
+bool
+amdgpu_svm_range_dequeue_locked(struct amdgpu_svm *svm,
+					struct list_head *work_list,
+					struct amdgpu_svm_range_op_ctx *op_ctx)
+{
+	struct amdgpu_svm_range *range;
+
+	lockdep_assert_held(&svm->work_lock);
+
+	range = list_first_entry_or_null(work_list, struct amdgpu_svm_range,
+					work_node);
+	if (!range)
+		return false;
+
+	list_del_init(&range->work_node);
+	range->queue_state = AMDGPU_SVM_RANGE_PROCESSING;
+
+	op_ctx->range = range;
+	op_ctx->start_page = range->pending_start_page;
+	op_ctx->last_page = range->pending_last_page;
+	op_ctx->pending_ops = range->pending_ops;
+
+	range->pending_start_page = ULONG_MAX;
+	range->pending_last_page = 0;
+	range->pending_ops = AMDGPU_SVM_RANGE_OP_NONE;
+
+	return true;
+}
+
+/**
+ * amdgpu_svm_range_put_if_dequeued() - Finish processing a dequeued range
+ * @svm: The SVM context.
+ * @range: The range previously moved to PROCESSING.
+ *
+ * Called after a PROCESSING range has been handled. If new UNMAP work was
+ * enqueued while it was being processed, re-queue it on the GC list and kick
+ * the worker again; otherwise return it to NOT_QUEUED and drop the reference
+ * taken at enqueue time.
+ */
+void
+amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm,
+				     struct amdgpu_svm_range *range)
+{
+	bool release_kref = false;
+	bool queue_gc = false;
+
+	spin_lock(&svm->work_lock);
+
+	if (range->queue_state != AMDGPU_SVM_RANGE_PROCESSING) {
+		spin_unlock(&svm->work_lock);
+		return;
+	}
+
+	if (range->pending_ops & AMDGPU_SVM_RANGE_OP_UNMAP) {
+		list_add_tail(&range->work_node, &svm->gc.list);
+		range->queue_state = AMDGPU_SVM_RANGE_IN_GC;
+		queue_gc = true;
+	} else {
+		range->queue_state = AMDGPU_SVM_RANGE_NOT_QUEUED;
+		release_kref = true;
+	}
+
+	spin_unlock(&svm->work_lock);
+
+	if (queue_gc)
+		queue_work(svm->gc.wq, &svm->gc.work);
+	if (release_kref)
+		drm_gpusvm_range_put(&range->base);
+}
+
+void amdgpu_svm_range_unqueue(struct amdgpu_svm *svm,
+					struct amdgpu_svm_range *range)
+{
+	bool put = false;
+
+	amdgpu_svm_assert_locked(svm);
+
+	spin_lock(&svm->work_lock);
+	if (range->queue_state == AMDGPU_SVM_RANGE_IN_GC) {
+		list_del_init(&range->work_node);
+		range->queue_state = AMDGPU_SVM_RANGE_NOT_QUEUED;
+		range->pending_ops = AMDGPU_SVM_RANGE_OP_NONE;
+		put = true;
+	}
+	spin_unlock(&svm->work_lock);
+
+	if (put)
+		drm_gpusvm_range_put(&range->base);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
index e5a5f52166576..f423be277a85a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
@@ -149,6 +149,16 @@ int amdgpu_svm_range_update_mapping(struct amdgpu_svm *svm,
 				    bool read_only,
 				    bool intr, bool wait,
 				    bool flush_tlb);
+bool amdgpu_svm_range_dequeue_locked(struct amdgpu_svm *svm,
+				     struct list_head *work_list,
+				     struct amdgpu_svm_range_op_ctx *op_ctx);
+void amdgpu_svm_range_put_if_dequeued(struct amdgpu_svm *svm,
+				      struct amdgpu_svm_range *range);
+void amdgpu_svm_range_unqueue(struct amdgpu_svm *svm,
+					struct amdgpu_svm_range *range);
+void amdgpu_svm_range_remove(struct amdgpu_svm *svm,
+			     struct amdgpu_svm_range *range,
+			     struct drm_gpusvm_ctx *ctx);
 int amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm,
 			       const struct amdgpu_svm_attrs *attrs,
 			       unsigned long start, unsigned long end);
@@ -165,5 +175,11 @@ amdgpu_svm_range_find_or_insert(struct amdgpu_svm *svm, unsigned long addr,
 int amdgpu_svm_range_get_pages(struct amdgpu_svm *svm,
 			       struct drm_gpusvm_range *range,
 			       struct drm_gpusvm_ctx *ctx);
+bool amdgpu_svm_range_notifier_event_begin(struct amdgpu_svm *svm,
+					   struct drm_gpusvm_range *range,
+					   const struct mmu_notifier_range *mmu_range);
+int amdgpu_svm_range_invalidate_interval(struct amdgpu_svm *svm,
+					 unsigned long start_page,
+					 unsigned long last_page);
 
 #endif /* __AMDGPU_SVM_RANGE_H__ */
-- 
2.53.0