[PATCH v9 18/18] drm/amdgpu: integrate VRAM migration into SVM fault and prefetch paths

From: Junhua Shen <[email protected]>

- Call amdgpu_svm_range_migrate_to_vram() before GPU mapping in
  fault and prefetch paths when VRAM migration is needed
- Set device_private_page_owner in drm_gpusvm_ctx for devmem awareness
- Allow AMDGPU_INTERCONNECT_VRAM entries in update_gpu_range proto check
- Skip valid-mapping fast path in prefetch when VRAM migration is
  requested, so prefetch can actively move pages to VRAM
- Enable amdgpu_pagemap_capable() check in amdgpu_svm_devmem_possible()
- Move AMDGPU_INTERCONNECT_VRAM/P2P defines to migration header

Signed-off-by: Junhua Shen <[email protected]>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c       |  4 +++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm.h       |  3 ---
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c | 11 ++++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 23 ++++++++++++++++---
 4 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c
index ac101574bd61d..8b3747e444f02 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c
@@ -35,6 +35,7 @@
 #include "amdgpu_svm_attr.h"
 #include "amdgpu_svm_fault.h"
 #include "amdgpu_svm_range.h"
+#include "amdgpu_svm_range_migrate.h"
 #include "amdgpu_vm.h"
 
 #if IS_ENABLED(CONFIG_DRM_AMDGPU_SVM)
@@ -314,7 +315,8 @@ bool amdgpu_svm_devmem_possible(struct amdgpu_svm *svm)
 	if (svm->adev->apu_prefer_gtt)
 		return false;
 
-	/* TODO: add amdgpu_pagemap_capable() */
+	if (amdgpu_pagemap_capable(svm))
+		return true;
 
 	return false;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.h
index c67f01b3b5d3b..a28cce16cd9ac 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.h
@@ -45,9 +45,6 @@ struct amdgpu_svm_attrs;
 struct drm_device;
 struct drm_file;
 
-#define AMDGPU_INTERCONNECT_VRAM	DRM_INTERCONNECT_DRIVER
-#define AMDGPU_INTERCONNECT_P2P		(AMDGPU_INTERCONNECT_VRAM + 1)
-
 enum amdgpu_svm_xnack_mode {
 	AMDGPU_SVM_XNACK_OFF,
 	AMDGPU_SVM_XNACK_ON,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c
index f28b3eb14500d..6ac7e583c9bc5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_fault.c
@@ -26,6 +26,8 @@
 #include "amdgpu_svm_attr.h"
 #include "amdgpu_svm_fault.h"
 #include "amdgpu_svm_range.h"
+#include "amdgpu_svm_range_migrate.h"
+#include "amdgpu_migrate.h"
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
 #include "amdgpu_gmc.h"
@@ -186,6 +188,8 @@ static int fault_map_range(struct amdgpu_svm *svm,
 		.check_pages_threshold = devmem_possible ? SZ_64K : 0,
 		.devmem_only = need_vram_migration,
 		.timeslice_ms = need_vram_migration ? 5 : 0,
+		.device_private_page_owner = devmem_possible ?
+			AMDGPU_PGMAP_OWNER(svm->adev) : NULL,
 	};
 	struct amdgpu_svm_range *range;
 	ktime_t timestamp = ktime_get_boottime();
@@ -255,7 +259,12 @@ static int fault_map_range(struct amdgpu_svm *svm,
 	}
 
 	AMDGPU_SVM_RANGE_TRACE(range, "PAGE FAULT");
-	/* TODO: add migration*/
+	if (need_vram_migration) {
+		AMDGPU_SVM_RANGE_TRACE(range, "MIGRATE PAGES");
+		ret = amdgpu_svm_range_migrate_to_vram(svm, &range->base);
+		if (ret)
+			return ret;
+	}
 
 	AMDGPU_SVM_RANGE_TRACE(range, "GET PAGES");
 	ret = amdgpu_svm_range_get_pages(svm, &range->base, &map_ctx);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
index ed057545d0964..8ee83904fd532 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
@@ -26,6 +26,8 @@
 #include "amdgpu_svm_attr.h"
 #include "amdgpu_svm_range.h"
 #include "amdgpu_svm_fault.h"
+#include "amdgpu_svm_range_migrate.h"
+#include "amdgpu_migrate.h"
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
 #include "amdgpu_trace.h"
@@ -198,7 +200,8 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm,
 		unsigned long start_page, last_page;
 		bool is_last_seg;
 
-		if (entry->proto != DRM_INTERCONNECT_SYSTEM)
+		if (entry->proto != DRM_INTERCONNECT_SYSTEM &&
+		    entry->proto != AMDGPU_INTERCONNECT_VRAM)
 			return -EOPNOTSUPP;
 
 		pte_flags = amdgpu_svm_range_attr_pte_flags(svm, attrs,
@@ -358,6 +361,8 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm,
 		.devmem_possible = devmem_possible,
 		.devmem_only = need_vram_migration,
 		.check_pages_threshold = devmem_possible ? SZ_64K : 0,
+		.device_private_page_owner = devmem_possible ?
+			AMDGPU_PGMAP_OWNER(svm->adev) : NULL,
 	};
 
 	while (addr < end) {
@@ -380,12 +385,24 @@ amdgpu_svm_range_map_attrs(struct amdgpu_svm *svm,
 		if (next_addr <= addr)
 			return -EINVAL;
 
-		if (amdgpu_svm_range_is_valid(range, attrs)) {
+		/*
+		 * Prefetch to device memory should still run migration logic even if
+		 * the current GPU mapping is already valid. This allows prefetch
+		 * requests to actively move backing pages to VRAM instead of being
+		 * skipped by the valid-mapping fast path.
+		 */
+		if (amdgpu_svm_range_is_valid(range, attrs) &&
+		    !need_vram_migration) {
 			addr = next_addr;
 			continue;
 		}
 
-		/* TODO: add migration */
+		if (need_vram_migration) {
+			AMDGPU_SVM_RANGE_TRACE(range, "MIGRATE PAGES");
+			ret = amdgpu_svm_range_migrate_to_vram(svm, &range->base);
+			if (ret)
+				return ret;
+		}
 
 		AMDGPU_SVM_RANGE_TRACE(range, "GET PAGES");
 
-- 
2.53.0