[PATCH v9 06/18] drm/amdgpu/gmc: add get_svm_pte_flags callback

From: Honglei Huang <[email protected]>

Add a gmc_funcs callback for computing the GPU PTE flags of an SVM
mapping, plus a HW-agnostic input struct (amdgpu_svm_pte_flags_params)
and the amdgpu_gmc_get_svm_pte_flags() wrapper. This lets the per-IP
MTYPE/PTE selection live in the HW IP specific gmc_vX_0.c files instead
of the SVM core, mirroring how get_vm_pte handles BO mappings.

Signed-off-by: Honglei Huang <[email protected]>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  24 +++++
 drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c  |  69 ++++++++++++++
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c   | 119 ++++++++++++++++++++++++
 3 files changed, 212 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index ddb0d500e0faa..448cfda67719d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -147,6 +147,24 @@ struct amdgpu_vmhub {
 	const struct amdgpu_vmhub_funcs *vmhub_funcs;
 };
 
+/* Decoded inputs for the SVM PTE flag selection, gmc_funcs::get_svm_pte_flags */
+struct amdgpu_svm_pte_flags_params {
+	/* SVM attribute: coherent */
+	bool coherent;
+	/* SVM attribute: extended coherent */
+	bool ext_coherent;
+	/* SVM attribute: GPU read only */
+	bool gpu_ro;
+	/* SVM attribute: GPU executable */
+	bool gpu_exec;
+	/* mapping is in VRAM local device or remote P2P */
+	bool is_vram;
+	/* mapping is in this GPU's local VRAM */
+	bool is_local;
+	/* remote VRAM BO is on a GPU in the same XGMI hive as this GPU. */
+	bool same_hive;
+};
+
 /*
  * GPU MC structures, functions & helpers
  */
@@ -175,6 +193,10 @@ struct amdgpu_gmc_funcs {
 			   struct amdgpu_bo *bo,
 			   uint32_t vm_flags,
 			   uint64_t *pte_flags);
+	/* get the full pte flags for an SVM mapping */
+	uint64_t (*get_svm_pte_flags)(struct amdgpu_device *adev,
+				      struct amdgpu_vm *vm,
+				      const struct amdgpu_svm_pte_flags_params *params);
 	/* override per-page pte flags */
 	void (*override_vm_pte_flags)(struct amdgpu_device *dev,
 				      struct amdgpu_vm *vm,
@@ -376,6 +398,8 @@ struct amdgpu_gmc {
 #define amdgpu_gmc_get_vm_pte(adev, vm, bo, vm_flags, pte_flags) \
 	((adev)->gmc.gmc_funcs->get_vm_pte((adev), (vm), (bo), (vm_flags), \
 					   (pte_flags)))
+#define amdgpu_gmc_get_svm_pte_flags(adev, vm, params) \
+	((adev)->gmc.gmc_funcs->get_svm_pte_flags((adev), (vm), (params)))
 #define amdgpu_gmc_override_vm_pte_flags(adev, vm, addr, pte_flags)	\
 	(adev)->gmc.gmc_funcs->override_vm_pte_flags			\
 		((adev), (vm), (addr), (pte_flags))
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index 84c93364d2201..2e599c0212748 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -548,6 +548,74 @@ static void gmc_v12_0_get_vm_pte(struct amdgpu_device *adev,
 		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC);
 }
 
+/*
+ * Mirror of the GFX12 cases of svm_range_get_pte_flags(), with the
+ * per IP logic kept here in the HW specific file.
+ */
+static uint64_t
+gmc_v12_0_get_svm_pte_flags(struct amdgpu_device *adev,
+			    struct amdgpu_vm *vm,
+			    const struct amdgpu_svm_pte_flags_params *params)
+{
+	uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
+	bool coherent = params->coherent;
+	bool ext_coherent = params->ext_coherent;
+	bool is_local = params->is_local;
+	bool is_vram = params->is_vram;
+	bool snoop = !is_vram;
+	uint32_t mapping_flags = 0;
+	unsigned int mtype_local, mtype_remote;
+	bool is_aid_a1;
+	uint64_t pte_flags;
+
+	switch (gc_ip_version) {
+	case IP_VERSION(12, 0, 0):
+	case IP_VERSION(12, 0, 1):
+		mapping_flags |= AMDGPU_VM_MTYPE_NC;
+		break;
+	case IP_VERSION(12, 1, 0):
+		is_aid_a1 = (adev->rev_id & 0x10);
+		mtype_local = amdgpu_mtype_local == 0 ? AMDGPU_VM_MTYPE_RW :
+				amdgpu_mtype_local == 1 ? AMDGPU_VM_MTYPE_NC :
+				is_aid_a1 ? AMDGPU_VM_MTYPE_RW : AMDGPU_VM_MTYPE_NC;
+		mtype_remote = is_aid_a1 ? AMDGPU_VM_MTYPE_NC : AMDGPU_VM_MTYPE_UC;
+		snoop = true;
+
+		if (is_local) {
+			mapping_flags |= mtype_local;
+		} else if (ext_coherent) {
+			mapping_flags |= AMDGPU_VM_MTYPE_UC;
+		} else {
+			/* system memory or remote VRAM */
+			mapping_flags |= mtype_remote;
+		}
+		break;
+	default:
+		mapping_flags |= coherent ?
+			AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
+		break;
+	}
+
+	if (params->gpu_exec)
+		mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
+
+	pte_flags = AMDGPU_PTE_VALID;
+	pte_flags |= is_vram ? 0 : AMDGPU_PTE_SYSTEM;
+	pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
+	pte_flags |= AMDGPU_PTE_IS_PTE;
+
+	gmc_v12_0_get_vm_pte(adev, vm, NULL, mapping_flags, &pte_flags);
+	pte_flags |= AMDGPU_PTE_READABLE;
+	if (!params->gpu_ro)
+		pte_flags |= AMDGPU_PTE_WRITEABLE;
+
+	if (gc_ip_version == IP_VERSION(12, 1, 0) &&
+	    adev->have_atomics_support)
+		pte_flags |= AMDGPU_PTE_BUS_ATOMICS;
+
+	return pte_flags;
+}
+
 static unsigned gmc_v12_0_get_vbios_fb_size(struct amdgpu_device *adev)
 {
 	return 0;
@@ -577,6 +645,7 @@ static const struct amdgpu_gmc_funcs gmc_v12_0_gmc_funcs = {
 	.emit_pasid_mapping = gmc_v12_0_emit_pasid_mapping,
 	.get_vm_pde = gmc_v12_0_get_vm_pde,
 	.get_vm_pte = gmc_v12_0_get_vm_pte,
+	.get_svm_pte_flags = gmc_v12_0_get_svm_pte_flags,
 	.get_vbios_fb_size = gmc_v12_0_get_vbios_fb_size,
 	.get_dcc_alignment = gmc_v12_0_get_dcc_alignment,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 8a5c44810ba1e..b85a83ce2ce02 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -1198,6 +1198,124 @@ static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
 		gmc_v9_0_get_coherence_flags(adev, vm, bo, vm_flags, flags);
 }
 
+/*
+ * Mirror of the GFX9 cases of svm_range_get_pte_flags(), with the
+ * per IP logic kept here in the HW specific file.
+ */
+static uint64_t
+gmc_v9_0_get_svm_pte_flags(struct amdgpu_device *adev,
+			   struct amdgpu_vm *vm,
+			   const struct amdgpu_svm_pte_flags_params *params)
+{
+	uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
+	bool coherent = params->coherent;
+	bool ext_coherent = params->ext_coherent;
+	bool is_local = params->is_local;
+	bool is_vram = params->is_vram;
+	bool same_hive = params->same_hive;
+	bool snoop = !is_vram;
+	uint32_t mapping_flags = 0;
+	unsigned int mtype_local;
+	uint64_t pte_flags;
+
+	switch (gc_ip_version) {
+	case IP_VERSION(9, 4, 1):
+		if (is_vram) {
+			if (is_local) {
+				mapping_flags |= coherent ?
+					AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;
+			} else {
+				mapping_flags |= coherent ?
+					AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
+				if (same_hive)
+					snoop = true;
+			}
+		} else {
+			mapping_flags |= coherent ?
+				AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
+		}
+		break;
+	case IP_VERSION(9, 4, 2):
+		if (is_vram) {
+			if (is_local) {
+				mapping_flags |= coherent ?
+					AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;
+				if (adev->gmc.xgmi.connected_to_cpu)
+					snoop = true;
+			} else {
+				mapping_flags |= coherent ?
+					AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
+				if (same_hive)
+					snoop = true;
+			}
+		} else {
+			mapping_flags |= coherent ?
+				AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
+		}
+		break;
+	case IP_VERSION(9, 4, 3):
+	case IP_VERSION(9, 4, 4):
+	case IP_VERSION(9, 5, 0):
+		if (ext_coherent)
+			mtype_local = AMDGPU_VM_MTYPE_CC;
+		else
+			mtype_local = amdgpu_mtype_local == 1 ? AMDGPU_VM_MTYPE_NC :
+				amdgpu_mtype_local == 2 ? AMDGPU_VM_MTYPE_CC :
+				AMDGPU_VM_MTYPE_RW;
+		snoop = true;
+		if (is_vram) {
+			/* local HBM region close to partition */
+			if (is_local)
+				mapping_flags |= mtype_local;
+			/* local HBM region far from partition or remote XGMI GPU
+			 * with regular system scope coherence
+			 */
+			else if (same_hive && !ext_coherent)
+				mapping_flags |= AMDGPU_VM_MTYPE_NC;
+			/* PCIe P2P on GPUs pre-9.5.0 */
+			else if (gc_ip_version < IP_VERSION(9, 5, 0) && !same_hive)
+				mapping_flags |= AMDGPU_VM_MTYPE_UC;
+			/* Other remote memory */
+			else
+				mapping_flags |= ext_coherent ?
+					AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
+		} else if (adev->flags & AMD_IS_APU) {
+			/* On NUMA systems, locality is determined per-page
+			 * in gmc_v9_0_override_vm_pte_flags.
+			 */
+			if (num_possible_nodes() <= 1)
+				mapping_flags |= mtype_local;
+			else
+				mapping_flags |= ext_coherent ?
+					AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
+		} else {
+			if (gc_ip_version < IP_VERSION(9, 5, 0) || ext_coherent)
+				mapping_flags |= AMDGPU_VM_MTYPE_UC;
+			else
+				mapping_flags |= AMDGPU_VM_MTYPE_NC;
+		}
+		break;
+	default:
+		mapping_flags |= coherent ?
+			AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
+		break;
+	}
+
+	if (params->gpu_exec)
+		mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
+
+	pte_flags = AMDGPU_PTE_VALID;
+	pte_flags |= is_vram ? 0 : AMDGPU_PTE_SYSTEM;
+	pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
+
+	gmc_v9_0_get_vm_pte(adev, vm, NULL, mapping_flags, &pte_flags);
+	pte_flags |= AMDGPU_PTE_READABLE;
+	if (!params->gpu_ro)
+		pte_flags |= AMDGPU_PTE_WRITEABLE;
+
+	return pte_flags;
+}
+
 static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
 					   struct amdgpu_vm *vm,
 					   uint64_t addr, uint64_t *flags)
@@ -1315,6 +1433,7 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
 	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
 	.get_vm_pde = gmc_v9_0_get_vm_pde,
 	.get_vm_pte = gmc_v9_0_get_vm_pte,
+	.get_svm_pte_flags = gmc_v9_0_get_svm_pte_flags,
 	.override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,
 	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
 	.query_mem_partition_mode = &amdgpu_gmc_query_memory_partition,
-- 
2.53.0