[PATCH 74/95] drm/amdkfd: program compute MQD coherent_aql_mtype on GFX 12.1

From: Mukul Joshi <[email protected]>

Name the compute MQD dword at offset 509 (previously reserved_509) as
coherent_aql_mtype in v12_1_compute_mqd and program it when initializing
a compute queue MQD.

The field must be set to 0 whenever the driver maps local or remote
memory as MTYPE_NC, and to 1 in all other cases. Add a shared helper
gmc_v12_1_get_mtypes() as the single source of truth for the local and
remote MTYPE decision (used by both the PTE coherence flags and the MQD
programming), and gmc_v12_1_get_coherent_aql_mtype() which derives the
0/1 value from it.

Signed-off-by: Mukul Joshi <[email protected]>
Reviewed-by: Felix Kuehling <[email protected]>
Signed-off-by: Alex Deucher <[email protected]>
---
 drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c        | 63 +++++++++++++------
 drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h        |  1 +
 .../drm/amd/amdkfd/kfd_mqd_manager_v12_1.c    |  7 +++
 drivers/gpu/drm/amd/include/v12_structs.h     |  2 +-
 4 files changed, 54 insertions(+), 19 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
index fd24d66290f55..a3b7e240abaed 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
@@ -616,6 +616,50 @@ static void gmc_v12_1_get_npa_flags(struct amdgpu_device *adev,
 	*flags &= ~AMDGPU_PTE_EXECUTABLE;
 }
 
+/*
+ * Resolve the MTYPEs used for local and remote memory accesses on GFX 12.1.
+ * Remote memory always uses MTYPE_UC; local memory depends on the AID stepping
+ * and the amdgpu_mtype_local module parameter.
+ */
+static void gmc_v12_1_get_mtypes(struct amdgpu_device *adev,
+				 unsigned int *mtype_local,
+				 unsigned int *mtype_remote)
+{
+	bool is_aid_a1 = (adev->rev_id & 0x10);
+
+	*mtype_local = is_aid_a1 ? MTYPE_RW : MTYPE_NC;
+	/* Remote memory always uses MTYPE_UC on GFX 12.1. */
+	*mtype_remote = MTYPE_UC;
+
+	if (amdgpu_mtype_local == 0) {
+		DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
+		*mtype_local = MTYPE_RW;
+	} else if (amdgpu_mtype_local == 1) {
+		DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
+		*mtype_local = MTYPE_NC;
+	} else if (amdgpu_mtype_local == 2) {
+		DRM_INFO_ONCE("MTYPE_CC not supported, using %s for local memory\n",
+			      is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC");
+	} else {
+		DRM_INFO_ONCE("Using %s for local memory and MTYPE_UC for remote memory\n",
+			      is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC");
+	}
+}
+
+/*
+ * The compute MQD coherent_aql_mtype field (offset 509) must be programmed to
+ * 0 whenever the driver maps local or remote memory as MTYPE_NC, and to 1 in
+ * all other cases.
+ */
+u32 gmc_v12_1_get_coherent_aql_mtype(struct amdgpu_device *adev)
+{
+	unsigned int mtype_local, mtype_remote;
+
+	gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote);
+
+	return (mtype_local == MTYPE_NC || mtype_remote == MTYPE_NC) ? 0 : 1;
+}
+
 static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev,
 					  struct amdgpu_bo *bo,
 					  uint64_t *flags)
@@ -631,27 +675,10 @@ static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev,
 	unsigned int mtype, mtype_local, mtype_remote;
 	bool snoop = false;
 	bool is_local = false;
-	bool is_aid_a1;
 
 	switch (gc_ip_version) {
 	case IP_VERSION(12, 1, 0):
-		is_aid_a1 = (adev->rev_id & 0x10);
-
-		mtype_local = is_aid_a1 ? MTYPE_RW : MTYPE_NC;
-		/* Remote memory always uses MTYPE_UC on GFX 12.1. */
-		mtype_remote = MTYPE_UC;
-		if (amdgpu_mtype_local == 0) {
-			DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
-			mtype_local = MTYPE_RW;
-		} else if (amdgpu_mtype_local == 1) {
-			DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
-			mtype_local = MTYPE_NC;
-		} else if (amdgpu_mtype_local == 2) {
-			DRM_INFO_ONCE("MTYPE_CC not supported, using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC");
-		} else {
-			DRM_INFO_ONCE("Using %s for local memory and MTYPE_UC for remote memory\n",
-					is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC");
-		}
+		gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote);
 
 		is_local = (is_vram && adev == bo_adev);
 		snoop = true;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h
index 22da84a1d5185..1c7aad7a1791d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h
@@ -28,4 +28,5 @@ void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev);
 void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev);
 void gmc_v12_1_init_vram_info(struct amdgpu_device *adev);
 void gmc_v12_1_init_nps_details(struct amdgpu_device *adev);
+u32 gmc_v12_1_get_coherent_aql_mtype(struct amdgpu_device *adev);
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
index 13a9ce02f3918..92ec081842df6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
@@ -31,6 +31,7 @@
 #include "gc/gc_12_1_0_sh_mask.h"
 #include "amdgpu_amdkfd.h"
 #include "kfd_device_queue_manager.h"
+#include "gmc_v12_1.h"
 
 static void update_mqd(struct mqd_manager *mm, void *mqd,
 		       struct queue_properties *q,
@@ -235,6 +236,12 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
 	}
 
+	/*
+	 * coherent_aql_mtype (offset 509): program 0 when the driver maps local
+	 * or remote memory as MTYPE_NC, and 1 in all other cases.
+	 */
+	m->coherent_aql_mtype = gmc_v12_1_get_coherent_aql_mtype(mm->dev->adev);
+
 	*mqd = m;
 	if (gart_addr)
 		*gart_addr = addr;
diff --git a/drivers/gpu/drm/amd/include/v12_structs.h b/drivers/gpu/drm/amd/include/v12_structs.h
index 28f4b01326811..7fa0c07b51c67 100644
--- a/drivers/gpu/drm/amd/include/v12_structs.h
+++ b/drivers/gpu/drm/amd/include/v12_structs.h
@@ -1696,7 +1696,7 @@ struct v12_1_compute_mqd {
     uint32_t reserved_506; // offset: 506  (0x1FA)
     uint32_t reserved_507; // offset: 507  (0x1FB)
     uint32_t reserved_508; // offset: 508  (0x1FC)
-    uint32_t reserved_509; // offset: 509  (0x1FD)
+    uint32_t coherent_aql_mtype; // offset: 509  (0x1FD)
     uint32_t reserved_510; // offset: 510  (0x1FE)
     uint32_t reserved_511; // offset: 511  (0x1FF)
     uint32_t reserved_512; // offset: 512  (0x200)
-- 
2.55.0