[PATCH] drm/amdkfd: zero SDMA mid-command save state before restoring user queues

From: Andrew Martin <[email protected]>

Applies to gfx10 (RDNA2) and gfx11 (RDNA3).

The SDMA firmware's saved mid-command state for a preempted COPY_LINEAR
(sdmax_rlcx_midcmd_data0 = source byte offset) has been observed to
overshoot the packet's byte count on gfx10.3 (Navi21, XNACK off) after
rapid quiesce/resume cycles driven by userptr/SVM eviction. On
MAP_QUEUES the engine resumes the copy at src+offset past the buffer:
an unmapped VA turns into a UTCL2 'Page not present' fault, a mapped
one into silent data corruption.

The silent corruption variant produces no dmesg output and is only
detectable through application-level data integrity checks (checksums,
validation). This makes diagnosis extremely difficult as there is no
kernel-level indication of the failure.

Zero the mid-command save area before remapping so the pending packet
re-executes from its packet boundary (idempotent for COPY/GCR/FENCE).
Workaround for field crashes of ROCm workloads on RDNA2/RDNA3.

Signed-off-by: Andrew Martin <[email protected]>
Assisted-by: Claude:Sonnet 4-5
---
 .../drm/amd/amdkfd/kfd_device_queue_manager.c | 37 ++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 51ee9c39104b..1507d371f741 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -40,6 +40,8 @@
 #include "amdgpu_ring.h"
 #include "amdgpu_mes.h"
 #include "kfd_debug.h"
+#include "v10_structs.h"
+#include "v11_structs.h"
 
 /* Size of the per-pipe EOP queue */
 #define CIK_HPD_EOP_BYTES_LOG2 11
@@ -1447,6 +1449,37 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
 	return ret;
 }
 
+/*
+ * Zero SDMA mid-command state before MAP_QUEUES. Firmware bug: preempted
+ * COPY_LINEAR can resume at corrupted offset past buffer (silent corruption
+ * or VM fault). Re-executing from packet start is idempotent and safe.
+ */
+static void clamp_sdma_midcmd_state(struct device_queue_manager *dqm,
+				    struct qcm_process_device *qpd)
+{
+	struct queue *q;
+	uint32_t gc_ver = KFD_GC_VERSION(dqm->dev);
+
+	if (gc_ver < IP_VERSION(10, 1, 1) || gc_ver >= IP_VERSION(12, 0, 0))
+		return;
+
+	list_for_each_entry(q, &qpd->queues_list, list) {
+		if ((q->properties.type != KFD_QUEUE_TYPE_SDMA &&
+		     q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI) || !q->mqd)
+			continue;
+
+		if (gc_ver < IP_VERSION(11, 0, 0)) {
+			struct v10_sdma_mqd *m = q->mqd;
+
+			memset(&m->sdmax_rlcx_midcmd_data0, 0, 10 * sizeof(uint32_t));
+		} else {
+			struct v11_sdma_mqd *m = q->mqd;
+
+			memset(&m->sdmax_rlcx_midcmd_data0, 0, 12 * sizeof(uint32_t));
+		}
+	}
+}
+
 static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
@@ -1498,9 +1531,11 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
 			}
 		}
 	}
-	if (!dqm->dev->kfd->shared_resources.enable_mes)
+	if (!dqm->dev->kfd->shared_resources.enable_mes) {
+		clamp_sdma_midcmd_state(dqm, qpd);
 		retval = execute_queues_cpsch(dqm,
 					      KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
+	}
 	eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
 	atomic64_add(eviction_duration, &pdd->evict_duration_counter);
 vm_not_acquired:
-- 
2.43.0