[PATCH 15/15] drm/amdkfd: set CP_IQ_WAIT_TIME2.QUE_SLEEP on GFX 12.1

Alex Deucher <[email protected]>
Newsgroups org.freedesktop.lists.amd-gfx
Message-ID <[email protected]> 
From: Mukul Joshi <[email protected]>

The optimized QUE_SLEEP was only programmed on the HWS packet-manager
path via a PM4 WRITE_DATA packet, which is skipped when MES is enabled.

Add gfx12.1 get_iq_wait_times/build_dequeue_wait_counts_packet_info
callbacks and program CP_IQ_WAIT_TIME2 through the MES WRITE_REG op at
start_cpsch, per XCC.

Signed-off-by: Mukul Joshi <[email protected]>
Reviewed-by: Harish Kasiviswanathan <[email protected]>
Signed-off-by: Alex Deucher <[email protected]>
---
 .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c    |  3 +-
 .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h    |  3 +-
 .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c  | 36 ++++++++++++++++-
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c |  3 +-
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h |  3 +-
 .../drm/amd/amdkfd/kfd_device_queue_manager.c | 40 +++++++++++++++++++
 .../drm/amd/amdkfd/kfd_packet_manager_v9.c    |  3 +-
 .../gpu/drm/amd/include/kgd_kfd_interface.h   |  3 +-
 8 files changed, 87 insertions(+), 7 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
index 88acf75f0edd6..bc48b2ba30d11 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
@@ -1026,7 +1026,8 @@ void kgd_gfx_v10_build_dequeue_wait_counts_packet_info(struct amdgpu_device *ade
 						uint32_t sch_wave,
 						uint32_t que_sleep,
 						uint32_t *reg_offset,
-						uint32_t *reg_data)
+						uint32_t *reg_data,
+						uint32_t inst)
 {
 	*reg_data = wait_times;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h
index a4c607c881783..df350ea67c306 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h
@@ -56,7 +56,8 @@ void kgd_gfx_v10_build_dequeue_wait_counts_packet_info(struct amdgpu_device *ade
 					       uint32_t sch_wave,
 					       uint32_t que_sleep,
 					       uint32_t *reg_offset,
-					       uint32_t *reg_data);
+					       uint32_t *reg_data,
+					       uint32_t inst);
 uint64_t kgd_gfx_v10_hqd_get_pq_addr(struct amdgpu_device *adev,
 				    uint32_t pipe_id,
 				    uint32_t queue_id,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
index 070001fd34b0f..0694bff871d2d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
@@ -518,6 +518,37 @@ static void kgd_gfx_v12_1_get_cu_occupancy(struct amdgpu_device *adev,
 				adev->gfx.cu_info.max_waves_per_simd;
 }
 
+static void kgd_gfx_v12_1_get_iq_wait_times(struct amdgpu_device *adev,
+					uint32_t *wait_times,
+					uint32_t inst)
+{
+	*wait_times = RREG32_SOC15(GC, GET_INST(GC, inst), regCP_IQ_WAIT_TIME2);
+}
+
+static void kgd_gfx_v12_1_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev,
+					uint32_t wait_times,
+					uint32_t sch_wave,
+					uint32_t que_sleep,
+					uint32_t *reg_offset,
+					uint32_t *reg_data,
+					uint32_t inst)
+{
+	*reg_data = wait_times;
+
+	if (sch_wave)
+		*reg_data = REG_SET_FIELD(*reg_data,
+				CP_IQ_WAIT_TIME2,
+				SCH_WAVE,
+				sch_wave);
+	if (que_sleep)
+		*reg_data = REG_SET_FIELD(*reg_data,
+				CP_IQ_WAIT_TIME2,
+				QUE_SLEEP,
+				que_sleep);
+
+	*reg_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, inst), regCP_IQ_WAIT_TIME2);
+}
+
 const struct kfd2kgd_calls gfx_v12_1_kfd2kgd = {
 	.init_interrupts = init_interrupts_v12_1,
 	.hqd_dump = hqd_dump_v12_1,
@@ -532,5 +563,8 @@ const struct kfd2kgd_calls gfx_v12_1_kfd2kgd = {
 	.set_address_watch = kgd_gfx_v12_1_set_address_watch,
 	.clear_address_watch = kgd_gfx_v12_1_clear_address_watch,
 	.hqd_sdma_get_doorbell = kgd_gfx_v12_1_hqd_sdma_get_doorbell,
-	.get_cu_occupancy = kgd_gfx_v12_1_get_cu_occupancy
+	.get_cu_occupancy = kgd_gfx_v12_1_get_cu_occupancy,
+	.get_iq_wait_times = kgd_gfx_v12_1_get_iq_wait_times,
+	.build_dequeue_wait_counts_packet_info =
+				kgd_gfx_v12_1_build_dequeue_wait_counts_packet_info,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index 2e116c06d5be3..b30ad9701bcef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -1082,7 +1082,8 @@ void kgd_gfx_v9_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev
 		uint32_t sch_wave,
 		uint32_t que_sleep,
 		uint32_t *reg_offset,
-		uint32_t *reg_data)
+		uint32_t *reg_data,
+		uint32_t inst)
 {
 	*reg_data = wait_times;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
index 704452ca62f8e..9f43f16a1acea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
@@ -102,7 +102,8 @@ void kgd_gfx_v9_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev
 					       uint32_t sch_wave,
 					       uint32_t que_sleep,
 					       uint32_t *reg_offset,
-					       uint32_t *reg_data);
+					       uint32_t *reg_data,
+					       uint32_t inst);
 uint64_t kgd_gfx_v9_hqd_get_pq_addr(struct amdgpu_device *adev,
 				    uint32_t pipe_id,
 				    uint32_t queue_id,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index a233845711937..f6daef0e5ec09 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -2046,6 +2046,43 @@ static int unhalt_cpsch(struct device_queue_manager *dqm)
 	return ret;
 }
 
+/* Program CP_IQ_WAIT_TIME2 via the MES WRITE_REG op (MES has no HIQ). */
+static int init_dequeue_wait_counts_mes(struct device_queue_manager *dqm)
+{
+	struct kfd_node *dev = dqm->dev;
+	struct amdgpu_device *adev = dev->adev;
+	uint32_t sch_wave = 0, que_sleep = 1;
+	int inst, ret;
+
+	if (!dev->kfd2kgd->build_dequeue_wait_counts_packet_info)
+		return 0;
+
+	if (KFD_GC_VERSION(dev) != IP_VERSION(12, 1, 0))
+		return 0;
+
+	/* CP_IQ_WAIT_TIME2 is per-XCC; the offset must encode the target
+	 * XCC so MES routes the write to that XCC's local register.
+	 */
+	for_each_inst(inst, dev->xcc_mask) {
+		uint32_t reg_offset = 0, reg_data = 0;
+
+		dev->kfd2kgd->build_dequeue_wait_counts_packet_info(
+				adev, dqm->wait_times, sch_wave, que_sleep,
+				&reg_offset, &reg_data, inst);
+
+		ret = amdgpu_mes_wreg(adev, reg_offset, reg_data, inst);
+		if (ret) {
+			dev_err(adev->dev,
+				"Failed to set optimized dequeue wait via MES on xcc %d\n",
+				inst);
+			return ret;
+		}
+	}
+
+	update_dqm_wait_times(dqm);
+	return 0;
+}
+
 static int start_cpsch(struct device_queue_manager *dqm)
 {
 	struct device *dev = dqm->dev->adev->dev;
@@ -2087,6 +2124,9 @@ static int start_cpsch(struct device_queue_manager *dqm)
 				KFD_DEQUEUE_WAIT_INIT, 0 /* unused */))
 			dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n");
 		execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
+	} else {
+		if (init_dequeue_wait_counts_mes(dqm))
+			dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n");
 	}
 
 	/* setup per-queue reset detection buffer  */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
index 54868b81086e7..55d573fa3de38 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
@@ -306,7 +306,8 @@ static inline void pm_build_dequeue_wait_counts_packet_info(struct packet_manage
 		sch_value,
 		que_sleep,
 		reg_offset,
-		reg_data);
+		reg_data,
+		0);
 }
 
 /* pm_grace_period_0_supported - whether firmware tolerates a CWSR grace
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 965b50c8ca303..01c2631bbdff8 100644
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -319,7 +319,8 @@ struct kfd2kgd_calls {
 			uint32_t sch_wave,
 			uint32_t que_sleep,
 			uint32_t *reg_offset,
-			uint32_t *reg_data);
+			uint32_t *reg_data,
+			uint32_t inst);
 	void (*get_cu_occupancy)(struct amdgpu_device *adev,
 				 struct kfd_cu_occupancy *cu_occupancy,
 				 int *max_waves_per_cu, uint32_t inst);
-- 
2.55.0
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.