[PATCH] drm/amdkfd: Enable SDMA queue oversubscription for GFX 12.0.x

When all HW SDMA slots are exhausted on GFX 12.0.x with MES enabled,
allow new queues to reuse HW queue IDs round-robin via a dedicated
monotonic rotor. Each oversubscribed queue gets a unique doorbell from
the per-process bitmap pool so MES can multiplex them independently.

Normal SDMA queues and all other GPU generations are unaffected.

Assisted-by: Claude:Sonnet 4.6
Signed-off-by: Sreekant Somasekharan <[email protected]>
---
 .../drm/amd/amdkfd/kfd_device_queue_manager.c | 54 +++++++++++++++----
 .../drm/amd/amdkfd/kfd_device_queue_manager.h |  4 ++
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h         |  1 +
 3 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index a23384571193..10a672b87ca5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -556,8 +556,9 @@ static int allocate_doorbell(struct qcm_process_device *qpd,
 			return -EINVAL;
 
 		q->doorbell_id = q->properties.queue_id;
-	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
-			q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
+	} else if ((q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+			q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) &&
+			!q->properties.is_oversubscribed) {
 		/* For SDMA queues on SOC15 with 8-byte doorbell, use static
 		 * doorbell assignments based on the engine and queue id.
 		 * The doobell index distance between RLC (2*i) and (2*i+1)
@@ -583,7 +584,10 @@ static int allocate_doorbell(struct qcm_process_device *qpd,
 			return -EINVAL;
 		q->doorbell_id = valid_id;
 	} else {
-		/* For CP queues on SOC15 */
+		/* For CP queues on SOC15, and oversubscribed SDMA queues which
+		 * reuse HW queue ids and so need a unique doorbell allocated
+		 * from the per-process bitmap pool.
+		 */
 		if (restore_id) {
 			if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
 				return -EINVAL;
@@ -621,9 +625,14 @@ static void deallocate_doorbell(struct qcm_process_device *qpd,
 	unsigned int old;
 	struct kfd_node *dev = qpd->dqm->dev;
 
+	/* Oversubscribed SDMA queues allocate a doorbell from the bitmap pool
+	 * (like CP queues), so they must be freed; normal SDMA queues use
+	 * static doorbell assignments and must not be.
+	 */
 	if (!KFD_IS_SOC15(dev) ||
-	    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
-	    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
+	    ((q->properties.type == KFD_QUEUE_TYPE_SDMA ||
+	      q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) &&
+	     !q->properties.is_oversubscribed))
 		return;
 
 	old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
@@ -1800,15 +1809,33 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
 {
 	struct device *dev = dqm->dev->adev->dev;
 	int bit;
+	bool is_oversubscribed = false;
 
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
 		if (bitmap_empty(dqm->sdma_bitmap, get_num_sdma_queues(dqm))) {
-			dev_warn(dev, "No more SDMA queue to allocate (%d total queues)\n",
-				 get_num_sdma_queues(dqm));
-			return -ENOMEM;
-		}
+			/* All HW queues are busy. On GFX 12.0.x under MES,
+			 * oversubscribe by reusing a HW queue id round-robin;
+			 * MES multiplexes the SW queues via their unique
+			 * doorbells. The non-MES HWS runlist path cannot express
+			 * duplicate sdma_ids, so it still fails with -ENOMEM.
+			 */
+			unsigned int num_reserved, num_available;
+
+			if (KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) ||
+			    KFD_GC_VERSION(dqm->dev) >= IP_VERSION(12, 1, 0) ||
+			    !dqm->dev->kfd->shared_resources.enable_mes) {
+				dev_warn(dev, "No more SDMA queue to allocate (%d total queues)\n",
+					 get_num_sdma_queues(dqm));
+				return -ENOMEM;
+			}
 
-		if (restore_sdma_id) {
+			num_reserved = kfd_get_num_sdma_engines(dqm->dev) *
+				dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine;
+			num_available = get_num_sdma_queues(dqm) - num_reserved;
+			q->sdma_id = num_reserved +
+				dqm->sdma_oversub_rotor++ % num_available;
+			is_oversubscribed = true;
+		} else if (restore_sdma_id) {
 			if (*restore_sdma_id >= get_num_sdma_queues(dqm))
 				return -EINVAL;
 
@@ -1901,6 +1928,7 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
 			return -ENOMEM;
 		}
 	}
+	q->properties.is_oversubscribed = is_oversubscribed;
 
 	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
 	pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
@@ -1914,7 +1942,11 @@ static void deallocate_sdma_queue(struct device_queue_manager *dqm,
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
 		if (q->sdma_id >= get_num_sdma_queues(dqm))
 			return;
-		set_bit(q->sdma_id, dqm->sdma_bitmap);
+		/* Don't return oversubscribed queue IDs to bitmap.
+		 * They are shared by multiple queues and managed via round-robin.
+		 */
+		if (!q->properties.is_oversubscribed)
+			set_bit(q->sdma_id, dqm->sdma_bitmap);
 	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
 		if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
 			return;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index c9f9f7a87111..81626e657055 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -255,6 +255,10 @@ struct device_queue_manager {
 	unsigned int		gws_queue_count;
 	unsigned int		total_queue_count;
 	unsigned int		next_pipe_to_allocate;
+	/* Monotonically increasing counter used to round-robin oversubscribed
+	 * SDMA queues across the available HW queue slots.
+	 */
+	unsigned int		sdma_oversub_rotor;
 	unsigned int		*allocated_queues;
 	DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES);
 	DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index d8631847f0eb..5fff5e55fb09 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -531,6 +531,7 @@ struct queue_properties {
 	bool is_dbg_wa;
 	bool is_user_cu_masked;
 	bool is_reset;
+	bool is_oversubscribed;
 	/* Not relevant for user mode queues in cp scheduling */
 	unsigned int vmid;
 	/* Relevant only for sdma queues*/
-- 
2.43.0