Re: [PATCH v2 2/2] drm/amdkfd: enable rs64mem for kfd queue
"Chen, Michael" <[email protected]> Mon, 3 Aug 2026 20:30:32 +0000
| Newsgroups | org.freedesktop.lists.amd-gfx |
|---|---|
| Message-ID | <DM6PR12MB3611DD14C882B3CBAA936FE194D52@DM6PR12MB3611.namprd12.prod.outlook.com> |
--_000_DM6PR12MB3611DD14C882B3CBAA936FE194D52DM6PR12MB3611namp_ Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable AMD General ________________________________ From: Liang, Prike <[email protected]> Sent: Wednesday, July 29, 2026 10:09 PM To: [email protected] <[email protected]> Cc: Deucher, Alexander <[email protected]>; Koenig, Christian <Chri= [email protected]>; Chen, Michael <[email protected]>; Liu, Shaoyun <= [email protected]>; Liang, Prike <[email protected]> Subject: [PATCH v2 2/2] drm/amdkfd: enable rs64mem for kfd queue Enabled RS64mem for KFD queues by integrating process and gang context index allocation in the per KFD device process and queue creation. Signed-off-by: Prike Liang <[email protected]> --- drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 3 +++ drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 3 +++ drivers/gpu/drm/amd/amdkfd/kfd_process.c | 5 ++++- drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c | 10 ++++++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/driver= s/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 51ee9c39104b..ea9d87450eae 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -226,8 +226,10 @@ static int add_queue_mes(struct device_queue_manager *= dqm, struct queue *q, /* MES unit for quantum is 100ns */ queue_input.process_quantum =3D KFD_MES_PROCESS_QUANTUM; /* Equiv= alent to 10ms. */ queue_input.process_context_addr =3D pdd->proc_ctx_gpu_addr; + queue_input.process_context_array_index =3D pdd->proc_ctx_array_ind= ex; queue_input.gang_quantum =3D KFD_MES_GANG_QUANTUM; /* Equivalent t= o 1ms */ queue_input.gang_context_addr =3D q->gang_ctx_gpu_addr; + queue_input.gang_context_array_index =3D q->gang_ctx_array_index; queue_input.inprocess_gang_priority =3D q->properties.priority; queue_input.gang_global_priority_level =3D AMDGPU_MES_PRIORITY_LEVEL_NORMAL; @@ -303,6 +305,7 @@ static int remove_queue_mes_on_reset_option(struct devi= ce_queue_manager *dqm, st queue_input.queue_type =3D convert_to_amdgpu_ring_type(q->properti= es.type); queue_input.remove_queue_after_reset =3D flush_mes_queue; queue_input.xcc_id =3D ffs(dqm->dev->xcc_mask) - 1; + queue_input.gang_context_array_index =3D q->gang_ctx_array_index; amdgpu_mes_lock(&adev->mes); r =3D adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/am= dkfd/kfd_priv.h index 88191a4c1657..3b89bcc625b3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -635,6 +635,7 @@ struct queue { void *gang_ctx_bo; uint64_t gang_ctx_gpu_addr; void *gang_ctx_cpu_ptr; + uint32_t gang_ctx_array_index; struct amdgpu_bo *wptr_bo_gart; }; @@ -871,6 +872,8 @@ struct kfd_process_device { uint64_t proc_ctx_gpu_addr; void *proc_ctx_cpu_ptr; + uint32_t proc_ctx_array_index; + /* Tracks queue reset status */ bool has_reset_queue; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd= /amdkfd/kfd_process.c index eb508fe3ded7..0a7c1900da95 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -1215,9 +1215,12 @@ static void kfd_process_destroy_pdds(struct kfd_proc= ess *p) kfd_free_process_doorbells(pdd->dev->kfd, pdd); if (pdd->dev->kfd->shared_resources.enable_mes && - pdd->proc_ctx_cpu_ptr) + pdd->proc_ctx_cpu_ptr) { + amdgpu_mes_free_proc_ctx_index(&pdd->dev->adev->mes= , + pdd->proc_ctx_array_= index); amdgpu_amdkfd_free_kernel_mem(pdd->dev->adev, &pdd->proc_ctx_bo); + } /* * before destroying pdd, make sure to report availability * for auto suspend diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drive= rs/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 8054e4fe0381..72913b5c8338 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -210,6 +210,7 @@ static void pqm_clean_queue_resource(struct process_que= ue_manager *pqm, } if (dev->kfd->shared_resources.enable_mes) { + amdgpu_mes_free_gang_ctx_index(&dev->adev->mes, pqn->q->gan= g_ctx_array_index); amdgpu_amdkfd_free_kernel_mem(dev->adev, &pqn->q->gang_ctx= _bo); amdgpu_amdkfd_free_kernel_mem(dev->adev, (void **)&pqn->q-= >wptr_bo_gart); } @@ -282,6 +283,9 @@ static int init_user_queue(struct process_queue_manager= *pqm, goto cleanup; } memset((*q)->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE= ); + /* Bind one MES gang context slot per queue (gang). */ + amdgpu_mes_alloc_gang_ctx_index(&dev->adev->mes, + &(*q)->gang_ctx_array_index= ); 1. Should check "mes->use_rs64mem" as you do in patch #1? 2. Check return error code (in oversubscription case, it may run out of index) /* Starting with GFX11, wptr BOs must be mapped to GART fo= r MES to determine work * on unmapped queues for usermode queue oversubscription = (no aggregated doorbell) @@ -304,6 +308,7 @@ static int init_user_queue(struct process_queue_manager= *pqm, return 0; free_gang_ctx_bo: + amdgpu_mes_free_gang_ctx_index(&dev->adev->mes, (*q)->gang_ctx_arra= y_index); amdgpu_amdkfd_free_kernel_mem(dev->adev, &(*q)->gang_ctx_bo); cleanup: uninit_queue(*q); @@ -386,6 +391,11 @@ int pqm_create_queue(struct process_queue_manager *pqm= , goto err_allocate_pqn; } memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE)= ; + /* Bind one MES process context slot to the whole process + * (per device); every queue of this process reuses it. + */ + amdgpu_mes_alloc_proc_ctx_index(&dev->adev->mes, + &pdd->proc_ctx_array_index)= ; Same comment as above. } pqn =3D kzalloc_obj(*pqn); -- 2.34.1 --_000_DM6PR12MB3611DD14C882B3CBAA936FE194D52DM6PR12MB3611namp_ Content-Type: text/html; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable <html> <head> <meta http-equiv=3D"Content-Type" content=3D"text/html; charset=3Diso-8859-= 1"> <style type=3D"text/css" style=3D"display:none;"> P {margin-top:0;margin-bo= ttom:0;} </style> </head> <body dir=3D"ltr"> <div class=3D"elementToProof" style=3D"font-family: Aptos, Aptos_EmbeddedFo= nt, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; c= olor: rgb(0, 0, 0);"> <div> <div style=3D"font-family: Calibri; text-align: left; color: rgb(0, 0, 255)= ; margin-left: 5pt; font-size: 10pt;"> AMD General</div> <br> </div> <br> </div> <div style=3D"font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, = Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);"> <br> </div> <hr style=3D"display: inline-block; width: 98%;"> <div id=3D"divRplyFwdMsg"> <div style=3D"direction: ltr; font-family: Calibri, sans-serif; font-size: = 11pt; color: rgb(0, 0, 0);"> <b>From:</b> Liang, Prike <[email protected]><br> <b>Sent:</b> Wednesday, July 29, 2026 10:09 PM<br> <b>To:</b> [email protected] <[email protected]&= gt;<br> <b>Cc:</b> Deucher, Alexander <[email protected]>; Koenig, Ch= ristian <[email protected]>; Chen, Michael <Michael.Chen@am= d.com>; Liu, Shaoyun <[email protected]>; Liang, Prike <Prike= [email protected]><br> <b>Subject:</b> [PATCH v2 2/2] drm/amdkfd: enable rs64mem for kfd queue</di= v> <div style=3D"direction: ltr;"> </div> </div> <div style=3D"font-size: 11pt;">Enabled RS64mem for KFD queues by integrati= ng<br> process and gang context index allocation in<br> the per KFD device process and queue creation.<br> <br> Signed-off-by: Prike Liang <[email protected]><br> ---<br> drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 3= +++<br> drivers/gpu/drm/amd/amdkfd/kfd_priv.h &n= bsp; |&nb= sp; 3 +++<br> drivers/gpu/drm/amd/amdkfd/kfd_process.c  = ; | 5 ++++-<br> drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c | 10 +++++++++= +<br> 4 files changed, 20 insertions(+), 1 deletion(-)<br> <br> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/driver= s/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c<br> index 51ee9c39104b..ea9d87450eae 100644<br> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c<br> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c<br> @@ -226,8 +226,10 @@ static int add_queue_mes(struct device_queue_manager *= dqm, struct queue *q,<br> /* MES unit for quantum is= 100ns */<br> queue_input.process_quantu= m =3D KFD_MES_PROCESS_QUANTUM; /* Equivalent to 10ms. */<br> queue_input.process_contex= t_addr =3D pdd->proc_ctx_gpu_addr;<br> + queue_input.process_context_array_ind= ex =3D pdd->proc_ctx_array_index;<br> queue_input.gang_quantum = =3D KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */<br> queue_input.gang_context_a= ddr =3D q->gang_ctx_gpu_addr;<br> + queue_input.gang_context_array_index = =3D q->gang_ctx_array_index;<br> queue_input.inprocess_gang= _priority =3D q->properties.priority;<br> queue_input.gang_global_pr= iority_level =3D<br> &nb= sp; = &nb= sp; AMDGPU_MES_PRIORITY_LEVEL_NORMAL;<br> @@ -303,6 +305,7 @@ static int remove_queue_mes_on_reset_option(struct devi= ce_queue_manager *dqm, st<br> queue_input.queue_type =3D= convert_to_amdgpu_ring_type(q->properties.type);<br> queue_input.remove_queue_a= fter_reset =3D flush_mes_queue;<br> queue_input.xcc_id =3D ffs= (dqm->dev->xcc_mask) - 1;<br> + queue_input.gang_context_array_index = =3D q->gang_ctx_array_index;<br> <br> amdgpu_mes_lock(&adev-= >mes);<br> r =3D adev->mes.funcs-&= gt;remove_hw_queue(&adev->mes, &queue_input);<br> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/am= dkfd/kfd_priv.h<br> index 88191a4c1657..3b89bcc625b3 100644<br> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h<br> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h<br> @@ -635,6 +635,7 @@ struct queue {<br> void *gang_ctx_bo;<br> uint64_t gang_ctx_gpu_addr= ;<br> void *gang_ctx_cpu_ptr;<br= > + uint32_t gang_ctx_array_index;<br> <br> struct amdgpu_bo *wptr_bo_= gart;<br> };<br> @@ -871,6 +872,8 @@ struct kfd_process_device {<br> uint64_t proc_ctx_gpu_addr= ;<br> void *proc_ctx_cpu_ptr;<br= > <br> + uint32_t proc_ctx_array_index;<br> +<br> /* Tracks queue reset stat= us */<br> bool has_reset_queue;<br> <br> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd= /amdkfd/kfd_process.c<br> index eb508fe3ded7..0a7c1900da95 100644<br> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c<br> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c<br> @@ -1215,9 +1215,12 @@ static void kfd_process_destroy_pdds(struct kfd_proc= ess *p)<br> &nb= sp; kfd_free_process_doorbells(pdd->dev->kfd, pdd);= <br> <br> &nb= sp; if (pdd->dev->kfd->shared_resources.enable_m= es &&<br> - &n= bsp; pdd->proc_ctx= _cpu_ptr)<br> + &n= bsp; pdd->proc_ctx= _cpu_ptr) {<br> + &n= bsp; amdgpu_mes_free_= proc_ctx_index(&pdd->dev->adev->mes,<br> + &n= bsp;  = ; &n= bsp;  = ; pdd->proc_ctx_array_index);<br> &nb= sp; amdgp= u_amdkfd_free_kernel_mem(pdd->dev->adev,<br> &nb= sp; = &nb= sp; = &pdd->proc_ctx_bo);<br> + &n= bsp; }<br> &nb= sp; /*<br> &nb= sp; * before destroying pdd, make sure to report av= ailability<br> &nb= sp; * for auto suspend<br> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drive= rs/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c<br> index 8054e4fe0381..72913b5c8338 100644<br> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c<br> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c<br> @@ -210,6 +210,7 @@ static void pqm_clean_queue_resource(struct process_que= ue_manager *pqm,<br> }<br> <br> if (dev->kfd->shared= _resources.enable_mes) {<br> + &n= bsp; amdgpu_mes_free_gang_ctx_index(&dev->adev->mes, pqn-&g= t;q->gang_ctx_array_index);<br> &nb= sp; amdgpu_amdkfd_free_kernel_mem(dev->adev, &pqn-= >q->gang_ctx_bo);<br> &nb= sp; amdgpu_amdkfd_free_kernel_mem(dev->adev, (void **)= &pqn->q->wptr_bo_gart);<br> }<br> @@ -282,6 +283,9 @@ static int init_user_queue(struct process_queue_manager= *pqm,<br> &nb= sp; goto = cleanup;<br> &nb= sp; }<br> &nb= sp; memset((*q)->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_= CTX_SIZE);<br> + &n= bsp; /* Bind one MES gang context slot per queue (gang). */<br> + &n= bsp; amdgpu_mes_alloc_gang_ctx_index(&dev->adev->mes,<br> + &n= bsp;  = ; &n= bsp; &(*q)->gang_ctx= _array_index);<br> </div> <ol start=3D"1" data-editing-info=3D"{"applyListStyleFromLevel":f= alse,"orderedStyleType":1}" style=3D"margin-top: 0px; margin-bott= om: 0px; list-style-type: decimal;"> <li style=3D"font-size: 11pt; margin-top: 0px; margin-bottom: 0px;"> <div class=3D"elementToProof" role=3D"presentation" style=3D"font-family: A= ptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-ser= if; color: rgb(0, 0, 0);"> Should check "mes->use_rs64mem" as you do in patch #1?</div> </li><li style=3D"font-size: 11pt; margin-top: 0px; margin-bottom: 0px;"> <div class=3D"elementToProof" role=3D"presentation" style=3D"font-family: A= ptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-ser= if; color: rgb(0, 0, 0);"> Check return error code (in oversubscription case, it may run out of index)= </div> </li></ol> <div style=3D"margin-top: 0px; margin-bottom: 0px; font-family: Aptos, Apto= s_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-s= ize: 11pt; color: rgb(0, 0, 0);"> <br> </div> <div style=3D"margin-top: 0px; margin-bottom: 0px; font-family: Aptos, Apto= s_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-s= ize: 11pt; color: rgb(0, 0, 0);"> <br> </div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> = &nb= sp; /* Starting with GFX11, wptr BOs must be mapped to GART for MES to dete= rmine work</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> = &nb= sp; * on unmapped queues for usermode queue oversubscription (no aggr= egated doorbell)</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;">@@ -304,6 +308,7 @= @ static int init_user_queue(struct process_queue_manager *pqm,</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> = return 0;</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> </div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> free_gang_ct= x_bo:</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;">+  = ; amdgpu_mes_free_gang_ctx_index(&dev->adev->me= s, (*q)->gang_ctx_array_index);</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> = amdgpu_amdkfd_free_kernel_mem(dev->adev, = &(*q)->gang_ctx_bo);</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> cleanup:</di= v> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> = uninit_queue(*q);</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;">@@ -386,6 +391,11 = @@ int pqm_create_queue(struct process_queue_manager *pqm,</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> = &nb= sp; goto err_allocate_pqn;<= /div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> = &nb= sp; }</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> = &nb= sp; memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;">+  = ; /* Bind= one MES process context slot to the whole process</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;">+  = ; *= (per device); every queue of this process reuses it.</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;">+  = ; *= /</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;">+  = ; amdgpu_= mes_alloc_proc_ctx_index(&dev->adev->mes,</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;">+  = ; &n= bsp;  = ; &n= bsp; &pdd->proc_ctx_array_index);</div= > <div class=3D"elementToProof" style=3D"font-size: 11pt;"><br> </div> <div class=3D"elementToProof" style=3D"font-family: Aptos, Aptos_EmbeddedFo= nt, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 11pt; c= olor: rgb(0, 0, 0);"> Same comment as above.</div> <div class=3D"elementToProof" style=3D"font-family: Aptos, Aptos_EmbeddedFo= nt, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 11pt; c= olor: rgb(0, 0, 0);"> <br> </div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> = }</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> </div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"> = pqn =3D kzalloc_obj(*pqn);</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;">--</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;">2.34.1</div> <div class=3D"elementToProof" style=3D"font-size: 11pt;"><br> </div> </body> </html> --_000_DM6PR12MB3611DD14C882B3CBAA936FE194D52DM6PR12MB3611namp_--