[PATCH] drm/amdkfd: fix NULL pointer dereference in GFX12 CRIU queue restore
"Marioukhine, Vladimir" <[email protected]> Tue, 4 Aug 2026 19:14:18 +0000
| Newsgroups | org.freedesktop.lists.amd-gfx |
|---|---|
| Message-ID | <SA1PR12MB8600E8B1821FA7D76923FC259FD42@SA1PR12MB8600.namprd12.prod.outlook.com> |
--_000_SA1PR12MB8600E8B1821FA7D76923FC259FD42SA1PR12MB8600namp_
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: quoted-printable
AMD General
kfd_mqd_manager_v12.c (GFX 12.0) and kfd_mqd_manager_v12_1.c (GFX 12.1)
initialize their MQD managers without assigning restore_mqd or checkpoint_m=
qd
callbacks, leaving both function pointers NULL.
Both create_queue_cpsch() and create_queue_nocpsch() in
kfd_device_queue_manager.c unconditionally call mqd_mgr->restore_mqd() when
a CRIU restore is in progress (qd !=3D NULL), with no NULL guard. On any sy=
stem
with a GFX 12.x GPU, a user holding CAP_CHECKPOINT_RESTORE can trigger a
kernel NULL pointer dereference and panic the machine by issuing
KFD_IOC_CRIU_OP_RESTORE with a crafted queue restore object.
Fix this by:
1. Implementing restore_mqd and checkpoint_mqd callbacks for both compute
and SDMA queues in kfd_mqd_manager_v12.c and kfd_mqd_manager_v12_1.c,
modeled after the GFX 11 implementation.
2. Adding a NULL guard at both call sites in kfd_device_queue_manager.c
as a defensive measure against future regressions.
Fixes: 48f0bdf4e38e ("drm/amdkfd: Added MQD manager files for GFX12.")
Fixes: 01bbc4a4b947 ("drm/amdkfd: Add MQD manager for GFX 12.1.0")
Signed-off-by: Vladimir Marioukhine <[email protected]>
---
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 10 ++++--
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c | 69 ++++++++++++++++=
+++++++++++++++++++++++
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c | 69 ++++++++++++++++=
++++++++++++++++++++++
3 files changed, 146 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/driver=
s/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 51ee9c39104b..31bf479212e5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -794,11 +794,16 @@ static int create_queue_nocpsch(struct device_queue_m=
anager *dqm,
goto out_deallocate_doorbell;
}
- if (qd)
+ if (qd) {
+ if (!mqd_mgr->restore_mqd) {
+ pr_err("amdkfd: restore_mqd not implemented for thi=
s GPU\n");
+ retval =3D -EINVAL;
+ goto out_deallocate_doorbell;
+ }
mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->=
gart_mqd_addr,
&q->properties, restore_mqd, restore_c=
tl_stack,
qd->ctl_stack_size);
- else
+ } else
mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
&q->gart_mqd_addr, &q->properties);
@@ -2153,11 +2158,16 @@ static int create_queue_cpsch(struct device_queue_m=
anager *dqm, struct queue *q,
q->properties.is_dbg_wa =3D qpd->pqm->process->debug_trap_enabled &=
&
kfd_dbg_has_cwsr_workaround(q->device);
- if (qd)
+ if (qd) {
+ if (!mqd_mgr->restore_mqd) {
+ pr_err("amdkfd: restore_mqd not implemented for thi=
s GPU\n");
+ retval =3D -EINVAL;
+ goto out;
+ }
mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->=
gart_mqd_addr,
&q->properties, restore_mqd, restore_c=
tl_stack,
qd->ctl_stack_size);
- else
+ } else
mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
&q->gart_mqd_addr, &q->properties);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c b/drivers/gpu=
/drm/amd/amdkfd/kfd_mqd_manager_v12.c
index 7c387fa90076..c6a8e99ad069 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
@@ -380,6 +380,75 @@ static int debugfs_show_mqd_sdma(struct seq_file *m, v=
oid *data)
#endif
+static void checkpoint_mqd(struct mqd_manager *mm, void *mqd,
+ void *mqd_dst, void *ctl_stack_dst)
+{
+ struct v12_compute_mqd *m;
+
+ m =3D get_mqd(mqd);
+
+ memcpy(mqd_dst, m, sizeof(struct v12_compute_mqd));
+}
+
+static void restore_mqd(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_ad=
dr,
+ struct queue_properties *qp, const void *mqd_src,
+ const void *ctl_stack_src, const u32 ctl_stack_siz=
e)
+{
+ uint64_t addr;
+ struct v12_compute_mqd *m;
+
+ m =3D (struct v12_compute_mqd *) mqd_mem_obj->cpu_ptr;
+ addr =3D mqd_mem_obj->gpu_addr;
+
+ memcpy(m, mqd_src, sizeof(*m));
+
+ *mqd =3D m;
+ if (gart_addr)
+ *gart_addr =3D addr;
+
+ m->cp_hqd_pq_doorbell_control =3D
+ qp->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OF=
FSET__SHIFT;
+ pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell=
_control);
+
+ qp->is_active =3D 0;
+}
+
+static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd,
+ void *mqd_dst, void *ctl_stack_dst)
+{
+ struct v12_sdma_mqd *m;
+
+ m =3D get_sdma_mqd(mqd);
+
+ memcpy(mqd_dst, m, sizeof(struct v12_sdma_mqd));
+}
+
+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *ga=
rt_addr,
+ struct queue_properties *qp,
+ const void *mqd_src,
+ const void *ctl_stack_src,
+ const u32 ctl_stack_size)
+{
+ uint64_t addr;
+ struct v12_sdma_mqd *m;
+
+ m =3D (struct v12_sdma_mqd *) mqd_mem_obj->cpu_ptr;
+ addr =3D mqd_mem_obj->gpu_addr;
+
+ memcpy(m, mqd_src, sizeof(*m));
+
+ m->sdmax_rlcx_doorbell_offset =3D
+ qp->doorbell_off << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__S=
HIFT;
+
+ *mqd =3D m;
+ if (gart_addr)
+ *gart_addr =3D addr;
+
+ qp->is_active =3D 0;
+}
+
struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type,
struct kfd_node *dev)
{
@@ -407,6 +476,8 @@ struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_T=
YPE type,
mqd->mqd_size =3D sizeof(struct v12_compute_mqd);
mqd->get_wave_state =3D get_wave_state;
mqd->mqd_stride =3D kfd_mqd_stride;
+ mqd->checkpoint_mqd =3D checkpoint_mqd;
+ mqd->restore_mqd =3D restore_mqd;
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd =3D debugfs_show_mqd;
#endif
@@ -453,6 +524,8 @@ struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_T=
YPE type,
mqd->is_occupied =3D kfd_is_occupied_sdma;
mqd->mqd_size =3D sizeof(struct v12_sdma_mqd);
mqd->mqd_stride =3D kfd_mqd_stride;
+ mqd->checkpoint_mqd =3D checkpoint_mqd_sdma;
+ mqd->restore_mqd =3D restore_mqd_sdma;
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd =3D debugfs_show_mqd_sdma;
#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/g=
pu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
index 431a940f91f3..7ee98ef084ea 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
@@ -641,6 +641,75 @@ static int debugfs_show_mqd_sdma(struct seq_file *m, v=
oid *data)
#endif
+static void checkpoint_mqd_v12_1(struct mqd_manager *mm, void *mqd,
+ void *mqd_dst, void *ctl_stack_dst)
+{
+ struct v12_1_compute_mqd *m;
+
+ m =3D get_mqd(mqd);
+
+ memcpy(mqd_dst, m, sizeof(struct v12_1_compute_mqd));
+}
+
+static void restore_mqd_v12_1(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64_t *g=
art_addr,
+ struct queue_properties *qp, const void *mqd=
_src,
+ const void *ctl_stack_src, const u32 ctl_sta=
ck_size)
+{
+ uint64_t addr;
+ struct v12_1_compute_mqd *m;
+
+ m =3D (struct v12_1_compute_mqd *) mqd_mem_obj->cpu_ptr;
+ addr =3D mqd_mem_obj->gpu_addr;
+
+ memcpy(m, mqd_src, sizeof(*m));
+
+ *mqd =3D m;
+ if (gart_addr)
+ *gart_addr =3D addr;
+
+ m->cp_hqd_pq_doorbell_control =3D
+ qp->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OF=
FSET__SHIFT;
+ pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell=
_control);
+
+ qp->is_active =3D 0;
+}
+
+static void checkpoint_mqd_sdma_v12_1(struct mqd_manager *mm, void *mqd,
+ void *mqd_dst, void *ctl_stack_dst)
+{
+ struct v12_sdma_mqd *m;
+
+ m =3D get_sdma_mqd(mqd);
+
+ memcpy(mqd_dst, m, sizeof(struct v12_sdma_mqd));
+}
+
+static void restore_mqd_sdma_v12_1(struct mqd_manager *mm, void **mqd,
+ struct kfd_mem_obj *mqd_mem_obj, uint64=
_t *gart_addr,
+ struct queue_properties *qp,
+ const void *mqd_src,
+ const void *ctl_stack_src,
+ const u32 ctl_stack_size)
+{
+ uint64_t addr;
+ struct v12_sdma_mqd *m;
+
+ m =3D (struct v12_sdma_mqd *) mqd_mem_obj->cpu_ptr;
+ addr =3D mqd_mem_obj->gpu_addr;
+
+ memcpy(m, mqd_src, sizeof(*m));
+
+ m->sdmax_rlcx_doorbell_offset =3D
+ qp->doorbell_off << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__S=
HIFT;
+
+ *mqd =3D m;
+ if (gart_addr)
+ *gart_addr =3D addr;
+
+ qp->is_active =3D 0;
+}
+
struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type,
struct kfd_node *dev)
{
@@ -668,6 +737,8 @@ struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD=
_TYPE type,
mqd->mqd_size =3D sizeof(struct v12_1_compute_mqd);
mqd->get_wave_state =3D get_wave_state_v12_1;
mqd->mqd_stride =3D kfd_mqd_stride;
+ mqd->checkpoint_mqd =3D checkpoint_mqd_v12_1;
+ mqd->restore_mqd =3D restore_mqd_v12_1;
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd =3D debugfs_show_mqd;
#endif
@@ -714,6 +785,8 @@ struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD=
_TYPE type,
mqd->is_occupied =3D kfd_is_occupied_sdma;
mqd->mqd_size =3D sizeof(struct v12_sdma_mqd);
mqd->mqd_stride =3D kfd_mqd_stride;
+ mqd->checkpoint_mqd =3D checkpoint_mqd_sdma_v12_1;
+ mqd->restore_mqd =3D restore_mqd_sdma_v12_1;
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd =3D debugfs_show_mqd_sdma;
#endif
--_000_SA1PR12MB8600E8B1821FA7D76923FC259FD42SA1PR12MB8600namp_
Content-Type: text/html; charset="us-ascii"
Content-Transfer-Encoding: quoted-printable
<html xmlns:v=3D"urn:schemas-microsoft-com:vml" xmlns:o=3D"urn:schemas-micr=
osoft-com:office:office" xmlns:w=3D"urn:schemas-microsoft-com:office:word" =
xmlns:m=3D"http://schemas.microsoft.com/office/2004/12/omml" xmlns=3D"http:=
//www.w3.org/TR/REC-html40">
<head>
<meta http-equiv=3D"Content-Type" content=3D"text/html; charset=3Dus-ascii"=
>
<meta name=3D"Generator" content=3D"Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Aptos;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
font-size:12.0pt;
font-family:"Aptos",sans-serif;
mso-ligatures:standardcontextual;}
span.EmailStyle17
{mso-style-type:personal-compose;
font-family:"Aptos",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext=3D"edit" spidmax=3D"1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext=3D"edit">
<o:idmap v:ext=3D"edit" data=3D"1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang=3D"EN-US" link=3D"#467886" vlink=3D"#96607D" style=3D"word-wrap:=
break-word">
<p style=3D"font-family:Calibri;font-size:10pt;color:#0000FF;margin:5pt;fon=
t-style:normal;font-weight:normal;text-decoration:none;" align=3D"Left">
AMD General<br>
</p>
<br>
<div>
<div class=3D"WordSection1">
<p class=3D"MsoNormal">kfd_mqd_manager_v12.c (GFX 12.0) and kfd_mqd_manager=
_v12_1.c (GFX 12.1)<o:p></o:p></p>
<p class=3D"MsoNormal">initialize their MQD managers without assigning rest=
ore_mqd or checkpoint_mqd<o:p></o:p></p>
<p class=3D"MsoNormal">callbacks, leaving both function pointers NULL.<o:p>=
</o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">Both create_queue_cpsch() and create_queue_nocpsch()=
in<o:p></o:p></p>
<p class=3D"MsoNormal">kfd_device_queue_manager.c unconditionally call mqd_=
mgr->restore_mqd() when<o:p></o:p></p>
<p class=3D"MsoNormal">a CRIU restore is in progress (qd !=3D NULL), with n=
o NULL guard. On any system<o:p></o:p></p>
<p class=3D"MsoNormal">with a GFX 12.x GPU, a user holding CAP_CHECKPOINT_R=
ESTORE can trigger a<o:p></o:p></p>
<p class=3D"MsoNormal">kernel NULL pointer dereference and panic the machin=
e by issuing<o:p></o:p></p>
<p class=3D"MsoNormal">KFD_IOC_CRIU_OP_RESTORE with a crafted queue restore=
object.<o:p></o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">Fix this by:<o:p></o:p></p>
<p class=3D"MsoNormal">1. Implementing restore_mqd and checkpoint_mqd callb=
acks for both compute<o:p></o:p></p>
<p class=3D"MsoNormal"> and SDMA queues in kfd_mqd_manager_v12.=
c and kfd_mqd_manager_v12_1.c,<o:p></o:p></p>
<p class=3D"MsoNormal"> modeled after the GFX 11 implementation=
.<o:p></o:p></p>
<p class=3D"MsoNormal">2. Adding a NULL guard at both call sites in kfd_dev=
ice_queue_manager.c<o:p></o:p></p>
<p class=3D"MsoNormal"> as a defensive measure against future r=
egressions.<o:p></o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">Fixes: 48f0bdf4e38e ("drm/amdkfd: Added MQD man=
ager files for GFX12.")<o:p></o:p></p>
<p class=3D"MsoNormal">Fixes: 01bbc4a4b947 ("drm/amdkfd: Add MQD manag=
er for GFX 12.1.0")<o:p></o:p></p>
<p class=3D"MsoNormal">Signed-off-by: Vladimir Marioukhine <Vladimir.Mar=
[email protected]><o:p></o:p></p>
<p class=3D"MsoNormal">---<o:p></o:p></p>
<p class=3D"MsoNormal">drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.=
c | 10 ++++--<o:p></o:p></p>
<p class=3D"MsoNormal">drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c&nbs=
p; | 69 +++++++++++++++++++++++++++++++++++++++<o:p=
></o:p></p>
<p class=3D"MsoNormal">drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c&n=
bsp; | 69 ++++++++++++++++++++++++++++++++++++++<o:p></o:p></p>
<p class=3D"MsoNormal">3 files changed, 146 insertions(+), 2 deletions(-)<o=
:p></o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_q=
ueue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c<o:p>=
</o:p></p>
<p class=3D"MsoNormal">index 51ee9c39104b..31bf479212e5 100644<o:p></o:p></=
p>
<p class=3D"MsoNormal">--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_ma=
nager.c<o:p></o:p></p>
<p class=3D"MsoNormal">+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_ma=
nager.c<o:p></o:p></p>
<p class=3D"MsoNormal">@@ -794,11 +794,16 @@ static int create_queue_nocpsc=
h(struct device_queue_manager *dqm,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; goto out_deallocate_doorbell;<o:p></=
o:p></p>
<p class=3D"MsoNormal"> }<o:p></o=
:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">- if (qd)<o:p></=
o:p></p>
<p class=3D"MsoNormal">+ if (qd) {<o:p>=
</o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; if (!mqd_mgr->restore_mqd) {<o:p></o:p=
></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
pr_err("amdkfd: restore_mqd not implemented for this GPU\n"=
;);<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
retval =3D -EINVAL;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
goto out_deallocate_doorbell;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; }<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd_mgr->restore_mqd(mqd_mgr, &am=
p;q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; &=
nbsp; &nbs=
p; &q->properties, restore_mqd, restore_ctl_stack,<o:p><=
/o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; &=
nbsp; &nbs=
p; qd->ctl_stack_size);<o:p></o:p></p>
<p class=3D"MsoNormal">- else<o:p></o:p=
></p>
<p class=3D"MsoNormal">+ } else<o:p></o=
:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd_mgr->init_mqd(mqd_mgr, &q=
->mqd, q->mqd_mem_obj,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; &=
nbsp; &nbs=
p; &q->gart_mqd_addr, &q->prope=
rties);<o:p></o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">@@ -2153,11 +2158,16 @@ static int create_queue_cpsc=
h(struct device_queue_manager *dqm, struct queue *q,<o:p></o:p></p>
<p class=3D"MsoNormal"> q->pro=
perties.is_dbg_wa =3D qpd->pqm->process->debug_trap_enabled &&=
amp;<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; &=
nbsp; kfd=
_dbg_has_cwsr_workaround(q->device);<o:p></o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">- if (qd)<o:p></=
o:p></p>
<p class=3D"MsoNormal">+ if (qd) {<o:p>=
</o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; if (!mqd_mgr->restore_mqd) {<o:p></o:p=
></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
pr_err("amdkfd: restore_mqd not implemented for this GPU\n"=
;);<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
retval =3D -EINVAL;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
goto out;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; }<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd_mgr->restore_mqd(mqd_mgr, &am=
p;q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; &=
nbsp; &nbs=
p; &q->properties, restore_mqd, restore_ctl_stack,<o:p><=
/o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; &=
nbsp; &nbs=
p; qd->ctl_stack_size);<o:p></o:p></p>
<p class=3D"MsoNormal">- else<o:p></o:p=
></p>
<p class=3D"MsoNormal">+ } else<o:p></o=
:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd_mgr->init_mqd(mqd_mgr, &q=
->mqd, q->mqd_mem_obj,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; &=
nbsp; &nb=
sp; &q->gart_mqd_addr, &q->prope=
rties);<o:p></o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_mana=
ger_v12.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c<o:p></o:p></p>
<p class=3D"MsoNormal">index 7c387fa90076..c6a8e99ad069 100644<o:p></o:p></=
p>
<p class=3D"MsoNormal">--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12=
.c<o:p></o:p></p>
<p class=3D"MsoNormal">+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12=
.c<o:p></o:p></p>
<p class=3D"MsoNormal">@@ -380,6 +380,75 @@ static int debugfs_show_mqd_sdm=
a(struct seq_file *m, void *data)<o:p></o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">#endif<o:p></o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">+static void checkpoint_mqd(struct mqd_manager *mm, =
void *mqd,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
void *mqd_dst, void *ctl_stack_dst)<o:p></o:=
p></p>
<p class=3D"MsoNormal">+{<o:p></o:p></p>
<p class=3D"MsoNormal">+ struct v12_com=
pute_mqd *m;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m =3D get_mqd(=
mqd);<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ memcpy(mqd_dst=
, m, sizeof(struct v12_compute_mqd));<o:p></o:p></p>
<p class=3D"MsoNormal">+}<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+static void restore_mqd(struct mqd_manager *mm, voi=
d **mqd,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,<o:p></o:=
p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
struct queue_properties *qp, const void *mqd_src,<o:p></o:p></=
p>
<p class=3D"MsoNormal">+ &nb=
sp; =
const void *ctl_stack_src, const u32 ctl_stack_size)<o:p></o:p=
></p>
<p class=3D"MsoNormal">+{<o:p></o:p></p>
<p class=3D"MsoNormal">+ uint64_t addr;=
<o:p></o:p></p>
<p class=3D"MsoNormal">+ struct v12_com=
pute_mqd *m;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m =3D (struct =
v12_compute_mqd *) mqd_mem_obj->cpu_ptr;<o:p></o:p></p>
<p class=3D"MsoNormal">+ addr =3D mqd_m=
em_obj->gpu_addr;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ memcpy(m, mqd_=
src, sizeof(*m));<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ *mqd =3D m;<o:=
p></o:p></p>
<p class=3D"MsoNormal">+ if (gart_addr)=
<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; *gart_addr =3D addr;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m->cp_hqd_p=
q_doorbell_control =3D<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; qp->doorbell_off << CP_HQD_PQ_DO=
ORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;<o:p></o:p></p>
<p class=3D"MsoNormal">+ pr_debug("=
;cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell_control)=
;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ qp->is_acti=
ve =3D 0;<o:p></o:p></p>
<p class=3D"MsoNormal">+}<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+static void checkpoint_mqd_sdma(struct mqd_manager =
*mm, void *mqd,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
void *mqd_dst,=
void *ctl_stack_dst)<o:p></o:p></p>
<p class=3D"MsoNormal">+{<o:p></o:p></p>
<p class=3D"MsoNormal">+ struct v12_sdm=
a_mqd *m;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m =3D get_sdma=
_mqd(mqd);<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ memcpy(mqd_dst=
, m, sizeof(struct v12_sdma_mqd));<o:p></o:p></p>
<p class=3D"MsoNormal">+}<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+static void restore_mqd_sdma(struct mqd_manager *mm=
, void **mqd,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
struct kfd_mem_obj *mqd_mem_obj,=
uint64_t *gart_addr,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
struct queue_properties *qp,<o:p=
></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
const void *mqd_src,<o:p></o:p><=
/p>
<p class=3D"MsoNormal">+ &nb=
sp; =
const void *ctl_stack_src,<o:p><=
/o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
const u32 ctl_stack_size)<o:p></=
o:p></p>
<p class=3D"MsoNormal">+{<o:p></o:p></p>
<p class=3D"MsoNormal">+ uint64_t addr;=
<o:p></o:p></p>
<p class=3D"MsoNormal">+ struct v12_sdm=
a_mqd *m;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m =3D (struct =
v12_sdma_mqd *) mqd_mem_obj->cpu_ptr;<o:p></o:p></p>
<p class=3D"MsoNormal">+ addr =3D mqd_m=
em_obj->gpu_addr;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ memcpy(m, mqd_=
src, sizeof(*m));<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m->sdmax_rl=
cx_doorbell_offset =3D<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; qp->doorbell_off << SDMA0_QUEUE0=
_DOORBELL_OFFSET__OFFSET__SHIFT;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ *mqd =3D m;<o:=
p></o:p></p>
<p class=3D"MsoNormal">+ if (gart_addr)=
<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; *gart_addr =3D addr;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ qp->is_acti=
ve =3D 0;<o:p></o:p></p>
<p class=3D"MsoNormal">+}<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">struct mqd_manager *mqd_manager_init_v12(enum KFD_MQ=
D_TYPE type,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; struct kfd_node *dev)<o:p></o:p></p>
<p class=3D"MsoNormal">{<o:p></o:p></p>
<p class=3D"MsoNormal">@@ -407,6 +476,8 @@ struct mqd_manager *mqd_manager_=
init_v12(enum KFD_MQD_TYPE type,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->mqd_size =3D sizeof(struct v=
12_compute_mqd);<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->get_wave_state =3D get_wave_=
state;<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->mqd_stride =3D kfd_mqd_strid=
e;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; mqd->checkpoint_mqd =3D checkpoint_mqd=
;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; mqd->restore_mqd =3D restore_mqd;<o:p>=
</o:p></p>
<p class=3D"MsoNormal">#if defined(CONFIG_DEBUG_FS)<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->debugfs_show_mqd =3D debugfs=
_show_mqd;<o:p></o:p></p>
<p class=3D"MsoNormal">#endif<o:p></o:p></p>
<p class=3D"MsoNormal">@@ -453,6 +524,8 @@ struct mqd_manager *mqd_manager_=
init_v12(enum KFD_MQD_TYPE type,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->is_occupied =3D kfd_is_occup=
ied_sdma;<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->mqd_size =3D sizeof(struct v=
12_sdma_mqd);<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->mqd_stride =3D kfd_mqd_strid=
e;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; mqd->checkpoint_mqd =3D checkpoint_mqd=
_sdma;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; mqd->restore_mqd =3D restore_mqd_sdma;=
<o:p></o:p></p>
<p class=3D"MsoNormal">#if defined(CONFIG_DEBUG_FS)<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->debugfs_show_mqd =3D debugfs=
_show_mqd_sdma;<o:p></o:p></p>
<p class=3D"MsoNormal">#endif<o:p></o:p></p>
<p class=3D"MsoNormal">diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_mana=
ger_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c<o:p></o:p>=
</p>
<p class=3D"MsoNormal">index 431a940f91f3..7ee98ef084ea 100644<o:p></o:p></=
p>
<p class=3D"MsoNormal">--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12=
_1.c<o:p></o:p></p>
<p class=3D"MsoNormal">+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12=
_1.c<o:p></o:p></p>
<p class=3D"MsoNormal">@@ -641,6 +641,75 @@ static int debugfs_show_mqd_sdm=
a(struct seq_file *m, void *data)<o:p></o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">#endif<o:p></o:p></p>
<p class=3D"MsoNormal"><o:p> </o:p></p>
<p class=3D"MsoNormal">+static void checkpoint_mqd_v12_1(struct mqd_manager=
*mm, void *mqd,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
void *mq=
d_dst, void *ctl_stack_dst)<o:p></o:p></p>
<p class=3D"MsoNormal">+{<o:p></o:p></p>
<p class=3D"MsoNormal">+ struct v12_1_c=
ompute_mqd *m;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m =3D get_mqd(=
mqd);<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ memcpy(mqd_dst=
, m, sizeof(struct v12_1_compute_mqd));<o:p></o:p></p>
<p class=3D"MsoNormal">+}<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+static void restore_mqd_v12_1(struct mqd_manager *m=
m, void **mqd,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
struct kfd_mem_obj *mqd_me=
m_obj, uint64_t *gart_addr,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
struct queue_properties *q=
p, const void *mqd_src,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &n=
bsp;  =
; const void *ctl_stack_src,=
const u32 ctl_stack_size)<o:p></o:p></p>
<p class=3D"MsoNormal">+{<o:p></o:p></p>
<p class=3D"MsoNormal">+ uint64_t addr;=
<o:p></o:p></p>
<p class=3D"MsoNormal">+ struct v12_1_c=
ompute_mqd *m;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m =3D (struct =
v12_1_compute_mqd *) mqd_mem_obj->cpu_ptr;<o:p></o:p></p>
<p class=3D"MsoNormal">+ addr =3D mqd_m=
em_obj->gpu_addr;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ memcpy(m, mqd_=
src, sizeof(*m));<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ *mqd =3D m;<o:=
p></o:p></p>
<p class=3D"MsoNormal">+ if (gart_addr)=
<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; *gart_addr =3D addr;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m->cp_hqd_p=
q_doorbell_control =3D<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; qp->doorbell_off << CP_HQD_PQ_DO=
ORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;<o:p></o:p></p>
<p class=3D"MsoNormal">+ pr_debug("=
;cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell_control)=
;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ qp->is_acti=
ve =3D 0;<o:p></o:p></p>
<p class=3D"MsoNormal">+}<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+static void checkpoint_mqd_sdma_v12_1(struct mqd_ma=
nager *mm, void *mqd,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
&nb=
sp; void *mqd_dst, void *ctl_stack_dst)<o:p></o:p></p>
<p class=3D"MsoNormal">+{<o:p></o:p></p>
<p class=3D"MsoNormal">+ struct v12_sdm=
a_mqd *m;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m =3D get_sdma=
_mqd(mqd);<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ memcpy(mqd_dst=
, m, sizeof(struct v12_sdma_mqd));<o:p></o:p></p>
<p class=3D"MsoNormal">+}<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+static void restore_mqd_sdma_v12_1(struct mqd_manag=
er *mm, void **mqd,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
&nb=
sp; struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
&nb=
sp; struct queue_properties *qp,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
&nb=
sp; const void *mqd_src,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
&nb=
sp; const void *ctl_stack_src,<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; =
&nb=
sp; const u32 ctl_stack_size)<o:p></o:p></p>
<p class=3D"MsoNormal">+{<o:p></o:p></p>
<p class=3D"MsoNormal">+ uint64_t addr;=
<o:p></o:p></p>
<p class=3D"MsoNormal">+ struct v12_sdm=
a_mqd *m;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m =3D (struct =
v12_sdma_mqd *) mqd_mem_obj->cpu_ptr;<o:p></o:p></p>
<p class=3D"MsoNormal">+ addr =3D mqd_m=
em_obj->gpu_addr;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ memcpy(m, mqd_=
src, sizeof(*m));<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ m->sdmax_rl=
cx_doorbell_offset =3D<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; qp->doorbell_off << SDMA0_QUEUE0=
_DOORBELL_OFFSET__OFFSET__SHIFT;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ *mqd =3D m;<o:=
p></o:p></p>
<p class=3D"MsoNormal">+ if (gart_addr)=
<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; *gart_addr =3D addr;<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">+ qp->is_acti=
ve =3D 0;<o:p></o:p></p>
<p class=3D"MsoNormal">+}<o:p></o:p></p>
<p class=3D"MsoNormal">+<o:p></o:p></p>
<p class=3D"MsoNormal">struct mqd_manager *mqd_manager_init_v12_1(enum KFD_=
MQD_TYPE type,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; struct kfd_node *dev)<o:p></o:p></p>
<p class=3D"MsoNormal">{<o:p></o:p></p>
<p class=3D"MsoNormal">@@ -668,6 +737,8 @@ struct mqd_manager *mqd_manager_=
init_v12_1(enum KFD_MQD_TYPE type,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->mqd_size =3D sizeof(struct v=
12_1_compute_mqd);<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->get_wave_state =3D get_wave_=
state_v12_1;<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->mqd_stride =3D kfd_mqd_strid=
e;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; mqd->checkpoint_mqd =3D checkpoint_mqd=
_v12_1;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; mqd->restore_mqd =3D restore_mqd_v12_1=
;<o:p></o:p></p>
<p class=3D"MsoNormal">#if defined(CONFIG_DEBUG_FS)<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->debugfs_show_mqd =3D debugfs=
_show_mqd;<o:p></o:p></p>
<p class=3D"MsoNormal">#endif<o:p></o:p></p>
<p class=3D"MsoNormal">@@ -714,6 +785,8 @@ struct mqd_manager *mqd_manager_=
init_v12_1(enum KFD_MQD_TYPE type,<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->is_occupied =3D kfd_is_occup=
ied_sdma;<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->mqd_size =3D sizeof(struct v=
12_sdma_mqd);<o:p></o:p></p>
<p class=3D"MsoNormal"> &nb=
sp; mqd->mqd_stride =3D kfd_mqd_strid=
e;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; mqd->checkpoint_mqd =3D checkpoint_mqd=
_sdma_v12_1;<o:p></o:p></p>
<p class=3D"MsoNormal">+ &nb=
sp; mqd->restore_mqd =3D restore_mqd_sdma_=
v12_1;<o:p></o:p></p>
<p class=3D"MsoNormal">#if defined(CONFIG_DEBUG_FS)<o:p></o:p></p>
<p class=3D"MsoNormal"> &nbs=
p; mqd->debugfs_show_mqd =3D debugfs=
_show_mqd_sdma;<o:p></o:p></p>
<p class=3D"MsoNormal">#endif<o:p></o:p></p>
</div>
</div>
</body>
</html>
--_000_SA1PR12MB8600E8B1821FA7D76923FC259FD42SA1PR12MB8600namp_--