[PATCH 26/95] drm/amdgpu: Rework VMID reservation logic
Alex Deucher <[email protected]>
| Newsgroups | org.freedesktop.lists.amd-gfx |
|---|---|
| Message-ID | <[email protected]> |
From: Mukul Joshi <[email protected]> Rework the VMID reservation logic by passing in the mask of VMIDs (for both GFXHUB and MMHUB) available for it to use. The VMID masks for both GFXHUB and MMHUB are initialized in the respective ASIC specific files. This also helps in reserving per-ASIC VMIDs. This is in preparation for reserving a VMID for NPA added in the next patch. Signed-off-by: Mukul Joshi <[email protected]> Reviewed-by: Christian König <[email protected]> Signed-off-by: Alex Deucher <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c | 33 ++++++++++++++----------- drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h | 27 +++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 2 +- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 4 +++ drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 5 ++++ drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 14 +++++++---- drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c | 6 +++++ drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 6 +++++ drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 6 +++++ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 6 +++++ 17 files changed, 95 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c index 8a2d64f0ebc3f..8c32c69437645 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c @@ -625,12 +625,27 @@ void amdgpu_vmid_reset_all(struct amdgpu_device *adev) for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[i]; - - for (j = 1; j < id_mgr->num_ids; ++j) + for_each_set_bit(j, id_mgr->vmid_mask, AMDGPU_NUM_VMID) amdgpu_vmid_reset(adev, i, j); } } +void amdgpu_vmid_mgr_set_vmid_mask(struct amdgpu_device *adev, + unsigned long vmid_mask, bool for_mmhub) +{ + unsigned int i; + + BUILD_BUG_ON(AMDGPU_NUM_VMID > BITS_PER_LONG); + + for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) { + bool is_mmhub = AMDGPU_IS_MMHUB0(i) || AMDGPU_IS_MMHUB1(i); + + if (is_mmhub == for_mmhub) + bitmap_copy(adev->vm_manager.id_mgr[i].vmid_mask, + &vmid_mask, AMDGPU_NUM_VMID); + } +} + /** * amdgpu_vmid_mgr_init - init the VMID manager * @@ -649,19 +664,8 @@ void amdgpu_vmid_mgr_init(struct amdgpu_device *adev) mutex_init(&id_mgr->lock); INIT_LIST_HEAD(&id_mgr->ids_lru); - /* for GC <10, SDMA uses MMHUB so use first_kfd_vmid for both GC and MM */ - if (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(10, 0, 0)) - /* manage only VMIDs not used by KFD */ - id_mgr->num_ids = adev->vm_manager.first_kfd_vmid; - else if (AMDGPU_IS_MMHUB0(i) || - AMDGPU_IS_MMHUB1(i)) - id_mgr->num_ids = 16; - else - /* manage only VMIDs not used by KFD */ - id_mgr->num_ids = adev->vm_manager.first_kfd_vmid; - /* skip over VMID 0, since it is the system VM */ - for (j = 1; j < id_mgr->num_ids; ++j) { + for_each_set_bit(j, id_mgr->vmid_mask, AMDGPU_NUM_VMID) { amdgpu_vmid_reset(adev, i, j); amdgpu_sync_create(&id_mgr->ids[j].active); list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru); @@ -685,6 +689,7 @@ void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev) &adev->vm_manager.id_mgr[i]; mutex_destroy(&id_mgr->lock); + for (j = 0; j < AMDGPU_NUM_VMID; ++j) { struct amdgpu_vmid *id = &id_mgr->ids[j]; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h index 4b55d0d9703a3..46b2c21601263 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h @@ -26,6 +26,8 @@ #include <linux/types.h> #include <linux/mutex.h> #include <linux/list.h> +#include <linux/bitops.h> +#include <linux/find.h> #include <linux/dma-fence.h> #include "amdgpu_sync.h" @@ -33,6 +35,27 @@ /* maximum number of VMIDs */ #define AMDGPU_NUM_VMID 16 +/** + * for_each_vmid_and_zero - iterate over VMID 0 plus every VMID set in a hub's + * vmid_mask + * @vmid: loop cursor (int / unsigned int) + * @adev: struct amdgpu_device * + * @hub: vmhub index into adev->vm_manager.id_mgr[] + * + * Equivalent to OR-ing BIT(0) into id_mgr[hub].vmid_mask and walking the + * result with for_each_set_bit(). Used by the gfx_v*_constants_init() paths + * which must always touch VMID 0 (system) in addition to the regular VMIDs + * owned by the hub. + * + * Relies on AMDGPU_NUM_VMID (16) fitting in a single unsigned long, i.e. + * vmid_mask[] being a 1-element bitmap. + */ +#define for_each_vmid_and_zero(vmid, adev, hub) \ + for (unsigned long __vmid_mask = \ + (adev)->vm_manager.id_mgr[(hub)].vmid_mask[0] | BIT(0); \ + __vmid_mask; __vmid_mask = 0) \ + for_each_set_bit((vmid), &__vmid_mask, AMDGPU_NUM_VMID) + struct amdgpu_device; struct amdgpu_fpriv; struct amdgpu_vm; @@ -65,10 +88,10 @@ struct amdgpu_vmid { struct amdgpu_vmid_mgr { struct mutex lock; - unsigned num_ids; struct list_head ids_lru; struct amdgpu_vmid ids[AMDGPU_NUM_VMID]; bool reserved_vmid; + DECLARE_BITMAP(vmid_mask, AMDGPU_NUM_VMID); }; int amdgpu_pasid_alloc(unsigned int bits, struct amdgpu_fpriv *fpriv); @@ -95,5 +118,7 @@ void amdgpu_vmid_reset_all(struct amdgpu_device *adev); void amdgpu_vmid_mgr_init(struct amdgpu_device *adev); void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev); +void amdgpu_vmid_mgr_set_vmid_mask(struct amdgpu_device *adev, + unsigned long vmid_mask, bool for_mmhub); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index f9154421c2d5f..f91e693de60a6 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -5364,7 +5364,7 @@ static void gfx_v10_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { nv_grbm_select(adev, 0, 0, 0, i); /* CP and shaders */ WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index a447562977abe..743fe1bd39e86 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -2213,7 +2213,7 @@ static void gfx_v11_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc21_grbm_select(adev, 0, 0, 0, i); /* CP and shaders */ WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index fdfee88e41e33..d65ff3e551189 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -1852,7 +1852,7 @@ static void gfx_v12_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc24_grbm_select(adev, 0, 0, 0, i); /* CP and shaders */ WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c index cc4e939cb3f37..aae5ab9c338c1 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c @@ -1725,7 +1725,7 @@ static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev, /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); /* CP and shaders */ WREG32_SOC15(GC, GET_INST(GC, xcc_id), diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index a1e00be54005d..fa209fd6d4ac9 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -1938,7 +1938,7 @@ static void gfx_v7_0_constants_init(struct amdgpu_device *adev) WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg); mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { + for_each_vmid_and_zero(i, adev, 0) { if (i == 0) sh_mem_base = 0; else diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index f7c2b149acd0a..4c79e594cf8be 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -3758,7 +3758,7 @@ static void gfx_v8_0_constants_init(struct amdgpu_device *adev) WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg); mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { + for_each_vmid_and_zero(i, adev, 0) { vi_srbm_select(adev, 0, 0, 0, i); /* CP and shaders */ if (i == 0) { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index a4365e26120a5..c220a1cea7d73 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -2682,7 +2682,7 @@ static void gfx_v9_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc15_grbm_select(adev, 0, 0, 0, i, 0); /* CP and shaders */ if (i == 0) { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index 99732056c58f6..4a7e7e1500300 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -1237,7 +1237,7 @@ static void gfx_v9_4_3_xcc_constants_init(struct amdgpu_device *adev, /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); /* CP and shaders */ if (i == 0) { diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index 75a552782c979..9a8a7921ce982 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -869,6 +869,10 @@ static int gmc_v10_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true); amdgpu_vm_manager_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c index f454aff831b03..ed75da48ce963 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c @@ -852,6 +852,11 @@ static int gmc_v11_0_sw_init(struct amdgpu_ip_block *ip_block) */ adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true); + amdgpu_vm_manager_init(adev); r = amdgpu_gmc_ras_sw_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c index 3add79c474f3b..cc4eb05215ea1 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c @@ -952,13 +952,17 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block) * number of VMs * VMID 0 is reserved for System * amdgpu graphics/compute will use VMIDs 1-7 - * amdkfd will use VMIDs 8-15 + * amdkfd will use VMIDs 8-15. + * On GFX 12.1, amdkfd will use VMIDs 3-15. */ adev->vm_manager.first_kfd_vmid = - amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) ? - 3 : 8; - adev->vm_manager.first_kfd_vmid = - adev->gfx.disable_kq ? 1 : (adev->vm_manager.first_kfd_vmid); + adev->gfx.disable_kq ? 1 : + (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) ? + 3 : 8); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true); amdgpu_vm_manager_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c index ca7afb9b7d0ae..cdd11587b12ae 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c @@ -864,6 +864,12 @@ static int gmc_v6_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); /* base offset of vram pages */ diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index 1c1eccfe383fd..c9cc55e18893b 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -1045,6 +1045,12 @@ static int gmc_v7_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); /* base offset of vram pages */ diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index bf089595e5dac..8d8a0775878ff 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -1122,6 +1122,12 @@ static int gmc_v8_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); /* base offset of vram pages */ diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index b46b87291c512..633772fa71fda 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -2038,6 +2038,12 @@ static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block) 3 : 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); gmc_v9_0_save_registers(adev); -- 2.55.0