[PATCH v2] amdgpu/vcn: vcn dec_msg integer overflow fix

if supplied msg[2] in the header is too large around
0x40000000, 4 times of this unsigned 32 bit value will
overflow and the test could pass.

v2: using kernel helpers to check the overflow.
    this needs 2 checks: multiplication and addition

Signed-off-by: David (Ming Qiang) Wu <[email protected]>
---
 drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c | 6 ++++--
 drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
index 81bba3ec2a93..7a301cfe91ee 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
@@ -1910,7 +1910,7 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
 	struct ttm_operation_ctx ctx = { false, false };
 	struct amdgpu_device *adev = p->adev;
 	struct amdgpu_bo_va_mapping *map;
-	uint32_t *msg, num_buffers, len_dw;
+	uint32_t *msg, num_buffers, len_dw, mul, total;
 	struct amdgpu_bo *bo;
 	uint64_t start, end;
 	unsigned int i;
@@ -1965,7 +1965,9 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
 	num_buffers = msg[2];
 
 	/* Verify that all indices fit within the claimed length. Each index is 4 DWORDs */
-	if (num_buffers > len_dw || 6 + num_buffers * 4 > len_dw) {
+	if (check_mul_overflow(num_buffers, 4u, &mul) ||
+	    check_add_overflow(6u, mul, &total) ||
+	    total > len_dw) {
 		DRM_ERROR("VCN message has too many buffers!\n");
 		r = -EINVAL;
 		goto out;
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
index 0cce78b205a8..413854fcf84a 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
@@ -1826,7 +1826,7 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
 	struct ttm_operation_ctx ctx = { false, false };
 	struct amdgpu_device *adev = p->adev;
 	struct amdgpu_bo_va_mapping *map;
-	uint32_t *msg, num_buffers, len_dw;
+	uint32_t *msg, num_buffers, len_dw, mul, total;
 	struct amdgpu_bo *bo;
 	uint64_t start, end;
 	unsigned int i;
@@ -1881,7 +1881,9 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
 	num_buffers = msg[2];
 
 	/* Verify that all indices fit within the claimed length. Each index is 4 DWORDs */
-	if (num_buffers > len_dw || 6 + num_buffers * 4 > len_dw) {
+	if (check_mul_overflow(num_buffers, 4u, &mul) ||
+	    check_add_overflow(6u, mul, &total) ||
+	    total > len_dw) {
 		DRM_ERROR("VCN message has too many buffers!\n");
 		r = -EINVAL;
 		goto out;
-- 
2.43.0