[PATCH v1 1/4] staging: atomisp: prevent integer overflow in DVS 6-axis allocation

[Siho Lee]

The width_y * height_y multiplication in alloc_dvs_6axis_table() can
overflow on 32-bit arithmetic when the user sets a large resolution via
VIDIOC_S_FMT, since ATOM_ISP_MAX_WIDTH = UINT_MAX imposes no limit.

For example, with frame_res width=16777216, width_y becomes 262145 and:
    262145 * 262145 = 524289 (32-bit, overflowed)
    262145 * 262145 = 68720001025 (64-bit, actual)

This causes kvmalloc() to allocate only 2 MB instead of the required
256 GB, leading to an out-of-bounds write in
init_dvs_6axis_table_from_default() that triggers a kernel panic.

Use array3_size() to prevent the overflow, consistent with the
CVE-2022-50399 fix in sh_css_set_black_frame().

Fixes: a49d25364dfb ("staging/atomisp: add support for DVS")
Cc: stable@xxxxxxxxxxxxxxx
Signed-off-by: Siho Lee <25esihoya@xxxxxxxxx>
---
 drivers/staging/media/atomisp/pci/sh_css_param_dvs.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c
b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c
index 9ccdb66de..3ea707528 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c
@@ -48,7 +48,7 @@ alloc_dvs_6axis_table(const struct ia_css_resolution
*frame_res,
 		}

 		/* Generate Y buffers  */
-		dvs_config->xcoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t),
+		dvs_config->xcoords_y = kvmalloc(array3_size(width_y, height_y,
sizeof(uint32_t)),
 						 GFP_KERNEL);
 		if (!dvs_config->xcoords_y) {
 			IA_CSS_ERROR("out of memory");
@@ -56,7 +56,7 @@ alloc_dvs_6axis_table(const struct ia_css_resolution
*frame_res,
 			goto exit;
 		}

-		dvs_config->ycoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t),
+		dvs_config->ycoords_y = kvmalloc(array3_size(width_y, height_y,
sizeof(uint32_t)),
 						 GFP_KERNEL);
 		if (!dvs_config->ycoords_y) {
 			IA_CSS_ERROR("out of memory");
@@ -67,7 +67,7 @@ alloc_dvs_6axis_table(const struct ia_css_resolution
*frame_res,
 		/* Generate UV buffers  */
 		IA_CSS_LOG("UV W %d H %d", width_uv, height_uv);

-		dvs_config->xcoords_uv = kvmalloc(width_uv * height_uv * sizeof(uint32_t),
+		dvs_config->xcoords_uv = kvmalloc(array3_size(width_uv, height_uv,
sizeof(uint32_t)),
 						  GFP_KERNEL);
 		if (!dvs_config->xcoords_uv) {
 			IA_CSS_ERROR("out of memory");
@@ -75,7 +75,7 @@ alloc_dvs_6axis_table(const struct ia_css_resolution
*frame_res,
 			goto exit;
 		}

-		dvs_config->ycoords_uv = kvmalloc(width_uv * height_uv * sizeof(uint32_t),
+		dvs_config->ycoords_uv = kvmalloc(array3_size(width_uv, height_uv,
sizeof(uint32_t)),
 						  GFP_KERNEL);
 		if (!dvs_config->ycoords_uv) {
 			IA_CSS_ERROR("out of memory");
-- 
2.43.0