From 2b8b7921c55a93049a86cfeb2fda9423d16f8ebe Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Fri, 24 Feb 2017 13:11:43 +0100
Subject: [PATCH] avcodec/vp3dsp: Fix multiple signed integer overflow: 46341 *
47523 cannot be represented in type 'int'
Fixes: 664/clusterfuzz-testcase-4917047475568640
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
libavcodec/vp3dsp.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libavcodec/vp3dsp.c b/libavcodec/vp3dsp.c
index 814c78e34a7..4b6c838b709 100644
--- a/libavcodec/vp3dsp.c
+++ b/libavcodec/vp3dsp.c
@@ -41,7 +41,7 @@
#define xC6S2 25080
#define xC7S1 12785
-#define M(a, b) (((a) * (b)) >> 16)
+#define M(a, b) ((int)((SUINT)(a) * (b)) >> 16)
static av_always_inline void idct(uint8_t *dst, int stride,
int16_t *input, int type)
--
GitLab