swscale: fix JPEG-range YUV scaling artifacts.

YUV planes were marked as uint16_t, but they contained signed data.
Fixes issue 1108 and 675.

Signed-off-by: Ronald S. Bultje <rsbultje@gmail.com>

libswscale/swscale.c

@@ -1701,7 +1701,7 @@ static void hScale_c(int16_t *dst, int dstW, const uint8_t *src,
 
 //FIXME all pal and rgb srcFormats could do this convertion as well
 //FIXME all scalers more complex than bilinear could do half of this transform
-static void chrRangeToJpeg_c(uint16_t *dstU, uint16_t *dstV, int width)
+static void chrRangeToJpeg_c(int16_t *dstU, int16_t *dstV, int width)
 {
     int i;
     for (i = 0; i < width; i++) {
@@ -1709,7 +1709,7 @@ static void chrRangeToJpeg_c(uint16_t *dstU, uint16_t *dstV, int width)
         dstV[i] = (FFMIN(dstV[i],30775)*4663 - 9289992)>>12; //-264
     }
 }
-static void chrRangeFromJpeg_c(uint16_t *dstU, uint16_t *dstV, int width)
+static void chrRangeFromJpeg_c(int16_t *dstU, int16_t *dstV, int width)
 {
     int i;
     for (i = 0; i < width; i++) {
@@ -1717,13 +1717,13 @@ static void chrRangeFromJpeg_c(uint16_t *dstU, uint16_t *dstV, int width)
         dstV[i] = (dstV[i]*1799 + 4081085)>>11; //1469
     }
 }
-static void lumRangeToJpeg_c(uint16_t *dst, int width)
+static void lumRangeToJpeg_c(int16_t *dst, int width)
 {
     int i;
     for (i = 0; i < width; i++)
         dst[i] = (FFMIN(dst[i],30189)*19077 - 39057361)>>14;
 }
-static void lumRangeFromJpeg_c(uint16_t *dst, int width)
+static void lumRangeFromJpeg_c(int16_t *dst, int width)
 {
     int i;
     for (i = 0; i < width; i++)
@@ -1752,7 +1752,7 @@ static av_always_inline void hyscale(SwsContext *c, uint16_t *dst, int dstWidth,
                                      uint32_t *pal, int isAlpha)
 {
     void (*toYV12)(uint8_t *, const uint8_t *, int, uint32_t *) = isAlpha ? c->alpToYV12 : c->lumToYV12;
-    void (*convertRange)(uint16_t *, int) = isAlpha ? NULL : c->lumConvertRange;
+    void (*convertRange)(int16_t *, int) = isAlpha ? NULL : c->lumConvertRange;
 
     if (toYV12) {
         toYV12(formatConvBuffer, src, srcW, pal);

libswscale/swscale_internal.h

@@ -310,8 +310,8 @@ typedef struct SwsContext {
                    int xInc, const int16_t *filter, const int16_t *filterPos,
                    int filterSize);
 
-    void (*lumConvertRange)(uint16_t *dst, int width); ///< Color range conversion function for luma plane if needed.
-    void (*chrConvertRange)(uint16_t *dst1, uint16_t *dst2, int width); ///< Color range conversion function for chroma planes if needed.
+    void (*lumConvertRange)(int16_t *dst, int width); ///< Color range conversion function for luma plane if needed.
+    void (*chrConvertRange)(int16_t *dst1, int16_t *dst2, int width); ///< Color range conversion function for chroma planes if needed.
 
     int needs_hcscale; ///< Set if there are chroma planes to be converted.
 

tests/ref/lavf/pixfmt

@@ -10,9 +10,9 @@ ac68f9fdd9d55efd0306d9b004038761 *./tests/data/pixfmt/yuyv422.yuv
 304128 ./tests/data/pixfmt/yuv410p.yuv
 8594ea0b8d7c2c964525b0801b5351de *./tests/data/pixfmt/yuv411p.yuv
 304128 ./tests/data/pixfmt/yuv411p.yuv
-66673539adf8cda28e3b76068d3aae61 *./tests/data/pixfmt/yuvj420p.yuv
+e176bd14185788110e055f945de7f95f *./tests/data/pixfmt/yuvj420p.yuv
 304128 ./tests/data/pixfmt/yuvj420p.yuv
-572bf387dd1e3f073cbfd082e055ca81 *./tests/data/pixfmt/yuvj422p.yuv
+472028e46a81c98d9b2477507def4723 *./tests/data/pixfmt/yuvj422p.yuv
 304128 ./tests/data/pixfmt/yuvj422p.yuv
 c10442da177c9f1d12be3c53be6fa12c *./tests/data/pixfmt/yuvj444p.yuv
 304128 ./tests/data/pixfmt/yuvj444p.yuv

tests/ref/lavfi/pixdesc_be

@@ -43,6 +43,6 @@ yuv444p16le         1262a0dc57ee147967fc896d04206313
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
 yuvj420p            32eec78ba51857b16ce9b813a49b7189
 yuvj422p            0dfa0ed434f73be51428758c69e082cb
-yuvj440p            9c3a093ff64a83ac4cf0b1e65390e236
-yuvj444p            ede1e5882d5c5bba48ea33cf1209d231
+yuvj440p            657501a28004e27a592757a7509f5189
+yuvj444p            98d3d054f2ec09a75eeed5d328dc75b7
 yuyv422             f2569f2b5069a0ee0cecae33de0455e3

tests/ref/lavfi/pixdesc_le

@@ -43,6 +43,6 @@ yuv444p16le         1262a0dc57ee147967fc896d04206313
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
 yuvj420p            32eec78ba51857b16ce9b813a49b7189
 yuvj422p            0dfa0ed434f73be51428758c69e082cb
-yuvj440p            9c3a093ff64a83ac4cf0b1e65390e236
-yuvj444p            ede1e5882d5c5bba48ea33cf1209d231
+yuvj440p            657501a28004e27a592757a7509f5189
+yuvj444p            98d3d054f2ec09a75eeed5d328dc75b7
 yuyv422             f2569f2b5069a0ee0cecae33de0455e3

tests/ref/lavfi/pixfmts_copy_le

@@ -43,6 +43,6 @@ yuv444p16le         1262a0dc57ee147967fc896d04206313
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
 yuvj420p            32eec78ba51857b16ce9b813a49b7189
 yuvj422p            0dfa0ed434f73be51428758c69e082cb
-yuvj440p            9c3a093ff64a83ac4cf0b1e65390e236
-yuvj444p            ede1e5882d5c5bba48ea33cf1209d231
+yuvj440p            657501a28004e27a592757a7509f5189
+yuvj444p            98d3d054f2ec09a75eeed5d328dc75b7
 yuyv422             f2569f2b5069a0ee0cecae33de0455e3

tests/ref/lavfi/pixfmts_hflip_le

@@ -34,5 +34,5 @@ yuv444p16le         70793e3d66d0c23a0cdedabe9c24c2a7
 yuva420p            d83ec0c01498189f179ec574918185f1
 yuvj420p            df3aaaec3bb157c3bde5f0365af30f4f
 yuvj422p            d113871528d510a192797af59df9c05c
-yuvj440p            e8f7ed76e57c892a1e9e27a3f29452db
-yuvj444p            2c3ae369607608c6dcb0d830f00f971a
+yuvj440p            07f5ff12ced85aba1b5cf51692fff4bb
+yuvj444p            8d95f6b4d4c9b4b0389d36df686bfa46

tests/ref/lavfi/pixfmts_null_le

@@ -43,6 +43,6 @@ yuv444p16le         1262a0dc57ee147967fc896d04206313
 yuva420p            a29884f3f3dfe1e00b961bc17bef3d47
 yuvj420p            32eec78ba51857b16ce9b813a49b7189
 yuvj422p            0dfa0ed434f73be51428758c69e082cb
-yuvj440p            9c3a093ff64a83ac4cf0b1e65390e236
-yuvj444p            ede1e5882d5c5bba48ea33cf1209d231
+yuvj440p            657501a28004e27a592757a7509f5189
+yuvj444p            98d3d054f2ec09a75eeed5d328dc75b7
 yuyv422             f2569f2b5069a0ee0cecae33de0455e3

tests/ref/lavfi/pixfmts_pad_le

@@ -13,5 +13,5 @@ yuv444p             45484f0411d336ce94636da0395f4692
 yuva420p            919722724765dc3a716c38fa53b20580
 yuvj420p            4f20e2799966c21a9d9e0788b0956925
 yuvj422p            e4d84b0683f77a76f1c17d976eff127c
-yuvj440p            9ccc1f03d8d9a00dc4a5888bd77093ad
-yuvj444p            7d202babcedf836e9ae9776163ee9425
+yuvj440p            33511c43339aa32533ab832861c150c3
+yuvj444p            82f0badd9d0c062bbfa0d9d73d7240a3

tests/ref/lavfi/pixfmts_scale_le

@@ -43,6 +43,6 @@ yuv444p16le         385d0cc5240d62da0871915be5d86f0a
 yuva420p            8673a9131fb47de69788863f93a50eb7
 yuvj420p            30427bd6caf5bda93a173dbebe759e09
 yuvj422p            fc8288f64fd149573f73cf8da05d8e6d
-yuvj440p            26d0b4713a87ab9637a4062c22e6e70d
-yuvj444p            894e6184d987a5ec4dc6f77bb75ef38c
+yuvj440p            508ac7a9ddeb6d1794a1100ba7a1664c
+yuvj444p            73aebe144085b22d1189caf6ca07e18c
 yuyv422             169e19ac91b257bd84ace0fdf56559ad

tests/ref/lavfi/pixfmts_vflip_le

@@ -43,6 +43,6 @@ yuv444p16le         8f31557bc52adfe00ae8b40a9b8c23f8
 yuva420p            c705d1cf061d8c6580ac690b55f92276
 yuvj420p            41fd02b204da0ab62452cd14b595e2e4
 yuvj422p            7f6ca9bc1812cde02036d7d29a7cce43
-yuvj440p            40591908cca457f51dee30a86c3e8ffd
-yuvj444p            77e5e095881c52a04fd9f5abd1d7b5ac
+yuvj440p            25711c3c0fd15ec19c59a10784fcfb96
+yuvj444p            e45dee2ac02276dfab92e8ebfbe52e00
 yuyv422             e944ff7316cd03c42c091717ce74f602