From bcff32d1b4eb583b8be96295fe63c575cef441f2 Mon Sep 17 00:00:00 2001
From: Kostya Shishkov <kostya.shishkov@gmail.com>
Date: Mon, 9 Mar 2009 17:25:43 +0000
Subject: [PATCH] Mask all unused bits for packed pixel format instead of green
and alpha mask only. That fixes the case when converting 15-bit RGB/BGR to
YUV and high bit is set for input value(s).
Originally committed as revision 28916 to svn://svn.mplayerhq.hu/mplayer/trunk/libswscale
---
libswscale/swscale_template.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libswscale/swscale_template.c b/libswscale/swscale_template.c
index 6f2e243052e..da6278d80c3 100644
--- a/libswscale/swscale_template.c
+++ b/libswscale/swscale_template.c
@@ -1663,7 +1663,7 @@ static inline void RENAME(name ## _half)(uint8_t *dstU, uint8_t *dstV, uint8_t *
{\
int pix0= ((type*)src)[2*i+0];\
int pix1= ((type*)src)[2*i+1];\
- int g= (pix0&(maskg|maska))+(pix1&(maskg|maska));\
+ int g= (pix0&~(maskr|maskb))+(pix1&~(maskr|maskb));\
int b= ((pix0+pix1-g)&(maskb|(2*maskb)))>>shb;\
int r= ((pix0+pix1-g)&(maskr|(2*maskr)))>>shr;\
g&= maskg|(2*maskg);\
--
GitLab