From bb4c9d0a8ead02f7d943c2bae3e36b30e605b30b Mon Sep 17 00:00:00 2001
From: Philip Langdale <philipl@overt.org>
Date: Mon, 13 Nov 2017 21:11:27 -0800
Subject: [PATCH] avcodec: Don't assume separate u and v planes in
ff_alloc_picture
alloc_frame_buffer in ff_alloc_picture asserts that the linesize
of planes 1 and 2 are the same. If the pixfmt has a single uv
plane, like NV12, this won't be true.
So, let's only do this check if there are more than 2 planes.
We never hit this with previous hw formats because they don't set
linesize to meaningful values, but the cuda hw format sets the
values based on the underlying data layout.
---
libavcodec/mpegpicture.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/libavcodec/mpegpicture.c b/libavcodec/mpegpicture.c
index 9811a778b75..2b72346fb24 100644
--- a/libavcodec/mpegpicture.c
+++ b/libavcodec/mpegpicture.c
@@ -22,6 +22,7 @@
#include "libavutil/avassert.h"
#include "libavutil/common.h"
+#include "libavutil/pixdesc.h"
#include "avcodec.h"
#include "motion_est.h"
@@ -155,7 +156,8 @@ static int alloc_frame_buffer(AVCodecContext *avctx, Picture *pic,
return -1;
}
- if (pic->f->linesize[1] != pic->f->linesize[2]) {
+ if (av_pix_fmt_count_planes(pic->f->format) > 2 &&
+ pic->f->linesize[1] != pic->f->linesize[2]) {
av_log(avctx, AV_LOG_ERROR,
"get_buffer() failed (uv stride mismatch)\n");
ff_mpeg_unref_picture(avctx, pic);
--
GitLab