From 9ea29985865720c24f6e7ac213a7dc597f1f0453 Mon Sep 17 00:00:00 2001
From: Pavel Koshevoy <pkoshevoy@gmail.com>
Date: Sun, 22 Jan 2017 16:20:05 -0700
Subject: [PATCH] avcodec/cuvid: fail early if GPU can't handle video
 resolution

CUVID on GeForce GT 730 and GeForce GTX 1060 does not report any error when
decoding 8K h264 packets. However, it does return an error during
cuvidCreateDecoder call if the indicated video resolution is not
supported.

Given that stream resolution is typically known as a result of probing
it is better to use this information during avcodec_open2 call to fail
immediately, rather than proceeding to decode and never receiving any
frames from the decoder nor receiving any indication of decode failure.

Signed-off-by: Timo Rothenpieler <timo@rothenpieler.org>
---
 libavcodec/cuvid.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c
index 8fc713d0816..9b35476f2a5 100644
--- a/libavcodec/cuvid.c
+++ b/libavcodec/cuvid.c
@@ -612,7 +612,10 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx)
     return 0;
 }
 
-static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo)
+static int cuvid_test_dummy_decoder(AVCodecContext *avctx,
+                                    const CUVIDPARSERPARAMS *cuparseinfo,
+                                    int probed_width,
+                                    int probed_height)
 {
     CuvidContext *ctx = avctx->priv_data;
     CUVIDDECODECREATEINFO cuinfo;
@@ -625,8 +628,8 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
     cuinfo.ChromaFormat = cudaVideoChromaFormat_420;
     cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12;
 
-    cuinfo.ulWidth = 1280;
-    cuinfo.ulHeight = 720;
+    cuinfo.ulWidth = probed_width;
+    cuinfo.ulHeight = probed_height;
     cuinfo.ulTargetWidth = cuinfo.ulWidth;
     cuinfo.ulTargetHeight = cuinfo.ulHeight;
 
@@ -669,6 +672,9 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
                                        AV_PIX_FMT_NV12,
                                        AV_PIX_FMT_NONE };
 
+    int probed_width = avctx->coded_width ? avctx->coded_width : 1280;
+    int probed_height = avctx->coded_height ? avctx->coded_height : 720;
+
     // Accelerated transcoding scenarios with 'ffmpeg' require that the
     // pix_fmt be set to AV_PIX_FMT_CUDA early. The sw_pix_fmt, and the
     // pix_fmt for non-accelerated transcoding, do not need to be correct
@@ -824,7 +830,9 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
     if (ret < 0)
         goto error;
 
-    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo);
+    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo,
+                                   probed_width,
+                                   probed_height);
     if (ret < 0)
         goto error;
 
-- 
GitLab