From 93e157f40f415119ea0f94b35596965e9870f863 Mon Sep 17 00:00:00 2001
From: Thomas Guillem <thomas@gllm.fr>
Date: Wed, 4 Jul 2018 09:05:22 +0200
Subject: [PATCH] avcodec/videotoolboxenc: fix undefined behavior with
rc_max_rate=0
On macOS, a zero rc_max_rate cause an error from
VTSessionSetProperty(kVTCompressionPropertyKey_DataRateLimits).
on iOS (depending on device/version), a zero rc_max_rate cause invalid
arguments from the vtenc_output_callback after few frames and then a crash
within the VideoToolbox library.
Signed-off-by: Aman Gupta <aman@tmm1.net>
---
libavcodec/videotoolboxenc.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
index ac847358ab3..f311149e40b 100644
--- a/libavcodec/videotoolboxenc.c
+++ b/libavcodec/videotoolboxenc.c
@@ -1017,7 +1017,7 @@ static int vtenc_create_encoder(AVCodecContext *avctx,
return AVERROR_EXTERNAL;
}
- if (vtctx->codec_id == AV_CODEC_ID_H264) {
+ if (vtctx->codec_id == AV_CODEC_ID_H264 && max_rate > 0) {
// kVTCompressionPropertyKey_DataRateLimits is not available for HEVC
bytes_per_second_value = max_rate >> 3;
bytes_per_second = CFNumberCreate(kCFAllocatorDefault,
@@ -1058,7 +1058,10 @@ static int vtenc_create_encoder(AVCodecContext *avctx,
av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status);
return AVERROR_EXTERNAL;
}
+ }
+ if (vtctx->codec_id == AV_CODEC_ID_H264) {
+ // kVTCompressionPropertyKey_ProfileLevel is not available for HEVC
if (profile_level) {
status = VTSessionSetProperty(vtctx->session,
kVTCompressionPropertyKey_ProfileLevel,
--
GitLab