diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 97bf588b512c1dddf0ffce48777cdacc62bdca2f..ad296753c3bd9f236037095f8b785d2c11d506dc 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -571,8 +571,7 @@ void ff_h264_remove_all_refs(H264Context *h)
 
     if (h->short_ref_count && !h->last_pic_for_ec.f->data[0]) {
         ff_h264_unref_picture(h, &h->last_pic_for_ec);
-        if (h->short_ref[0]->f->buf[0])
-            ff_h264_ref_picture(h, &h->last_pic_for_ec, h->short_ref[0]);
+        ff_h264_ref_picture(h, &h->last_pic_for_ec, h->short_ref[0]);
     }
 
     for (i = 0; i < h->short_ref_count; i++) {
diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c
index 41c0964392a47828b5de5deb3cf0bbc706246e85..35598ea7cd759d36ab4bff14124117eb4262c71f 100644
--- a/libavcodec/h264dec.c
+++ b/libavcodec/h264dec.c
@@ -850,7 +850,12 @@ static int output_frame(H264Context *h, AVFrame *dst, H264Picture *srcp)
     AVFrame *src = srcp->f;
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(src->format);
     int i;
-    int ret = av_frame_ref(dst, src);
+    int ret;
+
+    if (src->format == AV_PIX_FMT_VIDEOTOOLBOX && src->buf[0]->size == 1)
+        return AVERROR_EXTERNAL;
+
+    ret = av_frame_ref(dst, src);
     if (ret < 0)
         return ret;
 
diff --git a/libavcodec/version.h b/libavcodec/version.h
index 7647ad2e189e6e678894ce4525b19ae23af54bd7..cd9ed87225f1d39d298920885b267186a517d8c1 100644
--- a/libavcodec/version.h
+++ b/libavcodec/version.h
@@ -29,7 +29,7 @@
 
 #define LIBAVCODEC_VERSION_MAJOR  57
 #define LIBAVCODEC_VERSION_MINOR  82
-#define LIBAVCODEC_VERSION_MICRO 100
+#define LIBAVCODEC_VERSION_MICRO 101
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
                                                LIBAVCODEC_VERSION_MINOR, \
diff --git a/libavcodec/videotoolbox.c b/libavcodec/videotoolbox.c
index 9199b020556bd32b7d3562820b14048e71a0e4d8..67adad53ed8bb93d1cda3e7289aca98a8adfd1c6 100644
--- a/libavcodec/videotoolbox.c
+++ b/libavcodec/videotoolbox.c
@@ -352,8 +352,6 @@ static int videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame)
     AVVideotoolboxContext *videotoolbox = avctx->hwaccel_context;
     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 
-    av_buffer_unref(&frame->buf[0]);
-
     if (!videotoolbox->session || !vtctx->bitstream)
         return AVERROR_INVALIDDATA;