diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index accb6b3a945955d56d03638674a23a2409cbd3c0..3dddafca1064cf86d385addb96fb59c74316ff05 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -1804,6 +1804,9 @@ void avcodec_flush_buffers(AVCodecContext *avctx)
         ff_thread_flush(avctx);
     else if (avctx->codec->flush)
         avctx->codec->flush(avctx);
+
+    if (!avctx->refcounted_frames)
+        av_frame_unref(&avctx->internal->to_free);
 }
 
 int av_get_exact_bits_per_sample(enum AVCodecID codec_id)