diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 17f63c4b5b83fd16b816b61c52492b0bf5ec8151..3c99030ed2c7378bd35d4fad6a3fbee2f2f897f9 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -95,12 +95,6 @@
 #define IS_REF0(a)         ((a) & MB_TYPE_REF0)
 #define IS_8x8DCT(a)       ((a) & MB_TYPE_8x8DCT)
 
-/**
- * Value of Picture.reference when Picture is not a reference picture, but
- * is held for delayed output.
- */
-#define DELAYED_PIC_REF 4
-
 #define QP_MAX_NUM (51 + 2 * 6)           // The maximum supported qp
 
 /* NAL unit types */
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 42367e1e48160632ce3c15ac81ed64bb615fbfd7..773c9e258b0c0727623e664792c3eadc72f71acd 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -1008,9 +1008,6 @@ static int free_context_frame(MpegEncContext *s)
     for (i = 0; i < 3; i++)
         av_freep(&s->visualization_buffer[i]);
 
-    if (!(s->avctx->active_thread_type & FF_THREAD_FRAME))
-        avcodec_default_free_buffers(s->avctx);
-
     return 0;
 }
 
@@ -1126,6 +1123,9 @@ void ff_MPV_common_end(MpegEncContext *s)
 
     free_context_frame(s);
 
+    if (!(s->avctx->active_thread_type & FF_THREAD_FRAME))
+        avcodec_default_free_buffers(s->avctx);
+
     s->context_initialized      = 0;
     s->last_picture_ptr         =
     s->next_picture_ptr         =
@@ -1244,7 +1244,7 @@ static inline int pic_is_unused(MpegEncContext *s, Picture *pic)
 {
     if (pic->f.data[0] == NULL)
         return 1;
-    if (pic->needs_realloc)
+    if (pic->needs_realloc && !(pic->f.reference & DELAYED_PIC_REF))
         if (!pic->owner2 || pic->owner2 == s)
             return 1;
     return 0;
diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
index bd4caa3e7b216c4aa42723a7c8aef4925afbfb95..6c9a7cbda65d9d471e02eef80e457c0d3ac45cf8 100644
--- a/libavcodec/mpegvideo.h
+++ b/libavcodec/mpegvideo.h
@@ -79,6 +79,12 @@ enum OutputFormat {
 #define EXT_START_CODE          0x000001b5
 #define USER_START_CODE         0x000001b2
 
+/**
+ * Value of Picture.reference when Picture is not a reference picture, but
+ * is held for delayed output.
+ */
+#define DELAYED_PIC_REF 4
+
 struct MpegEncContext;
 
 /**