diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index 570799f6d7acb13e5197017a8f5eec4306fc8936..7601d1608289a90e1547a296a9607deac561c9f4 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -613,7 +613,6 @@ static int decode_receive_frame_internal(AVCodecContext *avctx, AVFrame *frame)
     if (ret == AVERROR_EOF)
         avci->draining_done = 1;
 
-    /* free the per-frame decode data */
     if (!ret) {
         /* the only case where decode data is not set should be decoders
          * that do not call ff_get_buffer() */
@@ -631,10 +630,11 @@ static int decode_receive_frame_internal(AVCodecContext *avctx, AVFrame *frame)
                 }
             }
         }
-
-        av_buffer_unref(&frame->private_ref);
     }
 
+    /* free the per-frame decode data */
+    av_buffer_unref(&frame->private_ref);
+
     return ret;
 }