diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
index 029e8f301ab4eaf99780c05a8c83762574e918da..948751daf4ab889e66fc4f7e8bf824081828502e 100644
--- a/libavcodec/qsvenc.c
+++ b/libavcodec/qsvenc.c
@@ -1026,6 +1026,9 @@ static void clear_unused_frames(QSVEncContext *q)
     while (cur) {
         if (cur->used && !cur->surface.Data.Locked) {
             free_encoder_ctrl_payloads(&cur->enc_ctrl);
+            if (cur->frame->format == AV_PIX_FMT_QSV) {
+                av_frame_unref(cur->frame);
+            }
             cur->used = 0;
         }
         cur = cur->next;