diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c
index dfe474c40cb2ad8a46f8407b468649a77548cb30..42c6b66b69a538b391a1143a58330a0e28c549ed 100644
--- a/libavfilter/vf_libvmaf.c
+++ b/libavfilter/vf_libvmaf.c
@@ -130,6 +130,8 @@ FRAMESYNC_DEFINE_CLASS(libvmaf, LIBVMAFContext, fs);
     \
     ret = !s->frame_set;                                                        \
     \
+    av_frame_unref(s->gref);                                                    \
+    av_frame_unref(s->gmain);                                                   \
     s->frame_set = 0;                                                           \
     \
     pthread_cond_signal(&s->cond);                                              \