diff --git a/avconv.c b/avconv.c
index 85ee6f7e251da8a934217d2164d9f78f87d06ef5..07bb86d4d1e2fd782aa4c7d369754ea53d81143b 100644
--- a/avconv.c
+++ b/avconv.c
@@ -1704,8 +1704,10 @@ static int poll_filters(void)
                                                    AV_TIME_BASE_Q,
                                                    ost->st->codec->time_base);
 
-            if (of->start_time && filtered_frame->pts < of->start_time)
-                return 0;
+            if (of->start_time && filtered_frame->pts < of->start_time) {
+                avfilter_unref_buffer(picref);
+                continue;
+            }
 
             switch (ost->filter->filter->inputs[0]->type) {
             case AVMEDIA_TYPE_VIDEO: