diff --git a/libswresample/resample_template.c b/libswresample/resample_template.c
index d519ec6b99a64890d7bc4cc571f8e3526113e1ac..2ce97fa6ecb57b95271c7d6ff96d2ed6b5b29c57 100644
--- a/libswresample/resample_template.c
+++ b/libswresample/resample_template.c
@@ -126,12 +126,13 @@ int RENAME(swri_resample)(ResampleContext *c, DELEM *dst, const DELEM *src, int
         }
         *consumed= FFMAX(sample_index, 0);
         index += FFMIN(sample_index, 0) << c->phase_shift;
-    }
 
-    if(compensation_distance){
-        compensation_distance -= dst_index;
-        av_assert1(compensation_distance > 0);
+        if(compensation_distance){
+            compensation_distance -= dst_index;
+            av_assert1(compensation_distance > 0);
+        }
     }
+
     if(update_ctx){
         c->frac= frac;
         c->index= index;