diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index d816835cbf16414fb966f2db54c3e4bb78e10f59..7d24df261e09a4fa58ff9bce84002aeca92919d6 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -3391,7 +3391,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
 
     if (h->ref_count[0] > max_refs || h->ref_count[1] > max_refs) {
         av_log(h->avctx, AV_LOG_ERROR, "reference overflow\n");
-        h->ref_count[0] = h->ref_count[1] = 1;
+        h->ref_count[0] = h->ref_count[1] = 0;
         return AVERROR_INVALIDDATA;
     }