diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
index 0cd966f662aeb729a8b5eef5eb5779ae8041c94b..020f8924e2fcf4bb4fb037fe83d3e8528bf27ed9 100644
--- a/libavcodec/vaapi_encode_h264.c
+++ b/libavcodec/vaapi_encode_h264.c
@@ -778,7 +778,7 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
 
         vseq->level_idc = avctx->level;
 
-        vseq->max_num_ref_frames = 2;
+        vseq->max_num_ref_frames = 1 + (avctx->max_b_frames > 0);
 
         vseq->picture_width_in_mbs  = priv->mb_width;
         vseq->picture_height_in_mbs = priv->mb_height;