diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index 3cc5f857151981ec6aefb993161b40a0827ac445..b83a279bc040d571359474128860e9c2f35a0183 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -48,7 +48,9 @@ static int get_scale_factor(H264Context * const h, int poc, int poc1, int i){
 }
 
 void ff_h264_direct_dist_scale_factor(H264Context * const h){
-    const int poc = h->cur_pic_ptr->field_poc[h->picture_structure == PICT_BOTTOM_FIELD];
+    const int poc = h->picture_structure == PICT_FRAME ?
+        h->cur_pic_ptr->poc :
+        h->cur_pic_ptr->field_poc[h->picture_structure == PICT_BOTTOM_FIELD];
     const int poc1 = h->ref_list[1][0].poc;
     int i, field;