diff --git a/libavformat/rtp_h264.c b/libavformat/rtp_h264.c
index f3ae6993f541ea5c94dbdfd3f229ebf0355c2f61..47cbf79db516fbfe5e5d47d1f50397a08a0b6a6d 100644
--- a/libavformat/rtp_h264.c
+++ b/libavformat/rtp_h264.c
@@ -310,6 +310,8 @@ static int h264_handle_packet(AVFormatContext *ctx,
         break;
     }
 
+    pkt->stream_index = st->index;
+
     return result;
 }
 
diff --git a/libavformat/rtpdec.c b/libavformat/rtpdec.c
index 4a5257bc6d8c38f1412ff6250172f24ec9d2980c..e1ba888cb1b672457c10f349cc003926c8c04bfe 100644
--- a/libavformat/rtpdec.c
+++ b/libavformat/rtpdec.c
@@ -382,7 +382,6 @@ static void finalize_packet(RTPDemuxContext *s, AVPacket *pkt, uint32_t timestam
         addend = av_rescale(s->last_rtcp_ntp_time - s->first_rtcp_ntp_time, s->st->time_base.den, (uint64_t)s->st->time_base.num << 32);
         pkt->pts = addend + delta_timestamp;
     }
-    pkt->stream_index = s->st->index;
 }
 
 /**
@@ -536,6 +535,8 @@ int rtp_parse_packet(RTPDemuxContext *s, AVPacket *pkt,
             memcpy(pkt->data, buf, len);
             break;
         }
+
+        pkt->stream_index = st->index;
     }
 
     // now perform timestamp things....