diff --git a/libavformat/movenchint.c b/libavformat/movenchint.c
index 579d040f64f2983b65b4755d06861581ffbafd92..5ef90f154ad420f8d1c3871036407f2ff6c47129 100644
--- a/libavformat/movenchint.c
+++ b/libavformat/movenchint.c
@@ -43,9 +43,9 @@ int ff_mov_init_hinting(AVFormatContext *s, int index, int src_index)
     track->enc->codec_type = AVMEDIA_TYPE_DATA;
     track->enc->codec_tag  = track->tag;
 
-    track->rtp_ctx = ff_rtp_chain_mux_open(s, src_st, NULL,
-                                           RTP_MAX_PACKET_SIZE);
-    if (!track->rtp_ctx)
+    ret = ff_rtp_chain_mux_open(&track->rtp_ctx, s, src_st, NULL,
+                                RTP_MAX_PACKET_SIZE);
+    if (ret < 0)
         goto fail;
 
     /* Copy the RTP AVStream timebase back to the hint AVStream */
diff --git a/libavformat/rtpenc_chain.c b/libavformat/rtpenc_chain.c
index 3b5ea6c586374731b77697e14e6aeafbedfcf426..374209931406bc560e5e900d82ea08d66466a7bf 100644
--- a/libavformat/rtpenc_chain.c
+++ b/libavformat/rtpenc_chain.c
@@ -25,8 +25,8 @@
 #include "avio_internal.h"
 #include "libavutil/opt.h"
 
-AVFormatContext *ff_rtp_chain_mux_open(AVFormatContext *s, AVStream *st,
-                                       URLContext *handle, int packet_size)
+int ff_rtp_chain_mux_open(AVFormatContext **out, AVFormatContext *s,
+                          AVStream *st, URLContext *handle, int packet_size)
 {
     AVFormatContext *rtpctx = NULL;
     int ret;
@@ -34,17 +34,23 @@ AVFormatContext *ff_rtp_chain_mux_open(AVFormatContext *s, AVStream *st,
     uint8_t *rtpflags;
     AVDictionary *opts = NULL;
 
-    if (!rtp_format)
+    if (!rtp_format) {
+        ret = AVERROR(ENOSYS);
         goto fail;
+    }
 
     /* Allocate an AVFormatContext for each output stream */
     rtpctx = avformat_alloc_context();
-    if (!rtpctx)
+    if (!rtpctx) {
+        ret = AVERROR(ENOMEM);
         goto fail;
+    }
 
     rtpctx->oformat = rtp_format;
-    if (!avformat_new_stream(rtpctx, NULL))
+    if (!avformat_new_stream(rtpctx, NULL)) {
+        ret = AVERROR(ENOMEM);
         goto fail;
+    }
     /* Pass the interrupt callback on */
     rtpctx->interrupt_callback = s->interrupt_callback;
     /* Copy the max delay setting; the rtp muxer reads this. */
@@ -76,14 +82,15 @@ AVFormatContext *ff_rtp_chain_mux_open(AVFormatContext *s, AVStream *st,
             av_free(ptr);
         }
         avformat_free_context(rtpctx);
-        return NULL;
+        return ret;
     }
 
-    return rtpctx;
+    *out = rtpctx;
+    return 0;
 
 fail:
     av_free(rtpctx);
     if (handle)
         ffurl_close(handle);
-    return NULL;
+    return ret;
 }
diff --git a/libavformat/rtpenc_chain.h b/libavformat/rtpenc_chain.h
index 6bdddcfe99e08f4be0fb0f244e6cb3589177a3c5..66b9e4cd0aeb2bf99b97e9768e432919df37d66b 100644
--- a/libavformat/rtpenc_chain.h
+++ b/libavformat/rtpenc_chain.h
@@ -25,7 +25,7 @@
 #include "avformat.h"
 #include "url.h"
 
-AVFormatContext *ff_rtp_chain_mux_open(AVFormatContext *s, AVStream *st,
-                                       URLContext *handle, int packet_size);
+int ff_rtp_chain_mux_open(AVFormatContext **out, AVFormatContext *s,
+                          AVStream *st, URLContext *handle, int packet_size);
 
 #endif /* AVFORMAT_RTPENC_CHAIN_H */
diff --git a/libavformat/rtsp.c b/libavformat/rtsp.c
index 31eb4befd650f47fec8e04ae4ee8f82ddaf585f1..f53aadf191f4a76bdf3b4e4d3511f4a3e8c85965 100644
--- a/libavformat/rtsp.c
+++ b/libavformat/rtsp.c
@@ -606,11 +606,13 @@ static int rtsp_open_transport_ctx(AVFormatContext *s, RTSPStream *rtsp_st)
         s->ctx_flags |= AVFMTCTX_NOHEADER;
 
     if (s->oformat && CONFIG_RTSP_MUXER) {
-        rtsp_st->transport_priv = ff_rtp_chain_mux_open(s, st,
-                                      rtsp_st->rtp_handle,
-                                      RTSP_TCP_MAX_PACKET_SIZE);
+        int ret = ff_rtp_chain_mux_open(&rtsp_st->transport_priv, s, st,
+                                        rtsp_st->rtp_handle,
+                                        RTSP_TCP_MAX_PACKET_SIZE);
         /* Ownership of rtp_handle is passed to the rtp mux context */
         rtsp_st->rtp_handle = NULL;
+        if (ret < 0)
+            return ret;
     } else if (rt->transport == RTSP_TRANSPORT_RDT && CONFIG_RTPDEC)
         rtsp_st->transport_priv = ff_rdt_parse_open(s, st->index,
                                             rtsp_st->dynamic_protocol_context,
diff --git a/libavformat/sapenc.c b/libavformat/sapenc.c
index 0c3e95edd67a8d3b493587016dc41587f7215012..7e84a3fb991bee39c5fec23fe9a052f3461f3ab7 100644
--- a/libavformat/sapenc.c
+++ b/libavformat/sapenc.c
@@ -150,8 +150,10 @@ static int sap_write_header(AVFormatContext *s)
             ret = AVERROR(EIO);
             goto fail;
         }
-        s->streams[i]->priv_data = contexts[i] =
-            ff_rtp_chain_mux_open(s, s->streams[i], fd, 0);
+        ret = ff_rtp_chain_mux_open(&contexts[i], s, s->streams[i], fd, 0);
+        if (ret < 0)
+            goto fail;
+        s->streams[i]->priv_data = contexts[i];
         av_strlcpy(contexts[i]->filename, url, sizeof(contexts[i]->filename));
     }