diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 63f063a7f3f1c6ea384ce303f4bdf9252e6d7d6d..4a19a2e35727680e157a3a5019e742c4651938f9 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -233,7 +233,8 @@ int ff_h264_check_intra_pred_mode(const H264Context *h, H264SliceContext *sl,
     return mode;
 }
 
-const uint8_t *ff_h264_decode_nal(H264Context *h, const uint8_t *src,
+const uint8_t *ff_h264_decode_nal(H264Context *h, H264SliceContext *sl,
+                                  const uint8_t *src,
                                   int *dst_length, int *consumed, int length)
 {
     int i, si, di;
@@ -293,8 +294,8 @@ const uint8_t *ff_h264_decode_nal(H264Context *h, const uint8_t *src,
     }
 #endif
 
-    av_fast_padded_malloc(&h->rbsp_buffer, &h->rbsp_buffer_size, length+MAX_MBPAIR_SIZE);
-    dst = h->rbsp_buffer;
+    av_fast_padded_malloc(&sl->rbsp_buffer, &sl->rbsp_buffer_size, length+MAX_MBPAIR_SIZE);
+    dst = sl->rbsp_buffer;
 
     if (!dst)
         return NULL;
@@ -402,10 +403,6 @@ void ff_h264_free_tables(H264Context *h, int free_rbsp)
         if (!hx)
             continue;
 
-        if (free_rbsp) {
-            av_freep(&hx->rbsp_buffer);
-            hx->rbsp_buffer_size = 0;
-        }
         if (i)
             av_freep(&h->thread_context[i]);
     }
@@ -427,6 +424,11 @@ void ff_h264_free_tables(H264Context *h, int free_rbsp)
         sl->edge_emu_buffer_allocated   = 0;
         sl->top_borders_allocated[0]    = 0;
         sl->top_borders_allocated[1]    = 0;
+
+        if (free_rbsp) {
+            av_freep(&sl->rbsp_buffer);
+            sl->rbsp_buffer_size            = 0;
+        }
     }
 }
 
@@ -745,8 +747,6 @@ static int decode_init_thread_copy(AVCodecContext *avctx)
         h->slice_ctx[i].h264 = h;
 
     h->avctx               = avctx;
-    h->rbsp_buffer         = NULL;
-    h->rbsp_buffer_size    = 0;
     h->context_initialized = 0;
 
     return 0;
@@ -1392,7 +1392,7 @@ static int get_last_needed_nal(H264Context *h, const uint8_t *buf, int buf_size)
                 continue;
         }
 
-        ptr = ff_h264_decode_nal(h, buf + buf_index, &dst_length, &consumed,
+        ptr = ff_h264_decode_nal(h, &h->slice_ctx[0], buf + buf_index, &dst_length, &consumed,
                                  next_avc - buf_index);
 
         if (!ptr || dst_length < 0)
@@ -1494,7 +1494,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size,
             hx = h->thread_context[context_count];
             sl = &h->slice_ctx[context_count];
 
-            ptr = ff_h264_decode_nal(hx, buf + buf_index, &dst_length,
+            ptr = ff_h264_decode_nal(hx, sl, buf + buf_index, &dst_length,
                                      &consumed, next_avc - buf_index);
             if (!ptr || dst_length < 0) {
                 ret = -1;
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 5793f0e471cafd209d52b1d74385cafe01371e9f..2df2dfa306ff0c0ee8efd011370ffdf02a913754 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -476,6 +476,10 @@ typedef struct H264SliceContext {
     CABACContext cabac;
     uint8_t cabac_state[1024];
     int cabac_init_idc;
+
+    // rbsp buffer used for this slice
+    uint8_t *rbsp_buffer;
+    unsigned int rbsp_buffer_size;
 } H264SliceContext;
 
 /**
@@ -585,8 +589,6 @@ typedef struct H264Context {
 
     int nal_ref_idc;
     int nal_unit_type;
-    uint8_t *rbsp_buffer;
-    unsigned int rbsp_buffer_size;
 
     /**
      * Used to parse AVC variant of h264
@@ -817,7 +819,7 @@ int ff_h264_decode_picture_parameter_set(H264Context *h, int bit_length);
  *                   or a decode rbsp tailing?
  * @return decoded bytes, might be src+1 if no escapes
  */
-const uint8_t *ff_h264_decode_nal(H264Context *h, const uint8_t *src,
+const uint8_t *ff_h264_decode_nal(H264Context *h, H264SliceContext *sl, const uint8_t *src,
                                   int *dst_length, int *consumed, int length);
 
 /**
diff --git a/libavcodec/h264_parser.c b/libavcodec/h264_parser.c
index 6aec473a9fa7223cca299980bc51cce4d4b18a6b..f4b97e6761aff797bc30840ab3704069e4bcfc83 100644
--- a/libavcodec/h264_parser.c
+++ b/libavcodec/h264_parser.c
@@ -272,7 +272,7 @@ static inline int parse_nal_units(AVCodecParserContext *s,
             }
             break;
         }
-        ptr = ff_h264_decode_nal(h, buf + buf_index, &dst_length,
+        ptr = ff_h264_decode_nal(h, sl, buf + buf_index, &dst_length,
                                  &consumed, src_length);
         if (!ptr || dst_length < 0)
             break;
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 2cce5547eaa204e14044a6dd140229268a44c004..fbabd1ece7dd04c7e578bef23d751050b51ddd0a 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -538,7 +538,6 @@ int ff_h264_update_thread_context(AVCodecContext *dst,
         for (i = 0; i < MAX_PPS_COUNT; i++)
             av_freep(h->pps_buffers + i);
 
-        av_freep(&h->rbsp_buffer);
         ff_h264_unref_picture(h, &h->last_pic_for_ec);
         memcpy(h, h1, sizeof(H264Context));
 
@@ -572,8 +571,6 @@ int ff_h264_update_thread_context(AVCodecContext *dst,
         h->list_counts       = NULL;
         h->mb2b_xy           = NULL;
         h->mb2br_xy          = NULL;
-        h->rbsp_buffer       = NULL;
-        h->rbsp_buffer_size  = 0;
 
         if (h1->context_initialized) {
         h->context_initialized = 0;
@@ -594,7 +591,6 @@ int ff_h264_update_thread_context(AVCodecContext *dst,
         }
         }
 
-
         h->thread_context[0] = h;
         h->context_initialized = h1->context_initialized;
     }