From c0c45188e56cfa3050bb39f8299025345b8a204c Mon Sep 17 00:00:00 2001
From: Tim Walker <tdskywalker@gmail.com>
Date: Thu, 6 Feb 2014 21:50:38 +0000
Subject: [PATCH] mlp: improve request_channel_layout behavior.

Don't decode further substreams if request_channel_layout
is a subset of the current substream's channel_layout.

Before, we would only discard further substreams if
request_channel_layout matched the substream's
channel_layout extactly, thus decoding additional
channels which the caller would probably end up downmixing.
---
 libavcodec/mlp_parser.c | 5 +++--
 libavcodec/mlpdec.c     | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/libavcodec/mlp_parser.c b/libavcodec/mlp_parser.c
index 8766bd0c1fe..a30cd05dc5a 100644
--- a/libavcodec/mlp_parser.c
+++ b/libavcodec/mlp_parser.c
@@ -362,8 +362,9 @@ FF_ENABLE_DEPRECATION_WARNINGS
                 mh.num_substreams > 1) {
                 avctx->channels       = 2;
                 avctx->channel_layout = AV_CH_LAYOUT_STEREO;
-            } else if (avctx->request_channel_layout == mh.channel_layout_thd_stream1 ||
-                       !mh.channels_thd_stream2) {
+            } else if (!mh.channels_thd_stream2 ||
+                       (mh.channel_layout_thd_stream1 & avctx->request_channel_layout) ==
+                       avctx->request_channel_layout) {
                 avctx->channels       = mh.channels_thd_stream1;
                 avctx->channel_layout = mh.channel_layout_thd_stream1;
             } else {
diff --git a/libavcodec/mlpdec.c b/libavcodec/mlpdec.c
index 76ce4c4367a..d8a76f554aa 100644
--- a/libavcodec/mlpdec.c
+++ b/libavcodec/mlpdec.c
@@ -504,8 +504,8 @@ FF_DISABLE_DEPRECATION_WARNINGS
     } else
 FF_ENABLE_DEPRECATION_WARNINGS
 #endif
-    if (m->avctx->request_channel_layout == s->ch_layout &&
-        m->max_decoded_substream > substr) {
+    if ((s->ch_layout & m->avctx->request_channel_layout) ==
+        m->avctx->request_channel_layout && m->max_decoded_substream > substr) {
         av_log(m->avctx, AV_LOG_DEBUG,
                "Extracting %d-channel downmix (0x%"PRIx64") from substream %d. "
                "Further substreams will be skipped.\n",
-- 
GitLab