matroskaenc: convert avstream stereo3d side data during encoding

Write the StereoMode Embl to bitstream.

libavformat/matroskaenc.c

@@ -43,6 +43,7 @@
 #include "libavutil/opt.h"
 #include "libavutil/random_seed.h"
 #include "libavutil/samplefmt.h"
+#include "libavutil/stereo3d.h"
 
 #include "libavcodec/xiph.h"
 #include "libavcodec/mpeg4audio.h"
@@ -624,25 +625,78 @@ static int mkv_write_codecprivate(AVFormatContext *s, AVIOContext *pb,
     return ret;
 }
 
-static void mkv_write_stereo_mode(AVIOContext *pb, uint8_t stereo_fmt,
-                                  int mode)
+static int mkv_write_stereo_mode(AVFormatContext *s, AVIOContext *pb,
+                                 AVStream *st, int mode)
 {
-    int valid_fmt = 0;
+    int i;
+    AVDictionaryEntry *tag;
+    MatroskaVideoStereoModeType format = MATROSKA_VIDEO_STEREOMODE_TYPE_NB;
+
+    // convert metadata into proper side data and add it to the stream
+    if ((tag = av_dict_get(s->metadata, "stereo_mode", NULL, 0))) {
+        int stereo_mode = atoi(tag->value);
+        if (stereo_mode < MATROSKA_VIDEO_STEREOMODE_TYPE_NB &&
+            stereo_mode != 10 && stereo_mode != 12) {
+            int ret = ff_mkv_stereo3d_conv(st, stereo_mode);
+            if (ret < 0)
+                return ret;
+        }
+    }
 
-    switch (mode) {
-    case MODE_WEBM:
-        if (stereo_fmt <= MATROSKA_VIDEO_STEREOMODE_TYPE_TOP_BOTTOM ||
-            stereo_fmt == MATROSKA_VIDEO_STEREOMODE_TYPE_RIGHT_LEFT)
-            valid_fmt = 1;
-        break;
-    case MODE_MATROSKAv2:
-        if (stereo_fmt <= MATROSKA_VIDEO_STEREOMODE_TYPE_BOTH_EYES_BLOCK_RL)
-            valid_fmt = 1;
-        break;
+    for (i = 0; i < st->nb_side_data; i++) {
+        AVPacketSideData sd = st->side_data[i];
+        if (sd.type == AV_PKT_DATA_STEREO3D) {
+            AVStereo3D *stereo = (AVStereo3D *)sd.data;
+
+            switch (stereo->type) {
+            case AV_STEREO3D_2D:
+                format = MATROSKA_VIDEO_STEREOMODE_TYPE_MONO;
+                break;
+            case AV_STEREO3D_SIDEBYSIDE:
+                format = (stereo->flags & AV_STEREO3D_FLAG_INVERT)
+                    ? MATROSKA_VIDEO_STEREOMODE_TYPE_RIGHT_LEFT
+                    : MATROSKA_VIDEO_STEREOMODE_TYPE_LEFT_RIGHT;
+                break;
+            case AV_STEREO3D_TOPBOTTOM:
+                format = MATROSKA_VIDEO_STEREOMODE_TYPE_TOP_BOTTOM;
+                if (stereo->flags & AV_STEREO3D_FLAG_INVERT)
+                    format--;
+                break;
+            case AV_STEREO3D_CHECKERBOARD:
+                format = MATROSKA_VIDEO_STEREOMODE_TYPE_CHECKERBOARD_LR;
+                if (stereo->flags & AV_STEREO3D_FLAG_INVERT)
+                    format--;
+                break;
+            case AV_STEREO3D_LINES:
+                format = MATROSKA_VIDEO_STEREOMODE_TYPE_ROW_INTERLEAVED_LR;
+                if (stereo->flags & AV_STEREO3D_FLAG_INVERT)
+                    format--;
+                break;
+            case AV_STEREO3D_COLUMNS:
+                format = MATROSKA_VIDEO_STEREOMODE_TYPE_COL_INTERLEAVED_LR;
+                if (stereo->flags & AV_STEREO3D_FLAG_INVERT)
+                    format--;
+                break;
+            case AV_STEREO3D_FRAMESEQUENCE:
+                format = MATROSKA_VIDEO_STEREOMODE_TYPE_BOTH_EYES_BLOCK_LR;
+                if (stereo->flags & AV_STEREO3D_FLAG_INVERT)
+                    format++;
+                break;
+            }
+
+            break;
+        }
     }
 
-    if (valid_fmt)
-        put_ebml_uint (pb, MATROSKA_ID_VIDEOSTEREOMODE, stereo_fmt);
+    if (mode == MODE_WEBM &&
+        (format > MATROSKA_VIDEO_STEREOMODE_TYPE_TOP_BOTTOM &&
+         format != MATROSKA_VIDEO_STEREOMODE_TYPE_RIGHT_LEFT))
+        format = MATROSKA_VIDEO_STEREOMODE_TYPE_NB;
+
+    if (format < MATROSKA_VIDEO_STEREOMODE_TYPE_NB)
+        put_ebml_uint(pb, MATROSKA_ID_VIDEOSTEREOMODE, format);
+
+    return 0;
 }
 
 static int mkv_write_track(AVFormatContext *s, MatroskaMuxContext *mkv,
@@ -743,9 +797,13 @@ static int mkv_write_track(AVFormatContext *s, MatroskaMuxContext *mkv,
         // XXX: interlace flag?
         put_ebml_uint (pb, MATROSKA_ID_VIDEOPIXELWIDTH , codec->width);
         put_ebml_uint (pb, MATROSKA_ID_VIDEOPIXELHEIGHT, codec->height);
-        if ((tag = av_dict_get(s->metadata, "stereo_mode", NULL, 0))) {
-            mkv_write_stereo_mode(pb, atoi(tag->value), mkv->mode);
-        }
+
+        // check both side data and metadata for stereo information,
+        // write the result to the bitstream if any is found
+        ret = mkv_write_stereo_mode(s, pb, st, mkv->mode);
+        if (ret < 0)
+            return ret;
+
         if (st->sample_aspect_ratio.num) {
             int d_width = codec->width*av_q2d(st->sample_aspect_ratio);
             put_ebml_uint(pb, MATROSKA_ID_VIDEODISPLAYWIDTH , d_width);