avfilter/vf_vectorscope: implement envelope support

Signed-off-by: Paul B Mahol <onemda@gmail.com>

doc/filters.texi

@@ -10603,6 +10603,22 @@ Set which color component will be represented on Y-axis. Default is @code{2}.
 @item intensity
 Set intensity, used by modes: gray, color and color3 for increasing brightness
 of color component which represents frequency of (X, Y) location in graph.
+
+@item envelope, e
+@table @samp
+@item none
+No envelope, this is default.
+
+@item instant
+Instant envelope, even darkest single pixel will be clearly highlighted.
+
+@item peak
+Hold maximum and minimum values presented in graph over time. This way you
+can still spot out of range values without constantly looking at vectorscope.
+
+@item peak+instant
+Peak and instant envelope combined together.
+@end table
 @end table
 
 @anchor{vidstabdetect}

libavfilter/vf_vectorscope.c

@@ -45,6 +45,8 @@ typedef struct VectorscopeContext {
     int planeheight[4];
     int x, y, pd;
     int is_yuv;
+    int envelope;
+    uint8_t peak[256][256];
 } VectorscopeContext;
 
 #define OFFSET(x) offsetof(VectorscopeContext, x)
@@ -53,15 +55,21 @@ typedef struct VectorscopeContext {
 static const AVOption vectorscope_options[] = {
     { "mode", "set vectorscope mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64=0}, 0, MODE_NB-1, FLAGS, "mode"},
     { "m",    "set vectorscope mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64=0}, 0, MODE_NB-1, FLAGS, "mode"},
-    { "gray",   0, 0, AV_OPT_TYPE_CONST, {.i64=GRAY},   0, 0, FLAGS, "mode" },
-    { "color",  0, 0, AV_OPT_TYPE_CONST, {.i64=COLOR},  0, 0, FLAGS, "mode" },
-    { "color2", 0, 0, AV_OPT_TYPE_CONST, {.i64=COLOR2}, 0, 0, FLAGS, "mode" },
-    { "color3", 0, 0, AV_OPT_TYPE_CONST, {.i64=COLOR3}, 0, 0, FLAGS, "mode" },
-    { "color4", 0, 0, AV_OPT_TYPE_CONST, {.i64=COLOR4}, 0, 0, FLAGS, "mode" },
+    {   "gray",   0, 0, AV_OPT_TYPE_CONST, {.i64=GRAY},   0, 0, FLAGS, "mode" },
+    {   "color",  0, 0, AV_OPT_TYPE_CONST, {.i64=COLOR},  0, 0, FLAGS, "mode" },
+    {   "color2", 0, 0, AV_OPT_TYPE_CONST, {.i64=COLOR2}, 0, 0, FLAGS, "mode" },
+    {   "color3", 0, 0, AV_OPT_TYPE_CONST, {.i64=COLOR3}, 0, 0, FLAGS, "mode" },
+    {   "color4", 0, 0, AV_OPT_TYPE_CONST, {.i64=COLOR4}, 0, 0, FLAGS, "mode" },
     { "x", "set color component on X axis", OFFSET(x), AV_OPT_TYPE_INT, {.i64=1}, 0, 2, FLAGS},
     { "y", "set color component on Y axis", OFFSET(y), AV_OPT_TYPE_INT, {.i64=2}, 0, 2, FLAGS},
     { "intensity", "set intensity", OFFSET(intensity), AV_OPT_TYPE_INT, {.i64=1}, 1, 255, FLAGS},
     { "i",         "set intensity", OFFSET(intensity), AV_OPT_TYPE_INT, {.i64=1}, 1, 255, FLAGS},
+    { "envelope",  "set envelope", OFFSET(envelope), AV_OPT_TYPE_INT, {.i64=0}, 0, 3, FLAGS, "envelope"},
+    { "e",         "set envelope", OFFSET(envelope), AV_OPT_TYPE_INT, {.i64=0}, 0, 3, FLAGS, "envelope"},
+    {   "none",         0, 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 0, FLAGS, "envelope" },
+    {   "instant",      0, 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 0, FLAGS, "envelope" },
+    {   "peak",         0, 0, AV_OPT_TYPE_CONST, {.i64=2}, 0, 0, FLAGS, "envelope" },
+    {   "peak+instant", 0, 0, AV_OPT_TYPE_CONST, {.i64=3}, 0, 0, FLAGS, "envelope" },
     { NULL }
 };
 
@@ -143,7 +151,9 @@ static int config_input(AVFilterLink *inlink)
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
     VectorscopeContext *s = inlink->dst->priv;
 
-    if (s->mode == GRAY)
+    s->is_yuv = !(desc->flags & AV_PIX_FMT_FLAG_RGB);
+
+    if (s->mode == GRAY && s->is_yuv)
         s->pd = 0;
     else {
         if ((s->x == 1 && s->y == 2) || (s->x == 2 && s->y == 1))
@@ -173,16 +183,75 @@ static int config_input(AVFilterLink *inlink)
 
 static int config_output(AVFilterLink *outlink)
 {
-    VectorscopeContext *s = outlink->src->priv;
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(outlink->format);
     int depth = desc->comp[0].depth_minus1 + 1;
 
-    s->is_yuv = !(desc->flags & AV_PIX_FMT_FLAG_RGB);
     outlink->h = outlink->w = 1 << depth;
     outlink->sample_aspect_ratio = (AVRational){1,1};
     return 0;
 }
 
+static void envelope_instant(VectorscopeContext *s, AVFrame *out)
+{
+    const int dlinesize = out->linesize[0];
+    uint8_t *dpd = s->mode == COLOR || !s->is_yuv ? out->data[s->pd] : out->data[0];
+    int i, j;
+
+    for (i = 0; i < out->height; i++) {
+        for (j = 0; j < out->width; j++) {
+            int pos = i * dlinesize + j;
+            int poa = (i - 1) * dlinesize + j;
+            int pob = (i + 1) * dlinesize + j;
+
+            if (dpd[pos] && (((!j || !dpd[pos - 1]) || ((j == (out->width - 1)) || !dpd[pos + 1]))
+                         || ((!i || !dpd[poa]) || ((i == (out->height - 1)) || !dpd[pob])))) {
+                dpd[pos] = 255;
+            }
+        }
+    }
+}
+
+static void envelope_peak(VectorscopeContext *s, AVFrame *out)
+{
+    const int dlinesize = out->linesize[0];
+    uint8_t *dpd = s->mode == COLOR || !s->is_yuv ? out->data[s->pd] : out->data[0];
+    int i, j;
+
+    for (i = 0; i < out->height; i++) {
+        for (j = 0; j < out->width; j++) {
+            int pos = i * dlinesize + j;
+
+            if (dpd[pos])
+                s->peak[i][j] = 255;
+        }
+    }
+
+    if (s->envelope == 3)
+        envelope_instant(s, out);
+
+    for (i = 0; i < out->height; i++) {
+        for (j = 0; j < out->width; j++) {
+            int pos = i * dlinesize + j;
+
+            if (s->peak[i][j] && (((!j || !s->peak[i][j-1]) || ((j == (out->width - 1)) || !s->peak[i][j + 1]))
+                              || ((!i || !s->peak[i-1][j]) || ((i == (out->height - 1)) || !s->peak[i + 1][j])))) {
+                dpd[pos] = 255;
+            }
+        }
+    }
+}
+
+static void envelope(VectorscopeContext *s, AVFrame *out)
+{
+    if (!s->envelope) {
+        return;
+    } else if (s->envelope == 1) {
+        envelope_instant(s, out);
+    } else {
+        envelope_peak(s, out);
+    }
+}
+
 static void vectorscope(VectorscopeContext *s, AVFrame *in, AVFrame *out, int pd)
 {
     const uint8_t * const *src = (const uint8_t * const *)in->data;
@@ -322,6 +391,8 @@ static void vectorscope(VectorscopeContext *s, AVFrame *in, AVFrame *out, int pd
     default:
         av_assert0(0);
     }
+
+    envelope(s, out);
 }
 
 static int filter_frame(AVFilterLink *inlink, AVFrame *in)