vf_colorspace: Add an option to clamp trc LUT output

Add a new flag to the vf_colorspace filter which provides the user an
option to clamp the linear and delinear transfer characteristics LUT
values to the [0, 1] represented range. This helps constrain the
potential value range when converting between colorspaces.

Certain colors when going through the conversion can result in out of
gamut colors after the rotation. The colorspace filter allows that with
the extended range. The added clamping just keeps the colors within the
[0, 1) range rather than using that extended range. I'm not enough of a
color scientist to say which is correct, but there are certain
situations where we would prefer to keep the colors in gamut.

The example I have is:

A solid color image of 8-bit YUV: Y=157, U=164, V=98.

Specify the input as:

Input range: MPEG
In color matrix: BT470BG
In color primaries: BT470M
In color transfer characteristics: Gamma 28

Output as:
Out color range: JPEG
Out color matrix: BT.709
Out color primaries: BT.709
Out color transfer characteristics: BT.709

During the calculation you get:

Input YUV:                             y=157,      u=164,      v-98
Post-yuv2rgb BT.470BG:                 r=0.456055, g=0.684152, b=0.928606
Post-apply gamma28 linear LUT:         r=0.110979, g=0.345494, b=0.812709
Post-color rotation BT.470M to BT.709: r=-0.04161, g=0.384626, b=0.852400
Post-apply Rec.709 delinear LUT:       r=-0.16382, g=0.615932, b=0.923793
Post-rgb2yuv Rec.709 matrix:           y=120,      u=190,      v=25

Where with this change, the delinear LUT output would be clamped to 0,
so the result would be:
r=0.000000, g=0.612390, b=0.918807 and a final output of
y=129, u=185, v=46

As for the long and av_clip64, this was just because lrint returned a
long, so I left it as that and then used av_clip64 to the [0,1) range to
avoid overflow. But re-reading, it looks like av_clip_int16 would
downcast that long to int anyway so the possibility of overflow already
existed there. I've put it back to int just to match the existing
behavior.

doc/filters.texi

@@ -10413,6 +10413,18 @@ von Kries whitepoint adaptation
 identity whitepoint adaptation (i.e. no whitepoint adaptation)
 @end table
 
+@item clipgamut
+Controls how to clip out-of-gamut colors that arise as a result of colorspace conversion.
+
+The accepted values are:
+@table @samp
+@item none
+No clipping of out of gamut colors.
+
+@item rgb
+Clips the RGB values to the [0, 1] range when building the gamma transfer LUTs.
+@end table
+
 @item iall
 Override all input properties at once. Same accepted values as @ref{all}.
 

libavfilter/vf_colorspace.c

@@ -66,6 +66,12 @@ enum WhitepointAdaptation {
     NB_WP_ADAPT,
 };
 
+enum ClipGamutMode {
+  CLIP_GAMUT_NONE,
+  CLIP_GAMUT_RGB,
+  NB_CLIP_GAMUT,
+};
+
 static const enum AVColorTransferCharacteristic default_trc[CS_NB + 1] = {
     [CS_UNSPECIFIED] = AVCOL_TRC_UNSPECIFIED,
     [CS_BT470M]      = AVCOL_TRC_GAMMA22,
@@ -123,6 +129,7 @@ typedef struct ColorSpaceContext {
     int fast_mode;
     enum DitherMode dither;
     enum WhitepointAdaptation wp_adapt;
+    enum ClipGamutMode clip_gamut;
 
     int16_t *rgb[3];
     ptrdiff_t rgb_stride;
@@ -199,6 +206,7 @@ static int fill_gamma_table(ColorSpaceContext *s)
     double in_ialpha = 1.0 / in_alpha, in_igamma = 1.0 / in_gamma, in_idelta = 1.0 / in_delta;
     double out_alpha = s->out_txchr->alpha, out_beta = s->out_txchr->beta;
     double out_gamma = s->out_txchr->gamma, out_delta = s->out_txchr->delta;
+    int clip_gamut = s->clip_gamut == CLIP_GAMUT_RGB;
 
     s->lin_lut = av_malloc(sizeof(*s->lin_lut) * 32768 * 2);
     if (!s->lin_lut)
@@ -215,7 +223,9 @@ static int fill_gamma_table(ColorSpaceContext *s)
         } else {
             d = out_alpha * pow(v, out_gamma) - (out_alpha - 1.0);
         }
-        s->delin_lut[n] = av_clip_int16(lrint(d * 28672.0));
+        int d_rounded = lrint(d * 28672.0);
+        s->delin_lut[n] = clip_gamut ? av_clip(d_rounded, 0, 28672)
+                                     : av_clip_int16(d_rounded);
 
         // linearize
         if (v <= -in_beta * in_delta) {
@@ -225,7 +235,9 @@ static int fill_gamma_table(ColorSpaceContext *s)
         } else {
             l = pow((v + in_alpha - 1.0) * in_ialpha, in_igamma);
         }
-        s->lin_lut[n] = av_clip_int16(lrint(l * 28672.0));
+        int l_rounded = lrint(l * 28672.0);
+        s->lin_lut[n] = clip_gamut ? av_clip(l_rounded, 0, 28672)
+                                   : av_clip_int16(l_rounded);
     }
 
     return 0;
@@ -1000,6 +1012,13 @@ static const AVOption colorspace_options[] = {
     ENUM("vonkries", WP_ADAPT_VON_KRIES, "wpadapt"),
     ENUM("identity", WP_ADAPT_IDENTITY, "wpadapt"),
 
+    { "clipgamut",
+      "Controls how to clip out-of-gamut colors that arise as a result of colorspace conversion.",
+      OFFSET(clip_gamut), AV_OPT_TYPE_INT,  { .i64 = CLIP_GAMUT_NONE },
+      CLIP_GAMUT_NONE, NB_CLIP_GAMUT - 1, FLAGS, .unit = "clipgamut" },
+    ENUM("none", CLIP_GAMUT_NONE, "clipgamut"),
+    ENUM("rgb", CLIP_GAMUT_RGB, "clipgamut"),
+
     { "iall",       "Set all input color properties together",
       OFFSET(user_iall),   AV_OPT_TYPE_INT, { .i64 = CS_UNSPECIFIED },
       CS_UNSPECIFIED, CS_NB - 1, FLAGS, .unit = "all" },