fate: add tests for psnr and ssim filter

Metadata filter output is passed through an Awk script comparing floats
against reference values with specified "fuzz" tolerance to account for
architectural differences (e.g. x86-32 vs. x86-64).

Signed-off-by: Tobias Rapp <t.rapp@noa-archive.com>

tests/fate-run.sh

@@ -234,6 +234,15 @@ lavftest(){
     ${base}/lavf-regression.sh $t lavf tests/vsynth1 "$target_exec" "$target_path" "$threads" "$thread_type" "$cpuflags" "$target_samples"
 }
 
+refcmp_metadata(){
+    refcmp=$1
+    pixfmt=$2
+    fuzz=${3:-0.001}
+    ffmpeg $FLAGS $ENC_OPTS \
+        -lavfi "testsrc2=size=300x200:rate=1:duration=5,format=${pixfmt},split[ref][tmp];[tmp]avgblur=4[enc];[enc][ref]${refcmp},metadata=print:file=-" \
+        -f null /dev/null | awk -v ref=${ref} -v fuzz=${fuzz} -f ${base}/refcmp-metadata.awk -
+}
+
 video_filter(){
     filters=$1
     shift

tests/fate/filter-video.mak

@@ -747,6 +747,20 @@ FATE_FILTER_SAMPLES-$(call ALLYES, MOV_DEMUXER H264_DECODER AAC_FIXED_DECODER PC
 fate-filter-meta-4560-rotate0: tests/data/file4560-override2rotate0.mov
 fate-filter-meta-4560-rotate0: CMD = framecrc -flags +bitexact -c:a aac_fixed -i $(TARGET_PATH)/tests/data/file4560-override2rotate0.mov
 
+REFCMP_DEPS = FFMPEG LAVFI_INDEV TESTSRC2_FILTER AVGBLUR_FILTER METADATA_FILTER
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-rgb
+fate-filter-refcmp-psnr-rgb: CMD = refcmp_metadata psnr rgb24
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-yuv
+fate-filter-refcmp-psnr-yuv: CMD = refcmp_metadata psnr yuv422p
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-rgb
+fate-filter-refcmp-ssim-rgb: CMD = refcmp_metadata ssim rgb24
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-yuv
+fate-filter-refcmp-ssim-yuv: CMD = refcmp_metadata ssim yuv422p
+
 FATE_SAMPLES_FFPROBE += $(FATE_METADATA_FILTER-yes)
 FATE_SAMPLES_FFMPEG += $(FATE_FILTER_SAMPLES-yes)
 FATE_FFMPEG += $(FATE_FILTER-yes)

tests/ref/fate/filter-refcmp-psnr-rgb

@@ -0,0 +1,45 @@
+frame:0    pts:0       pts_time:0
+lavfi.psnr.mse.r=1381.80
+lavfi.psnr.psnr.r=16.73
+lavfi.psnr.mse.g=896.00
+lavfi.psnr.psnr.g=18.61
+lavfi.psnr.mse.b=277.38
+lavfi.psnr.psnr.b=23.70
+lavfi.psnr.mse_avg=851.73
+lavfi.psnr.psnr_avg=18.83
+frame:1    pts:1       pts_time:1
+lavfi.psnr.mse.r=1380.37
+lavfi.psnr.psnr.r=16.73
+lavfi.psnr.mse.g=975.91
+lavfi.psnr.psnr.g=18.24
+lavfi.psnr.mse.b=435.72
+lavfi.psnr.psnr.b=21.74
+lavfi.psnr.mse_avg=930.67
+lavfi.psnr.psnr_avg=18.44
+frame:2    pts:2       pts_time:2
+lavfi.psnr.mse.r=1403.20
+lavfi.psnr.psnr.r=16.66
+lavfi.psnr.mse.g=954.05
+lavfi.psnr.psnr.g=18.34
+lavfi.psnr.mse.b=494.22
+lavfi.psnr.psnr.b=21.19
+lavfi.psnr.mse_avg=950.49
+lavfi.psnr.psnr_avg=18.35
+frame:3    pts:3       pts_time:3
+lavfi.psnr.mse.r=1452.80
+lavfi.psnr.psnr.r=16.51
+lavfi.psnr.mse.g=1001.02
+lavfi.psnr.psnr.g=18.13
+lavfi.psnr.mse.b=557.39
+lavfi.psnr.psnr.b=20.67
+lavfi.psnr.mse_avg=1003.74
+lavfi.psnr.psnr_avg=18.11
+frame:4    pts:4       pts_time:4
+lavfi.psnr.mse.r=1401.25
+lavfi.psnr.psnr.r=16.67
+lavfi.psnr.mse.g=1009.80
+lavfi.psnr.psnr.g=18.09
+lavfi.psnr.mse.b=602.42
+lavfi.psnr.psnr.b=20.33
+lavfi.psnr.mse_avg=1004.49
+lavfi.psnr.psnr_avg=18.11

tests/ref/fate/filter-refcmp-psnr-yuv

@@ -0,0 +1,45 @@
+frame:0    pts:0       pts_time:0
+lavfi.psnr.mse.y=222.06
+lavfi.psnr.psnr.y=24.67
+lavfi.psnr.mse.u=339.38
+lavfi.psnr.psnr.u=22.82
+lavfi.psnr.mse.v=705.41
+lavfi.psnr.psnr.v=19.65
+lavfi.psnr.mse_avg=372.23
+lavfi.psnr.psnr_avg=22.42
+frame:1    pts:1       pts_time:1
+lavfi.psnr.mse.y=236.74
+lavfi.psnr.psnr.y=24.39
+lavfi.psnr.mse.u=416.17
+lavfi.psnr.psnr.u=21.94
+lavfi.psnr.mse.v=704.98
+lavfi.psnr.psnr.v=19.65
+lavfi.psnr.mse_avg=398.66
+lavfi.psnr.psnr_avg=22.12
+frame:2    pts:2       pts_time:2
+lavfi.psnr.mse.y=234.79
+lavfi.psnr.psnr.y=24.42
+lavfi.psnr.mse.u=435.72
+lavfi.psnr.psnr.u=21.74
+lavfi.psnr.mse.v=699.60
+lavfi.psnr.psnr.v=19.68
+lavfi.psnr.mse_avg=401.23
+lavfi.psnr.psnr_avg=22.10
+frame:3    pts:3       pts_time:3
+lavfi.psnr.mse.y=250.88
+lavfi.psnr.psnr.y=24.14
+lavfi.psnr.mse.u=479.73
+lavfi.psnr.psnr.u=21.32
+lavfi.psnr.mse.v=707.55
+lavfi.psnr.psnr.v=19.63
+lavfi.psnr.mse_avg=422.26
+lavfi.psnr.psnr_avg=21.88
+frame:4    pts:4       pts_time:4
+lavfi.psnr.mse.y=241.05
+lavfi.psnr.psnr.y=24.31
+lavfi.psnr.mse.u=505.04
+lavfi.psnr.psnr.u=21.10
+lavfi.psnr.mse.v=716.00
+lavfi.psnr.psnr.v=19.58
+lavfi.psnr.mse_avg=425.79
+lavfi.psnr.psnr_avg=21.84

tests/ref/fate/filter-refcmp-ssim-rgb

@@ -0,0 +1,30 @@
+frame:0    pts:0       pts_time:0
+lavfi.ssim.R=0.72
+lavfi.ssim.G=0.76
+lavfi.ssim.B=0.89
+lavfi.ssim.All=0.79
+lavfi.ssim.dB=6.74
+frame:1    pts:1       pts_time:1
+lavfi.ssim.R=0.70
+lavfi.ssim.G=0.74
+lavfi.ssim.B=0.85
+lavfi.ssim.All=0.77
+lavfi.ssim.dB=6.31
+frame:2    pts:2       pts_time:2
+lavfi.ssim.R=0.71
+lavfi.ssim.G=0.75
+lavfi.ssim.B=0.84
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.29
+frame:3    pts:3       pts_time:3
+lavfi.ssim.R=0.70
+lavfi.ssim.G=0.73
+lavfi.ssim.B=0.83
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.11
+frame:4    pts:4       pts_time:4
+lavfi.ssim.R=0.71
+lavfi.ssim.G=0.74
+lavfi.ssim.B=0.80
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.05

tests/ref/fate/filter-refcmp-ssim-yuv

@@ -0,0 +1,30 @@
+frame:0    pts:0       pts_time:0
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.76
+lavfi.ssim.V=0.69
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.25
+frame:1    pts:1       pts_time:1
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.73
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.08
+frame:2    pts:2       pts_time:2
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.73
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.10
+frame:3    pts:3       pts_time:3
+lavfi.ssim.Y=0.79
+lavfi.ssim.U=0.72
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=5.94
+frame:4    pts:4       pts_time:4
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.72
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=5.97

tests/refcmp-metadata.awk

@@ -0,0 +1,64 @@
+# Compare metadata filter output containing float value strings to reference
+# output data. Returns the whole reference data if delta of each value is below
+# threshold, else returns the whole input data.
+
+function abs(val) {
+    return ((val < 0.0) ? -val : val);
+}
+
+function max(val1, val2) {
+    return ((val1 >= val2) ? val1 : val2);
+}
+
+function is_numeric_str(str) {
+    return (str ~ /^[+-]?[0-9]*\.?[0-9]+$/);
+}
+
+BEGIN {
+    FS = "=";
+    # check for "fuzz" (threshold) program parameter, else use default
+    if (fuzz <= 0.0) {
+        fuzz = 0.1;
+    }
+    # check for "ref" (reference file) program parameter
+    if (ref) {
+        ref_nr = 0;
+        while ((getline line < ref) > 0) {
+            ref_nr++;
+            ref_lines[ref_nr] = line;
+            if (split(line, fields) == 2 && is_numeric_str(fields[2])) {
+                ref_keys[ref_nr] = fields[1];
+                ref_vals[ref_nr] = fields[2] + 0;  # convert to number
+            }
+        }
+        close(ref);
+    }
+    delta_max = 0;
+    result = (ref ? 1 : 0);
+}
+
+{
+    cmp_lines[NR] = $0;
+    if (NF == 2 && is_numeric_str($2) && ref_vals[NR]) {
+        val = $2 + 0;  # convert to number
+        delta = abs((val / ref_vals[NR]) - 1);
+        delta_max = max(delta_max, delta);
+        result = result && ($1 == ref_keys[NR]) && (delta <= fuzz);
+    } else {
+        result = result && ($0 == ref_lines[NR]);
+    }
+}
+
+END {
+    if (result) {
+        for (i = 1; i <= ref_nr; i++)
+            print ref_lines[i];
+    } else {
+        for (i = 1; i <= NR; i++)
+            print cmp_lines[i];
+        if (NR != ref_nr)
+            print "[refcmp] lines: " NR " != " ref_nr > "/dev/stderr";
+        if (delta_max >= fuzz)
+            print "[refcmp] delta_max: " delta_max " >= " fuzz > "/dev/stderr";
+    }
+}