From 8b0575d7638f40730c020ada41a573fff715c1a7 Mon Sep 17 00:00:00 2001 From: Paul B Mahol Date: Sat, 16 May 2020 19:45:53 +0200 Subject: [PATCH] avfilter/vf_colorkey: fix formula for calculation of difference Also fixes colorhold filtering. --- libavfilter/vf_colorkey.c | 2 +- tests/ref/fate/ffmpeg-filter_colorkey | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libavfilter/vf_colorkey.c b/libavfilter/vf_colorkey.c index 7af426a61e..0ac847c633 100644 --- a/libavfilter/vf_colorkey.c +++ b/libavfilter/vf_colorkey.c @@ -45,7 +45,7 @@ static uint8_t do_colorkey_pixel(ColorkeyContext *ctx, uint8_t r, uint8_t g, uin int dg = (int)g - ctx->colorkey_rgba[1]; int db = (int)b - ctx->colorkey_rgba[2]; - double diff = sqrt((dr * dr + dg * dg + db * db) / (255.0 * 255.0)); + double diff = sqrt((dr * dr + dg * dg + db * db) / (255.0 * 255.0 * 3.0)); if (ctx->blend > 0.0001) { return av_clipd((diff - ctx->similarity) / ctx->blend, 0.0, 1.0) * 255.0; diff --git a/tests/ref/fate/ffmpeg-filter_colorkey b/tests/ref/fate/ffmpeg-filter_colorkey index 490a0f9aa6..25c3fc0912 100644 --- a/tests/ref/fate/ffmpeg-filter_colorkey +++ b/tests/ref/fate/ffmpeg-filter_colorkey @@ -3,13 +3,13 @@ #codec_id 0: rawvideo #dimensions 0: 720x576 #sar 0: 0/1 -0, 0, 0, 1, 622080, 0x78efb628 -0, 1, 1, 1, 622080, 0x641f2564 -0, 2, 2, 1, 622080, 0x348f25c3 -0, 3, 3, 1, 622080, 0x6afc485a -0, 4, 4, 1, 622080, 0xe949107f -0, 5, 5, 1, 622080, 0x171716e5 -0, 6, 6, 1, 622080, 0x2985a01f -0, 7, 7, 1, 622080, 0xc5ddabd7 -0, 8, 8, 1, 622080, 0xb4dd2b7f -0, 9, 9, 1, 622080, 0x6e75ba82 +0, 0, 0, 1, 622080, 0x5775bb12 +0, 1, 1, 1, 622080, 0x84d8395a +0, 2, 2, 1, 622080, 0x599d3a6b +0, 3, 3, 1, 622080, 0x1df1e9dc +0, 4, 4, 1, 622080, 0xf3255836 +0, 5, 5, 1, 622080, 0xf46bc26e +0, 6, 6, 1, 622080, 0x76d491da +0, 7, 7, 1, 622080, 0x2c9f3ca6 +0, 8, 8, 1, 622080, 0x5713e203 +0, 9, 9, 1, 622080, 0x47776493