From afbc6852b439005b546d8067f35cf65b1dddb8ac Mon Sep 17 00:00:00 2001 From: Mohammad Izadi Date: Mon, 23 Nov 2020 13:29:34 -0800 Subject: [PATCH] avcodec/hevc_sei: add support for HDR10+ metadata Signed-off-by: James Almer --- libavcodec/Makefile | 3 +- libavcodec/dynamic_hdr10_plus.c | 195 ++++++++++++++++++++++++++++++++ libavcodec/dynamic_hdr10_plus.h | 34 ++++++ libavcodec/hevc_sei.c | 50 ++++++-- libavcodec/hevc_sei.h | 5 + libavcodec/hevcdec.c | 15 +++ 6 files changed, 294 insertions(+), 8 deletions(-) create mode 100644 libavcodec/dynamic_hdr10_plus.c create mode 100644 libavcodec/dynamic_hdr10_plus.h diff --git a/libavcodec/Makefile b/libavcodec/Makefile index a6435c9e85..01d8c5ee46 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -96,7 +96,8 @@ OBJS-$(CONFIG_H264DSP) += h264dsp.o h264idct.o OBJS-$(CONFIG_H264PARSE) += h264_parse.o h2645_parse.o h264_ps.o OBJS-$(CONFIG_H264PRED) += h264pred.o OBJS-$(CONFIG_H264QPEL) += h264qpel.o -OBJS-$(CONFIG_HEVCPARSE) += hevc_parse.o h2645_parse.o hevc_ps.o hevc_sei.o hevc_data.o +OBJS-$(CONFIG_HEVCPARSE) += hevc_parse.o h2645_parse.o hevc_ps.o hevc_sei.o hevc_data.o \ + dynamic_hdr10_plus.o OBJS-$(CONFIG_HPELDSP) += hpeldsp.o OBJS-$(CONFIG_HUFFMAN) += huffman.o OBJS-$(CONFIG_HUFFYUVDSP) += huffyuvdsp.o diff --git a/libavcodec/dynamic_hdr10_plus.c b/libavcodec/dynamic_hdr10_plus.c new file mode 100644 index 0000000000..5684bfb2ef --- /dev/null +++ b/libavcodec/dynamic_hdr10_plus.c @@ -0,0 +1,195 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "dynamic_hdr10_plus.h" + +static const uint8_t usa_country_code = 0xB5; +static const uint16_t smpte_provider_code = 0x003C; +static const uint16_t smpte2094_40_provider_oriented_code = 0x0001; +static const uint16_t smpte2094_40_application_identifier = 0x04; +static const int64_t luminance_den = 1; +static const int32_t peak_luminance_den = 15; +static const int64_t rgb_den = 100000; +static const int32_t fraction_pixel_den = 1000; +static const int32_t knee_point_den = 4095; +static const int32_t bezier_anchor_den = 1023; +static const int32_t saturation_weight_den = 8; + +int ff_parse_itu_t_t35_to_dynamic_hdr10_plus(GetBitContext *gb, AVDynamicHDRPlus *s) +{ + if (!s) + return AVERROR(ENOMEM); + + s->application_version = get_bits(gb, 8); + + if (get_bits_left(gb) < 2) + return AVERROR_INVALIDDATA; + s->num_windows = get_bits(gb, 2); + + if (s->num_windows < 1 || s->num_windows > 3) { + return AVERROR_INVALIDDATA; + } + + if (get_bits_left(gb) < ((19 * 8 + 1) * (s->num_windows - 1))) + return AVERROR_INVALIDDATA; + + for (int w = 1; w < s->num_windows; w++) { + // The corners are set to absolute coordinates here. They should be + // converted to the relative coordinates (in [0, 1]) in the decoder. + AVHDRPlusColorTransformParams *params = &s->params[w]; + params->window_upper_left_corner_x = + (AVRational){get_bits(gb, 16), 1}; + params->window_upper_left_corner_y = + (AVRational){get_bits(gb, 16), 1}; + params->window_lower_right_corner_x = + (AVRational){get_bits(gb, 16), 1}; + params->window_lower_right_corner_y = + (AVRational){get_bits(gb, 16), 1}; + + params->center_of_ellipse_x = get_bits(gb, 16); + params->center_of_ellipse_y = get_bits(gb, 16); + params->rotation_angle = get_bits(gb, 8); + params->semimajor_axis_internal_ellipse = get_bits(gb, 16); + params->semimajor_axis_external_ellipse = get_bits(gb, 16); + params->semiminor_axis_external_ellipse = get_bits(gb, 16); + params->overlap_process_option = get_bits1(gb); + } + + if (get_bits_left(gb) < 28) + return AVERROR(EINVAL); + + s->targeted_system_display_maximum_luminance = + (AVRational){get_bits(gb, 27), luminance_den}; + s->targeted_system_display_actual_peak_luminance_flag = get_bits1(gb); + + if (s->targeted_system_display_actual_peak_luminance_flag) { + int rows, cols; + if (get_bits_left(gb) < 10) + return AVERROR(EINVAL); + rows = get_bits(gb, 5); + cols = get_bits(gb, 5); + if (((rows < 2) || (rows > 25)) || ((cols < 2) || (cols > 25))) { + return AVERROR_INVALIDDATA; + } + s->num_rows_targeted_system_display_actual_peak_luminance = rows; + s->num_cols_targeted_system_display_actual_peak_luminance = cols; + + if (get_bits_left(gb) < (rows * cols * 4)) + return AVERROR(EINVAL); + + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + s->targeted_system_display_actual_peak_luminance[i][j] = + (AVRational){get_bits(gb, 4), peak_luminance_den}; + } + } + } + for (int w = 0; w < s->num_windows; w++) { + AVHDRPlusColorTransformParams *params = &s->params[w]; + if (get_bits_left(gb) < (3 * 17 + 17 + 4)) + return AVERROR(EINVAL); + + for (int i = 0; i < 3; i++) { + params->maxscl[i] = + (AVRational){get_bits(gb, 17), rgb_den}; + } + params->average_maxrgb = + (AVRational){get_bits(gb, 17), rgb_den}; + params->num_distribution_maxrgb_percentiles = get_bits(gb, 4); + + if (get_bits_left(gb) < + (params->num_distribution_maxrgb_percentiles * 24)) + return AVERROR(EINVAL); + + for (int i = 0; i < params->num_distribution_maxrgb_percentiles; i++) { + params->distribution_maxrgb[i].percentage = get_bits(gb, 7); + params->distribution_maxrgb[i].percentile = + (AVRational){get_bits(gb, 17), rgb_den}; + } + + if (get_bits_left(gb) < 10) + return AVERROR(EINVAL); + + params->fraction_bright_pixels = (AVRational){get_bits(gb, 10), fraction_pixel_den}; + } + if (get_bits_left(gb) < 1) + return AVERROR(EINVAL); + s->mastering_display_actual_peak_luminance_flag = get_bits1(gb); + if (s->mastering_display_actual_peak_luminance_flag) { + int rows, cols; + if (get_bits_left(gb) < 10) + return AVERROR(EINVAL); + rows = get_bits(gb, 5); + cols = get_bits(gb, 5); + if (((rows < 2) || (rows > 25)) || ((cols < 2) || (cols > 25))) { + return AVERROR_INVALIDDATA; + } + s->num_rows_mastering_display_actual_peak_luminance = rows; + s->num_cols_mastering_display_actual_peak_luminance = cols; + + if (get_bits_left(gb) < (rows * cols * 4)) + return AVERROR(EINVAL); + + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + s->mastering_display_actual_peak_luminance[i][j] = + (AVRational){get_bits(gb, 4), peak_luminance_den}; + } + } + } + + for (int w = 0; w < s->num_windows; w++) { + AVHDRPlusColorTransformParams *params = &s->params[w]; + if (get_bits_left(gb) < 1) + return AVERROR(EINVAL); + + params->tone_mapping_flag = get_bits1(gb); + if (params->tone_mapping_flag) { + if (get_bits_left(gb) < 28) + return AVERROR(EINVAL); + + params->knee_point_x = + (AVRational){get_bits(gb, 12), knee_point_den}; + params->knee_point_y = + (AVRational){get_bits(gb, 12), knee_point_den}; + params->num_bezier_curve_anchors = get_bits(gb, 4); + + if (get_bits_left(gb) < (params->num_bezier_curve_anchors * 10)) + return AVERROR(EINVAL); + + for (int i = 0; i < params->num_bezier_curve_anchors; i++) { + params->bezier_curve_anchors[i] = + (AVRational){get_bits(gb, 10), bezier_anchor_den}; + } + } + + if (get_bits_left(gb) < 1) + return AVERROR(EINVAL); + params->color_saturation_mapping_flag = get_bits1(gb); + if (params->color_saturation_mapping_flag) { + if (get_bits_left(gb) < 6) + return AVERROR(EINVAL); + params->color_saturation_weight = + (AVRational){get_bits(gb, 6), saturation_weight_den}; + } + } + + skip_bits(gb, get_bits_left(gb)); + + return 0; +} diff --git a/libavcodec/dynamic_hdr10_plus.h b/libavcodec/dynamic_hdr10_plus.h new file mode 100644 index 0000000000..06053f88e7 --- /dev/null +++ b/libavcodec/dynamic_hdr10_plus.h @@ -0,0 +1,34 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_DYNAMIC_HDR10_PLUS_H +#define AVCODEC_DYNAMIC_HDR10_PLUS_H + +#include "libavutil/hdr_dynamic_metadata.h" +#include "get_bits.h" + +/** + * Parse the user data registered ITU-T T.35 to AVbuffer (AVDynamicHDRPlus). + * @param gb The bit content to be decoded. + * @param s A pointer containing the decoded AVDynamicHDRPlus structure. + * + * @return 0 if succeed. Otherwise, returns the appropriate AVERROR. + */ +int ff_parse_itu_t_t35_to_dynamic_hdr10_plus(GetBitContext *gb, AVDynamicHDRPlus *s); + +#endif /* AVCODEC_DYNAMIC_HDR10_PLUS_H */ diff --git a/libavcodec/hevc_sei.c b/libavcodec/hevc_sei.c index 06e7a3a10c..4a0bc538c2 100644 --- a/libavcodec/hevc_sei.c +++ b/libavcodec/hevc_sei.c @@ -23,6 +23,7 @@ */ #include "atsc_a53.h" +#include "dynamic_hdr10_plus.h" #include "golomb.h" #include "hevc_ps.h" #include "hevc_sei.h" @@ -209,8 +210,11 @@ static int decode_nal_sei_user_data_unregistered(HEVCSEIUnregistered *s, GetBitC static int decode_nal_sei_user_data_registered_itu_t_t35(HEVCSEI *s, GetBitContext *gb, int size) { - uint32_t country_code; - uint32_t user_identifier; + const uint8_t usa_country_code = 0xB5; + const uint16_t smpte_provider_code = 0x003C; + + uint8_t country_code = 0; + uint16_t provider_code = 0; if (size < 7) return AVERROR(EINVAL); @@ -222,18 +226,49 @@ static int decode_nal_sei_user_data_registered_itu_t_t35(HEVCSEI *s, GetBitConte size--; } - skip_bits(gb, 8); - skip_bits(gb, 8); + provider_code = get_bits(gb, 16); - user_identifier = get_bits_long(gb, 32); + if (country_code == usa_country_code && + provider_code == smpte_provider_code) { + // A/341 Amendment – 2094-40 + const uint16_t smpte2094_40_provider_oriented_code = 0x0001; + const uint8_t smpte2094_40_application_identifier = 0x04; - switch (user_identifier) { + uint16_t provider_oriented_code = get_bits(gb, 16); + uint8_t application_identifier = get_bits(gb, 8); + + if (provider_oriented_code == smpte2094_40_provider_oriented_code && + application_identifier == smpte2094_40_application_identifier) { + int err = 0; + size_t meta_size = 0; + AVDynamicHDRPlus *metadata = av_dynamic_hdr_plus_alloc(&meta_size); + if (!metadata) + return AVERROR(ENOMEM); + + err = ff_parse_itu_t_t35_to_dynamic_hdr10_plus(gb, metadata); + if (err < 0) { + av_free(metadata); + return err; + } + + av_buffer_unref(&s->dynamic_hdr_plus.info); + s->dynamic_hdr_plus.info = av_buffer_create((uint8_t *)metadata, + meta_size, NULL, NULL, 0); + if (!s->dynamic_hdr_plus.info) { + av_free(metadata); + return AVERROR(ENOMEM); + } + } + } else { + uint32_t user_identifier = get_bits_long(gb, 32); + switch (user_identifier) { case MKBETAG('G', 'A', '9', '4'): return decode_registered_user_data_closed_caption(&s->a53_caption, gb, size); default: - skip_bits_long(gb, size * 8); break; + } } + skip_bits_long(gb, size * 8); return 0; } @@ -420,4 +455,5 @@ void ff_hevc_reset_sei(HEVCSEI *s) av_buffer_unref(&s->unregistered.buf_ref[i]); s->unregistered.nb_buf_ref = 0; av_freep(&s->unregistered.buf_ref); + av_buffer_unref(&s->dynamic_hdr_plus.info); } diff --git a/libavcodec/hevc_sei.h b/libavcodec/hevc_sei.h index 5ee7a4796d..e9e2d46ed4 100644 --- a/libavcodec/hevc_sei.h +++ b/libavcodec/hevc_sei.h @@ -104,6 +104,10 @@ typedef struct HEVCSEIMasteringDisplay { uint32_t min_luminance; } HEVCSEIMasteringDisplay; +typedef struct HEVCSEIDynamicHDRPlus { + AVBufferRef *info; +} HEVCSEIDynamicHDRPlus; + typedef struct HEVCSEIContentLight { int present; uint16_t max_content_light_level; @@ -143,6 +147,7 @@ typedef struct HEVCSEI { HEVCSEIA53Caption a53_caption; HEVCSEIUnregistered unregistered; HEVCSEIMasteringDisplay mastering_display; + HEVCSEIDynamicHDRPlus dynamic_hdr_plus; HEVCSEIContentLight content_light; int active_seq_parameter_set_id; HEVCSEIAlternativeTransfer alternative_transfer; diff --git a/libavcodec/hevcdec.c b/libavcodec/hevcdec.c index 699c13bbcc..b2e196b294 100644 --- a/libavcodec/hevcdec.c +++ b/libavcodec/hevcdec.c @@ -2867,6 +2867,17 @@ static int set_side_data(HEVCContext *s) s->sei.timecode.num_clock_ts = 0; } + if (s->sei.dynamic_hdr_plus.info) { + AVBufferRef *info_ref = av_buffer_ref(s->sei.dynamic_hdr_plus.info); + if (!info_ref) + return AVERROR(ENOMEM); + + if (!av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_DYNAMIC_HDR_PLUS, info_ref)) { + av_buffer_unref(&info_ref); + return AVERROR(ENOMEM); + } + } + return 0; } @@ -3560,6 +3571,10 @@ static int hevc_update_thread_context(AVCodecContext *dst, } } + ret = av_buffer_replace(&s->sei.dynamic_hdr_plus.info, s0->sei.dynamic_hdr_plus.info); + if (ret < 0) + return ret; + s->sei.frame_packing = s0->sei.frame_packing; s->sei.display_orientation = s0->sei.display_orientation; s->sei.mastering_display = s0->sei.mastering_display;