diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 750f4a113d..94ab6163de 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -443,7 +443,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h) av_log(h->avctx, AV_LOG_ERROR, "top block unavailable for requested intra4x4 mode %d at %d %d\n", status, h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } else if (status) { h->intra4x4_pred_mode_cache[scan8[0] + i] = status; } @@ -459,7 +459,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h) av_log(h->avctx, AV_LOG_ERROR, "left block unavailable for requested intra4x4 mode %d at %d %d\n", status, h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } else if (status) { h->intra4x4_pred_mode_cache[scan8[0] + 8 * i] = status; } @@ -482,7 +482,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma) av_log(h->avctx, AV_LOG_ERROR, "out of range intra chroma pred mode at %d %d\n", h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } if (!(h->top_samples_available & 0x8000)) { @@ -491,7 +491,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma) av_log(h->avctx, AV_LOG_ERROR, "top block unavailable for requested intra mode at %d %d\n", h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } } @@ -507,7 +507,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma) av_log(h->avctx, AV_LOG_ERROR, "left block unavailable for requested intra mode at %d %d\n", h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } } @@ -1295,7 +1295,7 @@ int ff_h264_alloc_tables(H264Context *h) fail: free_tables(h, 1); - return -1; + return AVERROR(ENOMEM); } /** @@ -1394,7 +1394,7 @@ static int context_init(H264Context *h) return 0; fail: - return -1; // free_tables will clean up for us + return AVERROR(ENOMEM); // free_tables will clean up for us } static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size, @@ -1403,6 +1403,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size, int ff_h264_decode_extradata(H264Context *h, const uint8_t *buf, int size) { AVCodecContext *avctx = h->avctx; + int ret; if (!buf || size <= 0) return -1; @@ -1415,7 +1416,7 @@ int ff_h264_decode_extradata(H264Context *h, const uint8_t *buf, int size) if (size < 7) { av_log(avctx, AV_LOG_ERROR, "avcC too short\n"); - return -1; + return AVERROR_INVALIDDATA; } /* sps and pps in the avcC always have length coded with 2 bytes, * so put a fake nal_length_size = 2 while parsing them */ @@ -1426,11 +1427,12 @@ int ff_h264_decode_extradata(H264Context *h, const uint8_t *buf, int size) for (i = 0; i < cnt; i++) { nalsize = AV_RB16(p) + 2; if(nalsize > size - (p-buf)) - return -1; - if (decode_nal_units(h, p, nalsize, 1) < 0) { + return AVERROR_INVALIDDATA; + ret = decode_nal_units(h, p, nalsize, 1); + if (ret < 0) { av_log(avctx, AV_LOG_ERROR, "Decoding sps %d from avcC failed\n", i); - return -1; + return ret; } p += nalsize; } @@ -1439,11 +1441,12 @@ int ff_h264_decode_extradata(H264Context *h, const uint8_t *buf, int size) for (i = 0; i < cnt; i++) { nalsize = AV_RB16(p) + 2; if(nalsize > size - (p-buf)) - return -1; - if (decode_nal_units(h, p, nalsize, 1) < 0) { + return AVERROR_INVALIDDATA; + ret = decode_nal_units(h, p, nalsize, 1); + if (ret < 0) { av_log(avctx, AV_LOG_ERROR, "Decoding pps %d from avcC failed\n", i); - return -1; + return ret; } p += nalsize; } @@ -1451,8 +1454,9 @@ int ff_h264_decode_extradata(H264Context *h, const uint8_t *buf, int size) h->nal_length_size = (buf[4] & 0x03) + 1; } else { h->is_avc = 0; - if (decode_nal_units(h, buf, size, 1) < 0) - return -1; + ret = decode_nal_units(h, buf, size, 1); + if (ret < 0) + return ret; } return size; } @@ -1461,6 +1465,7 @@ av_cold int ff_h264_decode_init(AVCodecContext *avctx) { H264Context *h = avctx->priv_data; int i; + int ret; h->avctx = avctx; @@ -1523,10 +1528,12 @@ av_cold int ff_h264_decode_init(AVCodecContext *avctx) avctx->ticks_per_frame = 2; } - if (avctx->extradata_size > 0 && avctx->extradata && - ff_h264_decode_extradata(h, avctx->extradata, avctx->extradata_size) < 0) { - ff_h264_free_context(h); - return -1; + if (avctx->extradata_size > 0 && avctx->extradata) { + ret = ff_h264_decode_extradata(h, avctx->extradata, avctx->extradata_size); + if (ret < 0) { + ff_h264_free_context(h); + return ret; + } } if (h->sps.bitstream_restriction_flag && @@ -1697,9 +1704,10 @@ static int decode_update_thread_context(AVCodecContext *dst, avcodec_get_frame_defaults(&h->cur_pic.f); h->cur_pic.tf.f = &h->cur_pic.f; - if (ff_h264_alloc_tables(h) < 0) { + ret = ff_h264_alloc_tables(h); + if (ret < 0) { av_log(dst, AV_LOG_ERROR, "Could not allocate memory for h264\n"); - return AVERROR(ENOMEM); + return ret; } context_init(h); } @@ -3086,7 +3094,7 @@ static int h264_slice_header_init(H264Context *h, int reinit) int nb_slices = (HAVE_THREADS && h->avctx->active_thread_type & FF_THREAD_SLICE) ? h->avctx->thread_count : 1; - int i; + int i, ret; h->avctx->sample_aspect_ratio = h->sps.sar; av_assert0(h->avctx->sample_aspect_ratio.den); @@ -3109,10 +3117,11 @@ static int h264_slice_header_init(H264Context *h, int reinit) h->prev_interlaced_frame = 1; init_scan_tables(h); - if (ff_h264_alloc_tables(h) < 0) { + ret = ff_h264_alloc_tables(h); + if (ret < 0) { av_log(h->avctx, AV_LOG_ERROR, "Could not allocate memory for h264\n"); - return AVERROR(ENOMEM); + return ret; } if (nb_slices > MAX_THREADS || (nb_slices > h->mb_height && h->mb_height)) { @@ -3128,9 +3137,10 @@ static int h264_slice_header_init(H264Context *h, int reinit) h->slice_context_count = nb_slices; if (!HAVE_THREADS || !(h->avctx->active_thread_type & FF_THREAD_SLICE)) { - if (context_init(h) < 0) { + ret = context_init(h); + if (ret < 0) { av_log(h->avctx, AV_LOG_ERROR, "context_init() failed.\n"); - return -1; + return ret; } } else { for (i = 1; i < h->slice_context_count; i++) { @@ -3172,9 +3182,9 @@ static int h264_slice_header_init(H264Context *h, int reinit) } for (i = 0; i < h->slice_context_count; i++) - if (context_init(h->thread_context[i]) < 0) { + if ((ret = context_init(h->thread_context[i])) < 0) { av_log(h->avctx, AV_LOG_ERROR, "context_init() failed.\n"); - return -1; + return ret; } } @@ -3228,7 +3238,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) av_log(h->avctx, AV_LOG_ERROR, "slice type too large (%d) at %d %d\n", slice_type, h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } if (slice_type > 4) { slice_type -= 5; @@ -3246,13 +3256,13 @@ static int decode_slice_header(H264Context *h, H264Context *h0) pps_id = get_ue_golomb(&h->gb); if (pps_id >= MAX_PPS_COUNT) { av_log(h->avctx, AV_LOG_ERROR, "pps_id %d out of range\n", pps_id); - return -1; + return AVERROR_INVALIDDATA; } if (!h0->pps_buffers[pps_id]) { av_log(h->avctx, AV_LOG_ERROR, "non-existing PPS %u referenced\n", pps_id); - return -1; + return AVERROR_INVALIDDATA; } h->pps = *h0->pps_buffers[pps_id]; @@ -3260,7 +3270,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) av_log(h->avctx, AV_LOG_ERROR, "non-existing SPS %u referenced\n", h->pps.sps_id); - return -1; + return AVERROR_INVALIDDATA; } if (h->pps.sps_id != h->current_sps_id || @@ -3361,7 +3371,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) if (h != h0) { av_log(h->avctx, AV_LOG_ERROR, "Cannot (re-)initialize context during parallel decoding.\n"); - return -1; + return AVERROR_PATCHWELCOME; } if ((ret = get_pixel_format(h, 1)) < 0) @@ -3502,19 +3512,20 @@ static int decode_slice_header(H264Context *h, H264Context *h0) if (!h->sps.gaps_in_frame_num_allowed_flag) for(i=0; ilast_pocs); i++) h->last_pocs[i] = INT_MIN; - if (h264_frame_start(h) < 0) - return -1; + ret = h264_frame_start(h); + if (ret < 0) + return ret; h->prev_frame_num++; h->prev_frame_num %= 1 << h->sps.log2_max_frame_num; h->cur_pic_ptr->frame_num = h->prev_frame_num; ff_thread_report_progress(&h->cur_pic_ptr->tf, INT_MAX, 0); ff_thread_report_progress(&h->cur_pic_ptr->tf, INT_MAX, 1); - if ((ret = ff_generate_sliding_window_mmcos(h, 1)) < 0 && - h->avctx->err_recognition & AV_EF_EXPLODE) + ret = ff_generate_sliding_window_mmcos(h, 1); + if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE)) + return ret; + ret = ff_h264_execute_ref_pic_marking(h, h->mmco, h->mmco_index); + if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE)) return ret; - if (ff_h264_execute_ref_pic_marking(h, h->mmco, h->mmco_index) < 0 && - (h->avctx->err_recognition & AV_EF_EXPLODE)) - return AVERROR_INVALIDDATA; /* Error concealment: If a ref is missing, copy the previous ref * in its place. * FIXME: Avoiding a memcpy would be nice, but ref handling makes @@ -3573,7 +3584,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) if (!FIELD_PICTURE(h) || h0->first_field) { if (h264_frame_start(h) < 0) { h0->first_field = 0; - return -1; + return AVERROR_INVALIDDATA; } } else { release_unused_pictures(h, 0); @@ -3607,7 +3618,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) if (first_mb_in_slice << FIELD_OR_MBAFF_PICTURE(h) >= h->mb_num || first_mb_in_slice >= h->mb_num) { av_log(h->avctx, AV_LOG_ERROR, "first_mb_in_slice overflow\n"); - return -1; + return AVERROR_INVALIDDATA; } h->resync_mb_x = h->mb_x = first_mb_in_slice % h->mb_width; h->resync_mb_y = h->mb_y = (first_mb_in_slice / h->mb_width) << @@ -3688,10 +3699,12 @@ static int decode_slice_header(H264Context *h, H264Context *h0) ff_h264_fill_default_ref_list(h); } - if (h->slice_type_nos != AV_PICTURE_TYPE_I && - ff_h264_decode_ref_pic_list_reordering(h) < 0) { - h->ref_count[1] = h->ref_count[0] = 0; - return -1; + if (h->slice_type_nos != AV_PICTURE_TYPE_I) { + ret = ff_h264_decode_ref_pic_list_reordering(h); + if (ret < 0) { + h->ref_count[1] = h->ref_count[0] = 0; + return ret; + } } if ((h->pps.weighted_pred && h->slice_type_nos == AV_PICTURE_TYPE_P) || @@ -3714,12 +3727,13 @@ static int decode_slice_header(H264Context *h, H264Context *h0) // or h->mmco, which will cause ref list mix-ups and decoding errors // further down the line. This may break decoding if the first slice is // corrupt, thus we only do this if frame-mt is enabled. - if (h->nal_ref_idc && - ff_h264_decode_ref_pic_marking(h0, &h->gb, - !(h->avctx->active_thread_type & FF_THREAD_FRAME) || - h0->current_slice == 0) < 0 && - (h->avctx->err_recognition & AV_EF_EXPLODE)) - return AVERROR_INVALIDDATA; + if (h->nal_ref_idc) { + ret = ff_h264_decode_ref_pic_marking(h0, &h->gb, + !(h->avctx->active_thread_type & FF_THREAD_FRAME) || + h0->current_slice == 0); + if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE)) + return AVERROR_INVALIDDATA; + } if (FRAME_MBAFF(h)) { ff_h264_fill_mbaff_ref_list(h); @@ -3738,7 +3752,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) tmp = get_ue_golomb_31(&h->gb); if (tmp > 2) { av_log(h->avctx, AV_LOG_ERROR, "cabac_init_idc overflow\n"); - return -1; + return AVERROR_INVALIDDATA; } h->cabac_init_idc = tmp; } @@ -3747,7 +3761,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) tmp = h->pps.init_qp + get_se_golomb(&h->gb); if (tmp > 51 + 6 * (h->sps.bit_depth_luma - 8)) { av_log(h->avctx, AV_LOG_ERROR, "QP %u out of range\n", tmp); - return -1; + return AVERROR_INVALIDDATA; } h->qscale = tmp; h->chroma_qp[0] = get_chroma_qp(h, 0, h->qscale); @@ -3767,7 +3781,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) if (tmp > 2) { av_log(h->avctx, AV_LOG_ERROR, "deblocking_filter_idc %u out of range\n", tmp); - return -1; + return AVERROR_INVALIDDATA; } h->deblocking_filter = tmp; if (h->deblocking_filter < 2) @@ -3781,7 +3795,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) av_log(h->avctx, AV_LOG_ERROR, "deblocking filter parameters %d %d out of range\n", h->slice_alpha_c0_offset, h->slice_beta_offset); - return -1; + return AVERROR_INVALIDDATA; } } } @@ -3908,7 +3922,7 @@ int ff_h264_get_slice_type(const H264Context *h) case AV_PICTURE_TYPE_SI: return 4; default: - return -1; + return AVERROR_INVALIDDATA; } } @@ -4337,7 +4351,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg) h->cabac.bytestream_end - h->cabac.bytestream); er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x, h->mb_y, ER_MB_ERROR); - return -1; + return AVERROR_INVALIDDATA; } if (++h->mb_x >= h->mb_width) { @@ -4384,7 +4398,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg) "error while decoding MB %d %d\n", h->mb_x, h->mb_y); er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x, h->mb_y, ER_MB_ERROR); - return -1; + return ret; } if (++h->mb_x >= h->mb_width) { @@ -4413,7 +4427,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg) h->mb_x, h->mb_y, ER_MB_END); - return -1; + return AVERROR_INVALIDDATA; } } } @@ -4421,6 +4435,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg) if (get_bits_left(&h->gb) <= 0 && h->mb_skip_run <= 0) { tprintf(h->avctx, "slice end %d %d\n", get_bits_count(&h->gb), h->gb.size_in_bits); + if (get_bits_left(&h->gb) == 0) { er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x - 1, h->mb_y, @@ -4433,7 +4448,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg) er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x, h->mb_y, ER_MB_ERROR); - return -1; + return AVERROR_INVALIDDATA; } } } @@ -4499,6 +4514,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size, int nal_index; int idr_cleared=0; int first_slice = 0; + int ret = 0; h->nal_unit_type= 0; @@ -4568,7 +4584,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size, ptr = ff_h264_decode_nal(hx, buf + buf_index, &dst_length, &consumed, next_avc - buf_index); if (ptr == NULL || dst_length < 0) { - buf_index = -1; + ret = -1; goto end; } i = buf_index + consumed; @@ -4653,7 +4669,7 @@ again: if (first_slice != NAL_IDR_SLICE) { av_log(h->avctx, AV_LOG_ERROR, "Invalid mix of idr and non-idr slices\n"); - buf_index = -1; + ret = -1; goto end; } if(!idr_cleared) @@ -4698,8 +4714,8 @@ again: decode_postinit(h, nal_index >= nals_needed); if (h->avctx->hwaccel && - h->avctx->hwaccel->start_frame(h->avctx, NULL, 0) < 0) - return -1; + (ret = h->avctx->hwaccel->start_frame(h->avctx, NULL, 0)) < 0) + return ret; } if (hx->redundant_pic_count == 0 && @@ -4711,10 +4727,11 @@ again: hx->slice_type_nos == AV_PICTURE_TYPE_I) && avctx->skip_frame < AVDISCARD_ALL) { if (avctx->hwaccel) { - if (avctx->hwaccel->decode_slice(avctx, - &buf[buf_index - consumed], - consumed) < 0) - return -1; + ret = avctx->hwaccel->decode_slice(avctx, + &buf[buf_index - consumed], + consumed); + if (ret < 0) + return ret; } else context_count++; } @@ -4818,7 +4835,7 @@ end: h->picture_structure == PICT_BOTTOM_FIELD); } - return buf_index; + return (ret < 0) ? ret : buf_index; } /** @@ -4929,7 +4946,7 @@ not_extra: buf_index = decode_nal_units(h, buf, buf_size, 0); if (buf_index < 0) - return -1; + return AVERROR_INVALIDDATA; if (!h->cur_pic_ptr && h->nal_unit_type == NAL_END_SEQUENCE) { av_assert0(buf_index <= buf_size); @@ -4941,7 +4958,7 @@ not_extra: buf_size >= 4 && !memcmp("Q264", buf, 4)) return buf_size; av_log(avctx, AV_LOG_ERROR, "no frame!\n"); - return -1; + return AVERROR_INVALIDDATA; } if (!(avctx->flags2 & CODEC_FLAG2_CHUNKS) ||