From c341f734e5f9d6af4a8fdcceb6f5d12de6395c76 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 27 Sep 2013 20:13:59 -0400 Subject: [PATCH] Convert multiplier for MV from int to ptrdiff_t. This prevents emulated_edge_mc from not undoing mvy*stride-related integer overflows. Signed-off-by: Michael Niedermayer --- libavcodec/cavs.h | 2 +- libavcodec/diracdec.c | 2 +- libavcodec/h264.c | 2 +- libavcodec/h264.h | 6 +++--- libavcodec/mpegvideo.c | 9 +++++---- libavcodec/mpegvideo.h | 4 ++-- libavcodec/mpegvideo_enc.c | 4 ++-- libavcodec/mpegvideo_motion.c | 11 +++++++---- libavcodec/snow.c | 2 +- libavcodec/snow.h | 2 +- libavcodec/videodsp.h | 2 -- libavcodec/videodsp_template.c | 3 +-- libavcodec/vp3.c | 2 +- libavcodec/vp56.c | 2 +- libavcodec/vp8.c | 4 ++-- libavcodec/wmv2.c | 3 ++- libavcodec/x86/videodsp_init.c | 3 +-- 17 files changed, 32 insertions(+), 31 deletions(-) diff --git a/libavcodec/cavs.h b/libavcodec/cavs.h index b0cdb8f200..f3c05dc9f0 100644 --- a/libavcodec/cavs.h +++ b/libavcodec/cavs.h @@ -210,7 +210,7 @@ typedef struct AVSContext { 6: A3 X2 X3 */ int pred_mode_Y[3*3]; int *top_pred_Y; - int l_stride, c_stride; + ptrdiff_t l_stride, c_stride; int luma_scan[4]; int qp; int qp_fixed; diff --git a/libavcodec/diracdec.c b/libavcodec/diracdec.c index 7468280637..e5239a77f8 100644 --- a/libavcodec/diracdec.c +++ b/libavcodec/diracdec.c @@ -112,7 +112,7 @@ typedef struct SubBand { typedef struct Plane { int width; int height; - int stride; + ptrdiff_t stride; int idwt_width; int idwt_height; diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 51484edce1..acaab39f08 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -911,7 +911,7 @@ static av_always_inline void mc_dir_part(H264Context *h, Picture *pic, const int mx = h->mv_cache[list][scan8[n]][0] + src_x_offset * 8; int my = h->mv_cache[list][scan8[n]][1] + src_y_offset * 8; const int luma_xy = (mx & 3) + ((my & 3) << 2); - int offset = ((mx >> 2) << pixel_shift) + (my >> 2) * h->mb_linesize; + ptrdiff_t offset = ((mx >> 2) << pixel_shift) + (my >> 2) * h->mb_linesize; uint8_t *src_y = pic->f.data[0] + offset; uint8_t *src_cb, *src_cr; int extra_width = 0; diff --git a/libavcodec/h264.h b/libavcodec/h264.h index b2365bce83..f1db519e47 100644 --- a/libavcodec/h264.h +++ b/libavcodec/h264.h @@ -306,7 +306,7 @@ typedef struct H264Context { /* coded dimensions -- 16 * mb w/h */ int width, height; - int linesize, uvlinesize; + ptrdiff_t linesize, uvlinesize; int chroma_x_shift, chroma_y_shift; int qscale; @@ -379,8 +379,8 @@ typedef struct H264Context { uint32_t *mb2br_xy; int b_stride; // FIXME use s->b4_stride - int mb_linesize; ///< may be equal to s->linesize or s->linesize * 2, for mbaff - int mb_uvlinesize; + ptrdiff_t mb_linesize; ///< may be equal to s->linesize or s->linesize * 2, for mbaff + ptrdiff_t mb_uvlinesize; unsigned current_sps_id; ///< id of the current SPS SPS sps; ///< current sps diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c index 6cfd2a402f..96afca5d7c 100644 --- a/libavcodec/mpegvideo.c +++ b/libavcodec/mpegvideo.c @@ -2179,7 +2179,7 @@ static inline int hpel_motion_lowres(MpegEncContext *s, uint8_t *dest, uint8_t *src, int field_based, int field_select, int src_x, int src_y, - int width, int height, int stride, + int width, int height, ptrdiff_t stride, int h_edge_pos, int v_edge_pos, int w, int h, h264_chroma_mc_func *pix_op, int motion_x, int motion_y) @@ -2235,8 +2235,8 @@ static av_always_inline void mpeg_motion_lowres(MpegEncContext *s, int h, int mb_y) { uint8_t *ptr_y, *ptr_cb, *ptr_cr; - int mx, my, src_x, src_y, uvsrc_x, uvsrc_y, uvlinesize, linesize, sx, sy, - uvsx, uvsy; + int mx, my, src_x, src_y, uvsrc_x, uvsrc_y, sx, sy, uvsx, uvsy; + ptrdiff_t uvlinesize, linesize; const int lowres = s->avctx->lowres; const int op_index = FFMIN(lowres-1+s->chroma_x_shift, 3); const int block_s = 8>>lowres; @@ -2367,7 +2367,8 @@ static inline void chroma_4mv_motion_lowres(MpegEncContext *s, const int s_mask = (2 << lowres) - 1; const int h_edge_pos = s->h_edge_pos >> lowres + 1; const int v_edge_pos = s->v_edge_pos >> lowres + 1; - int emu = 0, src_x, src_y, offset, sx, sy; + int emu = 0, src_x, src_y, sx, sy; + ptrdiff_t offset; uint8_t *ptr; if (s->quarter_sample) { diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h index d17426cd4b..2152af2e10 100644 --- a/libavcodec/mpegvideo.h +++ b/libavcodec/mpegvideo.h @@ -284,8 +284,8 @@ typedef struct MpegEncContext { int b4_stride; ///< 4*mb_width+1 used for some 4x4 block arrays to allow simple addressing int h_edge_pos, v_edge_pos;///< horizontal / vertical position of the right/bottom edge (pixel replication) int mb_num; ///< number of MBs of a picture - int linesize; ///< line size, in bytes, may be different from width - int uvlinesize; ///< line size, for chroma in bytes, may be different from width + ptrdiff_t linesize; ///< line size, in bytes, may be different from width + ptrdiff_t uvlinesize; ///< line size, for chroma in bytes, may be different from width Picture *picture; ///< main picture buffer Picture **input_picture; ///< next pictures on display order for encoding Picture **reordered_input_picture; ///< pointer to the next pictures in codedorder for encoding diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c index a85cf563de..6f0d96168a 100644 --- a/libavcodec/mpegvideo_enc.c +++ b/libavcodec/mpegvideo_enc.c @@ -985,7 +985,7 @@ static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg) direct = 0; av_dlog(s->avctx, "%d %d %d %d\n", pic_arg->linesize[0], - pic_arg->linesize[1], s->linesize, s->uvlinesize); + pic_arg->linesize[1], (int) s->linesize, (int) s->uvlinesize); if (direct) { i = ff_find_unused_picture(s, 1); @@ -1771,7 +1771,7 @@ static av_always_inline void encode_mb_internal(MpegEncContext *s, int dct_offset = s->linesize * 8; // default for progressive frames int uv_dct_offset = s->uvlinesize * 8; uint8_t *ptr_y, *ptr_cb, *ptr_cr; - int wrap_y, wrap_c; + ptrdiff_t wrap_y, wrap_c; for (i = 0; i < mb_block_count; i++) skip_dct[i] = s->skipdct; diff --git a/libavcodec/mpegvideo_motion.c b/libavcodec/mpegvideo_motion.c index 3851739512..e5b7e4648a 100644 --- a/libavcodec/mpegvideo_motion.c +++ b/libavcodec/mpegvideo_motion.c @@ -38,7 +38,8 @@ static void gmc1_motion(MpegEncContext *s, uint8_t **ref_picture) { uint8_t *ptr; - int offset, src_x, src_y, linesize, uvlinesize; + int src_x, src_y; + ptrdiff_t offset, linesize, uvlinesize; int motion_x, motion_y; int emu=0; @@ -215,7 +216,7 @@ void mpeg_motion_internal(MpegEncContext *s, uint8_t *ptr_y, *ptr_cb, *ptr_cr; int dxy, uvdxy, mx, my, src_x, src_y, uvsrc_x, uvsrc_y, v_edge_pos; - emuedge_linesize_type uvlinesize, linesize; + ptrdiff_t uvlinesize, linesize; #if 0 if(s->quarter_sample) @@ -459,7 +460,8 @@ static inline void qpel_motion(MpegEncContext *s, int motion_x, int motion_y, int h) { uint8_t *ptr_y, *ptr_cb, *ptr_cr; - int dxy, uvdxy, mx, my, src_x, src_y, uvsrc_x, uvsrc_y, v_edge_pos, linesize, uvlinesize; + int dxy, uvdxy, mx, my, src_x, src_y, uvsrc_x, uvsrc_y, v_edge_pos; + ptrdiff_t linesize, uvlinesize; dxy = ((motion_y & 3) << 2) | (motion_x & 3); src_x = s->mb_x * 16 + (motion_x >> 2); @@ -552,7 +554,8 @@ static void chroma_4mv_motion(MpegEncContext *s, op_pixels_func *pix_op, int mx, int my) { - int dxy, emu=0, src_x, src_y, offset; + int dxy, emu=0, src_x, src_y; + ptrdiff_t offset; uint8_t *ptr; /* In case of 8X8, we construct a single chroma motion vector diff --git a/libavcodec/snow.c b/libavcodec/snow.c index 1125b25367..dc5c78e84a 100644 --- a/libavcodec/snow.c +++ b/libavcodec/snow.c @@ -297,7 +297,7 @@ static void mc_block(Plane *p, uint8_t *dst, const uint8_t *src, int stride, int } } -void ff_snow_pred_block(SnowContext *s, uint8_t *dst, uint8_t *tmp, int stride, int sx, int sy, int b_w, int b_h, BlockNode *block, int plane_index, int w, int h){ +void ff_snow_pred_block(SnowContext *s, uint8_t *dst, uint8_t *tmp, ptrdiff_t stride, int sx, int sy, int b_w, int b_h, BlockNode *block, int plane_index, int w, int h){ if(block->type & BLOCK_INTRA){ int x, y; const unsigned color = block->color[plane_index]; diff --git a/libavcodec/snow.h b/libavcodec/snow.h index c3e50adf5b..1222a776de 100644 --- a/libavcodec/snow.h +++ b/libavcodec/snow.h @@ -227,7 +227,7 @@ void ff_snow_release_buffer(AVCodecContext *avctx); void ff_snow_reset_contexts(SnowContext *s); int ff_snow_alloc_blocks(SnowContext *s); int ff_snow_frame_start(SnowContext *s); -void ff_snow_pred_block(SnowContext *s, uint8_t *dst, uint8_t *tmp, int stride, +void ff_snow_pred_block(SnowContext *s, uint8_t *dst, uint8_t *tmp, ptrdiff_t stride, int sx, int sy, int b_w, int b_h, BlockNode *block, int plane_index, int w, int h); /* common inline functions */ diff --git a/libavcodec/videodsp.h b/libavcodec/videodsp.h index 07519d71ba..e397720773 100644 --- a/libavcodec/videodsp.h +++ b/libavcodec/videodsp.h @@ -29,8 +29,6 @@ #include #include -typedef int emuedge_linesize_type; - #define EMULATED_EDGE(depth) \ void ff_emulated_edge_mc_ ## depth (uint8_t *buf, const uint8_t *src, ptrdiff_t linesize,\ int block_w, int block_h,\ diff --git a/libavcodec/videodsp_template.c b/libavcodec/videodsp_template.c index 1f709c499d..39aad4deec 100644 --- a/libavcodec/videodsp_template.c +++ b/libavcodec/videodsp_template.c @@ -21,13 +21,12 @@ #include "bit_depth_template.c" void FUNC(ff_emulated_edge_mc)(uint8_t *buf, const uint8_t *src, - ptrdiff_t linesize_arg, + ptrdiff_t linesize, int block_w, int block_h, int src_x, int src_y, int w, int h) { int x, y; int start_y, start_x, end_y, end_x; - emuedge_linesize_type linesize = linesize_arg; if (!w || !h) return; diff --git a/libavcodec/vp3.c b/libavcodec/vp3.c index a6e2d0eea7..78c094c038 100644 --- a/libavcodec/vp3.c +++ b/libavcodec/vp3.c @@ -1476,7 +1476,7 @@ static void render_slice(Vp3DecodeContext *s, int slice) uint8_t *output_plane = s->current_frame.f->data [plane] + s->data_offset[plane]; uint8_t * last_plane = s-> last_frame.f->data [plane] + s->data_offset[plane]; uint8_t *golden_plane = s-> golden_frame.f->data [plane] + s->data_offset[plane]; - int stride = s->current_frame.f->linesize[plane]; + ptrdiff_t stride = s->current_frame.f->linesize[plane]; int plane_width = s->width >> (plane && s->chroma_x_shift); int plane_height = s->height >> (plane && s->chroma_y_shift); int8_t (*motion_val)[2] = s->motion_val[!!plane]; diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c index d67eaa6b0d..b5d1884aa5 100644 --- a/libavcodec/vp56.c +++ b/libavcodec/vp56.c @@ -311,7 +311,7 @@ static void vp56_deblock_filter(VP56Context *s, uint8_t *yuv, } static void vp56_mc(VP56Context *s, int b, int plane, uint8_t *src, - int stride, int x, int y) + ptrdiff_t stride, int x, int y) { uint8_t *dst = s->frames[VP56_FRAME_CURRENT]->data[plane] + s->block_offset[b]; uint8_t *src_block; diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c index d7fac64411..48432c17c2 100644 --- a/libavcodec/vp8.c +++ b/libavcodec/vp8.c @@ -1180,7 +1180,7 @@ static av_always_inline void vp8_mc_luma(VP8Context *s, VP8ThreadData *td, uint8_t *dst, ThreadFrame *ref, const VP56mv *mv, int x_off, int y_off, int block_w, int block_h, - int width, int height, int linesize, + int width, int height, ptrdiff_t linesize, vp8_mc_func mc_func[3][3]) { uint8_t *src = ref->f->data[0]; @@ -1230,7 +1230,7 @@ void vp8_mc_luma(VP8Context *s, VP8ThreadData *td, uint8_t *dst, static av_always_inline void vp8_mc_chroma(VP8Context *s, VP8ThreadData *td, uint8_t *dst1, uint8_t *dst2, ThreadFrame *ref, const VP56mv *mv, int x_off, int y_off, - int block_w, int block_h, int width, int height, int linesize, + int block_w, int block_h, int width, int height, ptrdiff_t linesize, vp8_mc_func mc_func[3][3]) { uint8_t *src1 = ref->f->data[1], *src2 = ref->f->data[2]; diff --git a/libavcodec/wmv2.c b/libavcodec/wmv2.c index 918d438ec2..eea819bfdb 100644 --- a/libavcodec/wmv2.c +++ b/libavcodec/wmv2.c @@ -94,7 +94,8 @@ void ff_mspel_motion(MpegEncContext *s, { Wmv2Context * const w= (Wmv2Context*)s; uint8_t *ptr; - int dxy, offset, mx, my, src_x, src_y, v_edge_pos, linesize, uvlinesize; + int dxy, mx, my, src_x, src_y, v_edge_pos; + ptrdiff_t offset, linesize, uvlinesize; int emu=0; dxy = ((motion_y & 1) << 1) | (motion_x & 1); diff --git a/libavcodec/x86/videodsp_init.c b/libavcodec/x86/videodsp_init.c index 95c71a280f..17a90115cf 100644 --- a/libavcodec/x86/videodsp_init.c +++ b/libavcodec/x86/videodsp_init.c @@ -39,14 +39,13 @@ extern emu_edge_core_func ff_emu_edge_core_mmx; extern emu_edge_core_func ff_emu_edge_core_sse; static av_always_inline void emulated_edge_mc(uint8_t *buf, const uint8_t *src, - ptrdiff_t linesize_arg, + ptrdiff_t linesize, int block_w, int block_h, int src_x, int src_y, int w, int h, emu_edge_core_func *core_fn) { int start_y, start_x, end_y, end_x, src_y_add = 0; - emuedge_linesize_type linesize = linesize_arg; if(!w || !h) return;