virtualenv/FFmpeg
1
0
Fork 0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2024-11-26 19:01:44 +02:00
Code Issues Releases Activity

fftools/ffmpeg: move sub2video handling to ffmpeg_filter

Browse Source 
Make all relevant state per-filtergraph input, rather than per-input
stream. Refactor the code to make it work and avoid leaking memory when
a single subtitle stream is sent to multiple filters.
This commit is contained in:
Anton Khirnov 2023-05-24 10:08:33 +02:00
parent 20cacfe493
commit f8abab673c
5 changed files with 177 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
fftools/ffmpeg.c
View File

@ -147,139 +147,20 @@ static int restore_tty;
This is a temporary solution until libavfilter gets real subtitles support.
*/
static int sub2video_get_blank_frame(InputStream *ist)
{
int ret;
AVFrame *frame = ist->sub2video.frame;
av_frame_unref(frame);
frame->width = ist->sub2video.w;
frame->height = ist->sub2video.h;
frame->format = AV_PIX_FMT_RGB32;
if ((ret = av_frame_get_buffer(frame, 0)) < 0)
return ret;
memset(frame->data[0], 0, frame->height * frame->linesize[0]);
return 0;
}
static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
AVSubtitleRect *r)
{
uint32_t *pal, *dst2;
uint8_t *src, *src2;
int x, y;
if (r->type != SUBTITLE_BITMAP) {
av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
return;
}
if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n",
r->x, r->y, r->w, r->h, w, h
);
return;
}
dst += r->y * dst_linesize + r->x * 4;
src = r->data[0];
pal = (uint32_t *)r->data[1];
for (y = 0; y < r->h; y++) {
dst2 = (uint32_t *)dst;
src2 = src;
for (x = 0; x < r->w; x++)
*(dst2++) = pal[*(src2++)];
dst += dst_linesize;
src += r->linesize[0];
}
}
static void sub2video_push_ref(InputStream *ist, int64_t pts)
{
AVFrame *frame = ist->sub2video.frame;
int i;
int ret;
av_assert1(frame->data[0]);
ist->sub2video.last_pts = frame->pts = pts;
for (i = 0; i < ist->nb_filters; i++) {
ret = av_buffersrc_add_frame_flags(ist->filters[i]->filter, frame,
AV_BUFFERSRC_FLAG_KEEP_REF |
AV_BUFFERSRC_FLAG_PUSH);
if (ret != AVERROR_EOF && ret < 0)
av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n",
av_err2str(ret));
}
}
void sub2video_update(InputStream *ist, int64_t heartbeat_pts,
const AVSubtitle *sub)
{
AVFrame *frame = ist->sub2video.frame;
int8_t *dst;
int dst_linesize;
int num_rects, i;
int64_t pts, end_pts;
if (!frame)
return;
if (sub) {
pts = av_rescale_q(sub->pts + sub->start_display_time * 1000LL,
AV_TIME_BASE_Q, ist->st->time_base);
end_pts = av_rescale_q(sub->pts + sub->end_display_time * 1000LL,
AV_TIME_BASE_Q, ist->st->time_base);
num_rects = sub->num_rects;
} else {
/* If we are initializing the system, utilize current heartbeat
PTS as the start time, and show until the following subpicture
is received. Otherwise, utilize the previous subpicture's end time
as the fall-back value. */
pts = ist->sub2video.initialize ?
heartbeat_pts : ist->sub2video.end_pts;
end_pts = INT64_MAX;
num_rects = 0;
}
if (sub2video_get_blank_frame(ist) < 0) {
av_log(NULL, AV_LOG_ERROR,
"Impossible to get a blank canvas.\n");
return;
}
dst = frame->data [0];
dst_linesize = frame->linesize[0];
for (i = 0; i < num_rects; i++)
sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]);
sub2video_push_ref(ist, pts);
ist->sub2video.end_pts = end_pts;
ist->sub2video.initialize = 0;
}
static void sub2video_heartbeat(InputFile *infile, int64_t pts, AVRational tb)
{
int i, j, nb_reqs;
int64_t pts2;
/* When a frame is read from a file, examine all sub2video streams in
the same file and send the sub2video frame again. Otherwise, decoded
video frames could be accumulating in the filter graph while a filter
(possibly overlay) is desperately waiting for a subtitle frame. */
for (i = 0; i < infile->nb_streams; i++) {
InputStream *ist2 = infile->streams[i];
if (!ist2->sub2video.frame)
for (int i = 0; i < infile->nb_streams; i++) {
InputStream *ist = infile->streams[i];
if (ist->dec_ctx->codec_type != AVMEDIA_TYPE_SUBTITLE)
continue;
/* subtitles seem to be usually muxed ahead of other streams;
if not, subtracting a larger time here is necessary */
pts2 = av_rescale_q(pts, tb, ist2->st->time_base) - 1;
/* do not send the heartbeat frame if the subtitle is already ahead */
if (pts2 <= ist2->sub2video.last_pts)
continue;
if (pts2 >= ist2->sub2video.end_pts || ist2->sub2video.initialize)
/* if we have hit the end of the current displayed subpicture,
or if we need to initialize the system, update the
overlayed subpicture and its start/end times */
sub2video_update(ist2, pts2 + 1, NULL);
for (j = 0, nb_reqs = 0; j < ist2->nb_filters; j++)
nb_reqs += av_buffersrc_get_nb_failed_requests(ist2->filters[j]->filter);
if (nb_reqs)
sub2video_push_ref(ist2, pts2);
for (int j = 0; j < ist->nb_filters; j++)
ifilter_sub2video_heartbeat(ist->filters[j], pts, tb);
}
}

7
fftools/ffmpeg.h
View File

@ -372,11 +372,7 @@ typedef struct InputStream {
} prev_sub;
struct sub2video {
int64_t last_pts;
int64_t end_pts;
AVFrame *frame;
int w, h;
unsigned int initialize; ///< marks if sub2video_update should force an initialization
} sub2video;
/* decoded data from this stream goes into all those filters
@ -741,13 +737,12 @@ int init_simple_filtergraph(InputStream *ist, OutputStream *ost,
char *graph_desc);
int init_complex_filtergraph(FilterGraph *fg);
void sub2video_update(InputStream *ist, int64_t heartbeat_pts,
const AVSubtitle *sub);
int copy_av_subtitle(AVSubtitle *dst, const AVSubtitle *src);
int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame, int keep_reference);
int ifilter_send_eof(InputFilter *ifilter, int64_t pts, AVRational tb);
int ifilter_sub2video(InputFilter *ifilter, const AVSubtitle *sub);
void ifilter_sub2video_heartbeat(InputFilter *ifilter, int64_t pts, AVRational tb);
/**
* Set up fallback filtering parameters from a decoder context. They will only

9
fftools/ffmpeg_dec.c
View File

@ -321,13 +321,8 @@ static int video_frame_process(InputStream *ist, AVFrame *frame)
static void sub2video_flush(InputStream *ist)
{
int i;
int ret;
if (ist->sub2video.end_pts < INT64_MAX)
sub2video_update(ist, INT64_MAX, NULL);
for (i = 0; i < ist->nb_filters; i++) {
ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL);
for (int i = 0; i < ist->nb_filters; i++) {
int ret = ifilter_sub2video(ist->filters[i], NULL);
if (ret != AVERROR_EOF && ret < 0)
av_log(NULL, AV_LOG_WARNING, "Flush the frame error.\n");
}

1
fftools/ffmpeg_demux.c
View File

@ -817,7 +817,6 @@ static void ist_free(InputStream **pist)
av_dict_free(&ist->decoder_opts);
avsubtitle_free(&ist->prev_sub.subtitle);
av_frame_free(&ist->sub2video.frame);
av_freep(&ist->filters);
av_freep(&ist->outputs);
av_freep(&ist->hwaccel_device);

189
fftools/ffmpeg_filter.c
View File

@ -107,6 +107,14 @@ typedef struct InputFilterPriv {
struct {
///< queue of AVSubtitle* before filter init
AVFifo *queue;
AVFrame *frame;
int64_t last_pts;
int64_t end_pts;
///< marks if sub2video_update should force an initialization
unsigned int initialize;
} sub2video;
} InputFilterPriv;
@ -115,6 +123,111 @@ static InputFilterPriv *ifp_from_ifilter(InputFilter *ifilter)
return (InputFilterPriv*)ifilter;
}
static int sub2video_get_blank_frame(InputFilterPriv *ifp)
{
AVFrame *frame = ifp->sub2video.frame;
int ret;
av_frame_unref(frame);
frame->width = ifp->width;
frame->height = ifp->height;
frame->format = ifp->format;
ret = av_frame_get_buffer(frame, 0);
if (ret < 0)
return ret;
memset(frame->data[0], 0, frame->height * frame->linesize[0]);
return 0;
}
static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
AVSubtitleRect *r)
{
uint32_t *pal, *dst2;
uint8_t *src, *src2;
int x, y;
if (r->type != SUBTITLE_BITMAP) {
av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
return;
}
if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n",
r->x, r->y, r->w, r->h, w, h
);
return;
}
dst += r->y * dst_linesize + r->x * 4;
src = r->data[0];
pal = (uint32_t *)r->data[1];
for (y = 0; y < r->h; y++) {
dst2 = (uint32_t *)dst;
src2 = src;
for (x = 0; x < r->w; x++)
*(dst2++) = pal[*(src2++)];
dst += dst_linesize;
src += r->linesize[0];
}
}
static void sub2video_push_ref(InputFilterPriv *ifp, int64_t pts)
{
AVFrame *frame = ifp->sub2video.frame;
int ret;
av_assert1(frame->data[0]);
ifp->sub2video.last_pts = frame->pts = pts;
ret = av_buffersrc_add_frame_flags(ifp->ifilter.filter, frame,
AV_BUFFERSRC_FLAG_KEEP_REF |
AV_BUFFERSRC_FLAG_PUSH);
if (ret != AVERROR_EOF && ret < 0)
av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n",
av_err2str(ret));
}
static void sub2video_update(InputFilterPriv *ifp, int64_t heartbeat_pts,
const AVSubtitle *sub)
{
AVFrame *frame = ifp->sub2video.frame;
int8_t *dst;
int dst_linesize;
int num_rects, i;
int64_t pts, end_pts;
if (sub) {
pts = av_rescale_q(sub->pts + sub->start_display_time * 1000LL,
AV_TIME_BASE_Q, ifp->time_base);
end_pts = av_rescale_q(sub->pts + sub->end_display_time * 1000LL,
AV_TIME_BASE_Q, ifp->time_base);
num_rects = sub->num_rects;
} else {
/* If we are initializing the system, utilize current heartbeat
PTS as the start time, and show until the following subpicture
is received. Otherwise, utilize the previous subpicture's end time
as the fall-back value. */
pts = ifp->sub2video.initialize ?
heartbeat_pts : ifp->sub2video.end_pts;
end_pts = INT64_MAX;
num_rects = 0;
}
if (sub2video_get_blank_frame(ifp) < 0) {
av_log(NULL, AV_LOG_ERROR,
"Impossible to get a blank canvas.\n");
return;
}
dst = frame->data [0];
dst_linesize = frame->linesize[0];
for (i = 0; i < num_rects; i++)
sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]);
sub2video_push_ref(ifp, pts);
ifp->sub2video.end_pts = end_pts;
ifp->sub2video.initialize = 0;
}
// FIXME: YUV420P etc. are actually supported with full color range,
// yet the latter information isn't available here.
static const enum AVPixelFormat *get_compliance_normal_pix_fmts(const AVCodec *codec, const enum AVPixelFormat default_formats[])
@ -465,6 +578,12 @@ static int ifilter_bind_ist(InputFilter *ifilter, InputStream *ist)
if (ret < 0)
return ret;
if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE) {
ifp->sub2video.frame = av_frame_alloc();
if (!ifp->sub2video.frame)
return AVERROR(ENOMEM);
}
return 0;
}
@ -610,6 +729,7 @@ void fg_free(FilterGraph **pfg)
avsubtitle_free(&sub);
av_fifo_freep2(&ifp->sub2video.queue);
}
av_frame_free(&ifp->sub2video.frame);
av_channel_layout_uninit(&ifp->fallback.ch_layout);
@ -1108,20 +1228,15 @@ void check_filter_outputs(void)
}
}
static int sub2video_prepare(InputStream *ist, InputFilter *ifilter)
static void sub2video_prepare(InputFilterPriv *ifp)
{
ist->sub2video.frame = av_frame_alloc();
if (!ist->sub2video.frame)
return AVERROR(ENOMEM);
ist->sub2video.last_pts = INT64_MIN;
ist->sub2video.end_pts = INT64_MIN;
ifp->sub2video.last_pts = INT64_MIN;
ifp->sub2video.end_pts = INT64_MIN;
/* sub2video structure has been (re-)initialized.
Mark it as such so that the system will be
initialized with the first received heartbeat. */
ist->sub2video.initialize = 1;
return 0;
ifp->sub2video.initialize = 1;
}
static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
@ -1156,11 +1271,8 @@ static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
if (!fr.num)
fr = ist->framerate_guessed;
if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
ret = sub2video_prepare(ist, ifilter);
if (ret < 0)
goto fail;
}
if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE)
sub2video_prepare(ifp);
ifp->time_base = ist->framerate.num ? av_inv_q(ist->framerate) :
ist->st->time_base;
@ -1466,12 +1578,13 @@ int configure_filtergraph(FilterGraph *fg)
/* process queued up subtitle packets */
for (i = 0; i < fg->nb_inputs; i++) {
InputFilterPriv *ifp = ifp_from_ifilter(fg->inputs[i]);
InputStream *ist = ifp->ist;
if (ifp->sub2video.queue && ist->sub2video.frame) {
InputFilter *ifilter = fg->inputs[i];
InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE && ifp->sub2video.queue) {
AVSubtitle tmp;
while (av_fifo_read(ifp->sub2video.queue, &tmp, 1) >= 0) {
sub2video_update(ist, INT64_MIN, &tmp);
sub2video_update(ifp, INT64_MIN, &tmp);
avsubtitle_free(&tmp);
}
}
@ -1620,15 +1733,47 @@ int reap_filters(int flush)
return 0;
}
void ifilter_sub2video_heartbeat(InputFilter *ifilter, int64_t pts, AVRational tb)
{
InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
int64_t pts2;
if (!ifilter->graph->graph)
return;
/* subtitles seem to be usually muxed ahead of other streams;
if not, subtracting a larger time here is necessary */
pts2 = av_rescale_q(pts, tb, ifp->time_base) - 1;
/* do not send the heartbeat frame if the subtitle is already ahead */
if (pts2 <= ifp->sub2video.last_pts)
return;
if (pts2 >= ifp->sub2video.end_pts || ifp->sub2video.initialize)
/* if we have hit the end of the current displayed subpicture,
or if we need to initialize the system, update the
overlayed subpicture and its start/end times */
sub2video_update(ifp, pts2 + 1, NULL);
if (av_buffersrc_get_nb_failed_requests(ifilter->filter))
sub2video_push_ref(ifp, pts2);
}
int ifilter_sub2video(InputFilter *ifilter, const AVSubtitle *subtitle)
{
InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
InputStream *ist = ifp->ist;
int ret;
if (ist->sub2video.frame) {
sub2video_update(ist, INT64_MIN, subtitle);
} else {
if (ifilter->graph->graph) {
if (!subtitle) {
if (ifp->sub2video.end_pts < INT64_MAX)
sub2video_update(ifp, INT64_MAX, NULL);
return av_buffersrc_add_frame(ifilter->filter, NULL);
}
sub2video_update(ifp, INT64_MIN, subtitle);
} else if (subtitle) {
AVSubtitle sub;
if (!ifp->sub2video.queue)