1
0
mirror of https://github.com/FFmpeg/FFmpeg.git synced 2024-12-23 12:43:46 +02:00
FFmpeg/libavfilter/formats.c
Andreas Rheinhardt 2a471af50a avfilter/formats: Fix double frees and memleaks on error
The formats API deals with lists of channel layouts, sample rates,
pixel formats and sample formats. These lists are refcounted in a way in
which the list structure itself contains pointers to all of its owners.
Furthermore, it is possible for a list to be not owned by anyone yet;
this status is temporary until the list has been attached to an owner.
Adding an owner to a list involves reallocating the list's list of
owners and can therefore fail.

In order to reduce the amount of checks and cleanup code for the users
of this API, the API is supposed to be lenient when faced with input
lists that are NULL and it is supposed to clean up if adding an owner
to a list fails, so that a simple use case like

list = ff_make_format_list(foo_fmts);
if ((ret = ff_formats_ref(list, &ctx->inputs[0]->out_formats)) < 0)
    return ret;

needn't check whether list could be successfully allocated
(ff_formats_ref() return AVERROR(ENOMEM) if it couldn't) and it also
needn't free list if ff_formats_ref() couldn't add an owner for it.

But the cleaning up after itself was broken. The root cause was that
the refcount was decremented during unreferencing whether or not the
element to be unreferenced was actually an owner of the list or not.
This means that if the above sample code is continued by

if ((ret = ff_formats_ref(list, &ctx->inputs[1]->out_formats)) < 0)
    return ret;

and that if an error happens at the second ff_formats_ref() call, the
automatic cleaning of list will decrement the refcount from 1 (the sole
owner of list at this moment is ctx->input[0]->out_formats) to 0 and so
the list will be freed; yet ctx->input[0]->out_formats still points to
the list and this will lead to a double free/use-after-free when
ctx->input[0] is freed later.

Presumably in order to work around such an issue, commit
93afb338a4 restricted unreferencing to
lists with owners. This does not solve the root cause (the above example
is not fixed by this) at all, but it solves some crashs.

This commit fixes the API: The list's refcount is only decremented if
an owner is removed from the list of owners and not if the
unref-function is called with a pointer that is not among the owners of
the list. Furtermore, the requirement for the list to have owners is
dropped.

This implies that if the first call to ff_formats_ref() in the above
example fails, the refcount which is initially zero during unreferencing
is not modified, so that the list will be freed automatically in said
call to ff_formats_ref() as every list whose refcount reaches zero is.

If on the other hand, the second call to ff_formats_ref() is the first
to fail, the refcount would stay at one during the automatic
unreferencing in ff_formats_ref(). The list would later be freed when
its last (and in this case sole) owner (namely
ctx->inputs[0]->out_formats) gets unreferenced.

The issues described here for ff_formats_ref() also affected the other
functions of this API. E.g. ff_add_format() failed to clean up after
itself if adding an entry to an already existing list failed (the case
of a freshly allocated list was handled specially and this commit also
removes said code). E.g. ff_all_formats() inherited the flaw.

Reviewed-by: Nicolas George <george@nsup.org>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
2020-08-23 23:33:01 +02:00

665 lines
26 KiB
C

/*
* Filter layer - format negotiation
* Copyright (c) 2007 Bobby Bingham
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/avassert.h"
#include "libavutil/channel_layout.h"
#include "libavutil/common.h"
#include "libavutil/eval.h"
#include "libavutil/pixdesc.h"
#include "avfilter.h"
#include "internal.h"
#include "formats.h"
#define KNOWN(l) (!FF_LAYOUT2COUNT(l)) /* for readability */
/**
* Add all refs from a to ret and destroy a.
*/
#define MERGE_REF(ret, a, fmts, type, fail_statement) \
do { \
type ***tmp; \
int i; \
\
if (!(tmp = av_realloc_array(ret->refs, ret->refcount + a->refcount, \
sizeof(*tmp)))) \
{ fail_statement } \
ret->refs = tmp; \
\
for (i = 0; i < a->refcount; i ++) { \
ret->refs[ret->refcount] = a->refs[i]; \
*ret->refs[ret->refcount++] = ret; \
} \
\
av_freep(&a->refs); \
av_freep(&a->fmts); \
av_freep(&a); \
} while (0)
/**
* Add all formats common to a and b to a, add b's refs to a and destroy b.
* If check is set, nothing is modified and it is only checked whether
* the formats are compatible.
* If empty_allowed is set and one of a,b->nb is zero, the lists are
* merged; otherwise, it is treated as error.
*/
#define MERGE_FORMATS(a, b, fmts, nb, type, check, empty_allowed) \
do { \
int i, j, k = 0, skip = 0; \
\
if (empty_allowed) { \
if (!a->nb || !b->nb) { \
if (check) \
return 1; \
if (!a->nb) \
FFSWAP(type *, a, b); \
skip = 1; \
} \
} \
if (!skip) { \
for (i = 0; i < a->nb; i++) \
for (j = 0; j < b->nb; j++) \
if (a->fmts[i] == b->fmts[j]) { \
if (check) \
return 1; \
a->fmts[k++] = a->fmts[i]; \
break; \
} \
/* Check that there was at least one common format. \
* Notice that both a and b are unchanged if not. */ \
if (!k) \
return 0; \
av_assert2(!check); \
a->nb = k; \
} \
\
MERGE_REF(a, b, fmts, type, return AVERROR(ENOMEM);); \
} while (0)
static int merge_formats_internal(AVFilterFormats *a, AVFilterFormats *b,
enum AVMediaType type, int check)
{
int i, j;
int alpha1=0, alpha2=0;
int chroma1=0, chroma2=0;
if (a == b)
return 1;
/* Do not lose chroma or alpha in merging.
It happens if both lists have formats with chroma (resp. alpha), but
the only formats in common do not have it (e.g. YUV+gray vs.
RGB+gray): in that case, the merging would select the gray format,
possibly causing a lossy conversion elsewhere in the graph.
To avoid that, pretend that there are no common formats to force the
insertion of a conversion filter. */
if (type == AVMEDIA_TYPE_VIDEO)
for (i = 0; i < a->nb_formats; i++)
for (j = 0; j < b->nb_formats; j++) {
const AVPixFmtDescriptor *adesc = av_pix_fmt_desc_get(a->formats[i]);
const AVPixFmtDescriptor *bdesc = av_pix_fmt_desc_get(b->formats[j]);
alpha2 |= adesc->flags & bdesc->flags & AV_PIX_FMT_FLAG_ALPHA;
chroma2|= adesc->nb_components > 1 && bdesc->nb_components > 1;
if (a->formats[i] == b->formats[j]) {
alpha1 |= adesc->flags & AV_PIX_FMT_FLAG_ALPHA;
chroma1|= adesc->nb_components > 1;
}
}
// If chroma or alpha can be lost through merging then do not merge
if (alpha2 > alpha1 || chroma2 > chroma1)
return 0;
MERGE_FORMATS(a, b, formats, nb_formats, AVFilterFormats, check, 0);
return 1;
}
int ff_can_merge_formats(const AVFilterFormats *a, const AVFilterFormats *b,
enum AVMediaType type)
{
return merge_formats_internal((AVFilterFormats *)a,
(AVFilterFormats *)b, type, 1);
}
int ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b,
enum AVMediaType type)
{
av_assert2(a->refcount && b->refcount);
return merge_formats_internal(a, b, type, 0);
}
static int merge_samplerates_internal(AVFilterFormats *a,
AVFilterFormats *b, int check)
{
if (a == b) return 1;
MERGE_FORMATS(a, b, formats, nb_formats, AVFilterFormats, check, 1);
return 1;
}
int ff_can_merge_samplerates(const AVFilterFormats *a, const AVFilterFormats *b)
{
return merge_samplerates_internal((AVFilterFormats *)a, (AVFilterFormats *)b, 1);
}
int ff_merge_samplerates(AVFilterFormats *a, AVFilterFormats *b)
{
av_assert2(a->refcount && b->refcount);
return merge_samplerates_internal(a, b, 0);
}
int ff_merge_channel_layouts(AVFilterChannelLayouts *a,
AVFilterChannelLayouts *b)
{
uint64_t *channel_layouts;
unsigned a_all = a->all_layouts + a->all_counts;
unsigned b_all = b->all_layouts + b->all_counts;
int ret_max, ret_nb = 0, i, j, round;
av_assert2(a->refcount && b->refcount);
if (a == b) return 1;
/* Put the most generic set in a, to avoid doing everything twice */
if (a_all < b_all) {
FFSWAP(AVFilterChannelLayouts *, a, b);
FFSWAP(unsigned, a_all, b_all);
}
if (a_all) {
if (a_all == 1 && !b_all) {
/* keep only known layouts in b; works also for b_all = 1 */
for (i = j = 0; i < b->nb_channel_layouts; i++)
if (KNOWN(b->channel_layouts[i]))
b->channel_layouts[j++] = b->channel_layouts[i];
/* Not optimal: the unknown layouts of b may become known after
another merge. */
if (!j)
return 0;
b->nb_channel_layouts = j;
}
MERGE_REF(b, a, channel_layouts, AVFilterChannelLayouts, return AVERROR(ENOMEM););
return 1;
}
ret_max = a->nb_channel_layouts + b->nb_channel_layouts;
if (!(channel_layouts = av_malloc_array(ret_max, sizeof(*channel_layouts))))
return AVERROR(ENOMEM);
/* a[known] intersect b[known] */
for (i = 0; i < a->nb_channel_layouts; i++) {
if (!KNOWN(a->channel_layouts[i]))
continue;
for (j = 0; j < b->nb_channel_layouts; j++) {
if (a->channel_layouts[i] == b->channel_layouts[j]) {
channel_layouts[ret_nb++] = a->channel_layouts[i];
a->channel_layouts[i] = b->channel_layouts[j] = 0;
break;
}
}
}
/* 1st round: a[known] intersect b[generic]
2nd round: a[generic] intersect b[known] */
for (round = 0; round < 2; round++) {
for (i = 0; i < a->nb_channel_layouts; i++) {
uint64_t fmt = a->channel_layouts[i], bfmt;
if (!fmt || !KNOWN(fmt))
continue;
bfmt = FF_COUNT2LAYOUT(av_get_channel_layout_nb_channels(fmt));
for (j = 0; j < b->nb_channel_layouts; j++)
if (b->channel_layouts[j] == bfmt)
channel_layouts[ret_nb++] = a->channel_layouts[i];
}
/* 1st round: swap to prepare 2nd round; 2nd round: put it back */
FFSWAP(AVFilterChannelLayouts *, a, b);
}
/* a[generic] intersect b[generic] */
for (i = 0; i < a->nb_channel_layouts; i++) {
if (KNOWN(a->channel_layouts[i]))
continue;
for (j = 0; j < b->nb_channel_layouts; j++)
if (a->channel_layouts[i] == b->channel_layouts[j])
channel_layouts[ret_nb++] = a->channel_layouts[i];
}
if (!ret_nb) {
av_free(channel_layouts);
return 0;
}
if (a->refcount > b->refcount)
FFSWAP(AVFilterChannelLayouts *, a, b);
MERGE_REF(b, a, channel_layouts, AVFilterChannelLayouts,
{ av_free(channel_layouts); return AVERROR(ENOMEM); });
av_freep(&b->channel_layouts);
b->channel_layouts = channel_layouts;
b->nb_channel_layouts = ret_nb;
return 1;
}
int ff_fmt_is_in(int fmt, const int *fmts)
{
const int *p;
for (p = fmts; *p != -1; p++) {
if (fmt == *p)
return 1;
}
return 0;
}
#define MAKE_FORMAT_LIST(type, field, count_field) \
type *formats; \
int count = 0; \
if (fmts) \
for (count = 0; fmts[count] != -1; count++) \
; \
formats = av_mallocz(sizeof(*formats)); \
if (!formats) \
return NULL; \
formats->count_field = count; \
if (count) { \
formats->field = av_malloc_array(count, sizeof(*formats->field)); \
if (!formats->field) { \
av_freep(&formats); \
return NULL; \
} \
}
AVFilterFormats *ff_make_format_list(const int *fmts)
{
MAKE_FORMAT_LIST(AVFilterFormats, formats, nb_formats);
while (count--)
formats->formats[count] = fmts[count];
return formats;
}
AVFilterChannelLayouts *ff_make_format64_list(const int64_t *fmts)
{
MAKE_FORMAT_LIST(AVFilterChannelLayouts,
channel_layouts, nb_channel_layouts);
if (count)
memcpy(formats->channel_layouts, fmts,
sizeof(*formats->channel_layouts) * count);
return formats;
}
#if LIBAVFILTER_VERSION_MAJOR < 8
AVFilterChannelLayouts *avfilter_make_format64_list(const int64_t *fmts)
{
return ff_make_format64_list(fmts);
}
#endif
#define ADD_FORMAT(f, fmt, unref_fn, type, list, nb) \
do { \
type *fmts; \
\
if (!(*f) && !(*f = av_mallocz(sizeof(**f)))) { \
return AVERROR(ENOMEM); \
} \
\
fmts = av_realloc_array((*f)->list, (*f)->nb + 1, \
sizeof(*(*f)->list)); \
if (!fmts) { \
unref_fn(f); \
return AVERROR(ENOMEM); \
} \
\
(*f)->list = fmts; \
(*f)->list[(*f)->nb++] = fmt; \
} while (0)
int ff_add_format(AVFilterFormats **avff, int64_t fmt)
{
ADD_FORMAT(avff, fmt, ff_formats_unref, int, formats, nb_formats);
return 0;
}
int ff_add_channel_layout(AVFilterChannelLayouts **l, uint64_t channel_layout)
{
av_assert1(!(*l && (*l)->all_layouts));
ADD_FORMAT(l, channel_layout, ff_channel_layouts_unref, uint64_t, channel_layouts, nb_channel_layouts);
return 0;
}
AVFilterFormats *ff_all_formats(enum AVMediaType type)
{
AVFilterFormats *ret = NULL;
if (type == AVMEDIA_TYPE_VIDEO) {
const AVPixFmtDescriptor *desc = NULL;
while ((desc = av_pix_fmt_desc_next(desc))) {
if (ff_add_format(&ret, av_pix_fmt_desc_get_id(desc)) < 0)
return NULL;
}
} else if (type == AVMEDIA_TYPE_AUDIO) {
enum AVSampleFormat fmt = 0;
while (av_get_sample_fmt_name(fmt)) {
if (ff_add_format(&ret, fmt) < 0)
return NULL;
fmt++;
}
}
return ret;
}
int ff_formats_pixdesc_filter(AVFilterFormats **rfmts, unsigned want, unsigned rej)
{
unsigned nb_formats, fmt, flags;
AVFilterFormats *formats = NULL;
while (1) {
nb_formats = 0;
for (fmt = 0;; fmt++) {
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
if (!desc)
break;
flags = desc->flags;
if (!(desc->flags & AV_PIX_FMT_FLAG_HWACCEL) &&
!(desc->flags & AV_PIX_FMT_FLAG_PLANAR) &&
(desc->log2_chroma_w || desc->log2_chroma_h))
flags |= FF_PIX_FMT_FLAG_SW_FLAT_SUB;
if ((flags & (want | rej)) != want)
continue;
if (formats)
formats->formats[nb_formats] = fmt;
nb_formats++;
}
if (formats) {
av_assert0(formats->nb_formats == nb_formats);
*rfmts = formats;
return 0;
}
formats = av_mallocz(sizeof(*formats));
if (!formats)
return AVERROR(ENOMEM);
formats->nb_formats = nb_formats;
if (nb_formats) {
formats->formats = av_malloc_array(nb_formats, sizeof(*formats->formats));
if (!formats->formats) {
av_freep(&formats);
return AVERROR(ENOMEM);
}
}
}
}
AVFilterFormats *ff_planar_sample_fmts(void)
{
AVFilterFormats *ret = NULL;
int fmt;
for (fmt = 0; av_get_bytes_per_sample(fmt)>0; fmt++)
if (av_sample_fmt_is_planar(fmt))
if (ff_add_format(&ret, fmt) < 0)
return NULL;
return ret;
}
AVFilterFormats *ff_all_samplerates(void)
{
AVFilterFormats *ret = av_mallocz(sizeof(*ret));
return ret;
}
AVFilterChannelLayouts *ff_all_channel_layouts(void)
{
AVFilterChannelLayouts *ret = av_mallocz(sizeof(*ret));
if (!ret)
return NULL;
ret->all_layouts = 1;
return ret;
}
AVFilterChannelLayouts *ff_all_channel_counts(void)
{
AVFilterChannelLayouts *ret = av_mallocz(sizeof(*ret));
if (!ret)
return NULL;
ret->all_layouts = ret->all_counts = 1;
return ret;
}
#define FORMATS_REF(f, ref, unref_fn) \
void *tmp; \
\
if (!f || !ref) \
return AVERROR(ENOMEM); \
\
tmp = av_realloc_array(f->refs, sizeof(*f->refs), f->refcount + 1); \
if (!tmp) { \
unref_fn(&f); \
return AVERROR(ENOMEM); \
} \
f->refs = tmp; \
f->refs[f->refcount++] = ref; \
*ref = f; \
return 0
int ff_channel_layouts_ref(AVFilterChannelLayouts *f, AVFilterChannelLayouts **ref)
{
FORMATS_REF(f, ref, ff_channel_layouts_unref);
}
int ff_formats_ref(AVFilterFormats *f, AVFilterFormats **ref)
{
FORMATS_REF(f, ref, ff_formats_unref);
}
#define FIND_REF_INDEX(ref, idx) \
do { \
int i; \
for (i = 0; i < (*ref)->refcount; i ++) \
if((*ref)->refs[i] == ref) { \
idx = i; \
break; \
} \
} while (0)
#define FORMATS_UNREF(ref, list) \
do { \
int idx = -1; \
\
if (!ref || !*ref) \
return; \
\
FIND_REF_INDEX(ref, idx); \
\
if (idx >= 0) { \
memmove((*ref)->refs + idx, (*ref)->refs + idx + 1, \
sizeof(*(*ref)->refs) * ((*ref)->refcount - idx - 1)); \
--(*ref)->refcount; \
} \
if (!(*ref)->refcount) { \
av_free((*ref)->list); \
av_free((*ref)->refs); \
av_free(*ref); \
} \
*ref = NULL; \
} while (0)
void ff_formats_unref(AVFilterFormats **ref)
{
FORMATS_UNREF(ref, formats);
}
void ff_channel_layouts_unref(AVFilterChannelLayouts **ref)
{
FORMATS_UNREF(ref, channel_layouts);
}
#define FORMATS_CHANGEREF(oldref, newref) \
do { \
int idx = -1; \
\
FIND_REF_INDEX(oldref, idx); \
\
if (idx >= 0) { \
(*oldref)->refs[idx] = newref; \
*newref = *oldref; \
*oldref = NULL; \
} \
} while (0)
void ff_channel_layouts_changeref(AVFilterChannelLayouts **oldref,
AVFilterChannelLayouts **newref)
{
FORMATS_CHANGEREF(oldref, newref);
}
void ff_formats_changeref(AVFilterFormats **oldref, AVFilterFormats **newref)
{
FORMATS_CHANGEREF(oldref, newref);
}
#define SET_COMMON_FORMATS(ctx, fmts, in_fmts, out_fmts, ref_fn, unref_fn) \
int count = 0, i; \
\
if (!fmts) \
return AVERROR(ENOMEM); \
\
for (i = 0; i < ctx->nb_inputs; i++) { \
if (ctx->inputs[i] && !ctx->inputs[i]->out_fmts) { \
int ret = ref_fn(fmts, &ctx->inputs[i]->out_fmts); \
if (ret < 0) { \
return ret; \
} \
count++; \
} \
} \
for (i = 0; i < ctx->nb_outputs; i++) { \
if (ctx->outputs[i] && !ctx->outputs[i]->in_fmts) { \
int ret = ref_fn(fmts, &ctx->outputs[i]->in_fmts); \
if (ret < 0) { \
return ret; \
} \
count++; \
} \
} \
\
if (!count) { \
unref_fn(&fmts); \
} \
\
return 0;
int ff_set_common_channel_layouts(AVFilterContext *ctx,
AVFilterChannelLayouts *layouts)
{
SET_COMMON_FORMATS(ctx, layouts, in_channel_layouts, out_channel_layouts,
ff_channel_layouts_ref, ff_channel_layouts_unref);
}
int ff_set_common_samplerates(AVFilterContext *ctx,
AVFilterFormats *samplerates)
{
SET_COMMON_FORMATS(ctx, samplerates, in_samplerates, out_samplerates,
ff_formats_ref, ff_formats_unref);
}
/**
* A helper for query_formats() which sets all links to the same list of
* formats. If there are no links hooked to this filter, the list of formats is
* freed.
*/
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
{
SET_COMMON_FORMATS(ctx, formats, in_formats, out_formats,
ff_formats_ref, ff_formats_unref);
}
int ff_default_query_formats(AVFilterContext *ctx)
{
int ret;
enum AVMediaType type = ctx->inputs && ctx->inputs [0] ? ctx->inputs [0]->type :
ctx->outputs && ctx->outputs[0] ? ctx->outputs[0]->type :
AVMEDIA_TYPE_VIDEO;
ret = ff_set_common_formats(ctx, ff_all_formats(type));
if (ret < 0)
return ret;
if (type == AVMEDIA_TYPE_AUDIO) {
ret = ff_set_common_channel_layouts(ctx, ff_all_channel_counts());
if (ret < 0)
return ret;
ret = ff_set_common_samplerates(ctx, ff_all_samplerates());
if (ret < 0)
return ret;
}
return 0;
}
/* internal functions for parsing audio format arguments */
int ff_parse_pixel_format(enum AVPixelFormat *ret, const char *arg, void *log_ctx)
{
char *tail;
int pix_fmt = av_get_pix_fmt(arg);
if (pix_fmt == AV_PIX_FMT_NONE) {
pix_fmt = strtol(arg, &tail, 0);
if (*tail || !av_pix_fmt_desc_get(pix_fmt)) {
av_log(log_ctx, AV_LOG_ERROR, "Invalid pixel format '%s'\n", arg);
return AVERROR(EINVAL);
}
}
*ret = pix_fmt;
return 0;
}
int ff_parse_sample_rate(int *ret, const char *arg, void *log_ctx)
{
char *tail;
double srate = av_strtod(arg, &tail);
if (*tail || srate < 1 || (int)srate != srate || srate > INT_MAX) {
av_log(log_ctx, AV_LOG_ERROR, "Invalid sample rate '%s'\n", arg);
return AVERROR(EINVAL);
}
*ret = srate;
return 0;
}
int ff_parse_channel_layout(int64_t *ret, int *nret, const char *arg,
void *log_ctx)
{
int64_t chlayout;
int nb_channels;
if (av_get_extended_channel_layout(arg, &chlayout, &nb_channels) < 0) {
av_log(log_ctx, AV_LOG_ERROR, "Invalid channel layout '%s'\n", arg);
return AVERROR(EINVAL);
}
if (!chlayout && !nret) {
av_log(log_ctx, AV_LOG_ERROR, "Unknown channel layout '%s' is not supported.\n", arg);
return AVERROR(EINVAL);
}
*ret = chlayout;
if (nret)
*nret = nb_channels;
return 0;
}