Implement libavfilter audio framework.

Patch by S.N. Hemanth Meenakshisundaram * smeenaks * ucsd * edu *.

Originally committed as revision 24811 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavfilter/avfilter.c

@@ -21,6 +21,7 @@
 
 /* #define DEBUG */
 
+#include "libavcodec/audioconvert.c"
 #include "libavutil/pixdesc.h"
 #include "libavcore/imgutils.h"
 #include "avfilter.h"
@@ -58,6 +59,13 @@ AVFilterBufferRef *avfilter_ref_buffer(AVFilterBufferRef *ref, int pmask)
             return NULL;
         }
         *ret->video = *ref->video;
+    } else if (ref->type == AVMEDIA_TYPE_AUDIO) {
+        ret->audio = av_malloc(sizeof(AVFilterBufferRefAudioProps));
+        if (!ret->audio) {
+            av_free(ret);
+            return NULL;
+        }
+        *ret->audio = *ref->audio;
     }
     ret->perms &= pmask;
     ret->buf->refcount ++;
@@ -69,6 +77,7 @@ void avfilter_unref_buffer(AVFilterBufferRef *ref)
     if(!(--ref->buf->refcount))
         ref->buf->free(ref->buf);
     av_free(ref->video);
+    av_free(ref->audio);
     av_free(ref);
 }
 
@@ -225,6 +234,24 @@ AVFilterBufferRef *avfilter_get_video_buffer(AVFilterLink *link, int perms, int
     return ret;
 }
 
+AVFilterBufferRef *avfilter_get_audio_buffer(AVFilterLink *link, int perms,
+                                             enum SampleFormat sample_fmt, int size,
+                                             int64_t channel_layout, int planar)
+{
+    AVFilterBufferRef *ret = NULL;
+
+    if (link_dpad(link).get_audio_buffer)
+        ret = link_dpad(link).get_audio_buffer(link, perms, sample_fmt, size, channel_layout, planar);
+
+    if (!ret)
+        ret = avfilter_default_get_audio_buffer(link, perms, sample_fmt, size, channel_layout, planar);
+
+    if (ret)
+        ret->type = AVMEDIA_TYPE_AUDIO;
+
+    return ret;
+}
+
 int avfilter_request_frame(AVFilterLink *link)
 {
     FF_DPRINTF_START(NULL, request_frame); ff_dprintf_link(NULL, link, 1);
@@ -345,6 +372,40 @@ void avfilter_draw_slice(AVFilterLink *link, int y, int h, int slice_dir)
     draw_slice(link, y, h, slice_dir);
 }
 
+void avfilter_filter_samples(AVFilterLink *link, AVFilterBufferRef *samplesref)
+{
+    void (*filter_samples)(AVFilterLink *, AVFilterBufferRef *);
+    AVFilterPad *dst = &link_dpad(link);
+
+    if (!(filter_samples = dst->filter_samples))
+        filter_samples = avfilter_default_filter_samples;
+
+    /* prepare to copy the samples if the buffer has insufficient permissions */
+    if ((dst->min_perms & samplesref->perms) != dst->min_perms ||
+        dst->rej_perms & samplesref->perms) {
+
+        av_log(link->dst, AV_LOG_DEBUG,
+               "Copying audio data in avfilter (have perms %x, need %x, reject %x)\n",
+               samplesref->perms, link_dpad(link).min_perms, link_dpad(link).rej_perms);
+
+        link->cur_buf = avfilter_default_get_audio_buffer(link, dst->min_perms,
+                                                          samplesref->format,
+                                                          samplesref->audio->size,
+                                                          samplesref->audio->channel_layout,
+                                                          samplesref->audio->planar);
+        link->cur_buf->pts                = samplesref->pts;
+        link->cur_buf->audio->sample_rate = samplesref->audio->sample_rate;
+
+        /* Copy actual data into new samples buffer */
+        memcpy(link->cur_buf->data[0], samplesref->data[0], samplesref->audio->size);
+
+        avfilter_unref_buffer(samplesref);
+    } else
+        link->cur_buf = samplesref;
+
+    filter_samples(link, link->cur_buf);
+}
+
 #define MAX_REGISTERED_AVFILTERS_NB 64
 
 static AVFilter *registered_avfilters[MAX_REGISTERED_AVFILTERS_NB + 1];

libavfilter/avfilter.h

@@ -25,7 +25,7 @@
 #include "libavutil/avutil.h"
 
 #define LIBAVFILTER_VERSION_MAJOR  1
-#define LIBAVFILTER_VERSION_MINOR 35
+#define LIBAVFILTER_VERSION_MINOR 36
 #define LIBAVFILTER_VERSION_MICRO  0
 
 #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \
@@ -87,6 +87,19 @@ typedef struct AVFilterBuffer {
 #define AV_PERM_REUSE    0x08   ///< can output the buffer multiple times, with the same contents each time
 #define AV_PERM_REUSE2   0x10   ///< can output the buffer multiple times, modified each time
 
+/**
+ * Audio specific properties in a reference to an AVFilterBuffer. Since
+ * AVFilterBufferRef is common to different media formats, audio specific
+ * per reference properties must be separated out.
+ */
+typedef struct AVFilterBufferRefAudioProps {
+    int64_t channel_layout;     ///< channel layout of audio buffer
+    int samples_nb;             ///< number of audio samples
+    int size;                   ///< audio buffer size
+    uint32_t sample_rate;       ///< audio buffer sample rate
+    int planar;                 ///< audio buffer - planar or packed
+} AVFilterBufferRefAudioProps;
+
 /**
  * Video specific properties in a reference to an AVFilterBuffer. Since
  * AVFilterBufferRef is common to different media formats, video specific
@@ -110,7 +123,7 @@ typedef struct AVFilterBufferRefVideoProps {
  */
 typedef struct AVFilterBufferRef {
     AVFilterBuffer *buf;        ///< the buffer that this is a reference to
-    uint8_t *data[8];           ///< picture data for each plane
+    uint8_t *data[8];           ///< picture/audio data for each plane
     int linesize[8];            ///< number of bytes per line
     int format;                 ///< media format
 
@@ -121,11 +134,11 @@ typedef struct AVFilterBufferRef {
 
     enum AVMediaType type;      ///< media type of buffer data
     AVFilterBufferRefVideoProps *video; ///< video buffer specific properties
+    AVFilterBufferRefAudioProps *audio; ///< audio buffer specific properties
 } AVFilterBufferRef;
 
 /**
- * Copy properties of src to dst, without copying the actual video
+ * Copy properties of src to dst, without copying the actual data
- * data.
  */
 static inline void avfilter_copy_buffer_ref_props(AVFilterBufferRef *dst, AVFilterBufferRef *src)
 {
@@ -135,6 +148,7 @@ static inline void avfilter_copy_buffer_ref_props(AVFilterBufferRef *dst, AVFilt
 
     switch (src->type) {
     case AVMEDIA_TYPE_VIDEO: *dst->video = *src->video; break;
+    case AVMEDIA_TYPE_AUDIO: *dst->audio = *src->audio; break;
     }
 }
 
@@ -330,13 +344,23 @@ struct AVFilterPad {
     void (*start_frame)(AVFilterLink *link, AVFilterBufferRef *picref);
 
     /**
-     * Callback function to get a buffer. If NULL, the filter system will
+     * Callback function to get a video buffer. If NULL, the filter system will
      * use avfilter_default_get_video_buffer().
      *
      * Input video pads only.
      */
     AVFilterBufferRef *(*get_video_buffer)(AVFilterLink *link, int perms, int w, int h);
 
+    /**
+     * Callback function to get an audio buffer. If NULL, the filter system will
+     * use avfilter_default_get_audio_buffer().
+     *
+     * Input audio pads only.
+     */
+    AVFilterBufferRef *(*get_audio_buffer)(AVFilterLink *link, int perms,
+                                           enum SampleFormat sample_fmt, int size,
+                                           int64_t channel_layout, int planar);
+
     /**
      * Callback called after the slices of a frame are completely sent. If
      * NULL, the filter layer will default to releasing the reference stored
@@ -354,6 +378,14 @@ struct AVFilterPad {
      */
     void (*draw_slice)(AVFilterLink *link, int y, int height, int slice_dir);
 
+    /**
+     * Samples filtering callback. This is where a filter receives audio data
+     * and should do its processing.
+     *
+     * Input audio pads only.
+     */
+    void (*filter_samples)(AVFilterLink *link, AVFilterBufferRef *samplesref);
+
     /**
      * Frame poll callback. This returns the number of immediately available
      * frames. It should return a positive value if the next request_frame()
@@ -400,15 +432,24 @@ void avfilter_default_draw_slice(AVFilterLink *link, int y, int h, int slice_dir
 /** default handler for end_frame() for video inputs */
 void avfilter_default_end_frame(AVFilterLink *link);
 
-/** default handler for config_props() for video outputs */
+/** default handler for filter_samples() for audio inputs */
+void avfilter_default_filter_samples(AVFilterLink *link, AVFilterBufferRef *samplesref);
+
+/** default handler for config_props() for audio/video outputs */
 int avfilter_default_config_output_link(AVFilterLink *link);
 
-/** default handler for config_props() for video inputs */
+/** default handler for config_props() for audio/video inputs */
 int avfilter_default_config_input_link (AVFilterLink *link);
 
 /** default handler for get_video_buffer() for video inputs */
 AVFilterBufferRef *avfilter_default_get_video_buffer(AVFilterLink *link,
                                                      int perms, int w, int h);
+
+/** default handler for get_audio_buffer() for audio inputs */
+AVFilterBufferRef *avfilter_default_get_audio_buffer(AVFilterLink *link, int perms,
+                                                     enum SampleFormat sample_fmt, int size,
+                                                     int64_t channel_layout, int planar);
+
 /**
  * A helper for query_formats() which sets all links to the same list of
  * formats. If there are no links hooked to this filter, the list of formats is
@@ -428,10 +469,18 @@ void avfilter_null_draw_slice(AVFilterLink *link, int y, int h, int slice_dir);
 /** end_frame() handler for filters which simply pass video along */
 void avfilter_null_end_frame(AVFilterLink *link);
 
+/** filter_samples() handler for filters which simply pass audio along */
+void avfilter_null_filter_samples(AVFilterLink *link, AVFilterBufferRef *samplesref);
+
 /** get_video_buffer() handler for filters which simply pass video along */
 AVFilterBufferRef *avfilter_null_get_video_buffer(AVFilterLink *link,
                                                   int perms, int w, int h);
 
+/** get_audio_buffer() handler for filters which simply pass audio along */
+AVFilterBufferRef *avfilter_null_get_audio_buffer(AVFilterLink *link, int perms,
+                                                  enum SampleFormat sample_fmt, int size,
+                                                  int64_t channel_layout, int planar);
+
 /**
  * Filter definition. This defines the pads a filter contains, and all the
  * callback functions used to interact with the filter.
@@ -518,8 +567,13 @@ struct AVFilterLink {
 
     enum AVMediaType type;      ///< filter media type
 
+    /* These two parameters apply only to video */
     int w;                      ///< agreed upon image width
     int h;                      ///< agreed upon image height
+    /* These two parameters apply only to audio */
+    int64_t channel_layout;     ///< channel layout of current buffer (see avcodec.h)
+    int64_t sample_rate;        ///< samples per second
+
     int format;                 ///< agreed upon media format
 
     /**
@@ -574,6 +628,23 @@ int avfilter_config_links(AVFilterContext *filter);
 AVFilterBufferRef *avfilter_get_video_buffer(AVFilterLink *link, int perms,
                                           int w, int h);
 
+/**
+ * Request an audio samples buffer with a specific set of permissions.
+ *
+ * @param link           the output link to the filter from which the buffer will
+ *                       be requested
+ * @param perms          the required access permissions
+ * @param sample_fmt     the format of each sample in the buffer to allocate
+ * @param size           the buffer size in bytes
+ * @param channel_layout the number and type of channels per sample in the buffer to allocate
+ * @param planar         audio data layout - planar or packed
+ * @return               A reference to the samples. This must be unreferenced with
+ *                       avfilter_unref_samples when you are finished with it.
+ */
+AVFilterBufferRef *avfilter_get_audio_buffer(AVFilterLink *link, int perms,
+                                             enum SampleFormat sample_fmt, int size,
+                                             int64_t channel_layout, int planar);
+
 /**
  * Request an input frame from the filter at the other end of the link.
  * @param link the input link
@@ -622,6 +693,16 @@ void avfilter_end_frame(AVFilterLink *link);
  */
 void avfilter_draw_slice(AVFilterLink *link, int y, int h, int slice_dir);
 
+/**
+ * Send a buffer of audio samples to the next filter.
+ *
+ * @param link       the output link over which the audio samples are being sent
+ * @param samplesref a reference to the buffer of audio samples being sent. The
+ *                   receiving filter will free this reference when it no longer
+ *                   needs it or pass it on to the next filter.
+ */
+void avfilter_filter_samples(AVFilterLink *link, AVFilterBufferRef *samplesref);
+
 /** Initialize the filter system. Register all builtin filters. */
 void avfilter_register_all(void);
 

libavfilter/defaults.c

@@ -20,6 +20,7 @@
  */
 
 #include "libavcore/imgutils.h"
+#include "libavcodec/audioconvert.h"
 #include "avfilter.h"
 
 /* TODO: buffer pool.  see comment for avfilter_default_get_video_buffer() */
@@ -79,6 +80,85 @@ fail:
     return NULL;
 }
 
+AVFilterBufferRef *avfilter_default_get_audio_buffer(AVFilterLink *link, int perms,
+                                                     enum SampleFormat sample_fmt, int size,
+                                                     int64_t channel_layout, int planar)
+{
+    AVFilterBuffer *samples = av_mallocz(sizeof(AVFilterBuffer));
+    AVFilterBufferRef *ref = NULL;
+    int i, sample_size, chans_nb, bufsize, per_channel_size, step_size = 0;
+    char *buf;
+
+    if (!samples || !(ref = av_mallocz(sizeof(AVFilterBufferRef))))
+        goto fail;
+
+    ref->buf                   = samples;
+    ref->format                = sample_fmt;
+
+    ref->audio = av_mallocz(sizeof(AVFilterBufferRefAudioProps));
+    if (!ref->audio)
+        goto fail;
+
+    ref->audio->channel_layout = channel_layout;
+    ref->audio->size           = size;
+    ref->audio->planar         = planar;
+
+    /* make sure the buffer gets read permission or it's useless for output */
+    ref->perms = perms | AV_PERM_READ;
+
+    samples->refcount   = 1;
+    samples->free       = avfilter_default_free_buffer;
+
+    sample_size = av_get_bits_per_sample_format(sample_fmt) >>3;
+    chans_nb = avcodec_channel_layout_num_channels(channel_layout);
+
+    per_channel_size = size/chans_nb;
+    ref->audio->samples_nb = per_channel_size/sample_size;
+
+    /* Set the number of bytes to traverse to reach next sample of a particular channel:
+     * For planar, this is simply the sample size.
+     * For packed, this is the number of samples * sample_size.
+     */
+    for (i = 0; i < chans_nb; i++)
+        samples->linesize[i] = planar > 0 ? per_channel_size : sample_size;
+    memset(&samples->linesize[chans_nb], 0, (8-chans_nb) * sizeof(samples->linesize[0]));
+
+    /* Calculate total buffer size, round to multiple of 16 to be SIMD friendly */
+    bufsize = (size + 15)&~15;
+    buf = av_malloc(bufsize);
+    if (!buf)
+        goto fail;
+
+    /* For planar, set the start point of each channel's data within the buffer
+     * For packed, set the start point of the entire buffer only
+     */
+    samples->data[0] = buf;
+    if (buf && planar) {
+        for (i = 1; i < chans_nb; i++) {
+            step_size += per_channel_size;
+            samples->data[i] = buf + step_size;
+        }
+    } else {
+        for (i = 1; i < chans_nb; i++)
+            samples->data[i] = buf;
+    }
+
+    memset(&samples->data[chans_nb], 0, (8-chans_nb) * sizeof(samples->data[0]));
+
+    memcpy(ref->data,     samples->data,     sizeof(ref->data));
+    memcpy(ref->linesize, samples->linesize, sizeof(ref->linesize));
+
+    return ref;
+
+fail:
+    av_free(buf);
+    if (ref && ref->audio)
+        av_free(ref->audio);
+    av_free(ref);
+    av_free(samples);
+    return NULL;
+}
+
 void avfilter_default_start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
 {
     AVFilterLink *out = NULL;
@@ -123,14 +203,42 @@ void avfilter_default_end_frame(AVFilterLink *link)
     }
 }
 
+/* FIXME: samplesref is same as link->cur_buf. Need to consider removing the redundant parameter. */
+void avfilter_default_filter_samples(AVFilterLink *link, AVFilterBufferRef *samplesref)
+{
+    AVFilterLink *outlink = NULL;
+
+    if (link->dst->output_count)
+        outlink = link->dst->outputs[0];
+
+    if (outlink) {
+        outlink->out_buf = avfilter_default_get_audio_buffer(link, AV_PERM_WRITE, samplesref->format,
+                                                             samplesref->audio->size,
+                                                             samplesref->audio->channel_layout,
+                                                             samplesref->audio->planar);
+        outlink->out_buf->pts                = samplesref->pts;
+        outlink->out_buf->audio->sample_rate = samplesref->audio->sample_rate;
+        avfilter_filter_samples(outlink, avfilter_ref_buffer(outlink->out_buf, ~0));
+        avfilter_unref_buffer(outlink->out_buf);
+        outlink->out_buf = NULL;
+    }
+    avfilter_unref_buffer(samplesref);
+    link->cur_buf = NULL;
+}
+
 /**
  * default config_link() implementation for output video links to simplify
  * the implementation of one input one output video filters */
 int avfilter_default_config_output_link(AVFilterLink *link)
 {
     if(link->src->input_count && link->src->inputs[0]) {
-        link->w = link->src->inputs[0]->w;
+        if (link->type == AVMEDIA_TYPE_VIDEO) {
-        link->h = link->src->inputs[0]->h;
+            link->w = link->src->inputs[0]->w;
+            link->h = link->src->inputs[0]->h;
+        } else if (link->type == AVMEDIA_TYPE_AUDIO) {
+            link->channel_layout = link->src->inputs[0]->channel_layout;
+            link->sample_rate    = link->src->inputs[0]->sample_rate;
+        }
     } else {
         /* XXX: any non-simple filter which would cause this branch to be taken
          * really should implement its own config_props() for this link. */
@@ -197,8 +305,21 @@ void avfilter_null_end_frame(AVFilterLink *link)
     avfilter_end_frame(link->dst->outputs[0]);
 }
 
+void avfilter_null_filter_samples(AVFilterLink *link, AVFilterBufferRef *samplesref)
+{
+    avfilter_filter_samples(link->dst->outputs[0], samplesref);
+}
+
 AVFilterBufferRef *avfilter_null_get_video_buffer(AVFilterLink *link, int perms, int w, int h)
 {
     return avfilter_get_video_buffer(link->dst->outputs[0], perms, w, h);
 }
 
+AVFilterBufferRef *avfilter_null_get_audio_buffer(AVFilterLink *link, int perms,
+                                                  enum SampleFormat sample_fmt, int size,
+                                                  int64_t channel_layout, int packed)
+{
+    return avfilter_get_audio_buffer(link->dst->outputs[0], perms, sample_fmt,
+                                     size, channel_layout, packed);
+}
+

libavfilter/formats.c

@@ -107,7 +107,8 @@ AVFilterFormats *avfilter_all_formats(enum AVMediaType type)
 {
     AVFilterFormats *ret = NULL;
     int fmt;
-    int num_formats = type == AVMEDIA_TYPE_VIDEO ? PIX_FMT_NB : 0;
+    int num_formats = type == AVMEDIA_TYPE_VIDEO ? PIX_FMT_NB    :
+                      type == AVMEDIA_TYPE_AUDIO ? SAMPLE_FMT_NB : 0;
 
     for (fmt = 0; fmt < num_formats; fmt++)
         if ((type != AVMEDIA_TYPE_VIDEO) ||