pthread_frame: do not run hwaccel decoding asynchronously unless it's safe

Certain hardware decoding APIs are not guaranteed to be thread-safe, so
having the user access decoded hardware surfaces while the decoder is
running in another thread can cause failures (this is mainly known to
happen with DXVA2).

For such hwaccels, only allow the decoding thread to run while the user
is inside a lavc decode call (avcodec_send_packet/receive_frame).

libavcodec/avcodec.h

@@ -3375,6 +3375,11 @@ typedef struct AVHWAccel {
      * AVCodecInternal.hwaccel_priv_data.
      */
     int priv_data_size;
+
+    /**
+     * Internal hwaccel capabilities.
+     */
+    int caps_internal;
 } AVHWAccel;
 
 /**

libavcodec/hwaccel.h

@@ -0,0 +1,24 @@
+/*
+ * This file is part of Libav.
+ *
+ * Libav is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * Libav is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Libav; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_HWACCEL_H
+#define AVCODEC_HWACCEL_H
+
+#define HWACCEL_CAP_ASYNC_SAFE      (1 << 0)
+
+#endif /* AVCODEC_HWACCEL_H */

libavcodec/pthread_frame.c

@@ -34,6 +34,7 @@
 #endif
 
 #include "avcodec.h"
+#include "hwaccel.h"
 #include "internal.h"
 #include "pthread_internal.h"
 #include "thread.h"
@@ -101,6 +102,7 @@ typedef struct PerThreadContext {
     int die;                       ///< Set when the thread should exit.
 
     int hwaccel_serializing;
+    int async_serializing;
 } PerThreadContext;
 
 /**
@@ -116,6 +118,7 @@ typedef struct FrameThreadContext {
      * is used.
      */
     pthread_mutex_t hwaccel_mutex;
+    pthread_mutex_t async_mutex;
 
     int next_decoding;             ///< The next context to submit a packet to.
     int next_finished;             ///< The next context to return output from.
@@ -191,6 +194,11 @@ static attribute_align_arg void *frame_worker_thread(void *arg)
             pthread_mutex_unlock(&p->parent->hwaccel_mutex);
         }
 
+        if (p->async_serializing) {
+            p->async_serializing = 0;
+            pthread_mutex_unlock(&p->parent->async_mutex);
+        }
+
         atomic_store(&p->state, STATE_INPUT_READY);
 
         pthread_mutex_lock(&p->progress_mutex);
@@ -414,7 +422,11 @@ int ff_thread_decode_frame(AVCodecContext *avctx,
     FrameThreadContext *fctx = avctx->internal->thread_ctx;
     int finished = fctx->next_finished;
     PerThreadContext *p;
-    int err;
+    int err, ret;
+
+    /* release the async lock, permitting blocked hwaccel threads to
+     * go forward while we are in this function */
+    pthread_mutex_unlock(&fctx->async_mutex);
 
     /*
      * Submit a packet to the next decoding thread.
@@ -422,9 +434,11 @@ int ff_thread_decode_frame(AVCodecContext *avctx,
 
     p = &fctx->threads[fctx->next_decoding];
     err = update_context_from_user(p->avctx, avctx);
-    if (err) return err;
+    if (err)
+        goto finish;
     err = submit_packet(p, avpkt);
-    if (err) return err;
+    if (err)
+        goto finish;
 
     /*
      * If we're still receiving the initial packets, don't return a frame.
@@ -434,8 +448,10 @@ int ff_thread_decode_frame(AVCodecContext *avctx,
         if (fctx->next_decoding >= (avctx->thread_count-1)) fctx->delaying = 0;
 
         *got_picture_ptr=0;
-        if (avpkt->size)
+        if (avpkt->size) {
-            return avpkt->size;
+            ret = avpkt->size;
+            goto finish;
+        }
     }
 
     /*
@@ -477,7 +493,12 @@ int ff_thread_decode_frame(AVCodecContext *avctx,
     fctx->next_finished = finished;
 
     /* return the size of the consumed packet if no error occurred */
-    return (p->result >= 0) ? avpkt->size : p->result;
+    ret = (p->result >= 0) ? avpkt->size : p->result;
+finish:
+    pthread_mutex_lock(&fctx->async_mutex);
+    if (err < 0)
+        return err;
+    return ret;
 }
 
 void ff_thread_report_progress(ThreadFrame *f, int n, int field)
@@ -532,6 +553,13 @@ void ff_thread_finish_setup(AVCodecContext *avctx) {
         p->hwaccel_serializing = 1;
     }
 
+    /* this assumes that no hwaccel calls happen before ff_thread_finish_setup() */
+    if (avctx->hwaccel &&
+        !(avctx->hwaccel->caps_internal & HWACCEL_CAP_ASYNC_SAFE)) {
+        p->async_serializing = 1;
+        pthread_mutex_lock(&p->parent->async_mutex);
+    }
+
     pthread_mutex_lock(&p->progress_mutex);
 
     atomic_store(&p->state, STATE_SETUP_FINISHED);
@@ -545,6 +573,8 @@ static void park_frame_worker_threads(FrameThreadContext *fctx, int thread_count
 {
     int i;
 
+    pthread_mutex_unlock(&fctx->async_mutex);
+
     for (i = 0; i < thread_count; i++) {
         PerThreadContext *p = &fctx->threads[i];
 
@@ -555,6 +585,8 @@ static void park_frame_worker_threads(FrameThreadContext *fctx, int thread_count
             pthread_mutex_unlock(&p->progress_mutex);
         }
     }
+
+    pthread_mutex_lock(&fctx->async_mutex);
 }
 
 void ff_frame_thread_free(AVCodecContext *avctx, int thread_count)
@@ -613,6 +645,10 @@ void ff_frame_thread_free(AVCodecContext *avctx, int thread_count)
     av_freep(&fctx->threads);
     pthread_mutex_destroy(&fctx->buffer_mutex);
     pthread_mutex_destroy(&fctx->hwaccel_mutex);
+
+    pthread_mutex_unlock(&fctx->async_mutex);
+    pthread_mutex_destroy(&fctx->async_mutex);
+
     av_freep(&avctx->internal->thread_ctx);
 }
 
@@ -655,6 +691,10 @@ int ff_frame_thread_init(AVCodecContext *avctx)
 
     pthread_mutex_init(&fctx->buffer_mutex, NULL);
     pthread_mutex_init(&fctx->hwaccel_mutex, NULL);
+
+    pthread_mutex_init(&fctx->async_mutex, NULL);
+    pthread_mutex_lock(&fctx->async_mutex);
+
     fctx->delaying = 1;
 
     for (i = 0; i < thread_count; i++) {

libavcodec/vaapi_h264.c

@@ -22,6 +22,7 @@
 
 #include "h264dec.h"
 #include "h264_ps.h"
+#include "hwaccel.h"
 #include "vaapi_decode.h"
 
 /**
@@ -399,4 +400,5 @@ AVHWAccel ff_h264_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };

libavcodec/vaapi_mpeg2.c

@@ -20,6 +20,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include "hwaccel.h"
 #include "mpegutils.h"
 #include "mpegvideo.h"
 #include "internal.h"
@@ -184,4 +185,5 @@ AVHWAccel ff_mpeg2_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };

libavcodec/vaapi_mpeg4.c

@@ -21,6 +21,7 @@
  */
 
 #include "h263.h"
+#include "hwaccel.h"
 #include "internal.h"
 #include "mpeg4video.h"
 #include "mpegvideo.h"
@@ -200,6 +201,7 @@ AVHWAccel ff_mpeg4_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif
 
@@ -216,5 +218,6 @@ AVHWAccel ff_h263_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif

libavcodec/vaapi_vc1.c

@@ -20,6 +20,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include "hwaccel.h"
 #include "internal.h"
 #include "vaapi_decode.h"
 #include "vc1.h"
@@ -399,6 +400,7 @@ AVHWAccel ff_wmv3_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif
 
@@ -414,4 +416,5 @@ AVHWAccel ff_vc1_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };

libavcodec/vaapi_vp8.c

@@ -19,6 +19,7 @@
 #include <va/va.h>
 #include <va/va_dec_vp8.h>
 
+#include "hwaccel.h"
 #include "vaapi_decode.h"
 #include "vp8.h"
 
@@ -231,4 +232,5 @@ AVHWAccel ff_vp8_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };

libavcodec/vdpau_h264.c

@@ -27,6 +27,7 @@
 #include "internal.h"
 #include "h264dec.h"
 #include "h264_ps.h"
+#include "hwaccel.h"
 #include "mpegutils.h"
 #include "vdpau.h"
 #include "vdpau_internal.h"
@@ -273,4 +274,5 @@ AVHWAccel ff_h264_vdpau_hwaccel = {
     .init           = vdpau_h264_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };

libavcodec/vdpau_hevc.c

@@ -26,6 +26,7 @@
 #include "internal.h"
 #include "hevc_data.h"
 #include "hevcdec.h"
+#include "hwaccel.h"
 #include "vdpau.h"
 #include "vdpau_internal.h"
 
@@ -424,4 +425,5 @@ AVHWAccel ff_hevc_vdpau_hwaccel = {
     .init           = vdpau_hevc_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };

libavcodec/vdpau_mpeg12.c

@@ -24,6 +24,7 @@
 #include <vdpau/vdpau.h>
 
 #include "avcodec.h"
+#include "hwaccel.h"
 #include "mpegvideo.h"
 #include "vdpau.h"
 #include "vdpau_internal.h"
@@ -114,6 +115,7 @@ AVHWAccel ff_mpeg1_vdpau_hwaccel = {
     .init           = vdpau_mpeg1_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif
 
@@ -148,5 +150,6 @@ AVHWAccel ff_mpeg2_vdpau_hwaccel = {
     .init           = vdpau_mpeg2_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif

libavcodec/vdpau_mpeg4.c

@@ -24,6 +24,7 @@
 #include <vdpau/vdpau.h>
 
 #include "avcodec.h"
+#include "hwaccel.h"
 #include "mpeg4video.h"
 #include "vdpau.h"
 #include "vdpau_internal.h"
@@ -118,4 +119,5 @@ AVHWAccel ff_mpeg4_vdpau_hwaccel = {
     .init           = vdpau_mpeg4_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };

libavcodec/vdpau_vc1.c

@@ -24,6 +24,7 @@
 #include <vdpau/vdpau.h>
 
 #include "avcodec.h"
+#include "hwaccel.h"
 #include "vc1.h"
 #include "vdpau.h"
 #include "vdpau_internal.h"
@@ -143,6 +144,7 @@ AVHWAccel ff_wmv3_vdpau_hwaccel = {
     .init           = vdpau_vc1_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif
 
@@ -158,4 +160,5 @@ AVHWAccel ff_vc1_vdpau_hwaccel = {
     .init           = vdpau_vc1_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };

libavcodec/version.h

@@ -29,7 +29,7 @@
 
 #define LIBAVCODEC_VERSION_MAJOR 57
 #define LIBAVCODEC_VERSION_MINOR 30
-#define LIBAVCODEC_VERSION_MICRO  0
+#define LIBAVCODEC_VERSION_MICRO  1
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
                                                LIBAVCODEC_VERSION_MINOR, \