ffmpeg: use display matrix frame side data for autorotation

And give it priority over stream side data when present.
Fixes part of ticket #6945.

Signed-off-by: James Almer <jamrial@gmail.com>

fftools/cmdutils.c

@@ -2201,10 +2201,8 @@ void *grow_array(void *array, int elem_size, int *size, int new_size)
     return array;
 }
 
-double get_rotation(AVStream *st)
+double get_rotation(int32_t *displaymatrix)
 {
-    uint8_t* displaymatrix = av_stream_get_side_data(st,
-                                                     AV_PKT_DATA_DISPLAYMATRIX, NULL);
     double theta = 0;
     if (displaymatrix)
         theta = -round(av_display_rotation_get((int32_t*) displaymatrix));

fftools/cmdutils.h

@@ -648,6 +648,6 @@ void *grow_array(void *array, int elem_size, int *size, int new_size);
     char name[128];\
     av_get_channel_layout_string(name, sizeof(name), 0, ch_layout);
 
-double get_rotation(AVStream *st);
+double get_rotation(int32_t *displaymatrix);
 
 #endif /* FFTOOLS_CMDUTILS_H */

fftools/ffmpeg.c

@@ -535,6 +535,7 @@ static void ffmpeg_cleanup(int ret)
                 av_frame_free(&frame);
             }
             av_fifo_freep(&ifilter->frame_queue);
+            av_freep(&ifilter->displaymatrix);
             if (ist->sub2video.sub_queue) {
                 while (av_fifo_size(ist->sub2video.sub_queue)) {
                     AVSubtitle sub;
@@ -2183,6 +2184,7 @@ static int ifilter_has_all_input_formats(FilterGraph *fg)
 static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
 {
     FilterGraph *fg = ifilter->graph;
+    AVFrameSideData *sd;
     int need_reinit, ret, i;
 
     /* determine if the parameters for this input changed */
@@ -2207,6 +2209,12 @@ static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
         (ifilter->hw_frames_ctx && ifilter->hw_frames_ctx->data != frame->hw_frames_ctx->data))
         need_reinit = 1;
 
+    if (sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX)) {
+        if (!ifilter->displaymatrix || memcmp(sd->data, ifilter->displaymatrix, sizeof(int32_t) * 9))
+            need_reinit = 1;
+    } else if (ifilter->displaymatrix)
+        need_reinit = 1;
+
     if (need_reinit) {
         ret = ifilter_parameters_from_frame(ifilter, frame);
         if (ret < 0)

fftools/ffmpeg.h

@@ -258,6 +258,7 @@ typedef struct InputFilter {
     uint64_t channel_layout;
 
     AVBufferRef *hw_frames_ctx;
+    int32_t *displaymatrix;
 
     int eof;
 } InputFilter;

fftools/ffmpeg_filter.c

@@ -757,7 +757,12 @@ static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
     last_filter = ifilter->filter;
 
     if (ist->autorotate) {
-        double theta = get_rotation(ist->st);
+        int32_t *displaymatrix = ifilter->displaymatrix;
+        double theta;
+
+        if (!displaymatrix)
+            displaymatrix = (int32_t *)av_stream_get_side_data(ist->st, AV_PKT_DATA_DISPLAYMATRIX, NULL);
+        theta = get_rotation(displaymatrix);
 
         if (fabs(theta - 90) < 1.0) {
             ret = insert_filter(&last_filter, &pad_idx, "transpose", "clock");
@@ -1127,6 +1132,8 @@ fail:
 
 int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame)
 {
+    AVFrameSideData *sd;
+
     av_buffer_unref(&ifilter->hw_frames_ctx);
 
     ifilter->format = frame->format;
@@ -1139,6 +1146,11 @@ int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame)
     ifilter->channels            = frame->channels;
     ifilter->channel_layout      = frame->channel_layout;
 
+    av_freep(&ifilter->displaymatrix);
+    sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX);
+    if (sd)
+        ifilter->displaymatrix = av_memdup(sd->data, sizeof(int32_t) * 9);
+
     if (frame->hw_frames_ctx) {
         ifilter->hw_frames_ctx = av_buffer_ref(frame->hw_frames_ctx);
         if (!ifilter->hw_frames_ctx)

fftools/ffplay.c

@@ -1925,7 +1925,8 @@ static int configure_video_filters(AVFilterGraph *graph, VideoState *is, const c
 } while (0)
 
     if (autorotate) {
-        double theta  = get_rotation(is->video_st);
+        int32_t *displaymatrix = (int32_t *)av_stream_get_side_data(is->video_st, AV_PKT_DATA_DISPLAYMATRIX, NULL);
+        double theta = get_rotation(displaymatrix);
 
         if (fabs(theta - 90) < 1.0) {
             INSERT_FILT("transpose", "clock");