diff --git a/libavcodec/av1_frame_merge_bsf.c b/libavcodec/av1_frame_merge_bsf.c
index fce5bdb67e..19b9cd01a8 100644
--- a/libavcodec/av1_frame_merge_bsf.c
+++ b/libavcodec/av1_frame_merge_bsf.c
@@ -103,10 +103,15 @@ eof:
         err = AVERROR(EAGAIN);
     }
 
-    // Buffer packets with timestamps. There should be at most one per TU, be it split or not.
-    if (!buffer_pkt->data && in->pts != AV_NOPTS_VALUE)
+    /* Buffer packets with timestamps (there should be at most one per TU)
+     * or any packet if buffer_pkt is empty. The latter is needed to
+     * passthrough positions in case there are no timestamps like with
+     * the raw OBU demuxer. */
+    if (!buffer_pkt->data ||
+        in->pts != AV_NOPTS_VALUE && buffer_pkt->pts == AV_NOPTS_VALUE) {
+        av_packet_unref(buffer_pkt);
         av_packet_move_ref(buffer_pkt, in);
-    else
+    } else
         av_packet_unref(in);
 
     ff_cbs_fragment_reset(&ctx->frag[ctx->idx]);