diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 6445b1c2ce..1fe76671c2 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -2259,7 +2259,15 @@ static int frame_start(H264Context *h){
         memset(h->slice_table, -1, (s->mb_height*s->mb_stride-1) * sizeof(uint8_t));
 
 //    s->decode= (s->flags&CODEC_FLAG_PSNR) || !s->encoding || s->current_picture.reference /*|| h->contains_intra*/ || 1;
-    s->current_picture_ptr->reference= 0;
+
+    // We mark the current picture as non reference after allocating it, so
+    // that if we break out due to an error it can be released automatically
+    // in the next MPV_frame_start().
+    // SVQ3 as well as most other codecs have only last/next/current and thus
+    // get released even with set reference, besides SVQ3 and others do not
+    // mark frames as reference later "naturally".
+    if(s->codec_id != CODEC_ID_SVQ3)
+        s->current_picture_ptr->reference= 0;
     return 0;
 }