diff --git a/libavcodec/avcodec.c b/libavcodec/avcodec.c
index 2dd7dd84e0..97eb1ec431 100644
--- a/libavcodec/avcodec.c
+++ b/libavcodec/avcodec.c
@@ -364,9 +364,8 @@ int attribute_align_arg avcodec_open2(AVCodecContext *avctx, const AVCodec *code
             avctx->time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
 #endif
     }
-    if (codec->priv_data_size > 0 && avctx->priv_data && codec->priv_class) {
+    if (codec->priv_class)
         av_assert0(*(const AVClass **)avctx->priv_data == codec->priv_class);
-    }
 
 end:
     unlock_avcodec(codec);