diff --git a/libavcodec/pcm.c b/libavcodec/pcm.c
index 96a68f7fe8..12d1b3a2c3 100644
--- a/libavcodec/pcm.c
+++ b/libavcodec/pcm.c
@@ -235,13 +235,15 @@ static int pcm_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
 
 typedef struct PCMDecode {
     short   table[256];
-    AVFloatDSPContext *fdsp;
+    void (*vector_fmul_scalar)(float *dst, const float *src, float mul,
+                               int len);
     float   scale;
 } PCMDecode;
 
 static av_cold int pcm_decode_init(AVCodecContext *avctx)
 {
     PCMDecode *s = avctx->priv_data;
+    AVFloatDSPContext *fdsp;
     int i;
 
     if (avctx->channels <= 0) {
@@ -268,9 +270,11 @@ static av_cold int pcm_decode_init(AVCodecContext *avctx)
             return AVERROR_INVALIDDATA;
 
         s->scale = 1. / (1 << (avctx->bits_per_coded_sample - 1));
-        s->fdsp = avpriv_float_dsp_alloc(0);
-        if (!s->fdsp)
+        fdsp = avpriv_float_dsp_alloc(0);
+        if (!fdsp)
             return AVERROR(ENOMEM);
+        s->vector_fmul_scalar = fdsp->vector_fmul_scalar;
+        av_free(fdsp);
         break;
     default:
         break;
@@ -284,15 +288,6 @@ static av_cold int pcm_decode_init(AVCodecContext *avctx)
     return 0;
 }
 
-static av_cold int pcm_decode_close(AVCodecContext *avctx)
-{
-    PCMDecode *s = avctx->priv_data;
-
-    av_freep(&s->fdsp);
-
-    return 0;
-}
-
 /**
  * Read PCM samples macro
  * @param size   Data size of native machine format
@@ -529,9 +524,9 @@ static int pcm_decode_frame(AVCodecContext *avctx, void *data,
 
     if (avctx->codec_id == AV_CODEC_ID_PCM_F16LE ||
         avctx->codec_id == AV_CODEC_ID_PCM_F24LE) {
-        s->fdsp->vector_fmul_scalar((float *)frame->extended_data[0],
-                                    (const float *)frame->extended_data[0],
-                                    s->scale, FFALIGN(frame->nb_samples * avctx->channels, 4));
+        s->vector_fmul_scalar((float *)frame->extended_data[0],
+                              (const float *)frame->extended_data[0],
+                              s->scale, FFALIGN(frame->nb_samples * avctx->channels, 4));
         emms_c();
     }
 
@@ -570,7 +565,6 @@ AVCodec ff_ ## name_ ## _decoder = {                                        \
     .id             = AV_CODEC_ID_ ## id_,                                  \
     .priv_data_size = sizeof(PCMDecode),                                    \
     .init           = pcm_decode_init,                                      \
-    .close          = pcm_decode_close,                                     \
     .decode         = pcm_decode_frame,                                     \
     .capabilities   = AV_CODEC_CAP_DR1,                                     \
     .sample_fmts    = (const enum AVSampleFormat[]){ sample_fmt_,           \