diff --git a/libavcodec/atrac3.c b/libavcodec/atrac3.c
index ef2f8428dc..dc68e507aa 100644
--- a/libavcodec/atrac3.c
+++ b/libavcodec/atrac3.c
@@ -111,7 +111,8 @@ typedef struct ATRAC3Context {
 
     AtracGCContext    gainc_ctx;
     FFTContext        mdct_ctx;
-    AVFloatDSPContext *fdsp;
+    void (*vector_fmul)(float *dst, const float *src0, const float *src1,
+                        int len);
 } ATRAC3Context;
 
 static DECLARE_ALIGNED(32, float, mdct_window)[MDCT_SIZE];
@@ -144,7 +145,7 @@ static void imlt(ATRAC3Context *q, float *input, float *output, int odd_band)
     q->mdct_ctx.imdct_calc(&q->mdct_ctx, output, input);
 
     /* Perform windowing on the output. */
-    q->fdsp->vector_fmul(output, output, mdct_window, MDCT_SIZE);
+    q->vector_fmul(output, output, mdct_window, MDCT_SIZE);
 }
 
 /*
@@ -194,7 +195,6 @@ static av_cold int atrac3_decode_close(AVCodecContext *avctx)
 
     av_freep(&q->units);
     av_freep(&q->decoded_bytes_buffer);
-    av_freep(&q->fdsp);
 
     ff_mdct_end(&q->mdct_ctx);
 
@@ -874,6 +874,7 @@ static av_cold int atrac3_decode_init(AVCodecContext *avctx)
     int version, delay, samples_per_frame, frame_factor;
     const uint8_t *edata_ptr = avctx->extradata;
     ATRAC3Context *q = avctx->priv_data;
+    AVFloatDSPContext *fdsp;
 
     if (avctx->channels < MIN_CHANNELS || avctx->channels > MAX_CHANNELS) {
         av_log(avctx, AV_LOG_ERROR, "Channel configuration error!\n");
@@ -997,12 +998,15 @@ static av_cold int atrac3_decode_init(AVCodecContext *avctx)
     }
 
     ff_atrac_init_gain_compensation(&q->gainc_ctx, 4, 3);
-    q->fdsp = avpriv_float_dsp_alloc(avctx->flags & AV_CODEC_FLAG_BITEXACT);
+    fdsp = avpriv_float_dsp_alloc(avctx->flags & AV_CODEC_FLAG_BITEXACT);
+    if (!fdsp)
+        return AVERROR(ENOMEM);
+    q->vector_fmul = fdsp->vector_fmul;
+    av_free(fdsp);
 
     q->units = av_mallocz_array(avctx->channels, sizeof(*q->units));
-    if (!q->units || !q->fdsp) {
+    if (!q->units)
         return AVERROR(ENOMEM);
-    }
 
     return 0;
 }