feat(server): hardware video acceleration for Rockchip SOCs via RKMPP (#4645)

* feat(server): hardware video acceleration for Rockchip SOCs via RKMPP

* add tests

* use LD_LIBRARY_PATH for custom ffmpeg

* incorporate review feedback

* code re-use for ffmpeg call

* review feedback

docker/hwaccel-rkmpp.yml

@@ -0,0 +1,24 @@
+version: "3.8"
+
+# Hardware acceleration for transcoding using RKMPP for Rockchip SOCs
+# This is only needed if you want to use hardware acceleration for transcoding.
+# Supported host OS is Ubuntu Jammy 22.04 with custom ffmpeg from ppa:liujianfeng1994/rockchip-multimedia
+
+services:
+  hwaccel:
+    security_opt: # enables full access to /sys and /proc, still far better than privileged: true
+      - systempaths=unconfined
+      - apparmor=unconfined
+    group_add:
+      - video
+    devices:
+      - /dev/rga:/dev/rga
+      - /dev/dri:/dev/dri
+      - /dev/dma_heap:/dev/dma_heap
+      - /dev/mpp_service:/dev/mpp_service
+    volumes:
+      - /usr/bin/ffmpeg:/usr/bin/ffmpeg_mpp:ro
+      - /lib/aarch64-linux-gnu:/lib/ffmpeg-mpp:ro
+      - /lib/aarch64-linux-gnu/libblas.so.3:/lib/ffmpeg-mpp/libblas.so.3:ro # symlink is resolved by mounting
+      - /lib/aarch64-linux-gnu/liblapack.so.3:/lib/ffmpeg-mpp/liblapack.so.3:ro # symlink is resolved by mounting
+      - /lib/aarch64-linux-gnu/pulseaudio/libpulsecommon-15.99.so:/lib/ffmpeg-mpp/libpulsecommon-15.99.so:ro

server/immich-openapi-specs.json

@@ -8607,6 +8607,7 @@
           "nvenc",
           "qsv",
           "vaapi",
+          "rkmpp",
           "disabled"
         ],
         "type": "string"

server/src/domain/media/media.service.spec.ts

@@ -1508,6 +1508,83 @@ describe(MediaService.name, () => {
       await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false);
       expect(mediaMock.transcode).not.toHaveBeenCalled();
     });
+
+    it('should set vbr options for rkmpp when max bitrate is enabled', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },
+        { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: [
+            `-c:v hevc_rkmpp_encoder`,
+            '-c:a aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
+            '-v verbose',
+            '-level 153',
+            '-rc_mode 3',
+            '-quality_min 0',
+            '-quality_max 100',
+            '-b:v 10000k',
+            '-width 1280',
+            '-height 720',
+          ],
+          twoPass: false,
+          ffmpegPath: 'ffmpeg_mpp',
+          ldLibraryPath: '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp',
+        },
+      );
+    });
+
+    it('should set cqp options for rkmpp when max bitrate is disabled', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
+        { key: SystemConfigKey.FFMPEG_CRF, value: 30 },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: [],
+          outputOptions: [
+            `-c:v h264_rkmpp_encoder`,
+            '-c:a aac',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
+            '-v verbose',
+            '-level 51',
+            '-rc_mode 2',
+            '-quality_min 51',
+            '-quality_max 51',
+            '-width 1280',
+            '-height 720',
+          ],
+          twoPass: false,
+          ffmpegPath: 'ffmpeg_mpp',
+          ldLibraryPath: '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp',
+        },
+      );
+    });
   });
 
   it('should tonemap when policy is required and video is hdr', async () => {

server/src/domain/media/media.service.ts

@@ -26,7 +26,16 @@ import {
 import { StorageCore, StorageFolder } from '../storage';
 import { SystemConfigFFmpegDto } from '../system-config';
 import { SystemConfigCore } from '../system-config/system-config.core';
-import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util';
+import {
+  H264Config,
+  HEVCConfig,
+  NVENCConfig,
+  QSVConfig,
+  RKMPPConfig,
+  ThumbnailConfig,
+  VAAPIConfig,
+  VP9Config,
+} from './media.util';
 
 @Injectable()
 export class MediaService {
@@ -352,6 +361,10 @@ export class MediaService {
         devices = await this.storageRepository.readdir('/dev/dri');
         handler = new VAAPIConfig(config, devices);
         break;
+      case TranscodeHWAccel.RKMPP:
+        devices = await this.storageRepository.readdir('/dev/dri');
+        handler = new RKMPPConfig(config, devices);
+        break;
       default:
         throw new UnsupportedMediaTypeException(`${config.accel.toUpperCase()} acceleration is unsupported`);
     }

server/src/domain/media/media.util.ts

@@ -143,6 +143,13 @@ class BaseConfig implements VideoCodecSWConfig {
     return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
   }
 
+  getSize(videoStream: VideoStreamInfo) {
+    const smaller = this.getTargetResolution(videoStream);
+    const factor = Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width);
+    const larger = Math.round(smaller * factor);
+    return this.isVideoVertical(videoStream) ? { width: smaller, height: larger } : { width: larger, height: smaller };
+  }
+
   isVideoRotated(videoStream: VideoStreamInfo) {
     return Math.abs(videoStream.rotation) === 90;
   }
@@ -555,3 +562,68 @@ export class VAAPIConfig extends BaseHWConfig {
     return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9;
   }
 }
+
+export class RKMPPConfig extends BaseHWConfig {
+  getOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo): TranscodeOptions {
+    const options = super.getOptions(videoStream, audioStream);
+    options.ffmpegPath = 'ffmpeg_mpp';
+    options.ldLibraryPath = '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp';
+    options.outputOptions.push(...this.getSizeOptions(videoStream));
+    return options;
+  }
+
+  eligibleForTwoPass(): boolean {
+    return false;
+  }
+
+  getBaseInputOptions() {
+    if (this.devices.length === 0) {
+      throw Error('No RKMPP device found');
+    }
+    return [];
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo) {
+    return this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
+  }
+
+  getSizeOptions(videoStream: VideoStreamInfo) {
+    if (this.shouldScale(videoStream)) {
+      const { width, height } = this.getSize(videoStream);
+      return [`-width ${width}`, `-height ${height}`];
+    }
+    return [];
+  }
+
+  getPresetOptions() {
+    switch (this.config.targetVideoCodec) {
+      case VideoCodec.H264:
+        // from ffmpeg_mpp help, commonly referred to as H264 level 5.1
+        return ['-level 51'];
+      case VideoCodec.HEVC:
+        // from ffmpeg_mpp help, commonly referred to as HEVC level 5.1
+        return ['-level 153'];
+      default:
+        throw Error(`Incompatible video codec for RKMPP: ${this.config.targetVideoCodec}`);
+    }
+  }
+
+  getBitrateOptions() {
+    const bitrate = this.getMaxBitrateValue();
+    if (bitrate > 0) {
+      return ['-rc_mode 3', '-quality_min 0', '-quality_max 100', `-b:v ${bitrate}${this.getBitrateUnit()}`];
+    } else {
+      // convert CQP from 51-10 to 0-100, values below 10 are set to 10
+      const quality = Math.floor(125 - Math.max(this.config.crf, 10) * (125 / 51));
+      return ['-rc_mode 2', `-quality_min ${quality}`, `-quality_max ${quality}`];
+    }
+  }
+
+  getSupportedCodecs() {
+    return [VideoCodec.H264, VideoCodec.HEVC];
+  }
+
+  getVideoCodec(): string {
+    return `${this.config.targetVideoCodec}_rkmpp_encoder`;
+  }
+}

server/src/domain/repositories/media.repository.ts

@@ -51,6 +51,8 @@ export interface TranscodeOptions {
   inputOptions: string[];
   outputOptions: string[];
   twoPass: boolean;
+  ffmpegPath?: string;
+  ldLibraryPath?: string;
 }
 
 export interface BitrateDistribution {

server/src/infra/entities/system-config.entity.ts

@@ -119,6 +119,7 @@ export enum TranscodeHWAccel {
   NVENC = 'nvenc',
   QSV = 'qsv',
   VAAPI = 'vaapi',
+  RKMPP = 'rkmpp',
   DISABLED = 'disabled',
 }
 

server/src/infra/repositories/media.repository.ts

@@ -70,16 +70,18 @@ export class MediaRepository implements IMediaRepository {
   transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise<void> {
     if (!options.twoPass) {
       return new Promise((resolve, reject) => {
-        ffmpeg(input, { niceness: 10 })
-          .inputOptions(options.inputOptions)
-          .outputOptions(options.outputOptions)
-          .output(output)
-          .on('error', (err, stdout, stderr) => {
-            this.logger.error(stderr);
-            reject(err);
-          })
-          .on('end', resolve)
-          .run();
+        const oldLdLibraryPath = process.env.LD_LIBRARY_PATH;
+        if (options.ldLibraryPath) {
+          // fluent ffmpeg does not allow to set environment variables, so we do it manually
+          process.env.LD_LIBRARY_PATH = this.chainPath(oldLdLibraryPath || '', options.ldLibraryPath);
+        }
+        try {
+          this.configureFfmpegCall(input, output, options).on('error', reject).on('end', resolve).run();
+        } finally {
+          if (options.ldLibraryPath) {
+            process.env.LD_LIBRARY_PATH = oldLdLibraryPath;
+          }
+        }
       });
     }
 
@@ -90,29 +92,18 @@ export class MediaRepository implements IMediaRepository {
     // two-pass allows for precise control of bitrate at the cost of running twice
     // recommended for vp9 for better quality and compression
     return new Promise((resolve, reject) => {
-      ffmpeg(input, { niceness: 10 })
-        .inputOptions(options.inputOptions)
-        .outputOptions(options.outputOptions)
+      // first pass output is not saved as only the .log file is needed
+      this.configureFfmpegCall(input, '/dev/null', options)
         .addOptions('-pass', '1')
         .addOptions('-passlogfile', output)
         .addOptions('-f null')
-        .output('/dev/null') // first pass output is not saved as only the .log file is needed
-        .on('error', (err, stdout, stderr) => {
-          this.logger.error(stderr);
-          reject(err);
-        })
+        .on('error', reject)
         .on('end', () => {
           // second pass
-          ffmpeg(input, { niceness: 10 })
-            .inputOptions(options.inputOptions)
-            .outputOptions(options.outputOptions)
+          this.configureFfmpegCall(input, output, options)
             .addOptions('-pass', '2')
             .addOptions('-passlogfile', output)
-            .output(output)
-            .on('error', (err, stdout, stderr) => {
-              this.logger.error(stderr);
-              reject(err);
-            })
+            .on('error', reject)
             .on('end', () => fs.unlink(`${output}-0.log`))
             .on('end', () => fs.rm(`${output}-0.log.mbtree`, { force: true }))
             .on('end', resolve)
@@ -122,6 +113,20 @@ export class MediaRepository implements IMediaRepository {
     });
   }
 
+  configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) {
+    return ffmpeg(input, { niceness: 10 })
+      .setFfmpegPath(options.ffmpegPath || 'ffmpeg')
+      .inputOptions(options.inputOptions)
+      .outputOptions(options.outputOptions)
+      .output(output)
+      .on('error', (err, stdout, stderr) => this.logger.error(stderr || err));
+  }
+
+  chainPath(existing: string, path: string) {
+    const sep = existing.endsWith(':') ? '' : ':';
+    return `${existing}${sep}${path}`;
+  }
+
   async generateThumbhash(imagePath: string): Promise<Buffer> {
     const maxSize = 100;
 

web/src/api/open-api/api.ts

@@ -3964,6 +3964,7 @@ export const TranscodeHWAccel = {
     Nvenc: 'nvenc',
     Qsv: 'qsv',
     Vaapi: 'vaapi',
+    Rkmpp: 'rkmpp',
     Disabled: 'disabled'
 } as const;
 

web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte

@@ -281,6 +281,10 @@
                     value: TranscodeHWAccel.Vaapi,
                     text: 'VAAPI',
                   },
+                  {
+                    value: TranscodeHWAccel.Rkmpp,
+                    text: 'RKMPP (only on Rockchip SOCs)',
+                  },
                   {
                     value: TranscodeHWAccel.Disabled,
                     text: 'Disabled',