From 3f1d37e556d56a371df57fb79e8378384aeda770 Mon Sep 17 00:00:00 2001 From: Fynn Petersen-Frey <10599762+fyfrey@users.noreply.github.com> Date: Sat, 9 Mar 2024 03:17:26 +0100 Subject: [PATCH] feat(server): hardware HDR tonemapping for RKMPP (#7655) * feat(server): hardware HDR tonemapping for RKMPP * review feedback --- docker/hwaccel.transcoding.yml | 8 +++-- docs/docs/features/hardware-transcoding.md | 13 +++++++ server/src/domain/media/media.service.spec.ts | 36 +++++++++++++++++++ server/src/domain/media/media.service.ts | 14 +++++++- server/src/domain/media/media.util.ts | 35 +++++++++++++----- 5 files changed, 94 insertions(+), 12 deletions(-) diff --git a/docker/hwaccel.transcoding.yml b/docker/hwaccel.transcoding.yml index 2f6ae3ebde..ef9c0a5bb1 100644 --- a/docker/hwaccel.transcoding.yml +++ b/docker/hwaccel.transcoding.yml @@ -1,9 +1,9 @@ version: "3.8" -# Configurations for hardware-accelerated transcoding +# Configurations for hardware-accelerated transcoding # If using Unraid or another platform that doesn't allow multiple Compose files, -# you can inline the config for a backend by copying its contents +# you can inline the config for a backend by copying its contents # into the immich-microservices service in the docker-compose.yml file. # See https://immich.app/docs/features/hardware-transcoding for more info on using hardware transcoding. @@ -38,6 +38,10 @@ services: - /dev/dri:/dev/dri - /dev/dma_heap:/dev/dma_heap - /dev/mpp_service:/dev/mpp_service + #- /dev/mali0:/dev/mali0 # only required to enable OpenCL-accelerated HDR -> SDR tonemapping + volumes: + #- /etc/OpenCL:/etc/OpenCL:ro # only required to enable OpenCL-accelerated HDR -> SDR tonemapping + #- /usr/lib/aarch64-linux-gnu/libmali.so.1:/usr/lib/aarch64-linux-gnu/libmali.so.1:ro # only required to enable OpenCL-accelerated HDR -> SDR tonemapping vaapi: devices: diff --git a/docs/docs/features/hardware-transcoding.md b/docs/docs/features/hardware-transcoding.md index db3d1ba7d6..420cd2a43b 100644 --- a/docs/docs/features/hardware-transcoding.md +++ b/docs/docs/features/hardware-transcoding.md @@ -42,6 +42,18 @@ You do not need to redo any transcoding jobs after enabling hardware acceleratio - If you have an 11th gen CPU or older, then you may need to follow [these][jellyfin-lp] instructions as Low-Power mode is required - Additionally, if the server specifically has an 11th gen CPU and is running kernel 5.15 (shipped with Ubuntu 22.04 LTS), then you will need to upgrade this kernel (from [Jellyfin docs][jellyfin-kernel-bug]) +#### RKMPP + +For RKMPP to work: + +- You must have a supported Rockchip ARM SoC. +- Only RK3588 supports hardware tonemapping, other SoCs use slower software tonemapping while still using hardware encoding. +- Tonemapping requires `/usr/lib/aarch64-linux-gnu/libmali.so.1` to be present on your host system. Install [`libmali-valhall-g610-g6p0-gbm`][libmali-rockchip] and modify the [`hwaccel.transcoding.yml`][hw-file] file: + - under `rkmpp` uncomment the 3 lines required for OpenCL tonemapping by removing the `#` symbol at the beginning of each line + - `- /dev/mali0:/dev/mali0` + - `- /etc/OpenCL:/etc/OpenCL:ro` + - `- /usr/lib/aarch64-linux-gnu/libmali.so.1:/usr/lib/aarch64-linux-gnu/libmali.so.1:ro` + ## Setup #### Basic Setup @@ -106,3 +118,4 @@ Once this is done, you can continue to step 3 of "Basic Setup". [nvcr]: https://github.com/NVIDIA/nvidia-container-runtime/ [jellyfin-lp]: https://jellyfin.org/docs/general/administration/hardware-acceleration/intel/#configure-and-verify-lp-mode-on-linux [jellyfin-kernel-bug]: https://jellyfin.org/docs/general/administration/hardware-acceleration/intel/#known-issues-and-limitations +[libmali-rockchip]: https://github.com/tsukumijima/libmali-rockchip/releases diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index 244978d099..8a6eae4cc1 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -23,6 +23,7 @@ import { personStub, probeStub, } from '@test'; +import { Stats } from 'node:fs'; import { JobName } from '../job'; import { IAssetRepository, @@ -1853,6 +1854,41 @@ describe(MediaService.name, () => { }, ); }); + + it('should set OpenCL tonemapping options for rkmpp when OpenCL is available', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + storageMock.stat.mockResolvedValue({ ...new Stats(), isFile: () => true, isCharacterDevice: () => true }); + mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }, + { key: SystemConfigKey.FFMPEG_CRF, value: 30 }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga'], + outputOptions: [ + `-c:v h264_rkmpp`, + '-c:a copy', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-g 256', + '-v verbose', + '-vf scale_rkrga=-2:720:format=p010:afbc=1,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime', + '-level 51', + '-rc_mode CQP', + '-qp_init 30', + ], + twoPass: false, + }, + ); + }); }); it('should tonemap when policy is required and video is hdr', async () => { diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index 6a5c8ff9d3..5c8e777ad5 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -47,6 +47,7 @@ export class MediaService { private logger = new ImmichLogger(MediaService.name); private configCore: SystemConfigCore; private storageCore: StorageCore; + private hasOpenCL?: boolean = undefined; constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @@ -456,8 +457,19 @@ export class MediaService { break; } case TranscodeHWAccel.RKMPP: { + if (this.hasOpenCL === undefined) { + try { + const maliIcdStat = await this.storageRepository.stat('/etc/OpenCL/vendors/mali.icd'); + const maliDeviceStat = await this.storageRepository.stat('/dev/mali0'); + this.hasOpenCL = maliIcdStat.isFile() && maliDeviceStat.isCharacterDevice(); + } catch { + this.logger.warn('OpenCL not available for transcoding, using CPU instead.'); + this.hasOpenCL = false; + } + } + devices = await this.storageRepository.readdir('/dev/dri'); - handler = new RKMPPConfig(config, devices); + handler = new RKMPPConfig(config, devices, this.hasOpenCL); break; } default: { diff --git a/server/src/domain/media/media.util.ts b/server/src/domain/media/media.util.ts index d5f08ab0de..3acabb4356 100644 --- a/server/src/domain/media/media.util.ts +++ b/server/src/domain/media/media.util.ts @@ -608,6 +608,17 @@ export class VAAPIConfig extends BaseHWConfig { } export class RKMPPConfig extends BaseHWConfig { + private hasOpenCL: boolean; + + constructor( + protected config: SystemConfigFFmpegDto, + devices: string[] = [], + hasOpenCL: boolean = false, + ) { + super(config, devices); + this.hasOpenCL = hasOpenCL; + } + eligibleForTwoPass(): boolean { return false; } @@ -616,19 +627,25 @@ export class RKMPPConfig extends BaseHWConfig { if (this.devices.length === 0) { throw new Error('No RKMPP device found'); } - if (this.shouldToneMap(videoStream)) { - // disable hardware decoding - return []; - } - return ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga']; + return this.shouldToneMap(videoStream) && !this.hasOpenCL + ? [] // disable hardware decoding & filters + : ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga']; } getFilterOptions(videoStream: VideoStreamInfo) { if (this.shouldToneMap(videoStream)) { - // use software filter options - return super.getFilterOptions(videoStream); - } - if (this.shouldScale(videoStream)) { + if (!this.hasOpenCL) { + return super.getFilterOptions(videoStream); + } + const colors = this.getColors(); + return [ + `scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1`, + 'hwmap=derive_device=opencl:mode=read', + `tonemap_opencl=format=nv12:r=pc:p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:tonemap=${this.config.tonemap}:desat=0`, + 'hwmap=derive_device=rkmpp:mode=write:reverse=1', + 'format=drm_prime', + ]; + } else if (this.shouldScale(videoStream)) { return [`scale_rkrga=${this.getScaling(videoStream)}:format=nv12:afbc=1`]; } return [];