From 76f8d030ceb383c7abe78d92e5df1bb905ba5903 Mon Sep 17 00:00:00 2001 From: t4keda Date: Tue, 30 Jan 2024 02:40:02 +0100 Subject: [PATCH] added a configuration option to select the dri node in transcoding (#6376) * added a configuration option to select the dri node in transcoding * chore: open api * refactor: get hawrdware device --------- Co-authored-by: Jason Rasmussen --- mobile/openapi/doc/SystemConfigFFmpegDto.md | 1 + .../lib/model/system_config_f_fmpeg_dto.dart | 10 ++- .../test/system_config_f_fmpeg_dto_test.dart | 5 ++ open-api/immich-openapi-specs.json | 4 ++ open-api/typescript-sdk/client/api.ts | 6 ++ server/src/domain/media/media.service.spec.ts | 71 +++++++++++++++++++ server/src/domain/media/media.util.ts | 33 ++++++++- .../dto/system-config-ffmpeg.dto.ts | 3 + .../system-config/system-config.core.ts | 1 + .../system-config.service.spec.ts | 1 + .../infra/entities/system-config.entity.ts | 2 + .../settings/ffmpeg/ffmpeg-settings.svelte | 7 ++ 12 files changed, 140 insertions(+), 4 deletions(-) diff --git a/mobile/openapi/doc/SystemConfigFFmpegDto.md b/mobile/openapi/doc/SystemConfigFFmpegDto.md index 25726bb3da..05fe1c4437 100644 --- a/mobile/openapi/doc/SystemConfigFFmpegDto.md +++ b/mobile/openapi/doc/SystemConfigFFmpegDto.md @@ -17,6 +17,7 @@ Name | Type | Description | Notes **gopSize** | **int** | | **maxBitrate** | **String** | | **npl** | **int** | | +**preferredHwDevice** | **String** | | **preset** | **String** | | **refs** | **int** | | **targetAudioCodec** | [**AudioCodec**](AudioCodec.md) | | diff --git a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart index fca090bd3b..b1c0f278a9 100644 --- a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart +++ b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart @@ -22,6 +22,7 @@ class SystemConfigFFmpegDto { required this.gopSize, required this.maxBitrate, required this.npl, + required this.preferredHwDevice, required this.preset, required this.refs, required this.targetAudioCodec, @@ -52,6 +53,8 @@ class SystemConfigFFmpegDto { int npl; + String preferredHwDevice; + String preset; int refs; @@ -83,6 +86,7 @@ class SystemConfigFFmpegDto { other.gopSize == gopSize && other.maxBitrate == maxBitrate && other.npl == npl && + other.preferredHwDevice == preferredHwDevice && other.preset == preset && other.refs == refs && other.targetAudioCodec == targetAudioCodec && @@ -106,6 +110,7 @@ class SystemConfigFFmpegDto { (gopSize.hashCode) + (maxBitrate.hashCode) + (npl.hashCode) + + (preferredHwDevice.hashCode) + (preset.hashCode) + (refs.hashCode) + (targetAudioCodec.hashCode) + @@ -118,7 +123,7 @@ class SystemConfigFFmpegDto { (twoPass.hashCode); @override - String toString() => 'SystemConfigFFmpegDto[accel=$accel, acceptedAudioCodecs=$acceptedAudioCodecs, acceptedVideoCodecs=$acceptedVideoCodecs, bframes=$bframes, cqMode=$cqMode, crf=$crf, gopSize=$gopSize, maxBitrate=$maxBitrate, npl=$npl, preset=$preset, refs=$refs, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, temporalAQ=$temporalAQ, threads=$threads, tonemap=$tonemap, transcode=$transcode, twoPass=$twoPass]'; + String toString() => 'SystemConfigFFmpegDto[accel=$accel, acceptedAudioCodecs=$acceptedAudioCodecs, acceptedVideoCodecs=$acceptedVideoCodecs, bframes=$bframes, cqMode=$cqMode, crf=$crf, gopSize=$gopSize, maxBitrate=$maxBitrate, npl=$npl, preferredHwDevice=$preferredHwDevice, preset=$preset, refs=$refs, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, temporalAQ=$temporalAQ, threads=$threads, tonemap=$tonemap, transcode=$transcode, twoPass=$twoPass]'; Map toJson() { final json = {}; @@ -131,6 +136,7 @@ class SystemConfigFFmpegDto { json[r'gopSize'] = this.gopSize; json[r'maxBitrate'] = this.maxBitrate; json[r'npl'] = this.npl; + json[r'preferredHwDevice'] = this.preferredHwDevice; json[r'preset'] = this.preset; json[r'refs'] = this.refs; json[r'targetAudioCodec'] = this.targetAudioCodec; @@ -161,6 +167,7 @@ class SystemConfigFFmpegDto { gopSize: mapValueOfType(json, r'gopSize')!, maxBitrate: mapValueOfType(json, r'maxBitrate')!, npl: mapValueOfType(json, r'npl')!, + preferredHwDevice: mapValueOfType(json, r'preferredHwDevice')!, preset: mapValueOfType(json, r'preset')!, refs: mapValueOfType(json, r'refs')!, targetAudioCodec: AudioCodec.fromJson(json[r'targetAudioCodec'])!, @@ -227,6 +234,7 @@ class SystemConfigFFmpegDto { 'gopSize', 'maxBitrate', 'npl', + 'preferredHwDevice', 'preset', 'refs', 'targetAudioCodec', diff --git a/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart b/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart index 18a398fcd6..b0a4f2afb8 100644 --- a/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart +++ b/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart @@ -61,6 +61,11 @@ void main() { // TODO }); + // String preferredHwDevice + test('to test the property `preferredHwDevice`', () async { + // TODO + }); + // String preset test('to test the property `preset`', () async { // TODO diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index a593ac894a..a6d34c6e35 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -9422,6 +9422,9 @@ "npl": { "type": "integer" }, + "preferredHwDevice": { + "type": "string" + }, "preset": { "type": "string" }, @@ -9463,6 +9466,7 @@ "gopSize", "maxBitrate", "npl", + "preferredHwDevice", "preset", "refs", "targetAudioCodec", diff --git a/open-api/typescript-sdk/client/api.ts b/open-api/typescript-sdk/client/api.ts index 595187df44..2d7cac04a6 100644 --- a/open-api/typescript-sdk/client/api.ts +++ b/open-api/typescript-sdk/client/api.ts @@ -3712,6 +3712,12 @@ export interface SystemConfigFFmpegDto { * @memberof SystemConfigFFmpegDto */ 'npl': number; + /** + * + * @type {string} + * @memberof SystemConfigFFmpegDto + */ + 'preferredHwDevice': string; /** * * @type {string} diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index d76a793178..e4b6020174 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -1380,6 +1380,43 @@ describe(MediaService.name, () => { ); }); + it('should set options for qsv with custom dri node', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' }, + { key: SystemConfigKey.FFMPEG_PREFERRED_HW_DEVICE, value: '/dev/dri/renderD128' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: ['-init_hw_device qsv=hw,child_device=/dev/dri/renderD128', '-filter_hw_device hw'], + outputOptions: [ + `-c:v h264_qsv`, + '-c:a aac', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-bf 7', + '-refs 5', + '-g 256', + '-v verbose', + '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720', + '-preset 7', + '-global_quality 23', + '-maxrate 10000k', + '-bufsize 20000k', + ], + twoPass: false, + }, + ); + }); + it('should omit preset for qsv if invalid', async () => { storageMock.readdir.mockResolvedValue(['renderD128']); mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); @@ -1613,6 +1650,40 @@ describe(MediaService.name, () => { ); }); + it('should select specific gpu node if selected', async () => { + storageMock.readdir.mockResolvedValue(['renderD129', 'card1', 'card0', 'renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }, + { key: SystemConfigKey.FFMPEG_PREFERRED_HW_DEVICE, value: '/dev/dri/renderD128' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], + outputOptions: [ + `-c:v h264_vaapi`, + '-c:a aac', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-g 256', + '-v verbose', + '-vf format=nv12,hwupload,scale_vaapi=-2:720', + '-compression_level 7', + '-qp 23', + '-global_quality 23', + '-rc_mode 1', + ], + twoPass: false, + }, + ); + }); + it('should fallback to sw transcoding if hw transcoding fails', async () => { storageMock.readdir.mockResolvedValue(['renderD128']); mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); diff --git a/server/src/domain/media/media.util.ts b/server/src/domain/media/media.util.ts index 6741be5df4..6166a6d5cf 100644 --- a/server/src/domain/media/media.util.ts +++ b/server/src/domain/media/media.util.ts @@ -285,6 +285,20 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { } return this.config.gopSize; } + + getPreferredHardwareDevice(): string | null { + const device = this.config.preferredHwDevice; + if (device === 'auto') { + return null; + } + + const deviceName = device.replace('/dev/dri/', ''); + if (!this.devices.includes(deviceName)) { + throw new Error(`Device '${device}' does not exist`); + } + + return device; + } } export class ThumbnailConfig extends BaseConfig { @@ -463,7 +477,14 @@ export class QSVConfig extends BaseHWConfig { if (!this.devices.length) { throw Error('No QSV device found'); } - return ['-init_hw_device qsv=hw', '-filter_hw_device hw']; + + let qsvString = ''; + const hwDevice = this.getPreferredHardwareDevice(); + if (hwDevice !== null) { + qsvString = `,child_device=${hwDevice}`; + } + + return [`-init_hw_device qsv=hw${qsvString}`, '-filter_hw_device hw']; } getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { @@ -527,9 +548,15 @@ export class QSVConfig extends BaseHWConfig { export class VAAPIConfig extends BaseHWConfig { getBaseInputOptions() { if (this.devices.length === 0) { - throw Error('No VAAPI device found'); + throw new Error('No VAAPI device found'); } - return [`-init_hw_device vaapi=accel:/dev/dri/${this.devices[0]}`, '-filter_hw_device accel']; + + let hwDevice = this.getPreferredHardwareDevice(); + if (hwDevice === null) { + hwDevice = `/dev/dri/${this.devices[0]}`; + } + + return [`-init_hw_device vaapi=accel:${hwDevice}`, '-filter_hw_device accel']; } getFilterOptions(videoStream: VideoStreamInfo) { diff --git a/server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts b/server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts index e82ce4d7e9..2783e35e67 100644 --- a/server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts +++ b/server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts @@ -78,6 +78,9 @@ export class SystemConfigFFmpegDto { @IsBoolean() twoPass!: boolean; + @IsString() + preferredHwDevice!: string; + @IsEnum(TranscodePolicy) @ApiProperty({ enumName: 'TranscodePolicy', enum: TranscodePolicy }) transcode!: TranscodePolicy; diff --git a/server/src/domain/system-config/system-config.core.ts b/server/src/domain/system-config/system-config.core.ts index 926703d0dd..6be0ee81a1 100644 --- a/server/src/domain/system-config/system-config.core.ts +++ b/server/src/domain/system-config/system-config.core.ts @@ -43,6 +43,7 @@ export const defaults = Object.freeze({ temporalAQ: false, cqMode: CQMode.AUTO, twoPass: false, + preferredHwDevice: 'auto', transcode: TranscodePolicy.REQUIRED, tonemap: ToneMapping.HABLE, accel: TranscodeHWAccel.DISABLED, diff --git a/server/src/domain/system-config/system-config.service.spec.ts b/server/src/domain/system-config/system-config.service.spec.ts index d32fcb82e5..469e118a9b 100644 --- a/server/src/domain/system-config/system-config.service.spec.ts +++ b/server/src/domain/system-config/system-config.service.spec.ts @@ -55,6 +55,7 @@ const updatedConfig = Object.freeze({ temporalAQ: false, cqMode: CQMode.AUTO, twoPass: false, + preferredHwDevice: 'auto', transcode: TranscodePolicy.REQUIRED, accel: TranscodeHWAccel.DISABLED, tonemap: ToneMapping.HABLE, diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index e280d0ce7f..f07dd760b9 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -30,6 +30,7 @@ export enum SystemConfigKey { FFMPEG_TEMPORAL_AQ = 'ffmpeg.temporalAQ', FFMPEG_CQ_MODE = 'ffmpeg.cqMode', FFMPEG_TWO_PASS = 'ffmpeg.twoPass', + FFMPEG_PREFERRED_HW_DEVICE = 'ffmpeg.preferredHwDevice', FFMPEG_TRANSCODE = 'ffmpeg.transcode', FFMPEG_ACCEL = 'ffmpeg.accel', FFMPEG_TONEMAP = 'ffmpeg.tonemap', @@ -176,6 +177,7 @@ export interface SystemConfig { temporalAQ: boolean; cqMode: CQMode; twoPass: boolean; + preferredHwDevice: string; transcode: TranscodePolicy; accel: TranscodeHWAccel; tonemap: ToneMapping; diff --git a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte index bac314d66b..0dc6a85a16 100644 --- a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte +++ b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte @@ -282,6 +282,13 @@ bind:checked={config.ffmpeg.temporalAQ} isEdited={config.ffmpeg.temporalAQ !== savedConfig.ffmpeg.temporalAQ} /> +