feat(server): tone-mapping (#3512)

* added tonemapping

* check for hdr in transcode policy

* merged video thumbnail and transcoding logic

* updated tests

* removed log

* added dashboard option, updated api

* `out_color_matrix` for sdr video thumbs, cleanup

* updated tests & styling

* refactored tonemapping setting

* fixed tests

* formatting

* added tests

* updated api

* set target npl higher for mobius and reinhard

* convert to nv12 before nvenc

* fix test

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

cli/src/api/open-api/api.ts

@@ -2480,6 +2480,12 @@ export interface SystemConfigFFmpegDto {
      * @memberof SystemConfigFFmpegDto
      */
     'threads': number;
+    /**
+     * 
+     * @type {ToneMapping}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'tonemap': ToneMapping;
     /**
      * 
      * @type {TranscodePolicy}
@@ -2805,6 +2811,22 @@ export const TimeBucketSize = {
 export type TimeBucketSize = typeof TimeBucketSize[keyof typeof TimeBucketSize];
 
 
+/**
+ * 
+ * @export
+ * @enum {string}
+ */
+
+export const ToneMapping = {
+    Hable: 'hable',
+    Mobius: 'mobius',
+    Reinhard: 'reinhard',
+    Disabled: 'disabled'
+} as const;
+
+export type ToneMapping = typeof ToneMapping[keyof typeof ToneMapping];
+
+
 /**
  * 
  * @export

mobile/openapi/.openapi-generator/FILES

@@ -110,6 +110,7 @@ doc/TagTypeEnum.md
 doc/ThumbnailFormat.md
 doc/TimeBucketResponseDto.md
 doc/TimeBucketSize.md
+doc/ToneMapping.md
 doc/TranscodeHWAccel.md
 doc/TranscodePolicy.md
 doc/UpdateAlbumDto.md
@@ -240,6 +241,7 @@ lib/model/tag_type_enum.dart
 lib/model/thumbnail_format.dart
 lib/model/time_bucket_response_dto.dart
 lib/model/time_bucket_size.dart
+lib/model/tone_mapping.dart
 lib/model/transcode_hw_accel.dart
 lib/model/transcode_policy.dart
 lib/model/update_album_dto.dart
@@ -359,6 +361,7 @@ test/tag_type_enum_test.dart
 test/thumbnail_format_test.dart
 test/time_bucket_response_dto_test.dart
 test/time_bucket_size_test.dart
+test/tone_mapping_test.dart
 test/transcode_hw_accel_test.dart
 test/transcode_policy_test.dart
 test/update_album_dto_test.dart

server/immich-openapi-specs.json

@@ -6627,6 +6627,9 @@
           "threads": {
             "type": "integer"
           },
+          "tonemap": {
+            "$ref": "#/components/schemas/ToneMapping"
+          },
           "transcode": {
             "$ref": "#/components/schemas/TranscodePolicy"
           },
@@ -6641,6 +6644,7 @@
           "targetAudioCodec",
           "transcode",
           "accel",
+          "tonemap",
           "preset",
           "targetResolution",
           "maxBitrate",
@@ -6884,6 +6888,15 @@
         ],
         "type": "string"
       },
+      "ToneMapping": {
+        "enum": [
+          "hable",
+          "mobius",
+          "reinhard",
+          "disabled"
+        ],
+        "type": "string"
+      },
       "TranscodeHWAccel": {
         "enum": [
           "nvenc",

server/src/domain/media/media.repository.ts

@@ -14,6 +14,7 @@ export interface VideoStreamInfo {
   codecName?: string;
   codecType?: string;
   frameCount: number;
+  isHDR: boolean;
 }
 
 export interface AudioStreamInfo {
@@ -68,7 +69,6 @@ export interface IMediaRepository {
   generateThumbhash(imagePath: string): Promise<Buffer>;
 
   // video
-  extractVideoThumbnail(input: string, output: string, size: number): Promise<void>;
   probe(input: string): Promise<VideoInfo>;
   transcode(input: string, output: string, options: TranscodeOptions): Promise<void>;
 }

server/src/domain/media/media.service.spec.ts

@@ -1,4 +1,11 @@
-import { AssetType, SystemConfigKey, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
+import {
+  AssetType,
+  SystemConfigKey,
+  ToneMapping,
+  TranscodeHWAccel,
+  TranscodePolicy,
+  VideoCodec,
+} from '@app/infra/entities';
 import {
   assetStub,
   newAssetRepositoryMock,
@@ -111,6 +118,14 @@ describe(MediaService.name, () => {
       expect(assetMock.save).not.toHaveBeenCalledWith();
     });
 
+    it('should skip video thumbnail generation if no video stream', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.noVideoStreams);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleGenerateJpegThumbnail({ id: assetStub.image.id });
+      expect(mediaMock.resize).not.toHaveBeenCalled();
+      expect(assetMock.save).not.toHaveBeenCalledWith();
+    });
+
     it('should generate a thumbnail for an image', async () => {
       assetMock.getByIds.mockResolvedValue([assetStub.image]);
       await sut.handleGenerateJpegThumbnail({ id: assetStub.image.id });
@@ -127,15 +142,43 @@ describe(MediaService.name, () => {
     });
 
     it('should generate a thumbnail for a video', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);
       assetMock.getByIds.mockResolvedValue([assetStub.video]);
       await sut.handleGenerateJpegThumbnail({ id: assetStub.video.id });
 
       expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
-      expect(mediaMock.extractVideoThumbnail).toHaveBeenCalledWith(
-        '/original/path.ext',
-        'upload/thumbs/user-id/asset-id.jpeg',
-        1440,
-      );
+      expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
+        inputOptions: [],
+        outputOptions: [
+          '-ss 00:00:00.000',
+          '-frames:v 1',
+          '-v verbose',
+          '-vf scale=-2:1440:out_color_matrix=bt601:out_range=pc,format=yuv420p',
+        ],
+        twoPass: false,
+      });
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: 'asset-id',
+        resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
+      });
+    });
+
+    it('should tonemap thumbnail for hdr video', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleGenerateJpegThumbnail({ id: assetStub.video.id });
+
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
+      expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
+        inputOptions: [],
+        outputOptions: [
+          '-ss 00:00:00.000',
+          '-frames:v 1',
+          '-v verbose',
+          '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt470bg:t=601:m=bt470bg:range=pc,format=yuv420p',
+        ],
+        twoPass: false,
+      });
       expect(assetMock.save).toHaveBeenCalledWith({
         id: 'asset-id',
         resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
@@ -273,6 +316,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
+            '-vf format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -311,6 +355,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
+            '-vf format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -334,7 +379,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -361,6 +406,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
+            '-vf format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -385,7 +431,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=720:-2',
+            '-vf scale=720:-2,format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -410,7 +456,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -435,7 +481,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -484,7 +530,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
             '-maxrate 4500k',
@@ -514,7 +560,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-b:v 3104k',
             '-minrate 1552k',
@@ -541,7 +587,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -570,7 +616,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-cpu-used 5',
             '-row-mt 1',
             '-b:v 3104k',
@@ -601,7 +647,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-cpu-used 2',
             '-row-mt 1',
             '-crf 23',
@@ -631,7 +677,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-row-mt 1',
             '-crf 23',
             '-b:v 0',
@@ -660,7 +706,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-cpu-used 5',
             '-row-mt 1',
             '-threads 2',
@@ -688,7 +734,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-threads 2',
             '-x264-params "pools=none"',
@@ -716,7 +762,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -744,7 +790,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-threads 2',
             '-x265-params "pools=none"',
@@ -775,7 +821,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -844,7 +890,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf hwupload_cuda,scale_cuda=-2:720',
+            '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
             '-preset p1',
             '-b:v 6897k',
             '-maxrate 10000k',
@@ -884,7 +930,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf hwupload_cuda,scale_cuda=-2:720',
+            '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
             '-preset p1',
             '-cq:v 23',
             '-maxrate 10000k',
@@ -920,7 +966,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf hwupload_cuda,scale_cuda=-2:720',
+            '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
             '-preset p1',
             '-cq:v 23',
           ],
@@ -957,7 +1003,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf hwupload_cuda,scale_cuda=-2:720',
+            '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
             '-cq:v 23',
           ],
           twoPass: false,
@@ -990,7 +1036,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf hwupload_cuda,scale_cuda=-2:720',
+            '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
             '-preset p1',
             '-cq:v 23',
           ],
@@ -1086,10 +1132,10 @@ describe(MediaService.name, () => {
             '-extbrc 1',
             '-refs 5',
             '-bf 7',
-            '-low_power 1',
             '-acodec aac',
             '-movflags faststart',
             '-fps_mode passthrough',
+            '-low_power 1',
             '-v verbose',
             '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
             '-preset 7',
@@ -1269,7 +1315,7 @@ describe(MediaService.name, () => {
             '-movflags faststart',
             '-fps_mode passthrough',
             '-v verbose',
-            '-vf scale=-2:720',
+            '-vf scale=-2:720,format=yuv420p',
             '-preset ultrafast',
             '-crf 23',
           ],
@@ -1287,4 +1333,79 @@ describe(MediaService.name, () => {
       expect(mediaMock.transcode).not.toHaveBeenCalled();
     });
   });
+
+  it('should tonemap when policy is required and video is hdr', async () => {
+    mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+    configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.REQUIRED }]);
+    assetMock.getByIds.mockResolvedValue([assetStub.video]);
+    await sut.handleVideoConversion({ id: assetStub.video.id });
+    expect(mediaMock.transcode).toHaveBeenCalledWith(
+      '/original/path.ext',
+      'upload/encoded-video/user-id/asset-id.mp4',
+      {
+        inputOptions: [],
+        outputOptions: [
+          '-vcodec h264',
+          '-acodec aac',
+          '-movflags faststart',
+          '-fps_mode passthrough',
+          '-v verbose',
+          '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:range=pc,format=yuv420p',
+          '-preset ultrafast',
+          '-crf 23',
+        ],
+        twoPass: false,
+      },
+    );
+  });
+
+  it('should tonemap when policy is optimal and video is hdr', async () => {
+    mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+    configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.OPTIMAL }]);
+    assetMock.getByIds.mockResolvedValue([assetStub.video]);
+    await sut.handleVideoConversion({ id: assetStub.video.id });
+    expect(mediaMock.transcode).toHaveBeenCalledWith(
+      '/original/path.ext',
+      'upload/encoded-video/user-id/asset-id.mp4',
+      {
+        inputOptions: [],
+        outputOptions: [
+          '-vcodec h264',
+          '-acodec aac',
+          '-movflags faststart',
+          '-fps_mode passthrough',
+          '-v verbose',
+          '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:range=pc,format=yuv420p',
+          '-preset ultrafast',
+          '-crf 23',
+        ],
+        twoPass: false,
+      },
+    );
+  });
+
+  it('should set npl to 250 for reinhard and mobius tone-mapping algorithms', async () => {
+    mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+    configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TONEMAP, value: ToneMapping.MOBIUS }]);
+    assetMock.getByIds.mockResolvedValue([assetStub.video]);
+    await sut.handleVideoConversion({ id: assetStub.video.id });
+    expect(mediaMock.transcode).toHaveBeenCalledWith(
+      '/original/path.ext',
+      'upload/encoded-video/user-id/asset-id.mp4',
+      {
+        inputOptions: [],
+        outputOptions: [
+          '-vcodec h264',
+          '-acodec aac',
+          '-movflags faststart',
+          '-fps_mode passthrough',
+          '-v verbose',
+          '-vf zscale=t=linear:npl=250,tonemap=mobius:desat=0,zscale=p=bt709:t=bt709:m=bt709:range=pc,format=yuv420p',
+          '-preset ultrafast',
+          '-crf 23',
+        ],
+        twoPass: false,
+      },
+    );
+  });
 });

server/src/domain/media/media.service.ts

@@ -9,7 +9,7 @@ import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config
 import { SystemConfigCore } from '../system-config/system-config.core';
 import { JPEG_THUMBNAIL_SIZE, WEBP_THUMBNAIL_SIZE } from './media.constant';
 import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from './media.repository';
-import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, VAAPIConfig, VP9Config } from './media.util';
+import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util';
 
 @Injectable()
 export class MediaService {
@@ -70,10 +70,19 @@ export class MediaService {
           size: JPEG_THUMBNAIL_SIZE,
           format: 'jpeg',
         });
+        this.logger.log(`Successfully generated image thumbnail ${asset.id}`);
         break;
       case AssetType.VIDEO:
-        this.logger.log('Generating video thumbnail');
-        await this.mediaRepository.extractVideoThumbnail(asset.originalPath, jpegThumbnailPath, JPEG_THUMBNAIL_SIZE);
+        const { videoStreams } = await this.mediaRepository.probe(asset.originalPath);
+        const mainVideoStream = this.getMainVideoStream(videoStreams);
+        if (!mainVideoStream) {
+          this.logger.error(`Could not extract thumbnail for asset ${asset.id}: no video streams found`);
+          return false;
+        }
+        const { ffmpeg } = await this.configCore.getConfig();
+        const config = { ...ffmpeg, targetResolution: JPEG_THUMBNAIL_SIZE.toString(), twoPass: false };
+        const options = new ThumbnailConfig(config).getOptions(mainVideoStream);
+        await this.mediaRepository.transcode(asset.originalPath, jpegThumbnailPath, options);
         this.logger.log(`Successfully generated video thumbnail ${asset.id}`);
         break;
     }
@@ -226,10 +235,10 @@ export class MediaService {
         return true;
 
       case TranscodePolicy.REQUIRED:
-        return !allTargetsMatching;
+        return !allTargetsMatching || videoStream.isHDR;
 
       case TranscodePolicy.OPTIMAL:
-        return !allTargetsMatching || isLargerThanTargetRes;
+        return !allTargetsMatching || isLargerThanTargetRes || videoStream.isHDR;
 
       default:
         return false;

server/src/domain/media/media.util.ts

@@ -1,4 +1,4 @@
-import { TranscodeHWAccel, VideoCodec } from '@app/infra/entities';
+import { ToneMapping, TranscodeHWAccel, VideoCodec } from '@app/infra/entities';
 import { SystemConfigFFmpegDto } from '../system-config/dto';
 import {
   BitrateDistribution,
@@ -13,14 +13,7 @@ class BaseConfig implements VideoCodecSWConfig {
   getOptions(stream: VideoStreamInfo) {
     const options = {
       inputOptions: this.getBaseInputOptions(),
-      outputOptions: this.getBaseOutputOptions().concat([
-        `-acodec ${this.config.targetAudioCodec}`,
-        // Makes a second pass moving the moov atom to the
-        // beginning of the file for improved playback speed.
-        '-movflags faststart',
-        '-fps_mode passthrough',
-        '-v verbose',
-      ]),
+      outputOptions: this.getBaseOutputOptions().concat('-v verbose'),
       twoPass: this.eligibleForTwoPass(),
     } as TranscodeOptions;
     const filters = this.getFilterOptions(stream);
@@ -39,7 +32,13 @@ class BaseConfig implements VideoCodecSWConfig {
   }
 
   getBaseOutputOptions() {
-    return [`-vcodec ${this.config.targetVideoCodec}`];
+    return [
+      `-acodec ${this.config.targetAudioCodec}`,
+      // Makes a second pass moving the moov atom to the
+      // beginning of the file for improved playback speed.
+      '-movflags faststart',
+      '-fps_mode passthrough',
+    ];
   }
 
   getFilterOptions(stream: VideoStreamInfo) {
@@ -48,6 +47,11 @@ class BaseConfig implements VideoCodecSWConfig {
       options.push(`scale=${this.getScaling(stream)}`);
     }
 
+    if (this.shouldToneMap(stream)) {
+      options.push(...this.getToneMapping());
+    }
+    options.push('format=yuv420p');
+
     return options;
   }
 
@@ -111,6 +115,10 @@ class BaseConfig implements VideoCodecSWConfig {
     return Math.min(stream.height, stream.width) > this.getTargetResolution(stream);
   }
 
+  shouldToneMap(stream: VideoStreamInfo) {
+    return stream.isHDR && this.config.tonemap !== ToneMapping.DISABLED;
+  }
+
   getScaling(stream: VideoStreamInfo) {
     const targetResolution = this.getTargetResolution(stream);
     const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1
@@ -142,6 +150,27 @@ class BaseConfig implements VideoCodecSWConfig {
     const presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
     return presets.indexOf(this.config.preset);
   }
+
+  getColors() {
+    return {
+      primaries: 'bt709',
+      transfer: 'bt709',
+      matrix: 'bt709',
+    };
+  }
+
+  getToneMapping() {
+    const colors = this.getColors();
+    // npl stands for nominal peak luminance
+    // lower npl values result in brighter output (compensating for dimmer screens)
+    // since hable already outputs a darker image, we use a lower npl value for it
+    const npl = this.config.tonemap === ToneMapping.HABLE ? 100 : 250;
+    return [
+      `zscale=t=linear:npl=${npl}`,
+      `tonemap=${this.config.tonemap}:desat=0`,
+      `zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`,
+    ];
+  }
 }
 
 export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
@@ -172,7 +201,42 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
   }
 }
 
+export class ThumbnailConfig extends BaseConfig {
+  getBaseOutputOptions() {
+    return ['-ss 00:00:00.000', '-frames:v 1'];
+  }
+
+  getPresetOptions() {
+    return [];
+  }
+
+  getBitrateOptions() {
+    return [];
+  }
+
+  getScaling(stream: VideoStreamInfo) {
+    let options = super.getScaling(stream);
+    if (!this.shouldToneMap(stream)) {
+      options += ':out_color_matrix=bt601:out_range=pc';
+    }
+    return options;
+  }
+
+  getColors() {
+    return {
+      // jpeg and webp only support bt.601, so we need to convert to that directly when tone-mapping to avoid color shifts
+      primaries: 'bt470bg',
+      transfer: '601',
+      matrix: 'bt470bg',
+    };
+  }
+}
+
 export class H264Config extends BaseConfig {
+  getBaseOutputOptions() {
+    return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()];
+  }
+
   getThreadOptions() {
     if (this.config.threads <= 0) {
       return [];
@@ -186,6 +250,10 @@ export class H264Config extends BaseConfig {
 }
 
 export class HEVCConfig extends BaseConfig {
+  getBaseOutputOptions() {
+    return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()];
+  }
+
   getThreadOptions() {
     if (this.config.threads <= 0) {
       return [];
@@ -199,6 +267,10 @@ export class HEVCConfig extends BaseConfig {
 }
 
 export class VP9Config extends BaseConfig {
+  getBaseOutputOptions() {
+    return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()];
+  }
+
   getPresetOptions() {
     const speed = Math.min(this.getPresetIndex(), 5); // values over 5 require realtime mode, which is its own can of worms since it overrides -crf and -threads
     if (speed >= 0) {
@@ -247,11 +319,13 @@ export class NVENCConfig extends BaseHWConfig {
       '-rc-lookahead 20',
       '-i_qfactor 0.75',
       '-b_qfactor 1.1',
+      ...super.getBaseOutputOptions(),
     ];
   }
 
   getFilterOptions(stream: VideoStreamInfo) {
-    const options = ['hwupload_cuda'];
+    const options = this.shouldToneMap(stream) ? this.getToneMapping() : [];
+    options.push('format=nv12', 'hwupload_cuda');
     if (this.shouldScale(stream)) {
       options.push(`scale_cuda=${this.getScaling(stream)}`);
     }
@@ -303,7 +377,14 @@ export class QSVConfig extends BaseHWConfig {
 
   getBaseOutputOptions() {
     // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
-    const options = [`-vcodec ${this.config.targetVideoCodec}_qsv`, '-g 256', '-extbrc 1', '-refs 5', '-bf 7'];
+    const options = [
+      `-vcodec ${this.config.targetVideoCodec}_qsv`,
+      '-g 256',
+      '-extbrc 1',
+      '-refs 5',
+      '-bf 7',
+      ...super.getBaseOutputOptions(),
+    ];
     // VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a
     if (this.config.targetVideoCodec === VideoCodec.VP9) {
       options.push('-low_power 1');
@@ -312,7 +393,8 @@ export class QSVConfig extends BaseHWConfig {
   }
 
   getFilterOptions(stream: VideoStreamInfo) {
-    const options = ['format=nv12', 'hwupload=extra_hw_frames=64'];
+    const options = this.shouldToneMap(stream) ? this.getToneMapping() : [];
+    options.push('format=nv12', 'hwupload=extra_hw_frames=64');
     if (this.shouldScale(stream)) {
       options.push(`scale_qsv=${this.getScaling(stream)}`);
     }
@@ -353,11 +435,12 @@ export class VAAPIConfig extends BaseHWConfig {
   }
 
   getBaseOutputOptions() {
-    return [`-vcodec ${this.config.targetVideoCodec}_vaapi`];
+    return [`-vcodec ${this.config.targetVideoCodec}_vaapi`, ...super.getBaseOutputOptions()];
   }
 
   getFilterOptions(stream: VideoStreamInfo) {
-    const options = ['format=nv12', 'hwupload'];
+    const options = this.shouldToneMap(stream) ? this.getToneMapping() : [];
+    options.push('format=nv12', 'hwupload');
     if (this.shouldScale(stream)) {
       options.push(`scale_vaapi=${this.getScaling(stream)}`);
     }

server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts

@@ -1,4 +1,4 @@
-import { AudioCodec, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
+import { AudioCodec, ToneMapping, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
 import { ApiProperty } from '@nestjs/swagger';
 import { Type } from 'class-transformer';
 import { IsBoolean, IsEnum, IsInt, IsString, Max, Min } from 'class-validator';
@@ -44,4 +44,8 @@ export class SystemConfigFFmpegDto {
   @IsEnum(TranscodeHWAccel)
   @ApiProperty({ enumName: 'TranscodeHWAccel', enum: TranscodeHWAccel })
   accel!: TranscodeHWAccel;
+
+  @IsEnum(ToneMapping)
+  @ApiProperty({ enumName: 'ToneMapping', enum: ToneMapping })
+  tonemap!: ToneMapping;
 }

server/src/domain/system-config/system-config.core.ts

@@ -4,6 +4,7 @@ import {
   SystemConfigEntity,
   SystemConfigKey,
   SystemConfigValue,
+  ToneMapping,
   TranscodeHWAccel,
   TranscodePolicy,
   VideoCodec,
@@ -28,6 +29,7 @@ export const defaults = Object.freeze<SystemConfig>({
     maxBitrate: '0',
     twoPass: false,
     transcode: TranscodePolicy.REQUIRED,
+    tonemap: ToneMapping.HABLE,
     accel: TranscodeHWAccel.DISABLED,
   },
   job: {

server/src/domain/system-config/system-config.service.spec.ts

@@ -3,6 +3,7 @@ import {
   SystemConfig,
   SystemConfigEntity,
   SystemConfigKey,
+  ToneMapping,
   TranscodeHWAccel,
   TranscodePolicy,
   VideoCodec,
@@ -43,6 +44,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
     twoPass: false,
     transcode: TranscodePolicy.REQUIRED,
     accel: TranscodeHWAccel.DISABLED,
+    tonemap: ToneMapping.HABLE,
   },
   oauth: {
     autoLaunch: true,

server/src/infra/entities/system-config.entity.ts

@@ -24,6 +24,7 @@ export enum SystemConfigKey {
   FFMPEG_TWO_PASS = 'ffmpeg.twoPass',
   FFMPEG_TRANSCODE = 'ffmpeg.transcode',
   FFMPEG_ACCEL = 'ffmpeg.accel',
+  FFMPEG_TONEMAP = 'ffmpeg.tonemap',
 
   JOB_THUMBNAIL_GENERATION_CONCURRENCY = 'job.thumbnailGeneration.concurrency',
   JOB_METADATA_EXTRACTION_CONCURRENCY = 'job.metadataExtraction.concurrency',
@@ -79,6 +80,13 @@ export enum TranscodeHWAccel {
   DISABLED = 'disabled',
 }
 
+export enum ToneMapping {
+  HABLE = 'hable',
+  MOBIUS = 'mobius',
+  REINHARD = 'reinhard',
+  DISABLED = 'disabled',
+}
+
 export interface SystemConfig {
   ffmpeg: {
     crf: number;
@@ -91,6 +99,7 @@ export interface SystemConfig {
     twoPass: boolean;
     transcode: TranscodePolicy;
     accel: TranscodeHWAccel;
+    tonemap: ToneMapping;
   };
   job: Record<QueueName, { concurrency: number }>;
   oauth: {

server/src/infra/repositories/media.repository.ts

@@ -23,41 +23,11 @@ export class MediaRepository implements IMediaRepository {
   }
 
   async resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void> {
-    switch (options.format) {
-      case 'webp':
     await sharp(input, { failOnError: false })
       .resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true })
-          .webp()
       .rotate()
+      .toFormat(options.format)
       .toFile(output);
-        return;
-
-      case 'jpeg':
-        await sharp(input, { failOnError: false })
-          .resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true })
-          .jpeg()
-          .rotate()
-          .toFile(output);
-        return;
-    }
-  }
-
-  extractVideoThumbnail(input: string, output: string, size: number) {
-    return new Promise<void>((resolve, reject) => {
-      ffmpeg(input)
-        .outputOptions([
-          '-ss 00:00:00.000',
-          '-frames:v 1',
-          `-vf scale='min(${size},iw)':'min(${size},ih)':force_original_aspect_ratio=increase`,
-        ])
-        .output(output)
-        .on('error', (err, stdout, stderr) => {
-          this.logger.error(stderr);
-          reject(err);
-        })
-        .on('end', resolve)
-        .run();
-    });
   }
 
   async probe(input: string): Promise<VideoInfo> {
@@ -78,6 +48,7 @@ export class MediaRepository implements IMediaRepository {
           codecType: stream.codec_type,
           frameCount: Number.parseInt(stream.nb_frames ?? '0'),
           rotation: Number.parseInt(`${stream.rotation ?? 0}`),
+          isHDR: stream.color_transfer === 'smpte2084' || stream.color_transfer === 'arib-std-b67',
         })),
       audioStreams: results.streams
         .filter((stream) => stream.codec_type === 'audio')

server/test/fixtures/media.stub.ts

@@ -7,7 +7,7 @@ const probeStubDefaultFormat: VideoFormat = {
 };
 
 const probeStubDefaultVideoStream: VideoStreamInfo[] = [
-  { height: 1080, width: 1920, codecName: 'hevc', codecType: 'video', frameCount: 100, rotation: 0 },
+  { height: 1080, width: 1920, codecName: 'hevc', codecType: 'video', frameCount: 100, rotation: 0, isHDR: false },
 ];
 
 const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ codecName: 'aac', codecType: 'audio' }];
@@ -31,6 +31,7 @@ export const probeStub = {
         codecType: 'video',
         frameCount: 100,
         rotation: 0,
+        isHDR: false,
       },
       {
         height: 1080,
@@ -39,6 +40,7 @@ export const probeStub = {
         codecType: 'video',
         frameCount: 99,
         rotation: 0,
+        isHDR: false,
       },
     ],
   }),
@@ -52,6 +54,7 @@ export const probeStub = {
         codecType: 'video',
         frameCount: 100,
         rotation: 0,
+        isHDR: false,
       },
     ],
   }),
@@ -65,6 +68,21 @@ export const probeStub = {
         codecType: 'video',
         frameCount: 100,
         rotation: 0,
+        isHDR: false,
+      },
+    ],
+  }),
+  videoStreamHDR: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    videoStreams: [
+      {
+        height: 480,
+        width: 480,
+        codecName: 'h264',
+        codecType: 'video',
+        frameCount: 100,
+        rotation: 0,
+        isHDR: true,
       },
     ],
   }),
@@ -78,6 +96,7 @@ export const probeStub = {
         codecType: 'video',
         frameCount: 100,
         rotation: 90,
+        isHDR: false,
       },
     ],
   }),

server/test/repositories/media.repository.mock.ts

@@ -2,7 +2,6 @@ import { IMediaRepository } from '@app/domain';
 
 export const newMediaRepositoryMock = (): jest.Mocked<IMediaRepository> => {
   return {
-    extractVideoThumbnail: jest.fn(),
     generateThumbhash: jest.fn(),
     resize: jest.fn(),
     crop: jest.fn(),

web/src/api/open-api/api.ts

@@ -2480,6 +2480,12 @@ export interface SystemConfigFFmpegDto {
      * @memberof SystemConfigFFmpegDto
      */
     'threads': number;
+    /**
+     * 
+     * @type {ToneMapping}
+     * @memberof SystemConfigFFmpegDto
+     */
+    'tonemap': ToneMapping;
     /**
      * 
      * @type {TranscodePolicy}
@@ -2805,6 +2811,22 @@ export const TimeBucketSize = {
 export type TimeBucketSize = typeof TimeBucketSize[keyof typeof TimeBucketSize];
 
 
+/**
+ * 
+ * @export
+ * @enum {string}
+ */
+
+export const ToneMapping = {
+    Hable: 'hable',
+    Mobius: 'mobius',
+    Reinhard: 'reinhard',
+    Disabled: 'disabled'
+} as const;
+
+export type ToneMapping = typeof ToneMapping[keyof typeof ToneMapping];
+
+
 /**
  * 
  * @export

web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte

@@ -3,7 +3,15 @@
     notificationController,
     NotificationType,
   } from '$lib/components/shared-components/notification/notification';
-  import { api, AudioCodec, SystemConfigFFmpegDto, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@api';
+  import {
+    api,
+    AudioCodec,
+    SystemConfigFFmpegDto,
+    ToneMapping,
+    TranscodeHWAccel,
+    TranscodePolicy,
+    VideoCodec,
+  } from '@api';
   import SettingButtonsRow from '../setting-buttons-row.svelte';
   import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
   import SettingSelect from '../setting-select.svelte';
@@ -212,6 +220,32 @@
             isEdited={!(ffmpegConfig.accel == savedConfig.accel)}
           />
 
+          <SettingSelect
+            label="TONE-MAPPING"
+            desc="Attempts to preserve the appearance of HDR videos when converted to SDR. Each algorithm makes different tradeoffs for color, detail and brightness. Hable preserves detail, Mobius preserves color, and Reinhard preserves brightness."
+            bind:value={ffmpegConfig.tonemap}
+            name="tonemap"
+            options={[
+              {
+                value: ToneMapping.Hable,
+                text: 'Hable',
+              },
+              {
+                value: ToneMapping.Mobius,
+                text: 'Mobius',
+              },
+              {
+                value: ToneMapping.Reinhard,
+                text: 'Reinhard',
+              },
+              {
+                value: ToneMapping.Disabled,
+                text: 'Disabled',
+              },
+            ]}
+            isEdited={!(ffmpegConfig.tonemap == savedConfig.tonemap)}
+          />
+
           <SettingSwitch
             title="TWO-PASS ENCODING"
             subtitle="Transcode in two passes to produce better encoded videos. When max bitrate is enabled (required for it to work with H.264 and HEVC), this mode uses a bitrate range based on the max bitrate and ignores CRF. For VP9, CRF can be used if max bitrate is disabled."