feat(server): accepted video containers (#11274)

* add accepted container config

* update api

* mp4 option makes no sense

* add to transcoding settings

* wording

* updated spec config

* formatting

mobile/openapi/README.md

@@ -450,6 +450,7 @@ Class | Method | HTTP request | Description
  - [ValidateLibraryImportPathResponseDto](doc//ValidateLibraryImportPathResponseDto.md)
  - [ValidateLibraryResponseDto](doc//ValidateLibraryResponseDto.md)
  - [VideoCodec](doc//VideoCodec.md)
+ - [VideoContainer](doc//VideoContainer.md)
 
 
 ## Documentation For Authorization

mobile/openapi/lib/api.dart

@@ -263,6 +263,7 @@ part 'model/validate_library_dto.dart';
 part 'model/validate_library_import_path_response_dto.dart';
 part 'model/validate_library_response_dto.dart';
 part 'model/video_codec.dart';
+part 'model/video_container.dart';
 
 
 /// An [ApiClient] instance that uses the default values obtained from

mobile/openapi/lib/api_client.dart

@@ -584,6 +584,8 @@ class ApiClient {
           return ValidateLibraryResponseDto.fromJson(value);
         case 'VideoCodec':
           return VideoCodecTypeTransformer().decode(value);
+        case 'VideoContainer':
+          return VideoContainerTypeTransformer().decode(value);
         default:
           dynamic match;
           if (value is List && (match = _regList.firstMatch(targetType)?.group(1)) != null) {

mobile/openapi/lib/api_helper.dart

@@ -148,6 +148,9 @@ String parameterToString(dynamic value) {
   if (value is VideoCodec) {
     return VideoCodecTypeTransformer().encode(value).toString();
   }
+  if (value is VideoContainer) {
+    return VideoContainerTypeTransformer().encode(value).toString();
+  }
   return value.toString();
 }
 

mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart

@@ -16,6 +16,7 @@ class SystemConfigFFmpegDto {
     required this.accel,
     required this.accelDecode,
     this.acceptedAudioCodecs = const [],
+    this.acceptedContainers = const [],
     this.acceptedVideoCodecs = const [],
     required this.bframes,
     required this.cqMode,
@@ -42,6 +43,8 @@ class SystemConfigFFmpegDto {
 
   List<AudioCodec> acceptedAudioCodecs;
 
+  List<VideoContainer> acceptedContainers;
+
   List<VideoCodec> acceptedVideoCodecs;
 
   /// Minimum value: -1
@@ -92,6 +95,7 @@ class SystemConfigFFmpegDto {
     other.accel == accel &&
     other.accelDecode == accelDecode &&
     _deepEquality.equals(other.acceptedAudioCodecs, acceptedAudioCodecs) &&
+    _deepEquality.equals(other.acceptedContainers, acceptedContainers) &&
     _deepEquality.equals(other.acceptedVideoCodecs, acceptedVideoCodecs) &&
     other.bframes == bframes &&
     other.cqMode == cqMode &&
@@ -117,6 +121,7 @@ class SystemConfigFFmpegDto {
     (accel.hashCode) +
     (accelDecode.hashCode) +
     (acceptedAudioCodecs.hashCode) +
+    (acceptedContainers.hashCode) +
     (acceptedVideoCodecs.hashCode) +
     (bframes.hashCode) +
     (cqMode.hashCode) +
@@ -137,13 +142,14 @@ class SystemConfigFFmpegDto {
     (twoPass.hashCode);
 
   @override
-  String toString() => 'SystemConfigFFmpegDto[accel=$accel, accelDecode=$accelDecode, acceptedAudioCodecs=$acceptedAudioCodecs, acceptedVideoCodecs=$acceptedVideoCodecs, bframes=$bframes, cqMode=$cqMode, crf=$crf, gopSize=$gopSize, maxBitrate=$maxBitrate, npl=$npl, preferredHwDevice=$preferredHwDevice, preset=$preset, refs=$refs, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, temporalAQ=$temporalAQ, threads=$threads, tonemap=$tonemap, transcode=$transcode, twoPass=$twoPass]';
+  String toString() => 'SystemConfigFFmpegDto[accel=$accel, accelDecode=$accelDecode, acceptedAudioCodecs=$acceptedAudioCodecs, acceptedContainers=$acceptedContainers, acceptedVideoCodecs=$acceptedVideoCodecs, bframes=$bframes, cqMode=$cqMode, crf=$crf, gopSize=$gopSize, maxBitrate=$maxBitrate, npl=$npl, preferredHwDevice=$preferredHwDevice, preset=$preset, refs=$refs, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, temporalAQ=$temporalAQ, threads=$threads, tonemap=$tonemap, transcode=$transcode, twoPass=$twoPass]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
       json[r'accel'] = this.accel;
       json[r'accelDecode'] = this.accelDecode;
       json[r'acceptedAudioCodecs'] = this.acceptedAudioCodecs;
+      json[r'acceptedContainers'] = this.acceptedContainers;
       json[r'acceptedVideoCodecs'] = this.acceptedVideoCodecs;
       json[r'bframes'] = this.bframes;
       json[r'cqMode'] = this.cqMode;
@@ -176,6 +182,7 @@ class SystemConfigFFmpegDto {
         accel: TranscodeHWAccel.fromJson(json[r'accel'])!,
         accelDecode: mapValueOfType<bool>(json, r'accelDecode')!,
         acceptedAudioCodecs: AudioCodec.listFromJson(json[r'acceptedAudioCodecs']),
+        acceptedContainers: VideoContainer.listFromJson(json[r'acceptedContainers']),
         acceptedVideoCodecs: VideoCodec.listFromJson(json[r'acceptedVideoCodecs']),
         bframes: mapValueOfType<int>(json, r'bframes')!,
         cqMode: CQMode.fromJson(json[r'cqMode'])!,
@@ -244,6 +251,7 @@ class SystemConfigFFmpegDto {
     'accel',
     'accelDecode',
     'acceptedAudioCodecs',
+    'acceptedContainers',
     'acceptedVideoCodecs',
     'bframes',
     'cqMode',

mobile/openapi/lib/model/video_container.dart

@@ -0,0 +1,91 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.18
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+
+class VideoContainer {
+  /// Instantiate a new enum with the provided [value].
+  const VideoContainer._(this.value);
+
+  /// The underlying value of this enum member.
+  final String value;
+
+  @override
+  String toString() => value;
+
+  String toJson() => value;
+
+  static const mov = VideoContainer._(r'mov');
+  static const mp4 = VideoContainer._(r'mp4');
+  static const ogg = VideoContainer._(r'ogg');
+  static const webm = VideoContainer._(r'webm');
+
+  /// List of all possible values in this [enum][VideoContainer].
+  static const values = <VideoContainer>[
+    mov,
+    mp4,
+    ogg,
+    webm,
+  ];
+
+  static VideoContainer? fromJson(dynamic value) => VideoContainerTypeTransformer().decode(value);
+
+  static List<VideoContainer> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <VideoContainer>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = VideoContainer.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+}
+
+/// Transformation class that can [encode] an instance of [VideoContainer] to String,
+/// and [decode] dynamic data back to [VideoContainer].
+class VideoContainerTypeTransformer {
+  factory VideoContainerTypeTransformer() => _instance ??= const VideoContainerTypeTransformer._();
+
+  const VideoContainerTypeTransformer._();
+
+  String encode(VideoContainer data) => data.value;
+
+  /// Decodes a [dynamic value][data] to a VideoContainer.
+  ///
+  /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
+  /// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
+  /// cannot be decoded successfully, then an [UnimplementedError] is thrown.
+  ///
+  /// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
+  /// and users are still using an old app with the old code.
+  VideoContainer? decode(dynamic data, {bool allowNull = true}) {
+    if (data != null) {
+      switch (data) {
+        case r'mov': return VideoContainer.mov;
+        case r'mp4': return VideoContainer.mp4;
+        case r'ogg': return VideoContainer.ogg;
+        case r'webm': return VideoContainer.webm;
+        default:
+          if (!allowNull) {
+            throw ArgumentError('Unknown enum value to decode: $data');
+          }
+      }
+    }
+    return null;
+  }
+
+  /// Singleton [VideoContainerTypeTransformer] instance.
+  static VideoContainerTypeTransformer? _instance;
+}
+

open-api/immich-openapi-specs.json

@@ -10691,6 +10691,12 @@
             },
             "type": "array"
           },
+          "acceptedContainers": {
+            "items": {
+              "$ref": "#/components/schemas/VideoContainer"
+            },
+            "type": "array"
+          },
           "acceptedVideoCodecs": {
             "items": {
               "$ref": "#/components/schemas/VideoCodec"
@@ -10762,6 +10768,7 @@
           "accel",
           "accelDecode",
           "acceptedAudioCodecs",
+          "acceptedContainers",
           "acceptedVideoCodecs",
           "bframes",
           "cqMode",
@@ -11847,6 +11854,15 @@
           "av1"
         ],
         "type": "string"
+      },
+      "VideoContainer": {
+        "enum": [
+          "mov",
+          "mp4",
+          "ogg",
+          "webm"
+        ],
+        "type": "string"
       }
     }
   }

open-api/typescript-sdk/src/fetch-client.ts

@@ -960,6 +960,7 @@ export type SystemConfigFFmpegDto = {
     accel: TranscodeHWAccel;
     accelDecode: boolean;
     acceptedAudioCodecs: AudioCodec[];
+    acceptedContainers: VideoContainer[];
     acceptedVideoCodecs: VideoCodec[];
     bframes: number;
     cqMode: CQMode;
@@ -3178,6 +3179,12 @@ export enum AudioCodec {
     Aac = "aac",
     Libopus = "libopus"
 }
+export enum VideoContainer {
+    Mov = "mov",
+    Mp4 = "mp4",
+    Ogg = "ogg",
+    Webm = "webm"
+}
 export enum VideoCodec {
     H264 = "h264",
     Hevc = "hevc",

server/src/config.ts

@@ -37,6 +37,13 @@ export enum AudioCodec {
   LIBOPUS = 'libopus',
 }
 
+export enum VideoContainer {
+  MOV = 'mov',
+  MP4 = 'mp4',
+  OGG = 'ogg',
+  WEBM = 'webm',
+}
+
 export enum TranscodeHWAccel {
   NVENC = 'nvenc',
   QSV = 'qsv',
@@ -86,6 +93,7 @@ export interface SystemConfig {
     acceptedVideoCodecs: VideoCodec[];
     targetAudioCodec: AudioCodec;
     acceptedAudioCodecs: AudioCodec[];
+    acceptedContainers: VideoContainer[];
     targetResolution: string;
     maxBitrate: string;
     bframes: number;
@@ -218,6 +226,7 @@ export const defaults = Object.freeze<SystemConfig>({
     acceptedVideoCodecs: [VideoCodec.H264],
     targetAudioCodec: AudioCodec.AAC,
     acceptedAudioCodecs: [AudioCodec.AAC, AudioCodec.MP3, AudioCodec.LIBOPUS],
+    acceptedContainers: [VideoContainer.MOV, VideoContainer.OGG, VideoContainer.WEBM],
     targetResolution: '720',
     maxBitrate: '0',
     bframes: -1,

server/src/dtos/system-config.dto.ts

@@ -29,6 +29,7 @@ import {
   TranscodeHWAccel,
   TranscodePolicy,
   VideoCodec,
+  VideoContainer,
 } from 'src/config';
 import { CLIPConfig, DuplicateDetectionConfig, FacialRecognitionConfig } from 'src/dtos/model-config.dto';
 import { ConcurrentQueueName, QueueName } from 'src/interfaces/job.interface';
@@ -79,6 +80,10 @@ export class SystemConfigFFmpegDto {
   @ApiProperty({ enumName: 'AudioCodec', enum: AudioCodec, isArray: true })
   acceptedAudioCodecs!: AudioCodec[];
 
+  @IsEnum(VideoContainer, { each: true })
+  @ApiProperty({ enumName: 'VideoContainer', enum: VideoContainer, isArray: true })
+  acceptedContainers!: VideoContainer[];
+
   @IsString()
   targetResolution!: string;
 

server/src/services/media.service.spec.ts

@@ -957,6 +957,21 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should remux when input is not an accepted container', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamAvi);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: expect.any(Array),
+          outputOptions: expect.arrayContaining(['-c:v copy', '-c:a copy']),
+          twoPass: false,
+        },
+      );
+    });
+
     it('should throw an exception if transcode value is invalid', async () => {
       mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);
       systemMock.get.mockResolvedValue({ ffmpeg: { transcode: 'invalid' as any } });
@@ -973,6 +988,14 @@ describe(MediaService.name, () => {
       expect(mediaMock.transcode).not.toHaveBeenCalled();
     });
 
+    it('should not remux when input is not an accepted container and transcoding is disabled', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+      systemMock.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.DISABLED } });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).not.toHaveBeenCalled();
+    });
+
     it('should not transcode if target codec is invalid', async () => {
       mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);
       systemMock.get.mockResolvedValue({ ffmpeg: { targetVideoCodec: 'invalid' as any } });

server/src/services/media.service.ts

@@ -8,6 +8,7 @@ import {
   TranscodePolicy,
   TranscodeTarget,
   VideoCodec,
+  VideoContainer,
 } from 'src/config';
 import { GeneratedImageType, StorageCore, StorageFolder } from 'src/cores/storage.core';
 import { SystemConfigCore } from 'src/cores/system-config.core';
@@ -27,7 +28,7 @@ import {
   QueueName,
 } from 'src/interfaces/job.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
-import { AudioStreamInfo, IMediaRepository, VideoStreamInfo } from 'src/interfaces/media.interface';
+import { AudioStreamInfo, IMediaRepository, VideoFormat, VideoStreamInfo } from 'src/interfaces/media.interface';
 import { IMoveRepository } from 'src/interfaces/move.interface';
 import { IPersonRepository } from 'src/interfaces/person.interface';
 import { IStorageRepository } from 'src/interfaces/storage.interface';
@@ -314,8 +315,7 @@ export class MediaService {
     const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input);
     const mainVideoStream = this.getMainStream(videoStreams);
     const mainAudioStream = this.getMainStream(audioStreams);
-    const containerExtension = format.formatName;
+    if (!mainVideoStream || !format.formatName) {
-    if (!mainVideoStream || !containerExtension) {
       return JobStatus.FAILED;
     }
 
@@ -326,7 +326,7 @@ export class MediaService {
 
     const { ffmpeg } = await this.configCore.getConfig({ withCache: true });
     const target = this.getTranscodeTarget(ffmpeg, mainVideoStream, mainAudioStream);
-    if (target === TranscodeTarget.NONE) {
+    if (target === TranscodeTarget.NONE && !this.isRemuxRequired(ffmpeg, format)) {
       if (asset.encodedVideoPath) {
         this.logger.log(`Transcoded video exists for asset ${asset.id}, but is no longer required. Deleting...`);
         await this.jobRepository.queue({ name: JobName.DELETE_FILES, data: { files: [asset.encodedVideoPath] } });
@@ -456,6 +456,15 @@ export class MediaService {
     }
   }
 
+  private isRemuxRequired(ffmpegConfig: SystemConfigFFmpegDto, { formatName, formatLongName }: VideoFormat): boolean {
+    if (ffmpegConfig.transcode === TranscodePolicy.DISABLED) {
+      return false;
+    }
+
+    const name = formatLongName === 'QuickTime / MOV' ? VideoContainer.MOV : (formatName as VideoContainer);
+    return name !== VideoContainer.MP4 && !ffmpegConfig.acceptedContainers.includes(name);
+  }
+
   isSRGB(asset: AssetEntity): boolean {
     const { colorspace, profileDescription, bitsPerSample } = asset.exifInfo ?? {};
     if (colorspace || profileDescription) {

server/src/services/system-config.service.spec.ts

@@ -10,6 +10,7 @@ import {
   TranscodeHWAccel,
   TranscodePolicy,
   VideoCodec,
+  VideoContainer,
   defaults,
 } from 'src/config';
 import { SystemMetadataKey } from 'src/entities/system-metadata.entity';
@@ -54,6 +55,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
     targetResolution: '720',
     targetVideoCodec: VideoCodec.H264,
     acceptedVideoCodecs: [VideoCodec.H264],
+    acceptedContainers: [VideoContainer.MOV, VideoContainer.OGG, VideoContainer.WEBM],
     maxBitrate: '0',
     bframes: -1,
     refs: 0,

server/test/fixtures/media.stub.ts

@@ -177,4 +177,14 @@ export const probeStub = {
     ...probeStubDefault,
     videoStreams: [{ ...probeStubDefaultVideoStream[0], codecName: 'h264' }],
   }),
+  videoStreamAvi: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    videoStreams: [{ ...probeStubDefaultVideoStream[0], codecName: 'h264' }],
+    format: {
+      formatName: 'avi',
+      formatLongName: 'AVI (Audio Video Interleaved)',
+      duration: 0,
+      bitrate: 0,
+    },
+  }),
 };

web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte

@@ -7,6 +7,7 @@
     TranscodeHWAccel,
     TranscodePolicy,
     VideoCodec,
+    VideoContainer,
     type SystemConfigDto,
   } from '@immich/sdk';
   import { mdiHelpCircleOutline } from '@mdi/js';
@@ -85,6 +86,22 @@
           isEdited={config.ffmpeg.preset !== savedConfig.ffmpeg.preset}
         />
 
+        <SettingSelect
+          label={$t('admin.transcoding_video_codec')}
+          {disabled}
+          desc={$t('admin.transcoding_video_codec_description')}
+          bind:value={config.ffmpeg.targetVideoCodec}
+          options={[
+            { value: VideoCodec.H264, text: 'h264' },
+            { value: VideoCodec.Hevc, text: 'hevc' },
+            { value: VideoCodec.Vp9, text: 'vp9' },
+            { value: VideoCodec.Av1, text: 'av1' },
+          ]}
+          name="vcodec"
+          isEdited={config.ffmpeg.targetVideoCodec !== savedConfig.ffmpeg.targetVideoCodec}
+          on:select={() => (config.ffmpeg.acceptedVideoCodecs = [config.ffmpeg.targetVideoCodec])}
+        />
+
         <SettingSelect
           label={$t('admin.transcoding_audio_codec')}
           {disabled}
@@ -103,6 +120,21 @@
               : config.ffmpeg.acceptedAudioCodecs.push(config.ffmpeg.targetAudioCodec)}
         />
 
+        <SettingCheckboxes
+          label={$t('admin.transcoding_accepted_video_codecs')}
+          {disabled}
+          desc={$t('admin.transcoding_accepted_video_codecs_description')}
+          bind:value={config.ffmpeg.acceptedVideoCodecs}
+          name="videoCodecs"
+          options={[
+            { value: VideoCodec.H264, text: 'H.264' },
+            { value: VideoCodec.Hevc, text: 'HEVC' },
+            { value: VideoCodec.Vp9, text: 'VP9' },
+            { value: VideoCodec.Av1, text: 'AV1' },
+          ]}
+          isEdited={!isEqual(sortBy(config.ffmpeg.acceptedVideoCodecs), sortBy(savedConfig.ffmpeg.acceptedVideoCodecs))}
+        />
+
         <SettingCheckboxes
           label={$t('admin.transcoding_accepted_audio_codecs')}
           {disabled}
@@ -117,35 +149,18 @@
           isEdited={!isEqual(sortBy(config.ffmpeg.acceptedAudioCodecs), sortBy(savedConfig.ffmpeg.acceptedAudioCodecs))}
         />
 
-        <SettingSelect
-          label={$t('admin.transcoding_video_codec')}
-          {disabled}
-          desc={$t('admin.transcoding_video_codec_description')}
-          bind:value={config.ffmpeg.targetVideoCodec}
-          options={[
-            { value: VideoCodec.H264, text: 'h264' },
-            { value: VideoCodec.Hevc, text: 'hevc' },
-            { value: VideoCodec.Vp9, text: 'vp9' },
-            { value: VideoCodec.Av1, text: 'av1' },
-          ]}
-          name="vcodec"
-          isEdited={config.ffmpeg.targetVideoCodec !== savedConfig.ffmpeg.targetVideoCodec}
-          on:select={() => (config.ffmpeg.acceptedVideoCodecs = [config.ffmpeg.targetVideoCodec])}
-        />
-
         <SettingCheckboxes
-          label={$t('admin.transcoding_accepted_video_codecs')}
+          label={$t('admin.transcoding_accepted_containers')}
           {disabled}
-          desc={$t('admin.transcoding_accepted_video_codecs_description')}
+          desc={$t('admin.transcoding_accepted_containers_description')}
-          bind:value={config.ffmpeg.acceptedVideoCodecs}
+          bind:value={config.ffmpeg.acceptedContainers}
-          name="videoCodecs"
+          name="videoContainers"
           options={[
-            { value: VideoCodec.H264, text: 'H.264' },
+            { value: VideoContainer.Mov, text: 'MOV' },
-            { value: VideoCodec.Hevc, text: 'HEVC' },
+            { value: VideoContainer.Ogg, text: 'Ogg' },
-            { value: VideoCodec.Vp9, text: 'VP9' },
+            { value: VideoContainer.Webm, text: 'WebM' },
-            { value: VideoCodec.Av1, text: 'AV1' },
           ]}
-          isEdited={!isEqual(sortBy(config.ffmpeg.acceptedVideoCodecs), sortBy(savedConfig.ffmpeg.acceptedVideoCodecs))}
+          isEdited={!isEqual(sortBy(config.ffmpeg.acceptedContainers), sortBy(savedConfig.ffmpeg.acceptedContainers))}
         />
 
         <SettingSelect

web/src/lib/i18n/en.json

@@ -246,6 +246,8 @@
     "transcoding_acceleration_vaapi": "VAAPI",
     "transcoding_accepted_audio_codecs": "Accepted audio codecs",
     "transcoding_accepted_audio_codecs_description": "Select which audio codecs do not need to be transcoded. Only used for certain transcode policies.",
+    "transcoding_accepted_containers": "Accepted containers",
+    "transcoding_accepted_containers_description": "Select which container formats do not need to be remuxed to MP4. Only used for certain transcode policies.",
     "transcoding_accepted_video_codecs": "Accepted video codecs",
     "transcoding_accepted_video_codecs_description": "Select which video codecs do not need to be transcoded. Only used for certain transcode policies.",
     "transcoding_advanced_options_description": "Options most users should not need to change",