1
0
mirror of https://github.com/immich-app/immich.git synced 2024-11-24 08:52:28 +02:00

feat(server): transcoding improvements (#1370)

* feat: support isEdited flag for SettingSwitch

* feat: add transcodeAll ffmpeg settings for extra transcoding control

* refactor: tidy up and rename current video transcoding code + transcode everything

* feat: better video transcoding with ffprobe

analyses video files to see if they are already in the desired format
allows admin to choose to transcode all videos regardless of the current format

* fix: always serve encoded video if it exists

* feat: change video codec option to a select box, limit options

removed previous video codec config option as it's incompatible with new options
removed mapping for encoder to codec as we now store the codec in the config

* feat: add video conversion job for transcoding previously missed videos

* chore: fix spelling of job messages to pluralise assets

* chore: fix prettier/eslint warnings

* feat: force switch targetAudioCodec default to aac to avoid iOS incompatibility

* chore: lint issues after rebase
This commit is contained in:
Zack Pollard 2023-01-22 02:09:02 +00:00 committed by GitHub
parent 8eb82836b9
commit 4e0fe27de3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 256 additions and 59 deletions

3
.gitignore vendored
View File

@ -4,4 +4,5 @@
.idea
docker/upload
coverage
uploads
coverage

BIN
mobile/openapi/README.md generated

Binary file not shown.

Binary file not shown.

View File

@ -39,6 +39,7 @@ export interface IAssetRepository {
getAssetByTimeBucket(userId: string, getAssetByTimeBucketDto: GetAssetByTimeBucketDto): Promise<AssetEntity[]>;
getAssetByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity>;
getAssetWithNoThumbnail(): Promise<AssetEntity[]>;
getAssetWithNoEncodedVideo(): Promise<AssetEntity[]>;
getAssetWithNoEXIF(): Promise<AssetEntity[]>;
getAssetWithNoSmartInfo(): Promise<AssetEntity[]>;
getExistingAssets(
@ -80,6 +81,15 @@ export class AssetRepository implements IAssetRepository {
});
}
async getAssetWithNoEncodedVideo(): Promise<AssetEntity[]> {
return await this.assetRepository.find({
where: [
{ type: AssetType.VIDEO, encodedVideoPath: IsNull() },
{ type: AssetType.VIDEO, encodedVideoPath: '' },
],
});
}
async getAssetWithNoEXIF(): Promise<AssetEntity[]> {
return await this.assetRepository
.createQueryBuilder('asset')

View File

@ -128,6 +128,7 @@ describe('AssetService', () => {
getAssetWithNoEXIF: jest.fn(),
getAssetWithNoThumbnail: jest.fn(),
getAssetWithNoSmartInfo: jest.fn(),
getAssetWithNoEncodedVideo: jest.fn(),
getExistingAssets: jest.fn(),
countByIdAndUser: jest.fn(),
};

View File

@ -37,13 +37,13 @@ import {
import { GetAssetCountByTimeBucketDto } from './dto/get-asset-count-by-time-bucket.dto';
import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
import { assetUtils, timeUtils } from '@app/common/utils';
import { timeUtils } from '@app/common/utils';
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
import { UpdateAssetDto } from './dto/update-asset.dto';
import { AssetFileUploadResponseDto } from './response-dto/asset-file-upload-response.dto';
import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
import { IAssetUploadedJob, IVideoTranscodeJob, QueueName, JobName } from '@app/domain';
import { IAssetUploadedJob, IVideoTranscodeJob, JobName, QueueName } from '@app/domain';
import { InjectQueue } from '@nestjs/bull';
import { Queue } from 'bull';
import { DownloadService } from '../../modules/download/download.service';
@ -122,7 +122,7 @@ export class AssetService {
await this.storageService.moveAsset(livePhotoAssetEntity, originalAssetData.originalname);
await this.videoConversionQueue.add(JobName.MP4_CONVERSION, { asset: livePhotoAssetEntity });
await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset: livePhotoAssetEntity });
}
const assetEntity = await this.createUserAsset(
@ -456,7 +456,7 @@ export class AssetService {
await fs.access(videoPath, constants.R_OK | constants.W_OK);
if (query.isWeb && !assetUtils.isWebPlayable(asset.mimeType)) {
if (asset.encodedVideoPath) {
videoPath = asset.encodedVideoPath == '' ? String(asset.originalPath) : String(asset.encodedVideoPath);
mimeType = asset.encodedVideoPath == '' ? asset.mimeType : 'video/mp4';
}

View File

@ -3,8 +3,8 @@ import {
IMetadataExtractionJob,
IThumbnailGenerationJob,
IVideoTranscodeJob,
QueueName,
JobName,
QueueName,
} from '@app/domain';
import { InjectQueue } from '@nestjs/bull';
import { Queue } from 'bull';
@ -53,7 +53,7 @@ export class JobService {
case JobId.METADATA_EXTRACTION:
return this.runMetadataExtractionJob();
case JobId.VIDEO_CONVERSION:
return 0;
return this.runVideoConversionJob();
case JobId.MACHINE_LEARNING:
return this.runMachineLearningPipeline();
case JobId.STORAGE_TEMPLATE_MIGRATION:
@ -79,7 +79,6 @@ export class JobService {
response.videoConversionQueueCount = videoConversionJobCount;
response.isMachineLearningActive = Boolean(machineLearningJobCount.waiting);
response.machineLearningQueueCount = machineLearningJobCount;
response.isStorageMigrationActive = Boolean(storageMigrationJobCount.active);
response.storageMigrationQueueCount = storageMigrationJobCount;
@ -188,6 +187,22 @@ export class JobService {
return assetWithNoSmartInfo.length;
}
private async runVideoConversionJob(): Promise<number> {
const jobCount = await this.videoConversionQueue.getJobCounts();
if (jobCount.waiting > 0) {
throw new BadRequestException('Video conversion job is already running');
}
const assetsWithNoConvertedVideo = await this._assetRepository.getAssetWithNoEncodedVideo();
for (const asset of assetsWithNoConvertedVideo) {
await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset });
}
return assetsWithNoConvertedVideo.length;
}
async runStorageMigration() {
const jobCount = await this.configQueue.getJobCounts();

View File

@ -69,7 +69,7 @@ export class ScheduleTasksService {
});
for (const asset of assets) {
await this.videoConversionQueue.add(JobName.MP4_CONVERSION, { asset });
await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset });
}
}

View File

@ -40,7 +40,7 @@ export class AssetUploadedProcessor {
// Video Conversion
if (asset.type == AssetType.VIDEO) {
await this.videoConversionQueue.add(JobName.MP4_CONVERSION, { asset });
await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset });
await this.metadataExtractionQueue.add(JobName.EXTRACT_VIDEO_METADATA, { asset, fileName });
} else {
// Extract Metadata/Exif for Images - Currently the EXIF library on the web cannot extract EXIF for video yet

View File

@ -1,14 +1,12 @@
import { APP_UPLOAD_LOCATION } from '@app/common/constants';
import { AssetEntity } from '@app/infra';
import { QueueName, JobName } from '@app/domain';
import { IMp4ConversionProcessor } from '@app/domain';
import { IVideoConversionProcessor, JobName, QueueName, SystemConfigService } from '@app/domain';
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Job } from 'bull';
import ffmpeg from 'fluent-ffmpeg';
import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
import { existsSync, mkdirSync } from 'fs';
import { SystemConfigService } from '@app/domain';
import { Repository } from 'typeorm';
@Processor(QueueName.VIDEO_CONVERSION)
@ -19,24 +17,60 @@ export class VideoTranscodeProcessor {
private systemConfigService: SystemConfigService,
) {}
@Process({ name: JobName.MP4_CONVERSION, concurrency: 2 })
async mp4Conversion(job: Job<IMp4ConversionProcessor>) {
@Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 })
async videoConversion(job: Job<IVideoConversionProcessor>) {
const { asset } = job.data;
if (asset.mimeType != 'video/mp4') {
const basePath = APP_UPLOAD_LOCATION;
const encodedVideoPath = `${basePath}/${asset.userId}/encoded-video`;
const basePath = APP_UPLOAD_LOCATION;
const encodedVideoPath = `${basePath}/${asset.userId}/encoded-video`;
if (!existsSync(encodedVideoPath)) {
mkdirSync(encodedVideoPath, { recursive: true });
}
if (!existsSync(encodedVideoPath)) {
mkdirSync(encodedVideoPath, { recursive: true });
}
const savedEncodedPath = encodedVideoPath + '/' + asset.id + '.mp4';
const savedEncodedPath = `${encodedVideoPath}/${asset.id}.mp4`;
if (asset.encodedVideoPath == '' || !asset.encodedVideoPath) {
// Put the processing into its own async function to prevent the job exist right away
await this.runFFMPEGPipeLine(asset, savedEncodedPath);
}
if (!asset.encodedVideoPath) {
// Put the processing into its own async function to prevent the job exist right away
await this.runVideoEncode(asset, savedEncodedPath);
}
}
async runFFProbePipeline(asset: AssetEntity): Promise<FfprobeData> {
return new Promise((resolve, reject) => {
ffmpeg.ffprobe(asset.originalPath, (err, data) => {
if (err || !data) {
Logger.error(`Cannot probe video ${err}`, 'mp4Conversion');
reject(err);
}
resolve(data);
});
});
}
async runVideoEncode(asset: AssetEntity, savedEncodedPath: string): Promise<void> {
const config = await this.systemConfigService.getConfig();
if (config.ffmpeg.transcodeAll) {
return this.runFFMPEGPipeLine(asset, savedEncodedPath);
}
const videoInfo = await this.runFFProbePipeline(asset);
const videoStreams = videoInfo.streams.filter((stream) => {
return stream.codec_type === 'video';
});
const longestVideoStream = videoStreams.sort((stream1, stream2) => {
const stream1Frames = Number.parseInt(stream1.nb_frames ?? '0');
const stream2Frames = Number.parseInt(stream2.nb_frames ?? '0');
return stream2Frames - stream1Frames;
})[0];
//TODO: If video or audio are already the correct format, don't re-encode, copy the stream
if (longestVideoStream.codec_name !== config.ffmpeg.targetVideoCodec) {
return this.runFFMPEGPipeLine(asset, savedEncodedPath);
}
}

View File

@ -2899,6 +2899,9 @@
},
"targetScaling": {
"type": "string"
},
"transcodeAll": {
"type": "boolean"
}
},
"required": [
@ -2906,7 +2909,8 @@
"preset",
"targetVideoCodec",
"targetAudioCodec",
"targetScaling"
"targetScaling",
"transcodeAll"
]
},
"SystemConfigOAuthDto": {

View File

@ -1,10 +1,10 @@
import { AssetEntity } from '@app/infra/db/entities';
export interface IMp4ConversionProcessor {
export interface IVideoConversionProcessor {
/**
* The Asset entity that was saved in the database
*/
asset: AssetEntity;
}
export type IVideoTranscodeJob = IMp4ConversionProcessor;
export type IVideoTranscodeJob = IVideoConversionProcessor;

View File

@ -12,7 +12,7 @@ export enum QueueName {
export enum JobName {
ASSET_UPLOADED = 'asset-uploaded',
MP4_CONVERSION = 'mp4-conversion',
VIDEO_CONVERSION = 'mp4-conversion',
GENERATE_JPEG_THUMBNAIL = 'generate-jpeg-thumbnail',
GENERATE_WEBP_THUMBNAIL = 'generate-webp-thumbnail',
EXIF_EXTRACTION = 'exif-extraction',

View File

@ -3,7 +3,7 @@ import {
IDeleteFileOnDiskJob,
IExifExtractionProcessor,
IMachineLearningJob,
IMp4ConversionProcessor,
IVideoConversionProcessor,
IReverseGeocodingProcessor,
IUserDeletionJob,
JpegGeneratorProcessor,
@ -13,7 +13,7 @@ import { JobName } from './job.constants';
export type JobItem =
| { name: JobName.ASSET_UPLOADED; data: IAssetUploadedJob }
| { name: JobName.MP4_CONVERSION; data: IMp4ConversionProcessor }
| { name: JobName.VIDEO_CONVERSION; data: IVideoConversionProcessor }
| { name: JobName.GENERATE_JPEG_THUMBNAIL; data: JpegGeneratorProcessor }
| { name: JobName.GENERATE_WEBP_THUMBNAIL; data: WebpGeneratorProcessor }
| { name: JobName.EXIF_EXTRACTION; data: IExifExtractionProcessor }

View File

@ -1,4 +1,4 @@
import { IsString } from 'class-validator';
import { IsBoolean, IsString } from 'class-validator';
export class SystemConfigFFmpegDto {
@IsString()
@ -15,4 +15,7 @@ export class SystemConfigFFmpegDto {
@IsString()
targetScaling!: string;
@IsBoolean()
transcodeAll!: boolean;
}

View File

@ -11,9 +11,10 @@ const defaults: SystemConfig = Object.freeze({
ffmpeg: {
crf: '23',
preset: 'ultrafast',
targetVideoCodec: 'libx264',
targetAudioCodec: 'mp3',
targetVideoCodec: 'h264',
targetAudioCodec: 'aac',
targetScaling: '1280:-2',
transcodeAll: false,
},
oauth: {
enabled: false,

View File

@ -15,9 +15,10 @@ const updatedConfig = Object.freeze({
ffmpeg: {
crf: 'a new value',
preset: 'ultrafast',
targetAudioCodec: 'mp3',
targetAudioCodec: 'aac',
targetScaling: '1280:-2',
targetVideoCodec: 'libx264',
targetVideoCodec: 'h264',
transcodeAll: false,
},
oauth: {
autoLaunch: true,

View File

@ -48,9 +48,10 @@ export const systemConfigStub = {
ffmpeg: {
crf: '23',
preset: 'ultrafast',
targetAudioCodec: 'mp3',
targetAudioCodec: 'aac',
targetScaling: '1280:-2',
targetVideoCodec: 'libx264',
targetVideoCodec: 'h264',
transcodeAll: false,
},
oauth: {
autoLaunch: false,

View File

@ -18,6 +18,7 @@ export enum SystemConfigKey {
FFMPEG_TARGET_VIDEO_CODEC = 'ffmpeg.targetVideoCodec',
FFMPEG_TARGET_AUDIO_CODEC = 'ffmpeg.targetAudioCodec',
FFMPEG_TARGET_SCALING = 'ffmpeg.targetScaling',
FFMPEG_TRANSCODE_ALL = 'ffmpeg.transcodeAll',
OAUTH_ENABLED = 'oauth.enabled',
OAUTH_ISSUER_URL = 'oauth.issuerUrl',
OAUTH_CLIENT_ID = 'oauth.clientId',
@ -39,6 +40,7 @@ export interface SystemConfig {
targetVideoCodec: string;
targetAudioCodec: string;
targetScaling: string;
transcodeAll: boolean;
};
oauth: {
enabled: boolean;

View File

@ -0,0 +1,12 @@
import {MigrationInterface, QueryRunner} from 'typeorm';
export class RemoveVideoCodecConfigOption1674263302006 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DELETE FROM "system_config" WHERE key = 'ffmpeg.targetVideoCodec'`);
await queryRunner.query(`DELETE FROM "system_config" WHERE key = 'ffmpeg.targetAudioCodec'`);
}
public async down(): Promise<void> {
// noop
}
}

View File

@ -4,7 +4,7 @@
* Immich
* Immich API
*
* The version of the OpenAPI document: 1.41.1
* The version of the OpenAPI document: 1.42.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
@ -13,13 +13,24 @@
*/
import { Configuration } from './configuration';
import globalAxios, { AxiosPromise, AxiosInstance, AxiosRequestConfig } from 'axios';
import {Configuration} from './configuration';
import globalAxios, {AxiosInstance, AxiosPromise, AxiosRequestConfig} from 'axios';
// Some imports not used depending on template conditions
// @ts-ignore
import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction } from './common';
import {
assertParamExists,
createRequestFunction,
DUMMY_BASE_URL,
serializeDataIfNeeded,
setApiKeyToObject,
setBasicAuthToObject,
setBearerAuthToObject,
setOAuthToObject,
setSearchParams,
toPathString
} from './common';
// @ts-ignore
import { BASE_PATH, COLLECTION_FORMATS, RequestArgs, BaseAPI, RequiredError } from './base';
import {BASE_PATH, BaseAPI, COLLECTION_FORMATS, RequestArgs, RequiredError} from './base';
/**
*
@ -1799,6 +1810,12 @@ export interface SystemConfigFFmpegDto {
* @memberof SystemConfigFFmpegDto
*/
'targetScaling': string;
/**
*
* @type {boolean}
* @memberof SystemConfigFFmpegDto
*/
'transcodeAll': boolean;
}
/**
*

View File

@ -4,7 +4,7 @@
* Immich
* Immich API
*
* The version of the OpenAPI document: 1.41.1
* The version of the OpenAPI document: 1.42.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

View File

@ -4,7 +4,7 @@
* Immich
* Immich API
*
* The version of the OpenAPI document: 1.41.1
* The version of the OpenAPI document: 1.42.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

View File

@ -4,7 +4,7 @@
* Immich
* Immich API
*
* The version of the OpenAPI document: 1.41.1
* The version of the OpenAPI document: 1.42.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

View File

@ -4,7 +4,7 @@
* Immich
* Immich API
*
* The version of the OpenAPI document: 1.41.1
* The version of the OpenAPI document: 1.42.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

View File

@ -33,7 +33,7 @@
if (data) {
notificationController.show({
message: `Thumbnail generation job started for ${data} asset`,
message: `Thumbnail generation job started for ${data} assets`,
type: NotificationType.Info
});
} else {
@ -60,7 +60,7 @@
if (data) {
notificationController.show({
message: `Extract EXIF job started for ${data} asset`,
message: `Extract EXIF job started for ${data} assets`,
type: NotificationType.Info
});
} else {
@ -87,7 +87,7 @@
if (data) {
notificationController.show({
message: `Object detection job started for ${data} asset`,
message: `Object detection job started for ${data} assets`,
type: NotificationType.Info
});
} else {
@ -101,6 +101,28 @@
}
};
const runVideoConversion = async () => {
try {
const { data } = await api.jobApi.sendJobCommand(JobId.VideoConversion, {
command: JobCommand.Start
});
if (data) {
notificationController.show({
message: `Video conversion job started for ${data} assets`,
type: NotificationType.Info
});
} else {
notificationController.show({
message: `No videos without an encoded version found`,
type: NotificationType.Info
});
}
} catch (error) {
handleError(error, `Error running video conversion job, check console for more detail`);
}
};
const runTemplateMigration = async () => {
try {
const { data } = await api.jobApi.sendJobCommand(JobId.StorageTemplateMigration, {
@ -159,6 +181,17 @@
Note that some assets may not have any objects detected, this is normal.
</JobTile>
<JobTile
title={'Video transcoding'}
subtitle={'Run video transcoding process to transcode videos not in the desired format'}
on:click={runVideoConversion}
jobStatus={allJobsStatus?.isVideoConversionActive}
waitingJobCount={allJobsStatus?.videoConversionQueueCount.waiting}
activeJobCount={allJobsStatus?.videoConversionQueueCount.active}
>
Note that some videos won't require transcoding, this is normal.
</JobTile>
<JobTile
title={'Storage migration'}
subtitle={''}

View File

@ -6,10 +6,13 @@
import { api, SystemConfigFFmpegDto } from '@api';
import SettingButtonsRow from '../setting-buttons-row.svelte';
import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
import SettingSelect from '../setting-select.svelte';
import SettingSwitch from '../setting-switch.svelte';
import _ from 'lodash';
export let ffmpegConfig: SystemConfigFFmpegDto; // this is the config that is being edited
import { fade } from 'svelte/transition';
export let ffmpegConfig: SystemConfigFFmpegDto; // this is the config that is being edited
let savedConfig: SystemConfigFFmpegDto;
let defaultConfig: SystemConfigFFmpegDto;
@ -99,11 +102,10 @@
isEdited={!(ffmpegConfig.targetAudioCodec == savedConfig.targetAudioCodec)}
/>
<SettingInputField
inputType={SettingInputFieldType.TEXT}
<SettingSelect
label="VIDEO CODEC (-vcodec)"
bind:value={ffmpegConfig.targetVideoCodec}
required={true}
options={['h264', 'hevc', 'vp9']}
isEdited={!(ffmpegConfig.targetVideoCodec == savedConfig.targetVideoCodec)}
/>
@ -114,6 +116,13 @@
required={true}
isEdited={!(ffmpegConfig.targetScaling == savedConfig.targetScaling)}
/>
<SettingSwitch
title="TRANSCODE ALL"
subtitle="Transcode all files, even if they already match the specified format?"
bind:checked={ffmpegConfig.transcodeAll}
isEdited={!(ffmpegConfig.transcodeAll == savedConfig.transcodeAll)}
/>
</div>
<div class="ml-4">

View File

@ -0,0 +1,39 @@
<script lang="ts">
import { quintOut } from 'svelte/easing';
import { fly } from 'svelte/transition';
export let value: string;
export let options: string[];
export let label = '';
export let isEdited = false;
const handleChange = (e: Event) => {
value = (e.target as HTMLInputElement).value;
};
</script>
<div class="w-full">
<div class={`flex place-items-center gap-1 h-[26px]`}>
<label class={`immich-form-label text-sm`} for={label}>{label}</label>
{#if isEdited}
<div
transition:fly={{ x: 10, duration: 200, easing: quintOut }}
class="bg-orange-100 px-2 rounded-full text-orange-900 text-[10px]"
>
Unsaved change
</div>
{/if}
</div>
<select
class="immich-form-input w-full"
name="presets"
id="preset-select"
bind:value
on:change={handleChange}
>
{#each options as option}
<option value={option}>{option}</option>
{/each}
</select>
</div>

View File

@ -1,15 +1,29 @@
<script lang="ts">
import { quintOut } from 'svelte/easing';
import { fly } from 'svelte/transition';
export let title: string;
export let subtitle = '';
export let checked = false;
export let disabled = false;
export let isEdited = false;
</script>
<div class="flex justify-between place-items-center">
<div>
<h2 class="immich-form-label text-sm">
{title}
</h2>
<div class="flex place-items-center gap-1 h-[26px]">
<label class="immich-form-label text-sm" for={title}>
{title}
</label>
{#if isEdited}
<div
transition:fly={{ x: 10, duration: 200, easing: quintOut }}
class="bg-orange-100 px-2 rounded-full text-orange-900 text-[10px]"
>
Unsaved change
</div>
{/if}
</div>
<p class="text-sm dark:text-immich-dark-fg">{subtitle}</p>
</div>