self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-25 10:43:13 +02:00
Code Issues Releases Activity

feat(web/server): Add options to rerun job on all assets (#1422)

Browse Source
This commit is contained in:
Alex 2023-01-26 22:50:22 -06:00 committed by GitHub
parent 6ea91b2dde
commit 788b435f9b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 217 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
mobile/openapi/doc/JobCommandDto.md generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/model/job_command_dto.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/test/job_command_dto_test.dart generated
View File

Binary file not shown.

18
server/apps/immich/src/api-v1/asset/asset-repository.ts
View File

@ -29,6 +29,8 @@ export interface IAssetRepository {
livePhotoAssetEntity?: AssetEntity,
): Promise<AssetEntity>;
update(userId: string, asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>;
getAll(): Promise<AssetEntity[]>;
getAllVideos(): Promise<AssetEntity[]>;
getAllByUserId(userId: string, dto: AssetSearchDto): Promise<AssetEntity[]>;
getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
getById(assetId: string): Promise<AssetEntity>;
@ -61,6 +63,22 @@ export class AssetRepository implements IAssetRepository {
@Inject(ITagRepository) private _tagRepository: ITagRepository,
) {}
async getAllVideos(): Promise<AssetEntity[]> {
return await this.assetRepository.find({
where: { type: AssetType.VIDEO },
});
}
async getAll(): Promise<AssetEntity[]> {
return await this.assetRepository.find({
where: { isVisible: true },
relations: {
exifInfo: true,
smartInfo: true,
},
});
}
async getAssetWithNoSmartInfo(): Promise<AssetEntity[]> {
return await this.assetRepository
.createQueryBuilder('asset')

2
server/apps/immich/src/api-v1/asset/asset.service.spec.ts
View File

@ -123,6 +123,8 @@ describe('AssetService', () => {
assetRepositoryMock = {
create: jest.fn(),
update: jest.fn(),
getAll: jest.fn(),
getAllVideos: jest.fn(),
getAllByUserId: jest.fn(),
getAllByDeviceId: jest.fn(),
getAssetCountByTimeBucket: jest.fn(),

6
server/apps/immich/src/api-v1/job/dto/job-command.dto.ts
View File

@ -1,5 +1,5 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsIn, IsNotEmpty } from 'class-validator';
import { IsBoolean, IsIn, IsNotEmpty, IsOptional } from 'class-validator';
export class JobCommandDto {
@IsNotEmpty()
@ -9,4 +9,8 @@ export class JobCommandDto {
enumName: 'JobCommand',
})
command!: string;
@IsOptional()
@IsBoolean()
includeAllAssets!: boolean;
}

8
server/apps/immich/src/api-v1/job/job.controller.ts
View File

@ -21,12 +21,12 @@ export class JobController {
@Put('/:jobId')
async sendJobCommand(
@Param(ValidationPipe) params: GetJobDto,
@Body(ValidationPipe) body: JobCommandDto,
@Body(ValidationPipe) dto: JobCommandDto,
): Promise<number> {
if (body.command === 'start') {
return await this.jobService.start(params.jobId);
if (dto.command === 'start') {
return await this.jobService.start(params.jobId, dto.includeAllAssets);
}
if (body.command === 'stop') {
if (dto.command === 'stop') {
return await this.jobService.stop(params.jobId);
}
return 0;

43
server/apps/immich/src/api-v1/job/job.service.ts
View File

@ -5,7 +5,7 @@ import { IAssetRepository } from '../asset/asset-repository';
import { AssetType } from '@app/infra';
import { JobId } from './dto/get-job.dto';
import { MACHINE_LEARNING_ENABLED } from '@app/common';
import { getFileNameWithoutExtension } from '../../utils/file-name.util';
const jobIds = Object.values(JobId) as JobId[];
@Injectable()
@ -19,8 +19,8 @@ export class JobService {
}
}
start(jobId: JobId): Promise<number> {
return this.run(this.asQueueName(jobId));
start(jobId: JobId, includeAllAssets: boolean): Promise<number> {
return this.run(this.asQueueName(jobId), includeAllAssets);
}
async stop(jobId: JobId): Promise<number> {
@ -36,7 +36,7 @@ export class JobService {
return response;
}
private async run(name: QueueName): Promise<number> {
private async run(name: QueueName, includeAllAssets: boolean): Promise<number> {
const isActive = await this.jobRepository.isActive(name);
if (isActive) {
throw new BadRequestException(`Job is already running`);
@ -44,7 +44,9 @@ export class JobService {
switch (name) {
case QueueName.VIDEO_CONVERSION: {
const assets = await this._assetRepository.getAssetWithNoEncodedVideo();
const assets = includeAllAssets
? await this._assetRepository.getAllVideos()
: await this._assetRepository.getAssetWithNoEncodedVideo();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
@ -61,7 +63,10 @@ export class JobService {
throw new BadRequestException('Machine learning is not enabled.');
}
const assets = await this._assetRepository.getAssetWithNoSmartInfo();
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoSmartInfo();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.IMAGE_TAGGING, data: { asset } });
await this.jobRepository.add({ name: JobName.OBJECT_DETECTION, data: { asset } });
@ -70,19 +75,37 @@ export class JobService {
}
case QueueName.METADATA_EXTRACTION: {
const assets = await this._assetRepository.getAssetWithNoEXIF();
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoEXIF();
for (const asset of assets) {
if (asset.type === AssetType.VIDEO) {
await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } });
await this.jobRepository.add({
name: JobName.EXTRACT_VIDEO_METADATA,
data: {
asset,
fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath),
},
});
} else {
await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } });
await this.jobRepository.add({
name: JobName.EXIF_EXTRACTION,
data: {
asset,
fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath),
},
});
}
}
return assets.length;
}
case QueueName.THUMBNAIL_GENERATION: {
const assets = await this._assetRepository.getAssetWithNoThumbnail();
const assets = includeAllAssets
? await this._assetRepository.getAll()
: await this._assetRepository.getAssetWithNoThumbnail();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
}

90
server/apps/immich/src/modules/schedule-tasks/schedule-tasks.service.ts
View File

@ -1,9 +1,8 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { InjectRepository } from '@nestjs/typeorm';
import { IsNull, Not, Repository } from 'typeorm';
import { AssetEntity, AssetType, ExifEntity, UserEntity } from '@app/infra';
import { ConfigService } from '@nestjs/config';
import { UserEntity } from '@app/infra';
import { userUtils } from '@app/common';
import { IJobRepository, JobName } from '@app/domain';
@ -13,93 +12,8 @@ export class ScheduleTasksService {
@InjectRepository(UserEntity)
private userRepository: Repository<UserEntity>,
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@InjectRepository(ExifEntity)
private exifRepository: Repository<ExifEntity>,
@Inject(IJobRepository) private jobRepository: IJobRepository,
private configService: ConfigService,
) {}
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
async webpConversion() {
const assets = await this.assetRepository.find({
where: {
webpPath: '',
},
});
if (assets.length == 0) {
Logger.log('All assets has webp file - aborting task', 'CronjobWebpGenerator');
return;
}
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { asset } });
}
}
@Cron(CronExpression.EVERY_DAY_AT_1AM)
async videoConversion() {
const assets = await this.assetRepository.find({
where: {
type: AssetType.VIDEO,
mimeType: 'video/quicktime',
encodedVideoPath: '',
},
order: {
createdAt: 'DESC',
},
});
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
}
@Cron(CronExpression.EVERY_DAY_AT_2AM)
async reverseGeocoding() {
const isGeocodingEnabled = this.configService.get('DISABLE_REVERSE_GEOCODING') !== 'true';
if (isGeocodingEnabled) {
const exifInfo = await this.exifRepository.find({
where: {
city: IsNull(),
longitude: Not(IsNull()),
latitude: Not(IsNull()),
},
});
for (const exif of exifInfo) {
await this.jobRepository.add({
name: JobName.REVERSE_GEOCODING,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
data: { exifId: exif.id, latitude: exif.latitude!, longitude: exif.longitude! },
});
}
}
}
@Cron(CronExpression.EVERY_DAY_AT_3AM)
async extractExif() {
const exifAssets = await this.assetRepository
.createQueryBuilder('asset')
.leftJoinAndSelect('asset.exifInfo', 'ei')
.where('ei."assetId" IS NULL')
.getMany();
for (const asset of exifAssets) {
if (asset.type === AssetType.VIDEO) {
await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } });
} else {
await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } });
}
}
}
@Cron(CronExpression.EVERY_DAY_AT_11PM)
async deleteUserAndRelatedAssets() {
const usersToDelete = await this.userRepository.find({ withDeleted: true, where: { deletedAt: Not(IsNull()) } });

5
server/apps/immich/src/utils/file-name.util.ts Normal file
View File

@ -0,0 +1,5 @@
import { basename, extname } from 'node:path';
export function getFileNameWithoutExtension(path: string): string {
return basename(path, extname(path));
}

4
server/apps/microservices/src/processors/metadata-extraction.processor.ts
View File

@ -216,7 +216,7 @@ export class MetadataExtractionProcessor {
}
}
await this.exifRepository.save(newExif);
await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
} catch (error: any) {
this.logger.error(`Error extracting EXIF ${error}`, error?.stack);
}
@ -327,7 +327,7 @@ export class MetadataExtractionProcessor {
}
}
await this.exifRepository.save(newExif);
await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
await this.assetRepository.update({ id: asset.id }, { duration: durationString, createdAt: createdAt });
} catch (err) {
// do nothing

15
server/apps/microservices/src/processors/video-transcode.processor.ts
View File

@ -11,6 +11,7 @@ import { Repository } from 'typeorm';
@Processor(QueueName.VIDEO_CONVERSION)
export class VideoTranscodeProcessor {
readonly logger = new Logger(VideoTranscodeProcessor.name);
constructor(
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@ -20,7 +21,6 @@ export class VideoTranscodeProcessor {
@Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 })
async videoConversion(job: Job<IVideoConversionProcessor>) {
const { asset } = job.data;
const basePath = APP_UPLOAD_LOCATION;
const encodedVideoPath = `${basePath}/${asset.userId}/encoded-video`;
@ -30,17 +30,14 @@ export class VideoTranscodeProcessor {
const savedEncodedPath = `${encodedVideoPath}/${asset.id}.mp4`;
if (!asset.encodedVideoPath) {
// Put the processing into its own async function to prevent the job exist right away
await this.runVideoEncode(asset, savedEncodedPath);
}
await this.runVideoEncode(asset, savedEncodedPath);
}
async runFFProbePipeline(asset: AssetEntity): Promise<FfprobeData> {
return new Promise((resolve, reject) => {
ffmpeg.ffprobe(asset.originalPath, (err, data) => {
if (err || !data) {
Logger.error(`Cannot probe video ${err}`, 'mp4Conversion');
this.logger.error(`Cannot probe video ${err}`, 'runFFProbePipeline');
reject(err);
}
@ -88,14 +85,14 @@ export class VideoTranscodeProcessor {
])
.output(savedEncodedPath)
.on('start', () => {
Logger.log('Start Converting Video', 'mp4Conversion');
this.logger.log('Start Converting Video');
})
.on('error', (error) => {
Logger.error(`Cannot Convert Video ${error}`, 'mp4Conversion');
this.logger.error(`Cannot Convert Video ${error}`);
reject();
})
.on('end', async () => {
Logger.log(`Converting Success ${asset.id}`, 'mp4Conversion');
this.logger.log(`Converting Success ${asset.id}`);
await this.assetRepository.update({ id: asset.id }, { encodedVideoPath: savedEncodedPath });
resolve();
})

6
server/immich-openapi-specs.json
View File

@ -4538,10 +4538,14 @@
"properties": {
"command": {
"$ref": "#/components/schemas/JobCommand"
},
"includeAllAssets": {
"type": "boolean"
}
},
"required": [
"command"
"command",
"includeAllAssets"
]
}
}

6
web/src/api/open-api/api.ts generated
View File

@ -1203,6 +1203,12 @@ export interface JobCommandDto {
* @memberof JobCommandDto
*/
'command': JobCommand;
/**
*
* @type {boolean}
* @memberof JobCommandDto
*/
'includeAllAssets': boolean;
}
/**
*

4
web/src/app.css
View File

@ -101,4 +101,8 @@ input:focus-visible {
display: none;
scrollbar-width: none;
}
.job-play-button {
@apply h-full flex flex-col place-items-center place-content-center px-8 text-gray-600 transition-all hover:bg-immich-primary hover:text-white dark:text-gray-200 dark:hover:bg-immich-dark-primary text-sm dark:hover:text-black w-[120px] gap-2;
}
}

122
web/src/lib/components/admin-page/jobs/job-tile.svelte
View File

@ -1,76 +1,102 @@
<script lang="ts">
import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte';
import SelectionSearch from 'svelte-material-icons/SelectionSearch.svelte';
import Play from 'svelte-material-icons/Play.svelte';
import AllInclusive from 'svelte-material-icons/AllInclusive.svelte';
import { createEventDispatcher } from 'svelte';
import { JobCounts } from '@api';
export let title: string;
export let subtitle: string;
export let buttonTitle = 'Run';
export let jobCounts: JobCounts;
/**
* Show options to run job on all assets of just missing ones
*/
export let showOptions = true;
$: isRunning = jobCounts.active > 0 || jobCounts.waiting > 0;
const dispatch = createEventDispatcher();
const run = (includeAllAssets: boolean) => {
dispatch('click', { includeAllAssets });
};
</script>
<div class="flex border-b pb-5 dark:border-b-immich-dark-gray">
<div class="w-[70%]">
<h1 class="text-immich-primary dark:text-immich-dark-primary text-sm font-semibold">
{title.toUpperCase()}
</h1>
<p class="text-sm mt-1 dark:text-immich-dark-fg">{subtitle}</p>
<p class="text-sm dark:text-immich-dark-fg">
<slot />
</p>
<table class="text-left w-full mt-5">
<!-- table header -->
<thead
class="border rounded-md mb-2 dark:bg-immich-dark-gray dark:border-immich-dark-gray bg-immich-primary/10 flex text-immich-primary dark:text-immich-dark-primary w-full h-12"
>
<tr class="flex w-full place-items-center">
<th class="text-center w-1/3 font-medium text-sm">Status</th>
<th class="text-center w-1/3 font-medium text-sm">Active</th>
<th class="text-center w-1/3 font-medium text-sm">Waiting</th>
</tr>
</thead>
<tbody
class="overflow-y-auto rounded-md w-full max-h-[320px] block border bg-white dark:border-immich-dark-gray dark:bg-immich-dark-gray/75 dark:text-immich-dark-fg"
>
<tr class="text-center flex place-items-center w-full h-[60px]">
<td class="text-sm px-2 w-1/3 text-ellipsis">
{#if jobCounts}
<span>{jobCounts.active > 0 || jobCounts.waiting > 0 ? 'Active' : 'Idle'}</span>
{:else}
<LoadingSpinner />
{/if}
</td>
<td class="flex justify-center text-sm px-2 w-1/3 text-ellipsis">
<div class="flex justify-between rounded-3xl bg-gray-100 dark:bg-immich-dark-gray">
<div id="job-info" class="w-[70%] p-9">
<div class="flex flex-col gap-2">
<div class="text-xl font-semibold text-immich-primary dark:text-immich-dark-primary">
{title.toUpperCase()}
</div>
{#if subtitle.length > 0}
<div class="text-sm dark:text-white">{subtitle}</div>
{/if}
<div class="text-sm dark:text-white"><slot /></div>
<div class="flex w-full mt-4">
<div
class="flex place-items-center justify-between bg-immich-primary dark:bg-immich-dark-primary text-white dark:text-immich-dark-gray w-full rounded-tl-lg rounded-bl-lg py-4 pl-4 pr-6"
>
<p>Active</p>
<p class="text-2xl">
{#if jobCounts.active !== undefined}
{jobCounts.active}
{:else}
<LoadingSpinner />
{/if}
</td>
<td class="flex justify-center text-sm px-2 w-1/3 text-ellipsis">
</p>
</div>
<div
class="flex place-items-center justify-between bg-gray-200 text-immich-dark-bg dark:bg-gray-700 dark:text-immich-gray w-full rounded-tr-lg rounded-br-lg py-4 pr-4 pl-6"
>
<p class="text-2xl">
{#if jobCounts.waiting !== undefined}
{jobCounts.waiting}
{:else}
<LoadingSpinner />
{/if}
</td>
</tr>
</tbody>
</table>
</p>
<p>Waiting</p>
</div>
</div>
</div>
</div>
<div class="w-[30%] flex place-items-center place-content-end">
<button
on:click={() => dispatch('click')}
class="px-6 py-3 text-sm bg-immich-primary dark:bg-immich-dark-primary font-medium rounded-2xl hover:bg-immich-primary/50 transition-all hover:cursor-pointer disabled:cursor-not-allowed shadow-sm text-immich-bg dark:text-immich-dark-gray"
disabled={jobCounts.active > 0 && jobCounts.waiting > 0}
>
{#if jobCounts.active > 0 || jobCounts.waiting > 0}
<div id="job-action" class="flex flex-col">
{#if isRunning}
<button
class="job-play-button bg-gray-300/90 dark:bg-gray-600/90 rounded-br-3xl rounded-tr-3xl disabled:cursor-not-allowed"
disabled
>
<LoadingSpinner />
</button>
{/if}
{#if !isRunning}
{#if showOptions}
<button
class="job-play-button bg-gray-300 dark:bg-gray-600 rounded-tr-3xl"
on:click={() => run(true)}
>
<AllInclusive size="18" /> ALL
</button>
<button
class="job-play-button bg-gray-300/90 dark:bg-gray-600/90 rounded-br-3xl"
on:click={() => run(false)}
>
<SelectionSearch size="18" /> MISSING
</button>
{:else}
{buttonTitle}
<button
class="job-play-button bg-gray-300/90 dark:bg-gray-600/90 rounded-br-3xl rounded-tr-3xl"
on:click={() => run(true)}
>
<Play size="48" />
</button>
{/if}
</button>
{/if}
</div>
</div>

70
web/src/lib/components/admin-page/jobs/jobs-panel.svelte
View File

@ -18,20 +18,28 @@
onMount(async () => {
await load();
timer = setInterval(async () => await load(), 5_000);
timer = setInterval(async () => await load(), 1_000);
});
onDestroy(() => {
clearInterval(timer);
});
const run = async (jobId: JobId, jobName: string, emptyMessage: string) => {
const run = async (
jobId: JobId,
jobName: string,
emptyMessage: string,
includeAllAssets: boolean
) => {
try {
const { data } = await api.jobApi.sendJobCommand(jobId, { command: JobCommand.Start });
const { data } = await api.jobApi.sendJobCommand(jobId, {
command: JobCommand.Start,
includeAllAssets
});
if (data) {
notificationController.show({
message: `Started ${jobName}`,
message: includeAllAssets ? `Started ${jobName} for all assets` : `Started ${jobName}`,
type: NotificationType.Info
});
} else {
@ -43,53 +51,77 @@
};
</script>
<div class="flex flex-col gap-10">
<div class="flex flex-col gap-7">
{#if jobs}
<JobTile
title={'Generate thumbnails'}
subtitle={'Regenerate missing thumbnail (JPEG, WEBP)'}
on:click={() =>
run(JobId.ThumbnailGeneration, 'thumbnail generation', 'No missing thumbnails found')}
subtitle={'Regenerate JPEG and WebP thumbnails'}
on:click={(e) => {
const { includeAllAssets } = e.detail;
run(
JobId.ThumbnailGeneration,
'thumbnail generation',
'No missing thumbnails found',
includeAllAssets
);
}}
jobCounts={jobs[JobId.ThumbnailGeneration]}
/>
<JobTile
title={'Extract EXIF'}
subtitle={'Extract missing EXIF information'}
on:click={() => run(JobId.MetadataExtraction, 'extract EXIF', 'No missing EXIF found')}
title={'EXTRACT METADATA'}
subtitle={'Extract metadata information i.e. GPS, resolution...etc'}
on:click={(e) => {
const { includeAllAssets } = e.detail;
run(JobId.MetadataExtraction, 'extract EXIF', 'No missing EXIF found', includeAllAssets);
}}
jobCounts={jobs[JobId.MetadataExtraction]}
/>
<JobTile
title={'Detect objects'}
subtitle={'Run machine learning process to detect and classify objects'}
on:click={() =>
run(JobId.MachineLearning, 'object detection', 'No missing object detection found')}
on:click={(e) => {
const { includeAllAssets } = e.detail;
run(
JobId.MachineLearning,
'object detection',
'No missing object detection found',
includeAllAssets
);
}}
jobCounts={jobs[JobId.MachineLearning]}
>
Note that some assets may not have any objects detected, this is normal.
Note that some assets may not have any objects detected
</JobTile>
<JobTile
title={'Video transcoding'}
subtitle={'Run video transcoding process to transcode videos not in the desired format'}
on:click={() =>
subtitle={'Transcode videos not in the desired format'}
on:click={(e) => {
const { includeAllAssets } = e.detail;
run(
JobId.VideoConversion,
'video conversion',
'No videos without an encoded version found'
)}
'No videos without an encoded version found',
includeAllAssets
);
}}
jobCounts={jobs[JobId.VideoConversion]}
/>
<JobTile
title={'Storage migration'}
showOptions={false}
subtitle={''}
on:click={() =>
run(
JobId.StorageTemplateMigration,
'storage template migration',
'All files have been migrated to the new storage template'
'All files have been migrated to the new storage template',
false
)}
jobCounts={jobs[JobId.StorageTemplateMigration]}
>