self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-25 10:43:13 +02:00
Code Issues Releases Activity

refactor(server): job repository (#1382)

Browse Source 
* refactor(server): job repository

* refactor: job repository

* chore: generate open-api

* fix: job panel

* Remove incorrect subtitle

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Jason Rasmussen 2023-01-21 23:13:36 -05:00 committed by GitHub
parent f4c90426a5
commit 4cfac47674
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 367 additions and 721 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
mobile/openapi/.openapi-generator/FILES generated
View File

@ -53,7 +53,6 @@ doc/JobCommand.md
doc/JobCommandDto.md
doc/JobCounts.md
doc/JobId.md
doc/JobStatusResponseDto.md
doc/LoginCredentialDto.md
doc/LoginResponseDto.md
doc/LogoutResponseDto.md
@ -162,7 +161,6 @@ lib/model/job_command.dart
lib/model/job_command_dto.dart
lib/model/job_counts.dart
lib/model/job_id.dart
lib/model/job_status_response_dto.dart
lib/model/login_credential_dto.dart
lib/model/login_response_dto.dart
lib/model/logout_response_dto.dart
@ -250,7 +248,6 @@ test/job_command_dto_test.dart
test/job_command_test.dart
test/job_counts_test.dart
test/job_id_test.dart
test/job_status_response_dto_test.dart
test/login_credential_dto_test.dart
test/login_response_dto_test.dart
test/logout_response_dto_test.dart

BIN
mobile/openapi/README.md generated
View File

Binary file not shown.

BIN
mobile/openapi/doc/AllJobStatusResponseDto.md generated
View File

Binary file not shown.

BIN
mobile/openapi/doc/JobApi.md generated
View File

Binary file not shown.

BIN
mobile/openapi/doc/JobStatusResponseDto.md generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/api.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/api/job_api.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/api_client.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/model/all_job_status_response_dto.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/model/job_status_response_dto.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/test/all_job_status_response_dto_test.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/test/job_api_test.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/test/job_status_response_dto_test.dart generated
View File

Binary file not shown.

2
server/apps/immich/src/api-v1/asset/asset.controller.ts
View File

@ -295,7 +295,7 @@ export class AssetController {
deleteAssetList.filter((a) => a.id == res.id && res.status == DeleteAssetStatusEnum.SUCCESS);
});
await this.backgroundTaskService.deleteFileOnDisk(deleteAssetList);
await this.backgroundTaskService.deleteFileOnDisk(deleteAssetList as any[]);
return result;
}

12
server/apps/immich/src/api-v1/asset/asset.service.spec.ts
View File

@ -9,11 +9,11 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
import { DownloadService } from '../../modules/download/download.service';
import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
import { IAssetUploadedJob, IVideoTranscodeJob } from '@app/domain';
import { Queue } from 'bull';
import { IAlbumRepository } from '../album/album-repository';
import { StorageService } from '@app/storage';
import { ISharedLinkRepository } from '../share/shared-link.repository';
import { IJobRepository } from '@app/domain';
import { newJobRepositoryMock } from '@app/domain/../test';
describe('AssetService', () => {
let sui: AssetService;
@ -22,10 +22,9 @@ describe('AssetService', () => {
let albumRepositoryMock: jest.Mocked<IAlbumRepository>;
let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
let backgroundTaskServiceMock: jest.Mocked<BackgroundTaskService>;
let assetUploadedQueueMock: jest.Mocked<Queue<IAssetUploadedJob>>;
let videoConversionQueueMock: jest.Mocked<Queue<IVideoTranscodeJob>>;
let storageSeriveMock: jest.Mocked<StorageService>;
let sharedLinkRepositoryMock: jest.Mocked<ISharedLinkRepository>;
let jobMock: jest.Mocked<IJobRepository>;
const authUser: AuthUserDto = Object.freeze({
id: 'user_id_1',
email: 'auth@test.com',
@ -148,16 +147,17 @@ describe('AssetService', () => {
getByIdAndUserId: jest.fn(),
};
jobMock = newJobRepositoryMock();
sui = new AssetService(
assetRepositoryMock,
albumRepositoryMock,
a,
backgroundTaskServiceMock,
assetUploadedQueueMock,
videoConversionQueueMock,
downloadServiceMock as DownloadService,
storageSeriveMock,
sharedLinkRepositoryMock,
jobMock,
);
});

27
server/apps/immich/src/api-v1/asset/asset.service.ts
View File

@ -43,9 +43,7 @@ import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-as
import { UpdateAssetDto } from './dto/update-asset.dto';
import { AssetFileUploadResponseDto } from './response-dto/asset-file-upload-response.dto';
import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
import { IAssetUploadedJob, IVideoTranscodeJob, JobName, QueueName } from '@app/domain';
import { InjectQueue } from '@nestjs/bull';
import { Queue } from 'bull';
import { IJobRepository, JobName } from '@app/domain';
import { DownloadService } from '../../modules/download/download.service';
import { DownloadDto } from './dto/download-library.dto';
import { IAlbumRepository } from '../album/album-repository';
@ -66,24 +64,14 @@ export class AssetService {
constructor(
@Inject(IAssetRepository) private _assetRepository: IAssetRepository,
@Inject(IAlbumRepository) private _albumRepository: IAlbumRepository,
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
private backgroundTaskService: BackgroundTaskService,
@InjectQueue(QueueName.ASSET_UPLOADED)
private assetUploadedQueue: Queue<IAssetUploadedJob>,
@InjectQueue(QueueName.VIDEO_CONVERSION)
private videoConversionQueue: Queue<IVideoTranscodeJob>,
private downloadService: DownloadService,
private storageService: StorageService,
@Inject(ISharedLinkRepository) sharedLinkRepository: ISharedLinkRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
) {
this.shareCore = new ShareCore(sharedLinkRepository);
}
@ -122,7 +110,7 @@ export class AssetService {
await this.storageService.moveAsset(livePhotoAssetEntity, originalAssetData.originalname);
await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset: livePhotoAssetEntity });
await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset: livePhotoAssetEntity } });
}
const assetEntity = await this.createUserAsset(
@ -146,11 +134,10 @@ export class AssetService {
const movedAsset = await this.storageService.moveAsset(assetEntity, originalAssetData.originalname);
await this.assetUploadedQueue.add(
JobName.ASSET_UPLOADED,
{ asset: movedAsset, fileName: originalAssetData.originalname },
{ jobId: movedAsset.id },
);
await this.jobRepository.add({
name: JobName.ASSET_UPLOADED,
data: { asset: movedAsset, fileName: originalAssetData.originalname },
});
return new AssetFileUploadResponseDto(movedAsset.id);
} catch (err) {

15
server/apps/immich/src/api-v1/job/job.controller.ts
View File

@ -1,11 +1,9 @@
import { Controller, Get, Body, ValidationPipe, Put, Param } from '@nestjs/common';
import { JobService } from './job.service';
import { Body, Controller, Get, Param, Put, ValidationPipe } from '@nestjs/common';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
import { Authenticated } from '../../decorators/authenticated.decorator';
import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto';
import { GetJobDto } from './dto/get-job.dto';
import { JobStatusResponseDto } from './response-dto/job-status-response.dto';
import { JobService } from './job.service';
import { JobCommandDto } from './dto/job-command.dto';
@Authenticated({ admin: true })
@ -20,21 +18,16 @@ export class JobController {
return this.jobService.getAllJobsStatus();
}
@Get('/:jobId')
getJobStatus(@Param(ValidationPipe) params: GetJobDto): Promise<JobStatusResponseDto> {
return this.jobService.getJobStatus(params);
}
@Put('/:jobId')
async sendJobCommand(
@Param(ValidationPipe) params: GetJobDto,
@Body(ValidationPipe) body: JobCommandDto,
): Promise<number> {
if (body.command === 'start') {
return await this.jobService.startJob(params);
return await this.jobService.start(params.jobId);
}
if (body.command === 'stop') {
return await this.jobService.stopJob(params);
return await this.jobService.stop(params.jobId);
}
return 0;
}

277
server/apps/immich/src/api-v1/job/job.service.ts
View File

@ -1,217 +1,118 @@
import {
IMachineLearningJob,
IMetadataExtractionJob,
IThumbnailGenerationJob,
IVideoTranscodeJob,
JobName,
QueueName,
} from '@app/domain';
import { InjectQueue } from '@nestjs/bull';
import { Queue } from 'bull';
import { JobName, IJobRepository, QueueName } from '@app/domain';
import { BadRequestException, Inject, Injectable } from '@nestjs/common';
import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto';
import { IAssetRepository } from '../asset/asset-repository';
import { AssetType } from '@app/infra';
import { GetJobDto, JobId } from './dto/get-job.dto';
import { JobStatusResponseDto } from './response-dto/job-status-response.dto';
import { StorageService } from '@app/storage';
import { JobId } from './dto/get-job.dto';
import { MACHINE_LEARNING_ENABLED } from '@app/common';
const jobIds = Object.values(JobId) as JobId[];
@Injectable()
export class JobService {
constructor(
@InjectQueue(QueueName.THUMBNAIL_GENERATION)
private thumbnailGeneratorQueue: Queue<IThumbnailGenerationJob>,
@InjectQueue(QueueName.METADATA_EXTRACTION)
private metadataExtractionQueue: Queue<IMetadataExtractionJob>,
@InjectQueue(QueueName.VIDEO_CONVERSION)
private videoConversionQueue: Queue<IVideoTranscodeJob>,
@InjectQueue(QueueName.MACHINE_LEARNING)
private machineLearningQueue: Queue<IMachineLearningJob>,
@InjectQueue(QueueName.CONFIG)
private configQueue: Queue,
@Inject(IAssetRepository)
private _assetRepository: IAssetRepository,
private storageService: StorageService,
@Inject(IAssetRepository) private _assetRepository: IAssetRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
) {
this.thumbnailGeneratorQueue.empty();
this.metadataExtractionQueue.empty();
this.videoConversionQueue.empty();
this.configQueue.empty();
for (const jobId of jobIds) {
this.jobRepository.empty(this.asQueueName(jobId));
}
}
async startJob(jobDto: GetJobDto): Promise<number> {
switch (jobDto.jobId) {
case JobId.THUMBNAIL_GENERATION:
return this.runThumbnailGenerationJob();
case JobId.METADATA_EXTRACTION:
return this.runMetadataExtractionJob();
case JobId.VIDEO_CONVERSION:
return this.runVideoConversionJob();
case JobId.MACHINE_LEARNING:
return this.runMachineLearningPipeline();
case JobId.STORAGE_TEMPLATE_MIGRATION:
return this.runStorageMigration();
default:
throw new BadRequestException('Invalid job id');
}
start(jobId: JobId): Promise<number> {
return this.run(this.asQueueName(jobId));
}
async stop(jobId: JobId): Promise<number> {
await this.jobRepository.empty(this.asQueueName(jobId));
return 0;
}
async getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
const thumbnailGeneratorJobCount = await this.thumbnailGeneratorQueue.getJobCounts();
const metadataExtractionJobCount = await this.metadataExtractionQueue.getJobCounts();
const videoConversionJobCount = await this.videoConversionQueue.getJobCounts();
const machineLearningJobCount = await this.machineLearningQueue.getJobCounts();
const storageMigrationJobCount = await this.configQueue.getJobCounts();
const response = new AllJobStatusResponseDto();
response.isThumbnailGenerationActive = Boolean(thumbnailGeneratorJobCount.waiting);
response.thumbnailGenerationQueueCount = thumbnailGeneratorJobCount;
response.isMetadataExtractionActive = Boolean(metadataExtractionJobCount.waiting);
response.metadataExtractionQueueCount = metadataExtractionJobCount;
response.isVideoConversionActive = Boolean(videoConversionJobCount.waiting);
response.videoConversionQueueCount = videoConversionJobCount;
response.isMachineLearningActive = Boolean(machineLearningJobCount.waiting);
response.machineLearningQueueCount = machineLearningJobCount;
response.isStorageMigrationActive = Boolean(storageMigrationJobCount.active);
response.storageMigrationQueueCount = storageMigrationJobCount;
for (const jobId of jobIds) {
response[jobId] = await this.jobRepository.getJobCounts(this.asQueueName(jobId));
}
return response;
}
async getJobStatus(query: GetJobDto): Promise<JobStatusResponseDto> {
const response = new JobStatusResponseDto();
if (query.jobId === JobId.THUMBNAIL_GENERATION) {
response.isActive = Boolean((await this.thumbnailGeneratorQueue.getJobCounts()).waiting);
response.queueCount = await this.thumbnailGeneratorQueue.getJobCounts();
private async run(name: QueueName): Promise<number> {
const isActive = await this.jobRepository.isActive(name);
if (isActive) {
throw new BadRequestException(`Job is already running`);
}
if (query.jobId === JobId.METADATA_EXTRACTION) {
response.isActive = Boolean((await this.metadataExtractionQueue.getJobCounts()).waiting);
response.queueCount = await this.metadataExtractionQueue.getJobCounts();
}
switch (name) {
case QueueName.VIDEO_CONVERSION: {
const assets = await this._assetRepository.getAssetWithNoEncodedVideo();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
if (query.jobId === JobId.VIDEO_CONVERSION) {
response.isActive = Boolean((await this.videoConversionQueue.getJobCounts()).waiting);
response.queueCount = await this.videoConversionQueue.getJobCounts();
}
if (query.jobId === JobId.STORAGE_TEMPLATE_MIGRATION) {
response.isActive = Boolean((await this.configQueue.getJobCounts()).waiting);
response.queueCount = await this.configQueue.getJobCounts();
}
return response;
}
async stopJob(query: GetJobDto): Promise<number> {
switch (query.jobId) {
case JobId.THUMBNAIL_GENERATION:
this.thumbnailGeneratorQueue.empty();
return 0;
case JobId.METADATA_EXTRACTION:
this.metadataExtractionQueue.empty();
return 0;
case JobId.VIDEO_CONVERSION:
this.videoConversionQueue.empty();
return 0;
case JobId.MACHINE_LEARNING:
this.machineLearningQueue.empty();
return 0;
case JobId.STORAGE_TEMPLATE_MIGRATION:
this.configQueue.empty();
return 0;
default:
throw new BadRequestException('Invalid job id');
}
}
private async runThumbnailGenerationJob(): Promise<number> {
const jobCount = await this.thumbnailGeneratorQueue.getJobCounts();
if (jobCount.waiting > 0) {
throw new BadRequestException('Thumbnail generation job is already running');
}
const assetsWithNoThumbnail = await this._assetRepository.getAssetWithNoThumbnail();
for (const asset of assetsWithNoThumbnail) {
await this.thumbnailGeneratorQueue.add(JobName.GENERATE_JPEG_THUMBNAIL, { asset });
}
return assetsWithNoThumbnail.length;
}
private async runMetadataExtractionJob(): Promise<number> {
const jobCount = await this.metadataExtractionQueue.getJobCounts();
if (jobCount.waiting > 0) {
throw new BadRequestException('Metadata extraction job is already running');
}
const assetsWithNoExif = await this._assetRepository.getAssetWithNoEXIF();
for (const asset of assetsWithNoExif) {
if (asset.type === AssetType.VIDEO) {
await this.metadataExtractionQueue.add(JobName.EXTRACT_VIDEO_METADATA, { asset, fileName: asset.id });
} else {
await this.metadataExtractionQueue.add(JobName.EXIF_EXTRACTION, { asset, fileName: asset.id });
return assets.length;
}
case QueueName.CONFIG:
await this.jobRepository.add({ name: JobName.TEMPLATE_MIGRATION });
return 1;
case QueueName.MACHINE_LEARNING: {
if (!MACHINE_LEARNING_ENABLED) {
throw new BadRequestException('Machine learning is not enabled.');
}
const assets = await this._assetRepository.getAssetWithNoSmartInfo();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.IMAGE_TAGGING, data: { asset } });
await this.jobRepository.add({ name: JobName.OBJECT_DETECTION, data: { asset } });
}
return assets.length;
}
case QueueName.METADATA_EXTRACTION: {
const assets = await this._assetRepository.getAssetWithNoEXIF();
for (const asset of assets) {
if (asset.type === AssetType.VIDEO) {
await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } });
} else {
await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } });
}
}
return assets.length;
}
case QueueName.THUMBNAIL_GENERATION: {
const assets = await this._assetRepository.getAssetWithNoThumbnail();
for (const asset of assets) {
await this.jobRepository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
}
return assets.length;
}
default:
return 0;
}
return assetsWithNoExif.length;
}
private async runMachineLearningPipeline(): Promise<number> {
if (!MACHINE_LEARNING_ENABLED) {
throw new BadRequestException('Machine learning is not enabled.');
private asQueueName(jobId: JobId) {
switch (jobId) {
case JobId.THUMBNAIL_GENERATION:
return QueueName.THUMBNAIL_GENERATION;
case JobId.METADATA_EXTRACTION:
return QueueName.METADATA_EXTRACTION;
case JobId.VIDEO_CONVERSION:
return QueueName.VIDEO_CONVERSION;
case JobId.STORAGE_TEMPLATE_MIGRATION:
return QueueName.CONFIG;
case JobId.MACHINE_LEARNING:
return QueueName.MACHINE_LEARNING;
default:
throw new BadRequestException(`Invalid job id: ${jobId}`);
}
const jobCount = await this.machineLearningQueue.getJobCounts();
if (jobCount.waiting > 0) {
throw new BadRequestException('Metadata extraction job is already running');
}
const assetWithNoSmartInfo = await this._assetRepository.getAssetWithNoSmartInfo();
for (const asset of assetWithNoSmartInfo) {
await this.machineLearningQueue.add(JobName.IMAGE_TAGGING, { asset });
await this.machineLearningQueue.add(JobName.OBJECT_DETECTION, { asset });
}
return assetWithNoSmartInfo.length;
}
private async runVideoConversionJob(): Promise<number> {
const jobCount = await this.videoConversionQueue.getJobCounts();
if (jobCount.waiting > 0) {
throw new BadRequestException('Video conversion job is already running');
}
const assetsWithNoConvertedVideo = await this._assetRepository.getAssetWithNoEncodedVideo();
for (const asset of assetsWithNoConvertedVideo) {
await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset });
}
return assetsWithNoConvertedVideo.length;
}
async runStorageMigration() {
const jobCount = await this.configQueue.getJobCounts();
if (jobCount.active > 0) {
throw new BadRequestException('Storage migration job is already running');
}
await this.configQueue.add(JobName.TEMPLATE_MIGRATION, {});
return 1;
}
}

38
server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts
View File

@ -1,4 +1,5 @@
import { ApiProperty } from '@nestjs/swagger';
import { JobId } from '../dto/get-job.dto';
export class JobCounts {
@ApiProperty({ type: 'integer' })
@ -12,35 +13,20 @@ export class JobCounts {
@ApiProperty({ type: 'integer' })
waiting!: number;
}
export class AllJobStatusResponseDto {
isThumbnailGenerationActive!: boolean;
isMetadataExtractionActive!: boolean;
isVideoConversionActive!: boolean;
isMachineLearningActive!: boolean;
isStorageMigrationActive!: boolean;
@ApiProperty({ type: JobCounts })
[JobId.THUMBNAIL_GENERATION]!: JobCounts;
@ApiProperty({
type: JobCounts,
})
thumbnailGenerationQueueCount!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.METADATA_EXTRACTION]!: JobCounts;
@ApiProperty({
type: JobCounts,
})
metadataExtractionQueueCount!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.VIDEO_CONVERSION]!: JobCounts;
@ApiProperty({
type: JobCounts,
})
videoConversionQueueCount!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.MACHINE_LEARNING]!: JobCounts;
@ApiProperty({
type: JobCounts,
})
machineLearningQueueCount!: JobCounts;
@ApiProperty({
type: JobCounts,
})
storageMigrationQueueCount!: JobCounts;
@ApiProperty({ type: JobCounts })
[JobId.STORAGE_TEMPLATE_MIGRATION]!: JobCounts;
}

6
server/apps/immich/src/api-v1/job/response-dto/job-status-response.dto.ts
View File

@ -1,6 +0,0 @@
import Bull from 'bull';
export class JobStatusResponseDto {
isActive!: boolean;
queueCount!: Bull.JobCounts;
}

5
server/apps/immich/src/modules/background-task/background-task.module.ts
View File

@ -1,12 +1,9 @@
import { BullModule } from '@nestjs/bull';
import { Module } from '@nestjs/common';
import { QueueName } from '@app/domain';
import { BackgroundTaskProcessor } from './background-task.processor';
import { BackgroundTaskService } from './background-task.service';
@Module({
imports: [BullModule.registerQueue({ name: QueueName.BACKGROUND_TASK })],
providers: [BackgroundTaskService, BackgroundTaskProcessor],
exports: [BackgroundTaskService, BullModule],
exports: [BackgroundTaskService],
})
export class BackgroundTaskModule {}

4
server/apps/immich/src/modules/background-task/background-task.processor.ts
View File

@ -2,12 +2,12 @@ import { assetUtils } from '@app/common/utils';
import { Process, Processor } from '@nestjs/bull';
import { Job } from 'bull';
import { JobName, QueueName } from '@app/domain';
import { AssetResponseDto } from '../../api-v1/asset/response-dto/asset-response.dto';
import { AssetEntity } from '@app/infra';
@Processor(QueueName.BACKGROUND_TASK)
export class BackgroundTaskProcessor {
@Process(JobName.DELETE_FILE_ON_DISK)
async deleteFileOnDisk(job: Job<{ assets: AssetResponseDto[] }>) {
async deleteFileOnDisk(job: Job<{ assets: AssetEntity[] }>) {
const { assets } = job.data;
for (const asset of assets) {

17
server/apps/immich/src/modules/background-task/background-task.service.ts
View File

@ -1,17 +1,12 @@
import { InjectQueue } from '@nestjs/bull/dist/decorators';
import { Injectable } from '@nestjs/common';
import { Queue } from 'bull';
import { JobName, QueueName } from '@app/domain';
import { AssetResponseDto } from '../../api-v1/asset/response-dto/asset-response.dto';
import { IJobRepository, JobName } from '@app/domain';
import { AssetEntity } from '@app/infra';
import { Inject, Injectable } from '@nestjs/common';
@Injectable()
export class BackgroundTaskService {
constructor(
@InjectQueue(QueueName.BACKGROUND_TASK)
private backgroundTaskQueue: Queue,
) {}
constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {}
async deleteFileOnDisk(assets: AssetResponseDto[]) {
await this.backgroundTaskQueue.add(JobName.DELETE_FILE_ON_DISK, { assets });
async deleteFileOnDisk(assets: AssetEntity[]) {
await this.jobRepository.add({ name: JobName.DELETE_FILE_ON_DISK, data: { assets } });
}
}

37
server/apps/immich/src/modules/schedule-tasks/schedule-tasks.service.ts
View File

@ -1,14 +1,11 @@
import { Injectable, Logger } from '@nestjs/common';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { InjectRepository } from '@nestjs/typeorm';
import { IsNull, Not, Repository } from 'typeorm';
import { AssetEntity, AssetType, ExifEntity, UserEntity } from '@app/infra';
import { InjectQueue } from '@nestjs/bull';
import { Queue } from 'bull';
import { IMetadataExtractionJob, IVideoTranscodeJob, QueueName, JobName } from '@app/domain';
import { ConfigService } from '@nestjs/config';
import { IUserDeletionJob } from '@app/domain';
import { userUtils } from '@app/common';
import { IJobRepository, JobName } from '@app/domain';
@Injectable()
export class ScheduleTasksService {
@ -22,17 +19,7 @@ export class ScheduleTasksService {
@InjectRepository(ExifEntity)
private exifRepository: Repository<ExifEntity>,
@InjectQueue(QueueName.THUMBNAIL_GENERATION)
private thumbnailGeneratorQueue: Queue,
@InjectQueue(QueueName.VIDEO_CONVERSION)
private videoConversionQueue: Queue<IVideoTranscodeJob>,
@InjectQueue(QueueName.METADATA_EXTRACTION)
private metadataExtractionQueue: Queue<IMetadataExtractionJob>,
@InjectQueue(QueueName.USER_DELETION)
private userDeletionQueue: Queue<IUserDeletionJob>,
@Inject(IJobRepository) private jobRepository: IJobRepository,
private configService: ConfigService,
) {}
@ -51,7 +38,7 @@ export class ScheduleTasksService {
}
for (const asset of assets) {
await this.thumbnailGeneratorQueue.add(JobName.GENERATE_WEBP_THUMBNAIL, { asset: asset });
await this.jobRepository.add({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { asset } });
}
}
@ -69,7 +56,7 @@ export class ScheduleTasksService {
});
for (const asset of assets) {
await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset });
await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
}
}
@ -87,11 +74,11 @@ export class ScheduleTasksService {
});
for (const exif of exifInfo) {
await this.metadataExtractionQueue.add(
JobName.REVERSE_GEOCODING,
await this.jobRepository.add({
name: JobName.REVERSE_GEOCODING,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
{ exifId: exif.id, latitude: exif.latitude!, longitude: exif.longitude! },
);
data: { exifId: exif.id, latitude: exif.latitude!, longitude: exif.longitude! },
});
}
}
}
@ -106,9 +93,9 @@ export class ScheduleTasksService {
for (const asset of exifAssets) {
if (asset.type === AssetType.VIDEO) {
await this.metadataExtractionQueue.add(JobName.EXTRACT_VIDEO_METADATA, { asset, fileName: asset.id });
await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } });
} else {
await this.metadataExtractionQueue.add(JobName.EXIF_EXTRACTION, { asset, fileName: asset.id });
await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } });
}
}
}
@ -118,7 +105,7 @@ export class ScheduleTasksService {
const usersToDelete = await this.userRepository.find({ withDeleted: true, where: { deletedAt: Not(IsNull()) } });
for (const user of usersToDelete) {
if (userUtils.isReadyForDeletion(user)) {
await this.userDeletionQueue.add(JobName.USER_DELETION, { user });
await this.jobRepository.add({ name: JobName.USER_DELETION, data: { user } });
}
}
}

17
server/apps/microservices/src/microservices.service.ts
View File

@ -1,17 +1,16 @@
import { QueueName } from '@app/domain';
import { InjectQueue } from '@nestjs/bull';
import { Injectable, OnModuleInit } from '@nestjs/common';
import { Queue } from 'bull';
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { IJobRepository, JobName } from '@app/domain';
const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(() => resolve(), ms));
@Injectable()
export class MicroservicesService implements OnModuleInit {
constructor(
@InjectQueue(QueueName.CHECKSUM_GENERATION)
private generateChecksumQueue: Queue,
) {}
constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {}
async onModuleInit() {
// wait for migration
await this.generateChecksumQueue.add({}, { delay: 10000 });
await sleep(10_000);
await this.jobRepository.add({ name: JobName.CHECKSUM_GENERATION });
}
}

4
server/apps/microservices/src/processors/generate-checksum.processor.ts
View File

@ -1,5 +1,5 @@
import { AssetEntity } from '@app/infra';
import { QueueName } from '@app/domain';
import { JobName, QueueName } from '@app/domain';
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
@ -15,7 +15,7 @@ export class GenerateChecksumProcessor {
private assetRepository: Repository<AssetEntity>,
) {}
@Process()
@Process(JobName.CHECKSUM_GENERATION)
async generateChecksum() {
const pageSize = 200;
let hasNext = true;

89
server/immich-openapi-specs.json
View File

@ -2721,40 +2721,6 @@
}
},
"/jobs/{jobId}": {
"get": {
"operationId": "getJobStatus",
"description": "",
"parameters": [
{
"name": "jobId",
"required": true,
"in": "path",
"schema": {
"$ref": "#/components/schemas/JobId"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/JobStatusResponseDto"
}
}
}
}
},
"tags": [
"Job"
],
"security": [
{
"bearer": []
}
]
},
"put": {
"operationId": "sendJobCommand",
"description": "",
@ -4569,48 +4535,28 @@
"AllJobStatusResponseDto": {
"type": "object",
"properties": {
"thumbnailGenerationQueueCount": {
"thumbnail-generation": {
"$ref": "#/components/schemas/JobCounts"
},
"metadataExtractionQueueCount": {
"metadata-extraction": {
"$ref": "#/components/schemas/JobCounts"
},
"videoConversionQueueCount": {
"video-conversion": {
"$ref": "#/components/schemas/JobCounts"
},
"machineLearningQueueCount": {
"machine-learning": {
"$ref": "#/components/schemas/JobCounts"
},
"storageMigrationQueueCount": {
"storage-template-migration": {
"$ref": "#/components/schemas/JobCounts"
},
"isThumbnailGenerationActive": {
"type": "boolean"
},
"isMetadataExtractionActive": {
"type": "boolean"
},
"isVideoConversionActive": {
"type": "boolean"
},
"isMachineLearningActive": {
"type": "boolean"
},
"isStorageMigrationActive": {
"type": "boolean"
}
},
"required": [
"thumbnailGenerationQueueCount",
"metadataExtractionQueueCount",
"videoConversionQueueCount",
"machineLearningQueueCount",
"storageMigrationQueueCount",
"isThumbnailGenerationActive",
"isMetadataExtractionActive",
"isVideoConversionActive",
"isMachineLearningActive",
"isStorageMigrationActive"
"thumbnail-generation",
"metadata-extraction",
"video-conversion",
"machine-learning",
"storage-template-migration"
]
},
"JobId": {
@ -4623,21 +4569,6 @@
"storage-template-migration"
]
},
"JobStatusResponseDto": {
"type": "object",
"properties": {
"isActive": {
"type": "boolean"
},
"queueCount": {
"type": "object"
}
},
"required": [
"isActive",
"queueCount"
]
},
"JobCommand": {
"type": "string",
"enum": [

1
server/libs/domain/src/job/job.constants.ts
View File

@ -24,4 +24,5 @@ export enum JobName {
OBJECT_DETECTION = 'detect-object',
IMAGE_TAGGING = 'tag-image',
DELETE_FILE_ON_DISK = 'delete-file-on-disk',
CHECKSUM_GENERATION = 'checksum-generation',
}

16
server/libs/domain/src/job/job.repository.ts
View File

@ -6,10 +6,19 @@ import {
IVideoConversionProcessor,
IReverseGeocodingProcessor,
IUserDeletionJob,
IVideoLengthExtractionProcessor,
JpegGeneratorProcessor,
WebpGeneratorProcessor,
} from './interfaces';
import { JobName } from './job.constants';
import { JobName, QueueName } from './job.constants';
export interface JobCounts {
active: number;
completed: number;
failed: number;
delayed: number;
waiting: number;
}
export type JobItem =
| { name: JobName.ASSET_UPLOADED; data: IAssetUploadedJob }
@ -21,6 +30,8 @@ export type JobItem =
| { name: JobName.USER_DELETION; data: IUserDeletionJob }
| { name: JobName.TEMPLATE_MIGRATION }
| { name: JobName.CONFIG_CHANGE }
| { name: JobName.CHECKSUM_GENERATION }
| { name: JobName.EXTRACT_VIDEO_METADATA; data: IVideoLengthExtractionProcessor }
| { name: JobName.OBJECT_DETECTION; data: IMachineLearningJob }
| { name: JobName.IMAGE_TAGGING; data: IMachineLearningJob }
| { name: JobName.DELETE_FILE_ON_DISK; data: IDeleteFileOnDiskJob };
@ -28,5 +39,8 @@ export type JobItem =
export const IJobRepository = 'IJobRepository';
export interface IJobRepository {
empty(name: QueueName): Promise<void>;
add(item: JobItem): Promise<void>;
isActive(name: QueueName): Promise<boolean>;
getJobCounts(name: QueueName): Promise<JobCounts>;
}

3
server/libs/domain/test/job.repository.mock.ts
View File

@ -2,6 +2,9 @@ import { IJobRepository } from '../src';
export const newJobRepositoryMock = (): jest.Mocked<IJobRepository> => {
return {
empty: jest.fn(),
add: jest.fn().mockImplementation(() => Promise.resolve()),
isActive: jest.fn(),
getJobCounts: jest.fn(),
};
};

99
server/libs/infra/src/job/job.repository.ts
View File

@ -1,21 +1,110 @@
import { IJobRepository, JobItem, JobName, QueueName } from '@app/domain';
import {
IAssetUploadedJob,
IJobRepository,
IMachineLearningJob,
IMetadataExtractionJob,
IUserDeletionJob,
IVideoTranscodeJob,
JobCounts,
JobItem,
JobName,
QueueName,
} from '@app/domain';
import { InjectQueue } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { BadRequestException, Logger } from '@nestjs/common';
import { Queue } from 'bull';
export class JobRepository implements IJobRepository {
private logger = new Logger(JobRepository.name);
constructor(@InjectQueue(QueueName.CONFIG) private configQueue: Queue) {}
constructor(
@InjectQueue(QueueName.ASSET_UPLOADED) private assetUploaded: Queue<IAssetUploadedJob>,
@InjectQueue(QueueName.BACKGROUND_TASK) private backgroundTask: Queue,
@InjectQueue(QueueName.CHECKSUM_GENERATION) private generateChecksum: Queue,
@InjectQueue(QueueName.MACHINE_LEARNING) private machineLearning: Queue<IMachineLearningJob>,
@InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue<IMetadataExtractionJob>,
@InjectQueue(QueueName.CONFIG) private storageMigration: Queue,
@InjectQueue(QueueName.THUMBNAIL_GENERATION) private thumbnail: Queue,
@InjectQueue(QueueName.USER_DELETION) private userDeletion: Queue<IUserDeletionJob>,
@InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue<IVideoTranscodeJob>,
) {}
async isActive(name: QueueName): Promise<boolean> {
const counts = await this.getJobCounts(name);
return !!counts.active;
}
empty(name: QueueName) {
return this.getQueue(name).empty();
}
getJobCounts(name: QueueName): Promise<JobCounts> {
return this.getQueue(name).getJobCounts();
}
async add(item: JobItem): Promise<void> {
switch (item.name) {
case JobName.CONFIG_CHANGE:
await this.configQueue.add(JobName.CONFIG_CHANGE, {});
case JobName.ASSET_UPLOADED:
await this.assetUploaded.add(item.name, item.data, { jobId: item.data.asset.id });
break;
case JobName.DELETE_FILE_ON_DISK:
await this.backgroundTask.add(item.name, item.data);
break;
case JobName.CHECKSUM_GENERATION:
await this.generateChecksum.add(item.name, {});
break;
case JobName.OBJECT_DETECTION:
case JobName.IMAGE_TAGGING:
await this.machineLearning.add(item.name, item.data);
break;
case JobName.EXIF_EXTRACTION:
case JobName.EXTRACT_VIDEO_METADATA:
case JobName.REVERSE_GEOCODING:
await this.metadataExtraction.add(item.name, item.data);
break;
case JobName.TEMPLATE_MIGRATION:
case JobName.CONFIG_CHANGE:
await this.storageMigration.add(item.name, {});
break;
case JobName.GENERATE_JPEG_THUMBNAIL:
case JobName.GENERATE_WEBP_THUMBNAIL:
await this.thumbnail.add(item.name, item.data);
break;
case JobName.USER_DELETION:
await this.userDeletion.add(item.name, item.data);
break;
case JobName.VIDEO_CONVERSION:
await this.videoTranscode.add(item.name, item.data);
break;
default:
// TODO inject remaining queues and map job to queue
this.logger.error('Invalid job', item);
}
}
private getQueue(name: QueueName) {
switch (name) {
case QueueName.THUMBNAIL_GENERATION:
return this.thumbnail;
case QueueName.METADATA_EXTRACTION:
return this.metadataExtraction;
case QueueName.VIDEO_CONVERSION:
return this.videoTranscode;
case QueueName.CONFIG:
return this.storageMigration;
case QueueName.MACHINE_LEARNING:
return this.machineLearning;
default:
throw new BadRequestException('Invalid job name');
}
}
}

145
web/src/api/open-api/api.ts generated
View File

@ -13,24 +13,13 @@
*/
import {Configuration} from './configuration';
import globalAxios, {AxiosInstance, AxiosPromise, AxiosRequestConfig} from 'axios';
import { Configuration } from './configuration';
import globalAxios, { AxiosPromise, AxiosInstance, AxiosRequestConfig } from 'axios';
// Some imports not used depending on template conditions
// @ts-ignore
import {
assertParamExists,
createRequestFunction,
DUMMY_BASE_URL,
serializeDataIfNeeded,
setApiKeyToObject,
setBasicAuthToObject,
setBearerAuthToObject,
setOAuthToObject,
setSearchParams,
toPathString
} from './common';
import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction } from './common';
// @ts-ignore
import {BASE_PATH, BaseAPI, COLLECTION_FORMATS, RequestArgs, RequiredError} from './base';
import { BASE_PATH, COLLECTION_FORMATS, RequestArgs, BaseAPI, RequiredError } from './base';
/**
*
@ -293,61 +282,31 @@ export interface AllJobStatusResponseDto {
* @type {JobCounts}
* @memberof AllJobStatusResponseDto
*/
'thumbnailGenerationQueueCount': JobCounts;
'thumbnail-generation': JobCounts;
/**
*
* @type {JobCounts}
* @memberof AllJobStatusResponseDto
*/
'metadataExtractionQueueCount': JobCounts;
'metadata-extraction': JobCounts;
/**
*
* @type {JobCounts}
* @memberof AllJobStatusResponseDto
*/
'videoConversionQueueCount': JobCounts;
'video-conversion': JobCounts;
/**
*
* @type {JobCounts}
* @memberof AllJobStatusResponseDto
*/
'machineLearningQueueCount': JobCounts;
'machine-learning': JobCounts;
/**
*
* @type {JobCounts}
* @memberof AllJobStatusResponseDto
*/
'storageMigrationQueueCount': JobCounts;
/**
*
* @type {boolean}
* @memberof AllJobStatusResponseDto
*/
'isThumbnailGenerationActive': boolean;
/**
*
* @type {boolean}
* @memberof AllJobStatusResponseDto
*/
'isMetadataExtractionActive': boolean;
/**
*
* @type {boolean}
* @memberof AllJobStatusResponseDto
*/
'isVideoConversionActive': boolean;
/**
*
* @type {boolean}
* @memberof AllJobStatusResponseDto
*/
'isMachineLearningActive': boolean;
/**
*
* @type {boolean}
* @memberof AllJobStatusResponseDto
*/
'isStorageMigrationActive': boolean;
'storage-template-migration': JobCounts;
}
/**
*
@ -1269,25 +1228,6 @@ export const JobId = {
export type JobId = typeof JobId[keyof typeof JobId];
/**
*
* @export
* @interface JobStatusResponseDto
*/
export interface JobStatusResponseDto {
/**
*
* @type {boolean}
* @memberof JobStatusResponseDto
*/
'isActive': boolean;
/**
*
* @type {object}
* @memberof JobStatusResponseDto
*/
'queueCount': object;
}
/**
*
* @export
@ -5772,43 +5712,6 @@ export const JobApiAxiosParamCreator = function (configuration?: Configuration)
setSearchParams(localVarUrlObj, localVarQueryParameter);
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
return {
url: toPathString(localVarUrlObj),
options: localVarRequestOptions,
};
},
/**
*
* @param {JobId} jobId
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
getJobStatus: async (jobId: JobId, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
// verify required parameter 'jobId' is not null or undefined
assertParamExists('getJobStatus', 'jobId', jobId)
const localVarPath = `/jobs/{jobId}`
.replace(`{${"jobId"}}`, encodeURIComponent(String(jobId)));
// use dummy base URL string because the URL constructor only accepts absolute URLs.
const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
let baseOptions;
if (configuration) {
baseOptions = configuration.baseOptions;
}
const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options};
const localVarHeaderParameter = {} as any;
const localVarQueryParameter = {} as any;
// authentication bearer required
// http bearer authentication required
await setBearerAuthToObject(localVarHeaderParameter, configuration)
setSearchParams(localVarUrlObj, localVarQueryParameter);
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
@ -5880,16 +5783,6 @@ export const JobApiFp = function(configuration?: Configuration) {
const localVarAxiosArgs = await localVarAxiosParamCreator.getAllJobsStatus(options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
*
* @param {JobId} jobId
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async getJobStatus(jobId: JobId, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<JobStatusResponseDto>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.getJobStatus(jobId, options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
*
* @param {JobId} jobId
@ -5919,15 +5812,6 @@ export const JobApiFactory = function (configuration?: Configuration, basePath?:
getAllJobsStatus(options?: any): AxiosPromise<AllJobStatusResponseDto> {
return localVarFp.getAllJobsStatus(options).then((request) => request(axios, basePath));
},
/**
*
* @param {JobId} jobId
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
getJobStatus(jobId: JobId, options?: any): AxiosPromise<JobStatusResponseDto> {
return localVarFp.getJobStatus(jobId, options).then((request) => request(axios, basePath));
},
/**
*
* @param {JobId} jobId
@ -5958,17 +5842,6 @@ export class JobApi extends BaseAPI {
return JobApiFp(this.configuration).getAllJobsStatus(options).then((request) => request(this.axios, this.basePath));
}
/**
*
* @param {JobId} jobId
* @param {*} [options] Override http request option.
* @throws {RequiredError}
* @memberof JobApi
*/
public getJobStatus(jobId: JobId, options?: AxiosRequestConfig) {
return JobApiFp(this.configuration).getJobStatus(jobId, options).then((request) => request(this.axios, this.basePath));
}
/**
*
* @param {JobId} jobId

27
web/src/lib/components/admin-page/jobs/job-tile.svelte
View File

@ -1,13 +1,12 @@
<script lang="ts">
import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte';
import { createEventDispatcher } from 'svelte';
import { JobCounts } from '@api';
export let title: string;
export let subtitle: string;
export let buttonTitle = 'Run';
export let jobStatus: boolean;
export let waitingJobCount: number;
export let activeJobCount: number;
export let jobCounts: JobCounts;
const dispatch = createEventDispatcher();
</script>
@ -36,17 +35,23 @@
class="overflow-y-auto rounded-md w-full max-h-[320px] block border bg-white dark:border-immich-dark-gray dark:bg-immich-dark-gray/75 dark:text-immich-dark-fg"
>
<tr class="text-center flex place-items-center w-full h-[60px]">
<td class="text-sm px-2 w-1/3 text-ellipsis">{jobStatus ? 'Active' : 'Idle'}</td>
<td class="flex justify-center text-sm px-2 w-1/3 text-ellipsis">
{#if activeJobCount !== undefined}
{activeJobCount}
<td class="text-sm px-2 w-1/3 text-ellipsis">
{#if jobCounts}
<span>{jobCounts.active > 0 || jobCounts.waiting > 0 ? 'Active' : 'Idle'}</span>
{:else}
<LoadingSpinner />
{/if}
</td>
<td class="flex justify-center text-sm px-2 w-1/3 text-ellipsis">
{#if waitingJobCount !== undefined}
{waitingJobCount}
{#if jobCounts.active !== undefined}
{jobCounts.active}
{:else}
<LoadingSpinner />
{/if}
</td>
<td class="flex justify-center text-sm px-2 w-1/3 text-ellipsis">
{#if jobCounts.waiting !== undefined}
{jobCounts.waiting}
{:else}
<LoadingSpinner />
{/if}
@ -59,9 +64,9 @@
<button
on:click={() => dispatch('click')}
class="px-6 py-3 text-sm bg-immich-primary dark:bg-immich-dark-primary font-medium rounded-2xl hover:bg-immich-primary/50 transition-all hover:cursor-pointer disabled:cursor-not-allowed shadow-sm text-immich-bg dark:text-immich-dark-gray"
disabled={jobStatus}
disabled={jobCounts.active > 0 && jobCounts.waiting > 0}
>
{#if jobStatus}
{#if jobCounts.active > 0 || jobCounts.waiting > 0}
<LoadingSpinner />
{:else}
{buttonTitle}

244
web/src/lib/components/admin-page/jobs/jobs-panel.svelte
View File

@ -8,203 +8,97 @@
import { onDestroy, onMount } from 'svelte';
import JobTile from './job-tile.svelte';
let allJobsStatus: AllJobStatusResponseDto;
let setIntervalHandler: NodeJS.Timer;
let jobs: AllJobStatusResponseDto;
let timer: NodeJS.Timer;
const load = async () => {
const { data } = await api.jobApi.getAllJobsStatus();
jobs = data;
};
onMount(async () => {
const { data } = await api.jobApi.getAllJobsStatus();
allJobsStatus = data;
setIntervalHandler = setInterval(async () => {
const { data } = await api.jobApi.getAllJobsStatus();
allJobsStatus = data;
}, 1000);
await load();
timer = setInterval(async () => await load(), 5_000);
});
onDestroy(() => {
clearInterval(setIntervalHandler);
clearInterval(timer);
});
const runThumbnailGeneration = async () => {
const run = async (jobId: JobId, jobName: string, emptyMessage: string) => {
try {
const { data } = await api.jobApi.sendJobCommand(JobId.ThumbnailGeneration, {
command: JobCommand.Start
});
const { data } = await api.jobApi.sendJobCommand(jobId, { command: JobCommand.Start });
if (data) {
notificationController.show({
message: `Thumbnail generation job started for ${data} assets`,
message: `Started ${jobName}`,
type: NotificationType.Info
});
} else {
notificationController.show({
message: `No missing thumbnails found`,
type: NotificationType.Info
});
}
} catch (e) {
console.log('[ERROR] runThumbnailGeneration', e);
notificationController.show({
message: `Error running thumbnail generation job, check console for more detail`,
type: NotificationType.Error
});
}
};
const runExtractEXIF = async () => {
try {
const { data } = await api.jobApi.sendJobCommand(JobId.MetadataExtraction, {
command: JobCommand.Start
});
if (data) {
notificationController.show({
message: `Extract EXIF job started for ${data} assets`,
type: NotificationType.Info
});
} else {
notificationController.show({
message: `No missing EXIF found`,
type: NotificationType.Info
});
}
} catch (e) {
console.log('[ERROR] runExtractEXIF', e);
notificationController.show({
message: `Error running extract EXIF job, check console for more detail`,
type: NotificationType.Error
});
}
};
const runMachineLearning = async () => {
try {
const { data } = await api.jobApi.sendJobCommand(JobId.MachineLearning, {
command: JobCommand.Start
});
if (data) {
notificationController.show({
message: `Object detection job started for ${data} assets`,
type: NotificationType.Info
});
} else {
notificationController.show({
message: `No missing object detection found`,
type: NotificationType.Info
});
notificationController.show({ message: emptyMessage, type: NotificationType.Info });
}
} catch (error) {
handleError(error, `Error running machine learning job, check console for more detail`);
}
};
const runVideoConversion = async () => {
try {
const { data } = await api.jobApi.sendJobCommand(JobId.VideoConversion, {
command: JobCommand.Start
});
if (data) {
notificationController.show({
message: `Video conversion job started for ${data} assets`,
type: NotificationType.Info
});
} else {
notificationController.show({
message: `No videos without an encoded version found`,
type: NotificationType.Info
});
}
} catch (error) {
handleError(error, `Error running video conversion job, check console for more detail`);
}
};
const runTemplateMigration = async () => {
try {
const { data } = await api.jobApi.sendJobCommand(JobId.StorageTemplateMigration, {
command: JobCommand.Start
});
if (data) {
notificationController.show({
message: `Storage migration started`,
type: NotificationType.Info
});
} else {
notificationController.show({
message: `All files have been migrated to the new storage template`,
type: NotificationType.Info
});
}
} catch (e) {
console.log('[ERROR] runTemplateMigration', e);
notificationController.show({
message: `Error running template migration job, check console for more detail`,
type: NotificationType.Error
});
handleError(error, `Unable to start ${jobName}`);
}
};
</script>
<div class="flex flex-col gap-10">
<JobTile
title={'Generate thumbnails'}
subtitle={'Regenerate missing thumbnail (JPEG, WEBP)'}
on:click={runThumbnailGeneration}
jobStatus={allJobsStatus?.isThumbnailGenerationActive}
waitingJobCount={allJobsStatus?.thumbnailGenerationQueueCount.waiting}
activeJobCount={allJobsStatus?.thumbnailGenerationQueueCount.active}
/>
{#if jobs}
<JobTile
title={'Generate thumbnails'}
subtitle={'Regenerate missing thumbnail (JPEG, WEBP)'}
on:click={() =>
run(JobId.ThumbnailGeneration, 'thumbnail generation', 'No missing thumbnails found')}
jobCounts={jobs[JobId.ThumbnailGeneration]}
/>
<JobTile
title={'Extract EXIF'}
subtitle={'Extract missing EXIF information'}
on:click={runExtractEXIF}
jobStatus={allJobsStatus?.isMetadataExtractionActive}
waitingJobCount={allJobsStatus?.metadataExtractionQueueCount.waiting}
activeJobCount={allJobsStatus?.metadataExtractionQueueCount.active}
/>
<JobTile
title={'Extract EXIF'}
subtitle={'Extract missing EXIF information'}
on:click={() => run(JobId.MetadataExtraction, 'extract EXIF', 'No missing EXIF found')}
jobCounts={jobs[JobId.MetadataExtraction]}
/>
<JobTile
title={'Detect objects'}
subtitle={'Run machine learning process to detect and classify objects'}
on:click={runMachineLearning}
jobStatus={allJobsStatus?.isMachineLearningActive}
waitingJobCount={allJobsStatus?.machineLearningQueueCount.waiting}
activeJobCount={allJobsStatus?.machineLearningQueueCount.active}
>
Note that some assets may not have any objects detected, this is normal.
</JobTile>
<JobTile
title={'Video transcoding'}
subtitle={'Run video transcoding process to transcode videos not in the desired format'}
on:click={runVideoConversion}
jobStatus={allJobsStatus?.isVideoConversionActive}
waitingJobCount={allJobsStatus?.videoConversionQueueCount.waiting}
activeJobCount={allJobsStatus?.videoConversionQueueCount.active}
>
Note that some videos won't require transcoding, this is normal.
</JobTile>
<JobTile
title={'Storage migration'}
subtitle={''}
on:click={runTemplateMigration}
jobStatus={allJobsStatus?.isStorageMigrationActive}
waitingJobCount={allJobsStatus?.storageMigrationQueueCount.waiting}
activeJobCount={allJobsStatus?.storageMigrationQueueCount.active}
>
Apply the current
<a
href="/admin/system-settings?open=storage-template"
class="text-immich-primary dark:text-immich-dark-primary">Storage template</a
<JobTile
title={'Detect objects'}
subtitle={'Run machine learning process to detect and classify objects'}
on:click={() =>
run(JobId.MachineLearning, 'object detection', 'No missing object detection found')}
jobCounts={jobs[JobId.MachineLearning]}
>
to previously uploaded assets
</JobTile>
Note that some assets may not have any objects detected, this is normal.
</JobTile>
<JobTile
title={'Video transcoding'}
subtitle={'Run video transcoding process to transcode videos not in the desired format'}
on:click={() =>
run(
JobId.VideoConversion,
'video conversion',
'No videos without an encoded version found'
)}
jobCounts={jobs[JobId.MachineLearning]}
/>
<JobTile
title={'Storage migration'}
subtitle={''}
on:click={() =>
run(
JobId.StorageTemplateMigration,
'storage template migration',
'All files have been migrated to the new storage template'
)}
jobCounts={jobs[JobId.StorageTemplateMigration]}
>
Apply the current
<a
href="/admin/system-settings?open=storage-template"
class="text-immich-primary dark:text-immich-dark-primary">Storage template</a
>
to previously uploaded assets
</JobTile>
{/if}
</div>