From e309647f1b5760d708872e0f0e745aa2d2740b08 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Sat, 18 Feb 2023 10:18:07 -0500 Subject: [PATCH] refactor(server): remove checksum job (#1786) --- .../microservices/src/microservices.module.ts | 4 - .../src/microservices.service.ts | 16 ---- .../processors/generate-checksum.processor.ts | 82 ------------------- server/libs/domain/src/job/job.constants.ts | 2 - server/libs/domain/src/job/job.repository.ts | 1 - server/libs/infra/src/infra.module.ts | 1 - server/libs/infra/src/job/job.repository.ts | 5 -- 7 files changed, 111 deletions(-) delete mode 100644 server/apps/microservices/src/microservices.service.ts delete mode 100644 server/apps/microservices/src/processors/generate-checksum.processor.ts diff --git a/server/apps/microservices/src/microservices.module.ts b/server/apps/microservices/src/microservices.module.ts index 8e88b3349a..0439f4e7f5 100644 --- a/server/apps/microservices/src/microservices.module.ts +++ b/server/apps/microservices/src/microservices.module.ts @@ -14,9 +14,7 @@ import { Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { TypeOrmModule } from '@nestjs/typeorm'; import { CommunicationModule } from '../../immich/src/api-v1/communication/communication.module'; -import { MicroservicesService } from './microservices.service'; import { AssetUploadedProcessor } from './processors/asset-uploaded.processor'; -import { GenerateChecksumProcessor } from './processors/generate-checksum.processor'; import { MachineLearningProcessor } from './processors/machine-learning.processor'; import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor'; import { StorageMigrationProcessor } from './processors/storage-migration.processor'; @@ -46,12 +44,10 @@ import { DomainModule } from '@app/domain'; ], controllers: [], providers: [ - MicroservicesService, AssetUploadedProcessor, ThumbnailGeneratorProcessor, MetadataExtractionProcessor, VideoTranscodeProcessor, - GenerateChecksumProcessor, MachineLearningProcessor, UserDeletionProcessor, StorageMigrationProcessor, diff --git a/server/apps/microservices/src/microservices.service.ts b/server/apps/microservices/src/microservices.service.ts deleted file mode 100644 index a52928e3ab..0000000000 --- a/server/apps/microservices/src/microservices.service.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; -import { IJobRepository, JobName } from '@app/domain'; - -const sleep = (ms: number) => new Promise((resolve) => setTimeout(() => resolve(), ms)); - -@Injectable() -export class MicroservicesService implements OnModuleInit { - constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {} - - async onModuleInit() { - // wait for migration - await sleep(10_000); - - await this.jobRepository.add({ name: JobName.CHECKSUM_GENERATION }); - } -} diff --git a/server/apps/microservices/src/processors/generate-checksum.processor.ts b/server/apps/microservices/src/processors/generate-checksum.processor.ts deleted file mode 100644 index fc9fafbfcf..0000000000 --- a/server/apps/microservices/src/processors/generate-checksum.processor.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { AssetEntity } from '@app/infra'; -import { JobName, QueueName } from '@app/domain'; -import { Process, Processor } from '@nestjs/bull'; -import { Logger } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import { createHash } from 'node:crypto'; -import fs from 'node:fs'; -import { FindOptionsWhere, IsNull, MoreThan, QueryFailedError, Repository } from 'typeorm'; - -// TODO: just temporary task to generate previous uploaded assets. -@Processor(QueueName.CHECKSUM_GENERATION) -export class GenerateChecksumProcessor { - constructor( - @InjectRepository(AssetEntity) - private assetRepository: Repository, - ) {} - - @Process(JobName.CHECKSUM_GENERATION) - async generateChecksum() { - const pageSize = 200; - let hasNext = true; - let lastErrAssetId: string | undefined = undefined; - - while (hasNext) { - const whereStat: FindOptionsWhere = { - checksum: IsNull(), - }; - - if (lastErrAssetId) { - whereStat.id = MoreThan(lastErrAssetId); - } - - const assets = await this.assetRepository.find({ - where: whereStat, - take: pageSize, - order: { id: 'ASC' }, - }); - - if (!assets?.length) { - hasNext = false; // avoid using break - } else { - for (const asset of assets) { - try { - await this.generateAssetChecksum(asset); - } catch (err: any) { - lastErrAssetId = asset.id; - - if (err instanceof QueryFailedError && (err as any).constraint === 'UQ_userid_checksum') { - Logger.error(`${asset.originalPath} duplicated`); - } else { - Logger.error(`checksum generation ${err}`); - } - } - } - - // break when reach to the last page - if (assets.length < pageSize) { - hasNext = false; - } - } - } - - Logger.log(`checksum generation done!`); - } - - private async generateAssetChecksum(asset: AssetEntity) { - if (!asset.originalPath) return; - if (!fs.existsSync(asset.originalPath)) return; - - const fileReadStream = fs.createReadStream(asset.originalPath); - const sha1Hash = createHash('sha1'); - const deferred = new Promise((resolve, reject) => { - sha1Hash.once('error', (err) => reject(err)); - sha1Hash.once('finish', () => resolve(sha1Hash.read())); - }); - - fileReadStream.pipe(sha1Hash); - const checksum = await deferred; - - await this.assetRepository.update(asset.id, { checksum }); - } -} diff --git a/server/libs/domain/src/job/job.constants.ts b/server/libs/domain/src/job/job.constants.ts index 992faf19f0..50b9e2f90b 100644 --- a/server/libs/domain/src/job/job.constants.ts +++ b/server/libs/domain/src/job/job.constants.ts @@ -2,7 +2,6 @@ export enum QueueName { THUMBNAIL_GENERATION = 'thumbnail-generation-queue', METADATA_EXTRACTION = 'metadata-extraction-queue', VIDEO_CONVERSION = 'video-conversion-queue', - CHECKSUM_GENERATION = 'generate-checksum-queue', ASSET_UPLOADED = 'asset-uploaded-queue', MACHINE_LEARNING = 'machine-learning-queue', USER_DELETION = 'user-deletion-queue', @@ -24,5 +23,4 @@ export enum JobName { OBJECT_DETECTION = 'detect-object', IMAGE_TAGGING = 'tag-image', DELETE_FILE_ON_DISK = 'delete-file-on-disk', - CHECKSUM_GENERATION = 'checksum-generation', } diff --git a/server/libs/domain/src/job/job.repository.ts b/server/libs/domain/src/job/job.repository.ts index 74cd7517d8..2ad4e33f30 100644 --- a/server/libs/domain/src/job/job.repository.ts +++ b/server/libs/domain/src/job/job.repository.ts @@ -34,7 +34,6 @@ export type JobItem = | { name: JobName.USER_DELETION; data: IUserDeletionJob } | { name: JobName.TEMPLATE_MIGRATION } | { name: JobName.CONFIG_CHANGE } - | { name: JobName.CHECKSUM_GENERATION } | { name: JobName.EXTRACT_VIDEO_METADATA; data: IVideoLengthExtractionProcessor } | { name: JobName.OBJECT_DETECTION; data: IMachineLearningJob } | { name: JobName.IMAGE_TAGGING; data: IMachineLearningJob } diff --git a/server/libs/infra/src/infra.module.ts b/server/libs/infra/src/infra.module.ts index b71373b1b0..720ca9d116 100644 --- a/server/libs/infra/src/infra.module.ts +++ b/server/libs/infra/src/infra.module.ts @@ -79,7 +79,6 @@ const providers: Provider[] = [ { name: QueueName.ASSET_UPLOADED }, { name: QueueName.METADATA_EXTRACTION }, { name: QueueName.VIDEO_CONVERSION }, - { name: QueueName.CHECKSUM_GENERATION }, { name: QueueName.MACHINE_LEARNING }, { name: QueueName.CONFIG }, { name: QueueName.BACKGROUND_TASK }, diff --git a/server/libs/infra/src/job/job.repository.ts b/server/libs/infra/src/job/job.repository.ts index dea3438de9..83a7bb1043 100644 --- a/server/libs/infra/src/job/job.repository.ts +++ b/server/libs/infra/src/job/job.repository.ts @@ -20,7 +20,6 @@ export class JobRepository implements IJobRepository { constructor( @InjectQueue(QueueName.ASSET_UPLOADED) private assetUploaded: Queue, @InjectQueue(QueueName.BACKGROUND_TASK) private backgroundTask: Queue, - @InjectQueue(QueueName.CHECKSUM_GENERATION) private generateChecksum: Queue, @InjectQueue(QueueName.MACHINE_LEARNING) private machineLearning: Queue, @InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue, @InjectQueue(QueueName.CONFIG) private storageMigration: Queue, @@ -52,10 +51,6 @@ export class JobRepository implements IJobRepository { await this.backgroundTask.add(item.name, item.data); break; - case JobName.CHECKSUM_GENERATION: - await this.generateChecksum.add(item.name, {}); - break; - case JobName.OBJECT_DETECTION: case JobName.IMAGE_TAGGING: await this.machineLearning.add(item.name, item.data);