1
0
mirror of https://github.com/immich-app/immich.git synced 2025-08-09 23:17:29 +02:00

Remove thumbnail generation on mobile app (#292)

* Remove thumbnail generation on mobile

* Remove tconditions for missing thumbnail on the backend

* Remove console.log

* Refactor queue systems

* Convert queue and processor name to constant

* Added corresponding interface to job queue
This commit is contained in:
Alex
2022-07-02 21:06:36 -05:00
committed by GitHub
parent 32b847c26e
commit 76bf1c0379
22 changed files with 270 additions and 141 deletions

View File

@@ -31,6 +31,9 @@ import { SearchAssetDto } from './dto/search-asset.dto';
import { CommunicationGateway } from '../communication/communication.gateway';
import { InjectQueue } from '@nestjs/bull';
import { Queue } from 'bull';
import { IAssetUploadedJob } from '@app/job/index';
import { assetUploadedQueueName } from '@app/job/constants/queue-name.constant';
import { assetUploadedProcessorName } from '@app/job/constants/job-name.constant';
@UseGuards(JwtAuthGuard)
@Controller('asset')
@@ -40,8 +43,8 @@ export class AssetController {
private assetService: AssetService,
private backgroundTaskService: BackgroundTaskService,
@InjectQueue('asset-uploaded-queue')
private assetUploadedQueue: Queue,
@InjectQueue(assetUploadedQueueName)
private assetUploadedQueue: Queue<IAssetUploadedJob>,
) {}
@Post('upload')
@@ -56,7 +59,7 @@ export class AssetController {
)
async uploadFile(
@GetAuthUser() authUser: AuthUserDto,
@UploadedFiles() uploadFiles: { assetData: Express.Multer.File[]; thumbnailData?: Express.Multer.File[] },
@UploadedFiles() uploadFiles: { assetData: Express.Multer.File[] },
@Body(ValidationPipe) assetInfo: CreateAssetDto,
): Promise<'ok' | undefined> {
for (const file of uploadFiles.assetData) {
@@ -66,28 +69,12 @@ export class AssetController {
if (!savedAsset) {
return;
}
if (uploadFiles.thumbnailData != null) {
const assetWithThumbnail = await this.assetService.updateThumbnailInfo(
savedAsset,
uploadFiles.thumbnailData[0].path,
);
await this.assetUploadedQueue.add(
'asset-uploaded',
{ asset: assetWithThumbnail, fileName: file.originalname, fileSize: file.size, hasThumbnail: true },
{ jobId: savedAsset.id },
);
this.wsCommunicateionGateway.server
.to(savedAsset.userId)
.emit('on_upload_success', JSON.stringify(assetWithThumbnail));
} else {
await this.assetUploadedQueue.add(
'asset-uploaded',
{ asset: savedAsset, fileName: file.originalname, fileSize: file.size, hasThumbnail: false },
{ jobId: savedAsset.id },
);
}
await this.assetUploadedQueue.add(
assetUploadedProcessorName,
{ asset: savedAsset, fileName: file.originalname, fileSize: file.size },
{ jobId: savedAsset.id },
);
} catch (e) {
Logger.error(`Error receiving upload file ${e}`);
}

View File

@@ -7,6 +7,7 @@ import { BullModule } from '@nestjs/bull';
import { BackgroundTaskModule } from '../../modules/background-task/background-task.module';
import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
import { CommunicationModule } from '../communication/communication.module';
import { assetUploadedQueueName } from '@app/job/constants/queue-name.constant';
@Module({
imports: [
@@ -14,7 +15,7 @@ import { CommunicationModule } from '../communication/communication.module';
BackgroundTaskModule,
TypeOrmModule.forFeature([AssetEntity]),
BullModule.registerQueue({
name: 'asset-uploaded-queue',
name: assetUploadedQueueName,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,

View File

@@ -6,7 +6,6 @@ import { extname } from 'path';
import { Request } from 'express';
import { APP_UPLOAD_LOCATION } from '../constants/upload_location.constant';
import { randomUUID } from 'crypto';
// import { CreateAssetDto } from '../api-v1/asset/dto/create-asset.dto';
export const assetUploadOption: MulterOptions = {
fileFilter: (req: Request, file: any, cb: any) => {
@@ -30,34 +29,20 @@ export const assetUploadOption: MulterOptions = {
return;
}
if (file.fieldname == 'assetData') {
const originalUploadFolder = `${basePath}/${req.user.id}/original/${req.body['deviceId']}`;
const originalUploadFolder = `${basePath}/${req.user.id}/original/${req.body['deviceId']}`;
if (!existsSync(originalUploadFolder)) {
mkdirSync(originalUploadFolder, { recursive: true });
}
// Save original to disk
cb(null, originalUploadFolder);
} else if (file.fieldname == 'thumbnailData') {
const thumbnailUploadFolder = `${basePath}/${req.user.id}/thumb/${req.body['deviceId']}`;
if (!existsSync(thumbnailUploadFolder)) {
mkdirSync(thumbnailUploadFolder, { recursive: true });
}
// Save thumbnail to disk
cb(null, thumbnailUploadFolder);
if (!existsSync(originalUploadFolder)) {
mkdirSync(originalUploadFolder, { recursive: true });
}
// Save original to disk
cb(null, originalUploadFolder);
},
filename: (req: Request, file: Express.Multer.File, cb: any) => {
const fileNameUUID = randomUUID();
if (file.fieldname == 'assetData') {
cb(null, `${fileNameUUID}${req.body['fileExtension'].toLowerCase()}`);
} else if (file.fieldname == 'thumbnailData') {
cb(null, `${fileNameUUID}.jpeg`);
}
cb(null, `${fileNameUUID}${req.body['fileExtension'].toLowerCase()}`);
},
}),
};

View File

@@ -3,12 +3,13 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { AssetEntity } from '@app/database/entities/asset.entity';
import { ScheduleTasksService } from './schedule-tasks.service';
import { thumbnailGeneratorQueueName, videoConversionQueueName } from '@app/job/constants/queue-name.constant';
@Module({
imports: [
TypeOrmModule.forFeature([AssetEntity]),
BullModule.registerQueue({
name: 'video-conversion-queue',
name: videoConversionQueueName,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
@@ -16,7 +17,7 @@ import { ScheduleTasksService } from './schedule-tasks.service';
},
}),
BullModule.registerQueue({
name: 'thumbnail-generator-queue',
name: thumbnailGeneratorQueueName,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,

View File

@@ -6,6 +6,9 @@ import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
import { InjectQueue } from '@nestjs/bull';
import { Queue } from 'bull';
import { randomUUID } from 'crypto';
import { generateWEBPThumbnailProcessorName, mp4ConversionProcessorName } from '@app/job/constants/job-name.constant';
import { thumbnailGeneratorQueueName, videoConversionQueueName } from '@app/job/constants/queue-name.constant';
import { IVideoTranscodeJob } from '@app/job/interfaces/video-transcode.interface';
@Injectable()
export class ScheduleTasksService {
@@ -13,11 +16,11 @@ export class ScheduleTasksService {
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@InjectQueue('thumbnail-generator-queue')
@InjectQueue(thumbnailGeneratorQueueName)
private thumbnailGeneratorQueue: Queue,
@InjectQueue('video-conversion-queue')
private videoConversionQueue: Queue,
@InjectQueue(videoConversionQueueName)
private videoConversionQueue: Queue<IVideoTranscodeJob>,
) {}
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
@@ -36,7 +39,11 @@ export class ScheduleTasksService {
}
for (const asset of assets) {
await this.thumbnailGeneratorQueue.add('generate-webp-thumbnail', { asset: asset }, { jobId: randomUUID() });
await this.thumbnailGeneratorQueue.add(
generateWEBPThumbnailProcessorName,
{ asset: asset },
{ jobId: randomUUID() },
);
}
}
@@ -54,7 +61,7 @@ export class ScheduleTasksService {
});
for (const asset of assets) {
await this.videoConversionQueue.add('mp4-conversion', { asset }, { jobId: randomUUID() });
await this.videoConversionQueue.add(mp4ConversionProcessorName, { asset }, { jobId: randomUUID() });
}
}
}

View File

@@ -12,6 +12,12 @@ import { ThumbnailGeneratorProcessor } from './processors/thumbnail.processor';
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
import { VideoTranscodeProcessor } from './processors/video-transcode.processor';
import { CommunicationModule } from '../../immich/src/api-v1/communication/communication.module';
import {
assetUploadedQueueName,
metadataExtractionQueueName,
thumbnailGeneratorQueueName,
videoConversionQueueName,
} from '@app/job/constants/queue-name.constant';
@Module({
imports: [
@@ -26,7 +32,7 @@ import { CommunicationModule } from '../../immich/src/api-v1/communication/commu
}),
}),
BullModule.registerQueue({
name: 'thumbnail-generator-queue',
name: thumbnailGeneratorQueueName,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
@@ -34,7 +40,7 @@ import { CommunicationModule } from '../../immich/src/api-v1/communication/commu
},
}),
BullModule.registerQueue({
name: 'asset-uploaded-queue',
name: assetUploadedQueueName,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
@@ -42,7 +48,7 @@ import { CommunicationModule } from '../../immich/src/api-v1/communication/commu
},
}),
BullModule.registerQueue({
name: 'metadata-extraction-queue',
name: metadataExtractionQueueName,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
@@ -50,7 +56,7 @@ import { CommunicationModule } from '../../immich/src/api-v1/communication/commu
},
}),
BullModule.registerQueue({
name: 'video-conversion-queue',
name: videoConversionQueueName,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,

View File

@@ -1,61 +1,58 @@
import { InjectQueue, Process, Processor } from '@nestjs/bull';
import { Job, Queue } from 'bull';
import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { AssetType } from '@app/database/entities/asset.entity';
import { randomUUID } from 'crypto';
import {
IAssetUploadedJob,
IMetadataExtractionJob,
IThumbnailGenerationJob,
IVideoTranscodeJob,
assetUploadedQueueName,
metadataExtractionQueueName,
thumbnailGeneratorQueueName,
videoConversionQueueName,
assetUploadedProcessorName,
exifExtractionProcessorName,
generateJPEGThumbnailProcessorName,
mp4ConversionProcessorName,
videoLengthExtractionProcessorName,
} from '@app/job';
@Processor('asset-uploaded-queue')
@Processor(assetUploadedQueueName)
export class AssetUploadedProcessor {
constructor(
@InjectQueue('thumbnail-generator-queue')
private thumbnailGeneratorQueue: Queue,
@InjectQueue(thumbnailGeneratorQueueName)
private thumbnailGeneratorQueue: Queue<IThumbnailGenerationJob>,
@InjectQueue('metadata-extraction-queue')
private metadataExtractionQueue: Queue,
@InjectQueue(metadataExtractionQueueName)
private metadataExtractionQueue: Queue<IMetadataExtractionJob>,
@InjectQueue('video-conversion-queue')
private videoConversionQueue: Queue,
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@InjectQueue(videoConversionQueueName)
private videoConversionQueue: Queue<IVideoTranscodeJob>,
) {}
/**
* Post processing uploaded asset to perform the following function if missing
* 1. Generate JPEG Thumbnail
* 2. Generate Webp Thumbnail <-> if JPEG thumbnail exist
* 2. Generate Webp Thumbnail
* 3. EXIF extractor
* 4. Reverse Geocoding
*
* @param job asset-uploaded
*/
@Process('asset-uploaded')
async processUploadedVideo(job: Job) {
const {
asset,
fileName,
fileSize,
hasThumbnail,
}: { asset: AssetEntity; fileName: string; fileSize: number; hasThumbnail: boolean } = job.data;
@Process(assetUploadedProcessorName)
async processUploadedVideo(job: Job<IAssetUploadedJob>) {
const { asset, fileName, fileSize } = job.data;
if (hasThumbnail) {
// The jobs below depends on the existence of jpeg thumbnail
await this.thumbnailGeneratorQueue.add('generate-webp-thumbnail', { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add('tag-image', { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add('detect-object', { asset }, { jobId: randomUUID() });
} else {
// Generate Thumbnail -> Then generate webp, tag image and detect object
await this.thumbnailGeneratorQueue.add('generate-jpeg-thumbnail', { asset }, { jobId: randomUUID() });
}
await this.thumbnailGeneratorQueue.add(generateJPEGThumbnailProcessorName, { asset }, { jobId: randomUUID() });
// Video Conversion
if (asset.type == AssetType.VIDEO) {
await this.videoConversionQueue.add('mp4-conversion', { asset }, { jobId: randomUUID() });
await this.videoConversionQueue.add(mp4ConversionProcessorName, { asset }, { jobId: randomUUID() });
} else {
// Extract Metadata/Exif for Images - Currently the library cannot extract EXIF for video yet
// Extract Metadata/Exif for Images - Currently the EXIF library on the web cannot extract EXIF for video yet
await this.metadataExtractionQueue.add(
'exif-extraction',
exifExtractionProcessorName,
{
asset,
fileName,
@@ -67,7 +64,7 @@ export class AssetUploadedProcessor {
// Extract video duration if uploaded from the web
if (asset.type == AssetType.VIDEO && asset.duration == '0:00:00.000000') {
await this.metadataExtractionQueue.add('extract-video-length', { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add(videoLengthExtractionProcessorName, { asset }, { jobId: randomUUID() });
}
}
}

View File

@@ -13,8 +13,17 @@ import axios from 'axios';
import { SmartInfoEntity } from '@app/database/entities/smart-info.entity';
import ffmpeg from 'fluent-ffmpeg';
import path from 'path';
import {
IExifExtractionProcessor,
IVideoLengthExtractionProcessor,
exifExtractionProcessorName,
imageTaggingProcessorName,
objectDetectionProcessorName,
videoLengthExtractionProcessorName,
metadataExtractionQueueName,
} from '@app/job';
@Processor('metadata-extraction-queue')
@Processor(metadataExtractionQueueName)
export class MetadataExtractionProcessor {
private geocodingClient?: GeocodeService;
@@ -35,8 +44,8 @@ export class MetadataExtractionProcessor {
}
}
@Process('exif-extraction')
async extractExifInfo(job: Job) {
@Process(exifExtractionProcessorName)
async extractExifInfo(job: Job<IExifExtractionProcessor>) {
try {
const { asset, fileName, fileSize }: { asset: AssetEntity; fileName: string; fileSize: number } = job.data;
@@ -89,7 +98,7 @@ export class MetadataExtractionProcessor {
}
}
@Process({ name: 'tag-image', concurrency: 2 })
@Process({ name: imageTaggingProcessorName, concurrency: 2 })
async tagImage(job: Job) {
const { asset }: { asset: AssetEntity } = job.data;
@@ -108,7 +117,7 @@ export class MetadataExtractionProcessor {
}
}
@Process({ name: 'detect-object', concurrency: 2 })
@Process({ name: objectDetectionProcessorName, concurrency: 2 })
async detectObject(job: Job) {
try {
const { asset }: { asset: AssetEntity } = job.data;
@@ -131,9 +140,9 @@ export class MetadataExtractionProcessor {
}
}
@Process({ name: 'extract-video-length', concurrency: 2 })
async extractVideoLength(job: Job) {
const { asset }: { asset: AssetEntity } = job.data;
@Process({ name: videoLengthExtractionProcessorName, concurrency: 2 })
async extractVideoLength(job: Job<IVideoLengthExtractionProcessor>) {
const { asset } = job.data;
ffmpeg.ffprobe(asset.originalPath, async (err, data) => {
if (!err) {

View File

@@ -9,25 +9,35 @@ import { randomUUID } from 'node:crypto';
import { CommunicationGateway } from '../../../immich/src/api-v1/communication/communication.gateway';
import ffmpeg from 'fluent-ffmpeg';
import { Logger } from '@nestjs/common';
import {
WebpGeneratorProcessor,
generateJPEGThumbnailProcessorName,
generateWEBPThumbnailProcessorName,
imageTaggingProcessorName,
objectDetectionProcessorName,
metadataExtractionQueueName,
thumbnailGeneratorQueueName,
JpegGeneratorProcessor,
} from '@app/job';
@Processor('thumbnail-generator-queue')
@Processor(thumbnailGeneratorQueueName)
export class ThumbnailGeneratorProcessor {
constructor(
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
@InjectQueue('thumbnail-generator-queue')
@InjectQueue(thumbnailGeneratorQueueName)
private thumbnailGeneratorQueue: Queue,
private wsCommunicateionGateway: CommunicationGateway,
@InjectQueue('metadata-extraction-queue')
@InjectQueue(metadataExtractionQueueName)
private metadataExtractionQueue: Queue,
) {}
@Process('generate-jpeg-thumbnail')
async generateJPEGThumbnail(job: Job) {
const { asset }: { asset: AssetEntity } = job.data;
@Process({ name: generateJPEGThumbnailProcessorName, concurrency: 3 })
async generateJPEGThumbnail(job: Job<JpegGeneratorProcessor>) {
const { asset } = job.data;
const resizePath = `upload/${asset.userId}/thumb/${asset.deviceId}/`;
@@ -43,6 +53,7 @@ export class ThumbnailGeneratorProcessor {
sharp(asset.originalPath)
.resize(1440, 2560, { fit: 'inside' })
.jpeg()
.rotate()
.toFile(jpegThumbnailPath, async (err) => {
if (!err) {
await this.assetRepository.update({ id: asset.id }, { resizePath: jpegThumbnailPath });
@@ -50,9 +61,13 @@ export class ThumbnailGeneratorProcessor {
// Update resize path to send to generate webp queue
asset.resizePath = jpegThumbnailPath;
await this.thumbnailGeneratorQueue.add('generate-webp-thumbnail', { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add('tag-image', { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add('detect-object', { asset }, { jobId: randomUUID() });
await this.thumbnailGeneratorQueue.add(
generateWEBPThumbnailProcessorName,
{ asset },
{ jobId: randomUUID() },
);
await this.metadataExtractionQueue.add(imageTaggingProcessorName, { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add(objectDetectionProcessorName, { asset }, { jobId: randomUUID() });
this.wsCommunicateionGateway.server.to(asset.userId).emit('on_upload_success', JSON.stringify(asset));
}
});
@@ -76,9 +91,13 @@ export class ThumbnailGeneratorProcessor {
// Update resize path to send to generate webp queue
asset.resizePath = jpegThumbnailPath;
await this.thumbnailGeneratorQueue.add('generate-webp-thumbnail', { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add('tag-image', { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add('detect-object', { asset }, { jobId: randomUUID() });
await this.thumbnailGeneratorQueue.add(
generateWEBPThumbnailProcessorName,
{ asset },
{ jobId: randomUUID() },
);
await this.metadataExtractionQueue.add(imageTaggingProcessorName, { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add(objectDetectionProcessorName, { asset }, { jobId: randomUUID() });
this.wsCommunicateionGateway.server.to(asset.userId).emit('on_upload_success', JSON.stringify(asset));
})
@@ -86,8 +105,8 @@ export class ThumbnailGeneratorProcessor {
}
}
@Process({ name: 'generate-webp-thumbnail', concurrency: 2 })
async generateWepbThumbnail(job: Job<{ asset: AssetEntity }>) {
@Process({ name: generateWEBPThumbnailProcessorName, concurrency: 3 })
async generateWepbThumbnail(job: Job<WebpGeneratorProcessor>) {
const { asset } = job.data;
if (!asset.resizePath) {
@@ -98,6 +117,7 @@ export class ThumbnailGeneratorProcessor {
sharp(asset.resizePath)
.resize(250)
.webp()
.rotate()
.toFile(webpPath, (err) => {
if (!err) {
this.assetRepository.update({ id: asset.id }, { webpPath: webpPath });

View File

@@ -1,3 +1,6 @@
import { mp4ConversionProcessorName } from '@app/job/constants/job-name.constant';
import { videoConversionQueueName } from '@app/job/constants/queue-name.constant';
import { IMp4ConversionProcessor } from '@app/job/interfaces/video-transcode.interface';
import { Process, Processor } from '@nestjs/bull';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
@@ -8,16 +11,16 @@ import { Repository } from 'typeorm';
import { AssetEntity } from '../../../../libs/database/src/entities/asset.entity';
import { APP_UPLOAD_LOCATION } from '../../../immich/src/constants/upload_location.constant';
@Processor('video-conversion-queue')
@Processor(videoConversionQueueName)
export class VideoTranscodeProcessor {
constructor(
@InjectRepository(AssetEntity)
private assetRepository: Repository<AssetEntity>,
) {}
@Process({ name: 'mp4-conversion', concurrency: 1 })
async mp4Conversion(job: Job) {
const { asset }: { asset: AssetEntity } = job.data;
@Process({ name: mp4ConversionProcessorName, concurrency: 1 })
async mp4Conversion(job: Job<IMp4ConversionProcessor>) {
const { asset } = job.data;
if (asset.mimeType != 'video/mp4') {
const basePath = APP_UPLOAD_LOCATION;