1
0
mirror of https://github.com/immich-app/immich.git synced 2024-12-25 10:43:13 +02:00

feat(server): missing exif extract nightly task (#754)

* fix: nightly reverse geocoding task checking for mapbox

* refactor: remove file size from image processor and queue data

* feat: add missing exif nightly job

* Remove filesize requirement in assetUploadedProcessorName queue insertion

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Zack Pollard 2022-09-28 11:41:50 +01:00 committed by GitHub
parent b0cd2522e0
commit c33775b944
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 46 additions and 35 deletions

View File

@ -97,7 +97,7 @@ export class AssetController {
await this.assetUploadedQueue.add(
assetUploadedProcessorName,
{ asset: savedAsset, fileName: file.originalname, fileSize: file.size },
{ asset: savedAsset, fileName: file.originalname },
{ jobId: savedAsset.id },
);

View File

@ -8,14 +8,16 @@ import { Queue } from 'bull';
import { randomUUID } from 'crypto';
import { ExifEntity } from '@app/database/entities/exif.entity';
import {
exifExtractionProcessorName,
generateWEBPThumbnailProcessorName,
IMetadataExtractionJob,
IVideoTranscodeJob,
metadataExtractionQueueName,
thumbnailGeneratorQueueName,
videoConversionQueueName,
generateWEBPThumbnailProcessorName,
mp4ConversionProcessorName,
reverseGeocodingProcessorName,
thumbnailGeneratorQueueName,
videoConversionQueueName,
videoMetadataExtractionProcessorName,
} from '@app/job';
import { ConfigService } from '@nestjs/config';
@ -82,9 +84,9 @@ export class ScheduleTasksService {
@Cron(CronExpression.EVERY_DAY_AT_2AM)
async reverseGeocoding() {
const isMapboxEnable = this.configService.get('ENABLE_MAPBOX');
const isGeocodingEnabled = this.configService.get('DISABLE_REVERSE_GEOCODING') !== 'true';
if (isMapboxEnable) {
if (isGeocodingEnabled) {
const exifInfo = await this.exifRepository.find({
where: {
city: IsNull(),
@ -96,11 +98,36 @@ export class ScheduleTasksService {
for (const exif of exifInfo) {
await this.metadataExtractionQueue.add(
reverseGeocodingProcessorName,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
{ exifId: exif.id, latitude: exif.latitude!, longitude: exif.longitude! },
{ jobId: randomUUID() },
);
}
}
}
@Cron(CronExpression.EVERY_DAY_AT_3AM)
async extractExif() {
const exifAssets = await this.assetRepository.find({
where: {
exifInfo: IsNull(),
},
});
for (const asset of exifAssets) {
if (asset.type === AssetType.VIDEO) {
await this.metadataExtractionQueue.add(
videoMetadataExtractionProcessorName,
{ asset, fileName: asset.id },
{ jobId: randomUUID() },
);
} else {
await this.metadataExtractionQueue.add(
exifExtractionProcessorName,
{ asset, fileName: asset.id },
{ jobId: randomUUID() },
);
}
}
}
}

View File

@ -42,13 +42,18 @@ export class AssetUploadedProcessor {
*/
@Process(assetUploadedProcessorName)
async processUploadedVideo(job: Job<IAssetUploadedJob>) {
const { asset, fileName, fileSize } = job.data;
const { asset, fileName } = job.data;
await this.thumbnailGeneratorQueue.add(generateJPEGThumbnailProcessorName, { asset }, { jobId: randomUUID() });
// Video Conversion
if (asset.type == AssetType.VIDEO) {
await this.videoConversionQueue.add(mp4ConversionProcessorName, { asset }, { jobId: randomUUID() });
await this.metadataExtractionQueue.add(
videoMetadataExtractionProcessorName,
{ asset, fileName },
{ jobId: randomUUID() },
);
} else {
// Extract Metadata/Exif for Images - Currently the EXIF library on the web cannot extract EXIF for video yet
await this.metadataExtractionQueue.add(
@ -56,19 +61,9 @@ export class AssetUploadedProcessor {
{
asset,
fileName,
fileSize,
},
{ jobId: randomUUID() },
);
}
// Extract video duration if uploaded from the web & CLI
if (asset.type == AssetType.VIDEO) {
await this.metadataExtractionQueue.add(
videoMetadataExtractionProcessorName,
{ asset, fileName, fileSize },
{ jobId: randomUUID() }
);
}
}
}

View File

@ -28,6 +28,7 @@ import geocoder, { InitOptions } from 'local-reverse-geocoder';
import { getName } from 'i18n-iso-countries';
import { find } from 'geo-tz';
import * as luxon from 'luxon';
import fs from 'node:fs';
function geocoderInit(init: InitOptions) {
return new Promise<void>(function (resolve) {
@ -139,7 +140,7 @@ export class MetadataExtractionProcessor {
@Process(exifExtractionProcessorName)
async extractExifInfo(job: Job<IExifExtractionProcessor>) {
try {
const { asset, fileName, fileSize }: { asset: AssetEntity; fileName: string; fileSize: number } = job.data;
const { asset, fileName }: { asset: AssetEntity; fileName: string } = job.data;
const exifData = await exifr.parse(asset.originalPath, {
tiff: true,
ifd0: true as any,
@ -160,6 +161,9 @@ export class MetadataExtractionProcessor {
const createdAt = new Date(exifData.DateTimeOriginal || exifData.CreateDate || new Date(asset.createdAt));
const fileStats = fs.statSync(asset.originalPath);
const fileSizeInBytes = fileStats.size;
const newExif = new ExifEntity();
newExif.assetId = asset.id;
newExif.make = exifData['Make'] || null;
@ -167,7 +171,7 @@ export class MetadataExtractionProcessor {
newExif.imageName = path.parse(fileName).name || null;
newExif.exifImageHeight = exifData['ExifImageHeight'] || exifData['ImageHeight'] || null;
newExif.exifImageWidth = exifData['ExifImageWidth'] || exifData['ImageWidth'] || null;
newExif.fileSizeInByte = fileSize || null;
newExif.fileSizeInByte = fileSizeInBytes || null;
newExif.orientation = exifData['Orientation'] || null;
newExif.dateTimeOriginal = createdAt;
newExif.modifyDate = exifData['ModifyDate'] || null;

View File

@ -10,9 +10,4 @@ export interface IAssetUploadedJob {
* Original file name
*/
fileName: string;
/**
* File size in byte
*/
fileSize: number;
}

View File

@ -10,11 +10,6 @@ export interface IExifExtractionProcessor {
* Original file name
*/
fileName: string;
/**
* File size in byte
*/
fileSize: number;
}
export interface IVideoLengthExtractionProcessor {
@ -27,11 +22,6 @@ export interface IVideoLengthExtractionProcessor {
* Original file name
*/
fileName: string;
/**
* File size in byte
*/
fileSize: number;
}
export interface IReverseGeocodingProcessor {