feat(server): missing exif extract nightly task (#754)

* fix: nightly reverse geocoding task checking for mapbox

* refactor: remove file size from image processor and queue data

* feat: add missing exif nightly job

* Remove filesize requirement in assetUploadedProcessorName queue insertion

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/apps/immich/src/api-v1/asset/asset.controller.ts

@@ -97,7 +97,7 @@ export class AssetController {
 
       await this.assetUploadedQueue.add(
         assetUploadedProcessorName,
-        { asset: savedAsset, fileName: file.originalname, fileSize: file.size },
+        { asset: savedAsset, fileName: file.originalname },
         { jobId: savedAsset.id },
       );
 

server/apps/immich/src/modules/schedule-tasks/schedule-tasks.service.ts

@@ -8,14 +8,16 @@ import { Queue } from 'bull';
 import { randomUUID } from 'crypto';
 import { ExifEntity } from '@app/database/entities/exif.entity';
 import {
+  exifExtractionProcessorName,
+  generateWEBPThumbnailProcessorName,
   IMetadataExtractionJob,
   IVideoTranscodeJob,
   metadataExtractionQueueName,
-  thumbnailGeneratorQueueName,
-  videoConversionQueueName,
-  generateWEBPThumbnailProcessorName,
   mp4ConversionProcessorName,
   reverseGeocodingProcessorName,
+  thumbnailGeneratorQueueName,
+  videoConversionQueueName,
+  videoMetadataExtractionProcessorName,
 } from '@app/job';
 import { ConfigService } from '@nestjs/config';
 
@@ -82,9 +84,9 @@ export class ScheduleTasksService {
 
   @Cron(CronExpression.EVERY_DAY_AT_2AM)
   async reverseGeocoding() {
-    const isMapboxEnable = this.configService.get('ENABLE_MAPBOX');
+    const isGeocodingEnabled = this.configService.get('DISABLE_REVERSE_GEOCODING') !== 'true';
 
-    if (isMapboxEnable) {
+    if (isGeocodingEnabled) {
       const exifInfo = await this.exifRepository.find({
         where: {
           city: IsNull(),
@@ -96,11 +98,36 @@ export class ScheduleTasksService {
       for (const exif of exifInfo) {
         await this.metadataExtractionQueue.add(
           reverseGeocodingProcessorName,
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
           { exifId: exif.id, latitude: exif.latitude!, longitude: exif.longitude! },
           { jobId: randomUUID() },
         );
       }
     }
   }
+
+  @Cron(CronExpression.EVERY_DAY_AT_3AM)
+  async extractExif() {
+    const exifAssets = await this.assetRepository.find({
+      where: {
+        exifInfo: IsNull(),
+      },
+    });
+
+    for (const asset of exifAssets) {
+      if (asset.type === AssetType.VIDEO) {
+        await this.metadataExtractionQueue.add(
+          videoMetadataExtractionProcessorName,
+          { asset, fileName: asset.id },
+          { jobId: randomUUID() },
+        );
+      } else {
+        await this.metadataExtractionQueue.add(
+          exifExtractionProcessorName,
+          { asset, fileName: asset.id },
+          { jobId: randomUUID() },
+        );
+      }
+    }
+  }
 }

server/apps/microservices/src/processors/asset-uploaded.processor.ts

@@ -42,13 +42,18 @@ export class AssetUploadedProcessor {
    */
   @Process(assetUploadedProcessorName)
   async processUploadedVideo(job: Job<IAssetUploadedJob>) {
-    const { asset, fileName, fileSize } = job.data;
+    const { asset, fileName } = job.data;
 
     await this.thumbnailGeneratorQueue.add(generateJPEGThumbnailProcessorName, { asset }, { jobId: randomUUID() });
 
     // Video Conversion
     if (asset.type == AssetType.VIDEO) {
       await this.videoConversionQueue.add(mp4ConversionProcessorName, { asset }, { jobId: randomUUID() });
+      await this.metadataExtractionQueue.add(
+        videoMetadataExtractionProcessorName,
+        { asset, fileName },
+        { jobId: randomUUID() },
+      );
     } else {
       // Extract Metadata/Exif for Images - Currently the EXIF library on the web cannot extract EXIF for video yet
       await this.metadataExtractionQueue.add(
@@ -56,19 +61,9 @@ export class AssetUploadedProcessor {
         {
           asset,
           fileName,
-          fileSize,
         },
         { jobId: randomUUID() },
       );
     }
-
-    // Extract video duration if uploaded from the web & CLI
-    if (asset.type == AssetType.VIDEO) {
-      await this.metadataExtractionQueue.add(
-        videoMetadataExtractionProcessorName,
-        { asset, fileName, fileSize },
-        { jobId: randomUUID() }
-      );
-    }
   }
 }

server/apps/microservices/src/processors/metadata-extraction.processor.ts

@@ -28,6 +28,7 @@ import geocoder, { InitOptions } from 'local-reverse-geocoder';
 import { getName } from 'i18n-iso-countries';
 import { find } from 'geo-tz';
 import * as luxon from 'luxon';
+import fs from 'node:fs';
 
 function geocoderInit(init: InitOptions) {
   return new Promise<void>(function (resolve) {
@@ -139,7 +140,7 @@ export class MetadataExtractionProcessor {
   @Process(exifExtractionProcessorName)
   async extractExifInfo(job: Job<IExifExtractionProcessor>) {
     try {
-      const { asset, fileName, fileSize }: { asset: AssetEntity; fileName: string; fileSize: number } = job.data;
+      const { asset, fileName }: { asset: AssetEntity; fileName: string } = job.data;
       const exifData = await exifr.parse(asset.originalPath, {
         tiff: true,
         ifd0: true as any,
@@ -160,6 +161,9 @@ export class MetadataExtractionProcessor {
 
       const createdAt = new Date(exifData.DateTimeOriginal || exifData.CreateDate || new Date(asset.createdAt));
 
+      const fileStats = fs.statSync(asset.originalPath);
+      const fileSizeInBytes = fileStats.size;
+
       const newExif = new ExifEntity();
       newExif.assetId = asset.id;
       newExif.make = exifData['Make'] || null;
@@ -167,7 +171,7 @@ export class MetadataExtractionProcessor {
       newExif.imageName = path.parse(fileName).name || null;
       newExif.exifImageHeight = exifData['ExifImageHeight'] || exifData['ImageHeight'] || null;
       newExif.exifImageWidth = exifData['ExifImageWidth'] || exifData['ImageWidth'] || null;
-      newExif.fileSizeInByte = fileSize || null;
+      newExif.fileSizeInByte = fileSizeInBytes || null;
       newExif.orientation = exifData['Orientation'] || null;
       newExif.dateTimeOriginal = createdAt;
       newExif.modifyDate = exifData['ModifyDate'] || null;

server/libs/job/src/interfaces/asset-uploaded.interface.ts

@@ -10,9 +10,4 @@ export interface IAssetUploadedJob {
    * Original file name
    */
   fileName: string;
-
-  /**
-   * File size in byte
-   */
-  fileSize: number;
 }

server/libs/job/src/interfaces/metadata-extraction.interface.ts

@@ -10,11 +10,6 @@ export interface IExifExtractionProcessor {
    * Original file name
    */
   fileName: string;
-
-  /**
-   * File size in byte
-   */
-  fileSize: number;
 }
 
 export interface IVideoLengthExtractionProcessor {
@@ -27,11 +22,6 @@ export interface IVideoLengthExtractionProcessor {
    * Original file name
    */
   fileName: string;
-
-  /**
-   * File size in byte
-   */
-  fileSize: number;
 }
 
 export interface IReverseGeocodingProcessor {