From a9caa407ec521870caa1cad43c1d5d39622d4422 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Sat, 7 Sep 2024 13:39:10 -0400 Subject: [PATCH] refactor: metadata extraction (#12359) --- server/src/interfaces/map.interface.ts | 2 +- server/src/interfaces/metadata.interface.ts | 2 +- server/src/repositories/map.repository.ts | 4 +- .../src/repositories/metadata.repository.ts | 6 +- server/src/services/metadata.service.spec.ts | 7 +- server/src/services/metadata.service.ts | 263 +++++++++--------- 6 files changed, 146 insertions(+), 138 deletions(-) diff --git a/server/src/interfaces/map.interface.ts b/server/src/interfaces/map.interface.ts index dce75ffd25..80b37c3a5f 100644 --- a/server/src/interfaces/map.interface.ts +++ b/server/src/interfaces/map.interface.ts @@ -26,7 +26,7 @@ export interface MapMarker extends ReverseGeocodeResult { export interface IMapRepository { init(): Promise; - reverseGeocode(point: GeoPoint): Promise; + reverseGeocode(point: GeoPoint): Promise; getMapMarkers(ownerIds: string[], albumIds: string[], options?: MapMarkerSearchOptions): Promise; fetchStyle(url: string): Promise; } diff --git a/server/src/interfaces/metadata.interface.ts b/server/src/interfaces/metadata.interface.ts index 04e7b89d1e..39ff6ab4af 100644 --- a/server/src/interfaces/metadata.interface.ts +++ b/server/src/interfaces/metadata.interface.ts @@ -50,7 +50,7 @@ export interface ImmichTags extends Omit { export interface IMetadataRepository { teardown(): Promise; - readTags(path: string): Promise; + readTags(path: string): Promise; writeTags(path: string, tags: Partial): Promise; extractBinaryTag(tagName: string, path: string): Promise; getCountries(userIds: string[]): Promise>; diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.ts index da4e30d47c..3508de720b 100644 --- a/server/src/repositories/map.repository.ts +++ b/server/src/repositories/map.repository.ts @@ -124,7 +124,7 @@ export class MapRepository implements IMapRepository { } } - async reverseGeocode(point: GeoPoint): Promise { + async reverseGeocode(point: GeoPoint): Promise { this.logger.debug(`Request: ${point.latitude},${point.longitude}`); const response = await this.geodataPlacesRepository @@ -159,7 +159,7 @@ export class MapRepository implements IMapRepository { `Response from database for natural earth reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`, ); - return null; + return { country: null, state: null, city: null }; } this.logger.verbose(`Raw: ${JSON.stringify(ne_response, ['id', 'admin', 'admin_a3', 'type'], 2)}`); diff --git a/server/src/repositories/metadata.repository.ts b/server/src/repositories/metadata.repository.ts index abffc1b785..9902f04d9b 100644 --- a/server/src/repositories/metadata.repository.ts +++ b/server/src/repositories/metadata.repository.ts @@ -36,11 +36,11 @@ export class MetadataRepository implements IMetadataRepository { await this.exiftool.end(); } - readTags(path: string): Promise { + readTags(path: string): Promise { return this.exiftool.read(path).catch((error) => { this.logger.warn(`Error reading exif data (${path}): ${error}`, error?.stack); - return null; - }) as Promise; + return {}; + }) as Promise; } extractBinaryTag(path: string, tagName: string): Promise { diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index 52f6609772..5b447c2355 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -522,13 +522,13 @@ describe(MetadataService.name, () => { it('should extract the correct video orientation', async () => { assetMock.getByIds.mockResolvedValue([assetStub.video]); mediaMock.probe.mockResolvedValue(probeStub.videoStreamVertical2160p); - metadataMock.readTags.mockResolvedValue(null); + metadataMock.readTags.mockResolvedValue({}); await sut.handleMetadataExtraction({ id: assetStub.video.id }); expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id]); expect(assetMock.upsertExif).toHaveBeenCalledWith( - expect.objectContaining({ orientation: Orientation.Rotate270CW }), + expect.objectContaining({ orientation: Orientation.Rotate270CW.toString() }), ); }); @@ -814,6 +814,9 @@ describe(MetadataService.name, () => { projectionType: 'EQUIRECTANGULAR', timeZone: tags.tz, rating: tags.Rating, + country: null, + state: null, + city: null, }); expect(assetMock.update).toHaveBeenCalledWith({ id: assetStub.image.id, diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index 58e7b99448..cf51a332f8 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -1,5 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; -import { ContainerDirectoryItem, ExifDateTime, Tags } from 'exiftool-vendored'; +import { ContainerDirectoryItem, ExifDateTime, Maybe, Tags } from 'exiftool-vendored'; import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime'; import _ from 'lodash'; import { Duration } from 'luxon'; @@ -11,7 +11,6 @@ import { SystemConfigCore } from 'src/cores/system-config.core'; import { OnEmit } from 'src/decorators'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity } from 'src/entities/asset.entity'; -import { ExifEntity } from 'src/entities/exif.entity'; import { PersonEntity } from 'src/entities/person.entity'; import { AssetType, SourceType } from 'src/enum'; import { IAlbumRepository } from 'src/interfaces/album.interface'; @@ -30,7 +29,7 @@ import { QueueName, } from 'src/interfaces/job.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; -import { IMapRepository } from 'src/interfaces/map.interface'; +import { IMapRepository, ReverseGeocodeResult } from 'src/interfaces/map.interface'; import { IMediaRepository } from 'src/interfaces/media.interface'; import { IMetadataRepository, ImmichTags } from 'src/interfaces/metadata.interface'; import { IMoveRepository } from 'src/interfaces/move.interface'; @@ -56,23 +55,16 @@ const EXIF_DATE_TAGS: Array = [ ]; export enum Orientation { - Horizontal = '1', - MirrorHorizontal = '2', - Rotate180 = '3', - MirrorVertical = '4', - MirrorHorizontalRotate270CW = '5', - Rotate90CW = '6', - MirrorHorizontalRotate90CW = '7', - Rotate270CW = '8', + Horizontal = 1, + MirrorHorizontal = 2, + Rotate180 = 3, + MirrorVertical = 4, + MirrorHorizontalRotate270CW = 5, + Rotate90CW = 6, + MirrorHorizontalRotate90CW = 7, + Rotate270CW = 8, } -type ExifEntityWithoutGeocodeAndTypeOrm = Omit & { - dateTimeOriginal: Date; -}; - -const exifDate = (dt: ExifDateTime | string | undefined) => (dt instanceof ExifDateTime ? dt?.toDate() : null); -const tzOffset = (dt: ExifDateTime | string | undefined) => (dt instanceof ExifDateTime ? dt?.tzoffsetMinutes : null); - const validate = (value: T): NonNullable | null => { // handle lists of numbers if (Array.isArray(value)) { @@ -218,36 +210,73 @@ export class MetadataService { } async handleMetadataExtraction({ id }: IEntityJob): Promise { - const { metadata } = await this.configCore.getConfig({ withCache: true }); + const { metadata, reverseGeocoding } = await this.configCore.getConfig({ withCache: true }); const [asset] = await this.assetRepository.getByIds([id]); if (!asset) { return JobStatus.FAILED; } - const { exifData, exifTags } = await this.exifData(asset); + const stats = await this.storageRepository.stat(asset.originalPath); - if (asset.type === AssetType.VIDEO) { - await this.applyVideoMetadata(asset, exifData); - } + const exifTags = await this.getExifTags(asset); + + this.logger.verbose('Exif Tags', exifTags); + + const { dateTimeOriginal, localDateTime, timeZone, modifyDate } = this.getDates(asset, exifTags); + const { latitude, longitude, country, state, city } = await this.getGeo(exifTags, reverseGeocoding); + + const exifData = { + assetId: asset.id, + + // dates + dateTimeOriginal, + modifyDate, + timeZone, + + // gps + latitude, + longitude, + country, + state, + city, + + // image/file + fileSizeInByte: stats.size, + exifImageHeight: validate(exifTags.ImageHeight), + exifImageWidth: validate(exifTags.ImageWidth), + orientation: validate(exifTags.Orientation)?.toString() ?? null, + projectionType: exifTags.ProjectionType ? String(exifTags.ProjectionType).toUpperCase() : null, + bitsPerSample: this.getBitsPerSample(exifTags), + colorspace: exifTags.ColorSpace ?? null, + + // camera + make: exifTags.Make ?? null, + model: exifTags.Model ?? null, + fps: validate(Number.parseFloat(exifTags.VideoFrameRate!)), + iso: validate(exifTags.ISO), + exposureTime: exifTags.ExposureTime ?? null, + lensModel: exifTags.LensModel ?? null, + fNumber: validate(exifTags.FNumber), + focalLength: validate(exifTags.FocalLength), + + // comments + description: String(exifTags.ImageDescription || exifTags.Description || '').trim(), + profileDescription: exifTags.ProfileDescription || null, + rating: exifTags.Rating ?? null, + + // grouping + livePhotoCID: (exifTags.ContentIdentifier || exifTags.MediaGroupUUID) ?? null, + autoStackId: this.getAutoStackId(exifTags), + }; - await this.applyMotionPhotos(asset, exifTags); - await this.applyReverseGeocoding(asset, exifData); await this.applyTagList(asset, exifTags); + await this.applyMotionPhotos(asset, exifTags); await this.assetRepository.upsertExif(exifData); - const dateTimeOriginal = exifData.dateTimeOriginal; - let localDateTime = dateTimeOriginal ?? undefined; - - const timeZoneOffset = tzOffset(firstDateTime(exifTags as Tags)) ?? 0; - - if (dateTimeOriginal && timeZoneOffset) { - localDateTime = new Date(dateTimeOriginal.getTime() + timeZoneOffset * 60_000); - } - await this.assetRepository.update({ id: asset.id, - duration: asset.duration, + duration: exifTags.Duration?.toString() ?? null, localDateTime, fileCreatedAt: exifData.dateTimeOriginal ?? undefined, }); @@ -338,25 +367,20 @@ export class MetadataService { return JobStatus.SUCCESS; } - private async applyReverseGeocoding(asset: AssetEntity, exifData: ExifEntityWithoutGeocodeAndTypeOrm) { - const { latitude, longitude } = exifData; - const { reverseGeocoding } = await this.configCore.getConfig({ withCache: true }); - if (!reverseGeocoding.enabled || !longitude || !latitude) { - return; + private async getExifTags(asset: AssetEntity): Promise { + const mediaTags = await this.repository.readTags(asset.originalPath); + const sidecarTags = asset.sidecarPath ? await this.repository.readTags(asset.sidecarPath) : {}; + const videoTags = asset.type === AssetType.VIDEO ? await this.getVideoTags(asset.originalPath) : {}; + + // make sure dates comes from sidecar + const sidecarDate = firstDateTime(sidecarTags as Tags, EXIF_DATE_TAGS); + if (sidecarDate) { + for (const tag of EXIF_DATE_TAGS) { + delete mediaTags[tag]; + } } - try { - const reverseGeocode = await this.mapRepository.reverseGeocode({ latitude, longitude }); - if (!reverseGeocode) { - return; - } - Object.assign(exifData, reverseGeocode); - } catch (error: Error | any) { - this.logger.warn( - `Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`, - error?.stack, - ); - } + return { ...mediaTags, ...videoTags, ...sidecarTags }; } private async applyTagList(asset: AssetEntity, exifTags: ImmichTags) { @@ -576,66 +600,65 @@ export class MetadataService { ); } - private async exifData( - asset: AssetEntity, - ): Promise<{ exifData: ExifEntityWithoutGeocodeAndTypeOrm; exifTags: ImmichTags }> { - const stats = await this.storageRepository.stat(asset.originalPath); - const mediaTags = await this.repository.readTags(asset.originalPath); - const sidecarTags = asset.sidecarPath ? await this.repository.readTags(asset.sidecarPath) : null; + private getDates(asset: AssetEntity, exifTags: ImmichTags) { + const dateTime = firstDateTime(exifTags as Maybe, EXIF_DATE_TAGS); + this.logger.debug(`Asset ${asset.id} date time is ${dateTime}`); - // ensure date from sidecar is used if present - const hasDateOverride = !!this.getDateTimeOriginal(sidecarTags); - if (mediaTags && hasDateOverride) { - for (const tag of EXIF_DATE_TAGS) { - delete mediaTags[tag]; - } + // created + let dateTimeOriginal = dateTime?.toDate(); + if (!dateTimeOriginal) { + this.logger.warn(`Asset ${asset.id} has no valid date (${dateTime}), falling back to asset.fileCreatedAt`); + dateTimeOriginal = asset.fileCreatedAt; } - const exifTags = { ...mediaTags, ...sidecarTags }; + // timezone + let timeZone = exifTags.tz ?? null; + if (timeZone == null && dateTime?.rawValue?.endsWith('+00:00')) { + // exiftool-vendored returns "no timezone" information even though "+00:00" might be set explicitly + // https://github.com/photostructure/exiftool-vendored.js/issues/203 + timeZone = 'UTC+0'; + } - this.logger.verbose('Exif Tags', exifTags); + if (timeZone) { + this.logger.debug(`Asset ${asset.id} timezone is ${timeZone} (via ${exifTags.tzSource})`); + } else { + this.logger.warn(`Asset ${asset.id} has no time zone information`); + } - const dateTimeOriginalWithRawValue = this.getDateTimeOriginalWithRawValue(exifTags); - const dateTimeOriginal = dateTimeOriginalWithRawValue.exifDate ?? asset.fileCreatedAt; - const timeZone = this.getTimeZone(exifTags, dateTimeOriginalWithRawValue.rawValue); + // offset minutes + const offsetMinutes = dateTime?.tzoffsetMinutes || 0; + let localDateTime = dateTimeOriginal; + if (offsetMinutes) { + localDateTime = new Date(dateTimeOriginal.getTime() + offsetMinutes * 60_000); + this.logger.debug(`Asset ${asset.id} local time is offset by ${offsetMinutes} minutes`); + } - const exifData = { - // altitude: tags.GPSAltitude ?? null, - assetId: asset.id, - bitsPerSample: this.getBitsPerSample(exifTags), - colorspace: exifTags.ColorSpace ?? null, + return { dateTimeOriginal, - description: String(exifTags.ImageDescription || exifTags.Description || '').trim(), - exifImageHeight: validate(exifTags.ImageHeight), - exifImageWidth: validate(exifTags.ImageWidth), - exposureTime: exifTags.ExposureTime ?? null, - fileSizeInByte: stats.size, - fNumber: validate(exifTags.FNumber), - focalLength: validate(exifTags.FocalLength), - fps: validate(Number.parseFloat(exifTags.VideoFrameRate!)), - iso: validate(exifTags.ISO), - latitude: validate(exifTags.GPSLatitude), - lensModel: exifTags.LensModel ?? null, - livePhotoCID: (exifTags.ContentIdentifier || exifTags.MediaGroupUUID) ?? null, - autoStackId: this.getAutoStackId(exifTags), - longitude: validate(exifTags.GPSLongitude), - make: exifTags.Make ?? null, - model: exifTags.Model ?? null, - modifyDate: exifDate(exifTags.ModifyDate) ?? asset.fileModifiedAt, - orientation: validate(exifTags.Orientation)?.toString() ?? null, - profileDescription: exifTags.ProfileDescription || null, - projectionType: exifTags.ProjectionType ? String(exifTags.ProjectionType).toUpperCase() : null, timeZone, - rating: exifTags.Rating ?? null, + localDateTime, + modifyDate: (exifTags.ModifyDate as ExifDateTime)?.toDate() ?? asset.fileModifiedAt, }; + } - if (exifData.latitude === 0 && exifData.longitude === 0) { - this.logger.warn('Exif data has latitude and longitude of 0, setting to null'); - exifData.latitude = null; - exifData.longitude = null; + private async getGeo(tags: ImmichTags, reverseGeocoding: SystemConfig['reverseGeocoding']) { + let latitude = validate(tags.GPSLatitude); + let longitude = validate(tags.GPSLongitude); + + // TODO take ref into account + + if (latitude === 0 && longitude === 0) { + this.logger.warn('Latitude and longitude of 0, setting to null'); + latitude = null; + longitude = null; } - return { exifData, exifTags }; + let result: ReverseGeocodeResult = { country: null, state: null, city: null }; + if (reverseGeocoding.enabled && longitude && latitude) { + result = await this.mapRepository.reverseGeocode({ latitude, longitude }); + } + + return { ...result, latitude, longitude }; } private getAutoStackId(tags: ImmichTags | null): string | null { @@ -645,28 +668,6 @@ export class MetadataService { return tags.BurstID ?? tags.BurstUUID ?? tags.CameraBurstID ?? tags.MediaUniqueID ?? null; } - private getDateTimeOriginal(tags: ImmichTags | Tags | null) { - return this.getDateTimeOriginalWithRawValue(tags).exifDate; - } - - private getDateTimeOriginalWithRawValue(tags: ImmichTags | Tags | null): { exifDate: Date | null; rawValue: string } { - if (!tags) { - return { exifDate: null, rawValue: '' }; - } - const first = firstDateTime(tags as Tags, EXIF_DATE_TAGS); - return { exifDate: exifDate(first), rawValue: first?.rawValue ?? '' }; - } - - private getTimeZone(exifTags: ImmichTags, rawValue: string) { - const timeZone = exifTags.tz ?? null; - if (timeZone == null && rawValue.endsWith('+00:00')) { - // exiftool-vendored returns "no timezone" information even though "+00:00" might be set explicitly - // https://github.com/photostructure/exiftool-vendored.js/issues/203 - return 'UTC+0'; - } - return timeZone; - } - private getBitsPerSample(tags: ImmichTags): number | null { const bitDepthTags = [ tags.BitsPerSample, @@ -685,33 +686,37 @@ export class MetadataService { return bitsPerSample; } - private async applyVideoMetadata(asset: AssetEntity, exifData: ExifEntityWithoutGeocodeAndTypeOrm) { - const { videoStreams, format } = await this.mediaRepository.probe(asset.originalPath); + private async getVideoTags(originalPath: string) { + const { videoStreams, format } = await this.mediaRepository.probe(originalPath); + + const tags: Pick = {}; if (videoStreams[0]) { switch (videoStreams[0].rotation) { case -90: { - exifData.orientation = Orientation.Rotate90CW; + tags.Orientation = Orientation.Rotate90CW; break; } case 0: { - exifData.orientation = Orientation.Horizontal; + tags.Orientation = Orientation.Horizontal; break; } case 90: { - exifData.orientation = Orientation.Rotate270CW; + tags.Orientation = Orientation.Rotate270CW; break; } case 180: { - exifData.orientation = Orientation.Rotate180; + tags.Orientation = Orientation.Rotate180; break; } } } if (format.duration) { - asset.duration = Duration.fromObject({ seconds: format.duration }).toFormat('hh:mm:ss.SSS'); + tags.Duration = Duration.fromObject({ seconds: format.duration }).toFormat('hh:mm:ss.SSS'); } + + return tags; } private async processSidecar(id: string, isSync: boolean): Promise {