self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-27 10:58:13 +02:00
Code Issues Releases Activity

refactor: metadata extraction (#12359)

Browse Source
This commit is contained in:
Jason Rasmussen 2024-09-07 13:39:10 -04:00 committed by GitHub
parent 00a5da0ebc
commit a9caa407ec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 146 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/interfaces/map.interface.ts
View File

@ -26,7 +26,7 @@ export interface MapMarker extends ReverseGeocodeResult {
export interface IMapRepository {
init(): Promise<void>;
reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null>;
reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
getMapMarkers(ownerIds: string[], albumIds: string[], options?: MapMarkerSearchOptions): Promise<MapMarker[]>;
fetchStyle(url: string): Promise<any>;
}

2
server/src/interfaces/metadata.interface.ts
View File

@ -50,7 +50,7 @@ export interface ImmichTags extends Omit<Tags, TagsWithWrongTypes> {
export interface IMetadataRepository {
teardown(): Promise<void>;
readTags(path: string): Promise<ImmichTags | null>;
readTags(path: string): Promise<ImmichTags>;
writeTags(path: string, tags: Partial<Tags>): Promise<void>;
extractBinaryTag(tagName: string, path: string): Promise<Buffer>;
getCountries(userIds: string[]): Promise<Array<string | null>>;

4
server/src/repositories/map.repository.ts
View File

@ -124,7 +124,7 @@ export class MapRepository implements IMapRepository {
}
}
async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null> {
async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {
this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
const response = await this.geodataPlacesRepository
@ -159,7 +159,7 @@ export class MapRepository implements IMapRepository {
`Response from database for natural earth reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`,
);
return null;
return { country: null, state: null, city: null };
}
this.logger.verbose(`Raw: ${JSON.stringify(ne_response, ['id', 'admin', 'admin_a3', 'type'], 2)}`);

6
server/src/repositories/metadata.repository.ts
View File

@ -36,11 +36,11 @@ export class MetadataRepository implements IMetadataRepository {
await this.exiftool.end();
}
readTags(path: string): Promise<ImmichTags | null> {
readTags(path: string): Promise<ImmichTags> {
return this.exiftool.read(path).catch((error) => {
this.logger.warn(`Error reading exif data (${path}): ${error}`, error?.stack);
return null;
}) as Promise<ImmichTags | null>;
return {};
}) as Promise<ImmichTags>;
}
extractBinaryTag(path: string, tagName: string): Promise<Buffer> {

7
server/src/services/metadata.service.spec.ts
View File

@ -522,13 +522,13 @@ describe(MetadataService.name, () => {
it('should extract the correct video orientation', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.video]);
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVertical2160p);
metadataMock.readTags.mockResolvedValue(null);
metadataMock.readTags.mockResolvedValue({});
await sut.handleMetadataExtraction({ id: assetStub.video.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.video.id]);
expect(assetMock.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ orientation: Orientation.Rotate270CW }),
expect.objectContaining({ orientation: Orientation.Rotate270CW.toString() }),
);
});
@ -814,6 +814,9 @@ describe(MetadataService.name, () => {
projectionType: 'EQUIRECTANGULAR',
timeZone: tags.tz,
rating: tags.Rating,
country: null,
state: null,
city: null,
});
expect(assetMock.update).toHaveBeenCalledWith({
id: assetStub.image.id,

263
server/src/services/metadata.service.ts
View File

@ -1,5 +1,5 @@
import { Inject, Injectable } from '@nestjs/common';
import { ContainerDirectoryItem, ExifDateTime, Tags } from 'exiftool-vendored';
import { ContainerDirectoryItem, ExifDateTime, Maybe, Tags } from 'exiftool-vendored';
import { firstDateTime } from 'exiftool-vendored/dist/FirstDateTime';
import _ from 'lodash';
import { Duration } from 'luxon';
@ -11,7 +11,6 @@ import { SystemConfigCore } from 'src/cores/system-config.core';
import { OnEmit } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { PersonEntity } from 'src/entities/person.entity';
import { AssetType, SourceType } from 'src/enum';
import { IAlbumRepository } from 'src/interfaces/album.interface';
@ -30,7 +29,7 @@ import {
QueueName,
} from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IMapRepository } from 'src/interfaces/map.interface';
import { IMapRepository, ReverseGeocodeResult } from 'src/interfaces/map.interface';
import { IMediaRepository } from 'src/interfaces/media.interface';
import { IMetadataRepository, ImmichTags } from 'src/interfaces/metadata.interface';
import { IMoveRepository } from 'src/interfaces/move.interface';
@ -56,23 +55,16 @@ const EXIF_DATE_TAGS: Array<keyof Tags> = [
];
export enum Orientation {
Horizontal = '1',
MirrorHorizontal = '2',
Rotate180 = '3',
MirrorVertical = '4',
MirrorHorizontalRotate270CW = '5',
Rotate90CW = '6',
MirrorHorizontalRotate90CW = '7',
Rotate270CW = '8',
Horizontal = 1,
MirrorHorizontal = 2,
Rotate180 = 3,
MirrorVertical = 4,
MirrorHorizontalRotate270CW = 5,
Rotate90CW = 6,
MirrorHorizontalRotate90CW = 7,
Rotate270CW = 8,
}
type ExifEntityWithoutGeocodeAndTypeOrm = Omit<ExifEntity, 'city' | 'state' | 'country' | 'description'> & {
dateTimeOriginal: Date;
};
const exifDate = (dt: ExifDateTime | string | undefined) => (dt instanceof ExifDateTime ? dt?.toDate() : null);
const tzOffset = (dt: ExifDateTime | string | undefined) => (dt instanceof ExifDateTime ? dt?.tzoffsetMinutes : null);
const validate = <T>(value: T): NonNullable<T> | null => {
// handle lists of numbers
if (Array.isArray(value)) {
@ -218,36 +210,73 @@ export class MetadataService {
}
async handleMetadataExtraction({ id }: IEntityJob): Promise<JobStatus> {
const { metadata } = await this.configCore.getConfig({ withCache: true });
const { metadata, reverseGeocoding } = await this.configCore.getConfig({ withCache: true });
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset) {
return JobStatus.FAILED;
}
const { exifData, exifTags } = await this.exifData(asset);
const stats = await this.storageRepository.stat(asset.originalPath);
if (asset.type === AssetType.VIDEO) {
await this.applyVideoMetadata(asset, exifData);
}
const exifTags = await this.getExifTags(asset);
this.logger.verbose('Exif Tags', exifTags);
const { dateTimeOriginal, localDateTime, timeZone, modifyDate } = this.getDates(asset, exifTags);
const { latitude, longitude, country, state, city } = await this.getGeo(exifTags, reverseGeocoding);
const exifData = {
assetId: asset.id,
// dates
dateTimeOriginal,
modifyDate,
timeZone,
// gps
latitude,
longitude,
country,
state,
city,
// image/file
fileSizeInByte: stats.size,
exifImageHeight: validate(exifTags.ImageHeight),
exifImageWidth: validate(exifTags.ImageWidth),
orientation: validate(exifTags.Orientation)?.toString() ?? null,
projectionType: exifTags.ProjectionType ? String(exifTags.ProjectionType).toUpperCase() : null,
bitsPerSample: this.getBitsPerSample(exifTags),
colorspace: exifTags.ColorSpace ?? null,
// camera
make: exifTags.Make ?? null,
model: exifTags.Model ?? null,
fps: validate(Number.parseFloat(exifTags.VideoFrameRate!)),
iso: validate(exifTags.ISO),
exposureTime: exifTags.ExposureTime ?? null,
lensModel: exifTags.LensModel ?? null,
fNumber: validate(exifTags.FNumber),
focalLength: validate(exifTags.FocalLength),
// comments
description: String(exifTags.ImageDescription || exifTags.Description || '').trim(),
profileDescription: exifTags.ProfileDescription || null,
rating: exifTags.Rating ?? null,
// grouping
livePhotoCID: (exifTags.ContentIdentifier || exifTags.MediaGroupUUID) ?? null,
autoStackId: this.getAutoStackId(exifTags),
};
await this.applyMotionPhotos(asset, exifTags);
await this.applyReverseGeocoding(asset, exifData);
await this.applyTagList(asset, exifTags);
await this.applyMotionPhotos(asset, exifTags);
await this.assetRepository.upsertExif(exifData);
const dateTimeOriginal = exifData.dateTimeOriginal;
let localDateTime = dateTimeOriginal ?? undefined;
const timeZoneOffset = tzOffset(firstDateTime(exifTags as Tags)) ?? 0;
if (dateTimeOriginal && timeZoneOffset) {
localDateTime = new Date(dateTimeOriginal.getTime() + timeZoneOffset * 60_000);
}
await this.assetRepository.update({
id: asset.id,
duration: asset.duration,
duration: exifTags.Duration?.toString() ?? null,
localDateTime,
fileCreatedAt: exifData.dateTimeOriginal ?? undefined,
});
@ -338,25 +367,20 @@ export class MetadataService {
return JobStatus.SUCCESS;
}
private async applyReverseGeocoding(asset: AssetEntity, exifData: ExifEntityWithoutGeocodeAndTypeOrm) {
const { latitude, longitude } = exifData;
const { reverseGeocoding } = await this.configCore.getConfig({ withCache: true });
if (!reverseGeocoding.enabled || !longitude || !latitude) {
return;
private async getExifTags(asset: AssetEntity): Promise<ImmichTags> {
const mediaTags = await this.repository.readTags(asset.originalPath);
const sidecarTags = asset.sidecarPath ? await this.repository.readTags(asset.sidecarPath) : {};
const videoTags = asset.type === AssetType.VIDEO ? await this.getVideoTags(asset.originalPath) : {};
// make sure dates comes from sidecar
const sidecarDate = firstDateTime(sidecarTags as Tags, EXIF_DATE_TAGS);
if (sidecarDate) {
for (const tag of EXIF_DATE_TAGS) {
delete mediaTags[tag];
}
}
try {
const reverseGeocode = await this.mapRepository.reverseGeocode({ latitude, longitude });
if (!reverseGeocode) {
return;
}
Object.assign(exifData, reverseGeocode);
} catch (error: Error | any) {
this.logger.warn(
`Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`,
error?.stack,
);
}
return { ...mediaTags, ...videoTags, ...sidecarTags };
}
private async applyTagList(asset: AssetEntity, exifTags: ImmichTags) {
@ -576,66 +600,65 @@ export class MetadataService {
);
}
private async exifData(
asset: AssetEntity,
): Promise<{ exifData: ExifEntityWithoutGeocodeAndTypeOrm; exifTags: ImmichTags }> {
const stats = await this.storageRepository.stat(asset.originalPath);
const mediaTags = await this.repository.readTags(asset.originalPath);
const sidecarTags = asset.sidecarPath ? await this.repository.readTags(asset.sidecarPath) : null;
private getDates(asset: AssetEntity, exifTags: ImmichTags) {
const dateTime = firstDateTime(exifTags as Maybe<Tags>, EXIF_DATE_TAGS);
this.logger.debug(`Asset ${asset.id} date time is ${dateTime}`);
// ensure date from sidecar is used if present
const hasDateOverride = !!this.getDateTimeOriginal(sidecarTags);
if (mediaTags && hasDateOverride) {
for (const tag of EXIF_DATE_TAGS) {
delete mediaTags[tag];
}
// created
let dateTimeOriginal = dateTime?.toDate();
if (!dateTimeOriginal) {
this.logger.warn(`Asset ${asset.id} has no valid date (${dateTime}), falling back to asset.fileCreatedAt`);
dateTimeOriginal = asset.fileCreatedAt;
}
const exifTags = { ...mediaTags, ...sidecarTags };
// timezone
let timeZone = exifTags.tz ?? null;
if (timeZone == null && dateTime?.rawValue?.endsWith('+00:00')) {
// exiftool-vendored returns "no timezone" information even though "+00:00" might be set explicitly
// https://github.com/photostructure/exiftool-vendored.js/issues/203
timeZone = 'UTC+0';
}
this.logger.verbose('Exif Tags', exifTags);
if (timeZone) {
this.logger.debug(`Asset ${asset.id} timezone is ${timeZone} (via ${exifTags.tzSource})`);
} else {
this.logger.warn(`Asset ${asset.id} has no time zone information`);
}
const dateTimeOriginalWithRawValue = this.getDateTimeOriginalWithRawValue(exifTags);
const dateTimeOriginal = dateTimeOriginalWithRawValue.exifDate ?? asset.fileCreatedAt;
const timeZone = this.getTimeZone(exifTags, dateTimeOriginalWithRawValue.rawValue);
// offset minutes
const offsetMinutes = dateTime?.tzoffsetMinutes || 0;
let localDateTime = dateTimeOriginal;
if (offsetMinutes) {
localDateTime = new Date(dateTimeOriginal.getTime() + offsetMinutes * 60_000);
this.logger.debug(`Asset ${asset.id} local time is offset by ${offsetMinutes} minutes`);
}
const exifData = {
// altitude: tags.GPSAltitude ?? null,
assetId: asset.id,
bitsPerSample: this.getBitsPerSample(exifTags),
colorspace: exifTags.ColorSpace ?? null,
return {
dateTimeOriginal,
description: String(exifTags.ImageDescription || exifTags.Description || '').trim(),
exifImageHeight: validate(exifTags.ImageHeight),
exifImageWidth: validate(exifTags.ImageWidth),
exposureTime: exifTags.ExposureTime ?? null,
fileSizeInByte: stats.size,
fNumber: validate(exifTags.FNumber),
focalLength: validate(exifTags.FocalLength),
fps: validate(Number.parseFloat(exifTags.VideoFrameRate!)),
iso: validate(exifTags.ISO),
latitude: validate(exifTags.GPSLatitude),
lensModel: exifTags.LensModel ?? null,
livePhotoCID: (exifTags.ContentIdentifier || exifTags.MediaGroupUUID) ?? null,
autoStackId: this.getAutoStackId(exifTags),
longitude: validate(exifTags.GPSLongitude),
make: exifTags.Make ?? null,
model: exifTags.Model ?? null,
modifyDate: exifDate(exifTags.ModifyDate) ?? asset.fileModifiedAt,
orientation: validate(exifTags.Orientation)?.toString() ?? null,
profileDescription: exifTags.ProfileDescription || null,
projectionType: exifTags.ProjectionType ? String(exifTags.ProjectionType).toUpperCase() : null,
timeZone,
rating: exifTags.Rating ?? null,
localDateTime,
modifyDate: (exifTags.ModifyDate as ExifDateTime)?.toDate() ?? asset.fileModifiedAt,
};
}
if (exifData.latitude === 0 && exifData.longitude === 0) {
this.logger.warn('Exif data has latitude and longitude of 0, setting to null');
exifData.latitude = null;
exifData.longitude = null;
private async getGeo(tags: ImmichTags, reverseGeocoding: SystemConfig['reverseGeocoding']) {
let latitude = validate(tags.GPSLatitude);
let longitude = validate(tags.GPSLongitude);
// TODO take ref into account
if (latitude === 0 && longitude === 0) {
this.logger.warn('Latitude and longitude of 0, setting to null');
latitude = null;
longitude = null;
}
return { exifData, exifTags };
let result: ReverseGeocodeResult = { country: null, state: null, city: null };
if (reverseGeocoding.enabled && longitude && latitude) {
result = await this.mapRepository.reverseGeocode({ latitude, longitude });
}
return { ...result, latitude, longitude };
}
private getAutoStackId(tags: ImmichTags | null): string | null {
@ -645,28 +668,6 @@ export class MetadataService {
return tags.BurstID ?? tags.BurstUUID ?? tags.CameraBurstID ?? tags.MediaUniqueID ?? null;
}
private getDateTimeOriginal(tags: ImmichTags | Tags | null) {
return this.getDateTimeOriginalWithRawValue(tags).exifDate;
}
private getDateTimeOriginalWithRawValue(tags: ImmichTags | Tags | null): { exifDate: Date | null; rawValue: string } {
if (!tags) {
return { exifDate: null, rawValue: '' };
}
const first = firstDateTime(tags as Tags, EXIF_DATE_TAGS);
return { exifDate: exifDate(first), rawValue: first?.rawValue ?? '' };
}
private getTimeZone(exifTags: ImmichTags, rawValue: string) {
const timeZone = exifTags.tz ?? null;
if (timeZone == null && rawValue.endsWith('+00:00')) {
// exiftool-vendored returns "no timezone" information even though "+00:00" might be set explicitly
// https://github.com/photostructure/exiftool-vendored.js/issues/203
return 'UTC+0';
}
return timeZone;
}
private getBitsPerSample(tags: ImmichTags): number | null {
const bitDepthTags = [
tags.BitsPerSample,
@ -685,33 +686,37 @@ export class MetadataService {
return bitsPerSample;
}
private async applyVideoMetadata(asset: AssetEntity, exifData: ExifEntityWithoutGeocodeAndTypeOrm) {
const { videoStreams, format } = await this.mediaRepository.probe(asset.originalPath);
private async getVideoTags(originalPath: string) {
const { videoStreams, format } = await this.mediaRepository.probe(originalPath);
const tags: Pick<ImmichTags, 'Duration' | 'Orientation'> = {};
if (videoStreams[0]) {
switch (videoStreams[0].rotation) {
case -90: {
exifData.orientation = Orientation.Rotate90CW;
tags.Orientation = Orientation.Rotate90CW;
break;
}
case 0: {
exifData.orientation = Orientation.Horizontal;
tags.Orientation = Orientation.Horizontal;
break;
}
case 90: {
exifData.orientation = Orientation.Rotate270CW;
tags.Orientation = Orientation.Rotate270CW;
break;
}
case 180: {
exifData.orientation = Orientation.Rotate180;
tags.Orientation = Orientation.Rotate180;
break;
}
}
}
if (format.duration) {
asset.duration = Duration.fromObject({ seconds: format.duration }).toFormat('hh:mm:ss.SSS');
tags.Duration = Duration.fromObject({ seconds: format.duration }).toFormat('hh:mm:ss.SSS');
}
return tags;
}
private async processSidecar(id: string, isSync: boolean): Promise<JobStatus> {