From 4cb74f0fe46a17b3a20bc9f22dc542de90104d9a Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Tue, 4 Apr 2023 18:23:07 -0400 Subject: [PATCH] refactor(server): reverse geocoding (#2167) * refactor(server): reverse geocoding * fix: nullable results --- .../metadata-extraction.processor.ts | 167 ++++-------------- server/libs/domain/src/index.ts | 1 + server/libs/domain/src/job/job.constants.ts | 1 - server/libs/domain/src/job/job.interface.ts | 8 - server/libs/domain/src/job/job.repository.ts | 2 - .../src/metadata/geocoding.repository.ts | 17 ++ server/libs/domain/src/metadata/index.ts | 1 + server/libs/infra/src/infra.config.ts | 19 ++ server/libs/infra/src/infra.module.ts | 5 +- .../src/repositories/geocoding.repository.ts | 44 +++++ server/libs/infra/src/repositories/index.ts | 1 + .../infra/src/repositories/job.repository.ts | 5 +- 12 files changed, 125 insertions(+), 146 deletions(-) create mode 100644 server/libs/domain/src/metadata/geocoding.repository.ts create mode 100644 server/libs/domain/src/metadata/index.ts create mode 100644 server/libs/infra/src/repositories/geocoding.repository.ts diff --git a/server/apps/microservices/src/processors/metadata-extraction.processor.ts b/server/apps/microservices/src/processors/metadata-extraction.processor.ts index 82975d80f2..43e1ac98d8 100644 --- a/server/apps/microservices/src/processors/metadata-extraction.processor.ts +++ b/server/apps/microservices/src/processors/metadata-extraction.processor.ts @@ -4,8 +4,8 @@ import { IAssetRepository, IAssetUploadedJob, IBaseJob, + IGeocodingRepository, IJobRepository, - IReverseGeocodingJob, JobName, QueueName, WithoutProperty, @@ -15,12 +15,10 @@ import { Process, Processor } from '@nestjs/bull'; import { Inject, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; +import tz_lookup from '@photostructure/tz-lookup'; import { Job } from 'bull'; import { ExifDateTime, exiftool, Tags } from 'exiftool-vendored'; -import tz_lookup from '@photostructure/tz-lookup'; import ffmpeg, { FfprobeData } from 'fluent-ffmpeg'; -import { getName } from 'i18n-iso-countries'; -import geocoder, { InitOptions } from 'local-reverse-geocoder'; import { Duration } from 'luxon'; import fs from 'node:fs'; import path from 'path'; @@ -34,123 +32,42 @@ interface ImmichTags extends Tags { ContentIdentifier?: string; } -function geocoderInit(init: InitOptions) { - return new Promise(function (resolve) { - geocoder.init(init, () => { - resolve(); - }); - }); -} - -function geocoderLookup(points: { latitude: number; longitude: number }[]) { - return new Promise(function (resolve) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - geocoder.lookUp(points, 1, (err, addresses) => { - resolve(addresses[0][0] as GeoData); - }); - }); -} - -const geocodingPrecisionLevels = ['cities15000', 'cities5000', 'cities1000', 'cities500']; - -export type AdminCode = { - name: string; - asciiName: string; - geoNameId: string; -}; - -export type GeoData = { - geoNameId: string; - name: string; - asciiName: string; - alternateNames: string; - latitude: string; - longitude: string; - featureClass: string; - featureCode: string; - countryCode: string; - cc2?: any; - admin1Code?: AdminCode | string; - admin2Code?: AdminCode | string; - admin3Code?: any; - admin4Code?: any; - population: string; - elevation: string; - dem: string; - timezone: string; - modificationDate: string; - distance: number; -}; - @Processor(QueueName.METADATA_EXTRACTION) export class MetadataExtractionProcessor { private logger = new Logger(MetadataExtractionProcessor.name); - private isGeocodeInitialized = false; private assetCore: AssetCore; + private reverseGeocodingEnabled: boolean; constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, - - @InjectRepository(ExifEntity) - private exifRepository: Repository, + @Inject(IGeocodingRepository) private geocodingRepository: IGeocodingRepository, + @InjectRepository(ExifEntity) private exifRepository: Repository, configService: ConfigService, ) { this.assetCore = new AssetCore(assetRepository, jobRepository); - - if (!configService.get('DISABLE_REVERSE_GEOCODING')) { - this.logger.log('Initializing Reverse Geocoding'); - geocoderInit({ - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - citiesFileOverride: geocodingPrecisionLevels[configService.get('REVERSE_GEOCODING_PRECISION')], - load: { - admin1: true, - admin2: true, - admin3And4: false, - alternateNames: false, - }, - countries: [], - dumpDirectory: - configService.get('REVERSE_GEOCODING_DUMP_DIRECTORY') || process.cwd() + '/.reverse-geocoding-dump/', - }).then(() => { - this.isGeocodeInitialized = true; - this.logger.log('Reverse Geocoding Initialised'); - }); - } + this.reverseGeocodingEnabled = !configService.get('DISABLE_REVERSE_GEOCODING'); + this.init(); } - private async reverseGeocodeExif( - latitude: number, - longitude: number, - ): Promise<{ country: string; state: string; city: string }> { - const geoCodeInfo = await geocoderLookup([{ latitude, longitude }]); - - const country = getName(geoCodeInfo.countryCode, 'en'); - const city = geoCodeInfo.name; - - let state = ''; - - if (geoCodeInfo.admin2Code) { - const adminCode2 = geoCodeInfo.admin2Code as AdminCode; - state += adminCode2.name; + private async init() { + this.logger.warn(`Reverse geocoding is ${this.reverseGeocodingEnabled ? 'enabled' : 'disabled'}`); + if (!this.reverseGeocodingEnabled) { + return; } - if (geoCodeInfo.admin1Code) { - const adminCode1 = geoCodeInfo.admin1Code as AdminCode; + try { + this.logger.log('Initializing Reverse Geocoding'); - if (geoCodeInfo.admin2Code) { - const adminCode2 = geoCodeInfo.admin2Code as AdminCode; - if (adminCode2.name) { - state += ', '; - } - } - state += adminCode1.name; + await this.jobRepository.pause(QueueName.METADATA_EXTRACTION); + await this.geocodingRepository.init(); + await this.jobRepository.resume(QueueName.METADATA_EXTRACTION); + + this.logger.log('Reverse Geocoding Initialized'); + } catch (error: any) { + this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack); } - - return { country, state, city }; } @Process(JobName.QUEUE_METADATA_EXTRACTION) @@ -241,18 +158,7 @@ export class MetadataExtractionProcessor { } } - /** - * Reverse Geocoding - * - * Get the city, state or region name of the asset - * based on lat/lon GPS coordinates. - */ - if (this.isGeocodeInitialized && newExif.latitude && newExif.longitude) { - const { country, state, city } = await this.reverseGeocodeExif(newExif.latitude, newExif.longitude); - newExif.country = country; - newExif.state = state; - newExif.city = city; - } + await this.applyReverseGeocoding(newExif); /** * IF the EXIF doesn't contain the width and height of the image, @@ -282,15 +188,6 @@ export class MetadataExtractionProcessor { } } - @Process({ name: JobName.REVERSE_GEOCODING }) - async reverseGeocoding(job: Job) { - if (this.isGeocodeInitialized) { - const { latitude, longitude } = job.data; - const { country, state, city } = await this.reverseGeocodeExif(latitude, longitude); - await this.exifRepository.update({ assetId: job.data.assetId }, { city, state, country }); - } - } - @Process({ name: JobName.EXTRACT_VIDEO_METADATA, concurrency: 2 }) async extractVideoMetadata(job: Job) { let asset = job.data.asset; @@ -377,13 +274,7 @@ export class MetadataExtractionProcessor { } } - // Reverse GeoCoding - if (this.isGeocodeInitialized && newExif.longitude && newExif.latitude) { - const { country, state, city } = await this.reverseGeocodeExif(newExif.latitude, newExif.longitude); - newExif.country = country; - newExif.state = state; - newExif.city = city; - } + await this.applyReverseGeocoding(newExif); for (const stream of data.streams) { if (stream.codec_type === 'video') { @@ -418,6 +309,20 @@ export class MetadataExtractionProcessor { } } + private async applyReverseGeocoding(newExif: ExifEntity) { + const { assetId, latitude, longitude } = newExif; + if (this.reverseGeocodingEnabled && longitude && latitude) { + try { + const { country, state, city } = await this.geocodingRepository.reverseGeocode({ latitude, longitude }); + newExif.country = country; + newExif.state = state; + newExif.city = city; + } catch (error: any) { + this.logger.warn(`Unable to run reverse geocoding for asset: ${assetId}, due to ${error}`, error?.stack); + } + } + } + private extractDuration(duration: number | string | null) { const videoDurationInSecond = Number(duration); if (!videoDurationInSecond) { diff --git a/server/libs/domain/src/index.ts b/server/libs/domain/src/index.ts index 2008d1a89a..82eacd737e 100644 --- a/server/libs/domain/src/index.ts +++ b/server/libs/domain/src/index.ts @@ -11,6 +11,7 @@ export * from './domain.module'; export * from './domain.util'; export * from './job'; export * from './media'; +export * from './metadata'; export * from './oauth'; export * from './search'; export * from './server-info'; diff --git a/server/libs/domain/src/job/job.constants.ts b/server/libs/domain/src/job/job.constants.ts index 9e08f7015a..ab272fff24 100644 --- a/server/libs/domain/src/job/job.constants.ts +++ b/server/libs/domain/src/job/job.constants.ts @@ -33,7 +33,6 @@ export enum JobName { QUEUE_METADATA_EXTRACTION = 'queue-metadata-extraction', EXIF_EXTRACTION = 'exif-extraction', EXTRACT_VIDEO_METADATA = 'extract-video-metadata', - REVERSE_GEOCODING = 'reverse-geocoding', // user deletion USER_DELETION = 'user-deletion', diff --git a/server/libs/domain/src/job/job.interface.ts b/server/libs/domain/src/job/job.interface.ts index 6b758b0916..912afb0c3f 100644 --- a/server/libs/domain/src/job/job.interface.ts +++ b/server/libs/domain/src/job/job.interface.ts @@ -28,11 +28,3 @@ export interface IDeleteFilesJob extends IBaseJob { export interface IUserDeletionJob extends IBaseJob { user: UserEntity; } - -export interface IReverseGeocodingJob extends IBaseJob { - assetId: string; - latitude: number; - longitude: number; -} - -export type IMetadataExtractionJob = IAssetUploadedJob | IReverseGeocodingJob; diff --git a/server/libs/domain/src/job/job.repository.ts b/server/libs/domain/src/job/job.repository.ts index 6f3a5c396c..f4a0509627 100644 --- a/server/libs/domain/src/job/job.repository.ts +++ b/server/libs/domain/src/job/job.repository.ts @@ -5,7 +5,6 @@ import { IBaseJob, IBulkEntityJob, IDeleteFilesJob, - IReverseGeocodingJob, IUserDeletionJob, } from './job.interface'; @@ -49,7 +48,6 @@ export type JobItem = | { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob } | { name: JobName.EXIF_EXTRACTION; data: IAssetUploadedJob } | { name: JobName.EXTRACT_VIDEO_METADATA; data: IAssetUploadedJob } - | { name: JobName.REVERSE_GEOCODING; data: IReverseGeocodingJob } // Object Tagging | { name: JobName.QUEUE_OBJECT_TAGGING; data: IBaseJob } diff --git a/server/libs/domain/src/metadata/geocoding.repository.ts b/server/libs/domain/src/metadata/geocoding.repository.ts new file mode 100644 index 0000000000..4bdc3ff8db --- /dev/null +++ b/server/libs/domain/src/metadata/geocoding.repository.ts @@ -0,0 +1,17 @@ +export const IGeocodingRepository = 'IGeocodingRepository'; + +export interface GeoPoint { + latitude: number; + longitude: number; +} + +export interface ReverseGeocodeResult { + country: string | null; + state: string | null; + city: string | null; +} + +export interface IGeocodingRepository { + init(): Promise; + reverseGeocode(point: GeoPoint): Promise; +} diff --git a/server/libs/domain/src/metadata/index.ts b/server/libs/domain/src/metadata/index.ts new file mode 100644 index 0000000000..14ac8b01e9 --- /dev/null +++ b/server/libs/domain/src/metadata/index.ts @@ -0,0 +1 @@ +export * from './geocoding.repository'; diff --git a/server/libs/infra/src/infra.config.ts b/server/libs/infra/src/infra.config.ts index fe593b15dd..5a7b70884b 100644 --- a/server/libs/infra/src/infra.config.ts +++ b/server/libs/infra/src/infra.config.ts @@ -1,6 +1,7 @@ import { QueueName } from '@app/domain'; import { BullModuleOptions } from '@nestjs/bull'; import { RedisOptions } from 'ioredis'; +import { InitOptions } from 'local-reverse-geocoder'; import { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration'; function parseRedisConfig(): RedisOptions { @@ -69,3 +70,21 @@ function parseTypeSenseConfig(): ConfigurationOptions { } export const typesenseConfig: ConfigurationOptions = parseTypeSenseConfig(); + +function parseLocalGeocodingConfig(): InitOptions { + const precision = Number(process.env.REVERSE_GEOCODING_PRECISION); + + return { + citiesFileOverride: precision ? ['cities15000', 'cities5000', 'cities1000', 'cities500'][precision] : undefined, + load: { + admin1: true, + admin2: true, + admin3And4: false, + alternateNames: false, + }, + countries: [], + dumpDirectory: process.env.REVERSE_GEOCODING_DUMP_DIRECTORY || process.cwd() + '/.reverse-geocoding-dump/', + }; +} + +export const localGeocodingConfig: InitOptions = parseLocalGeocodingConfig(); diff --git a/server/libs/infra/src/infra.module.ts b/server/libs/infra/src/infra.module.ts index 34347c1502..a95c920e1f 100644 --- a/server/libs/infra/src/infra.module.ts +++ b/server/libs/infra/src/infra.module.ts @@ -4,6 +4,7 @@ import { ICommunicationRepository, ICryptoRepository, IDeviceInfoRepository, + IGeocodingRepository, IJobRepository, IKeyRepository, IMachineLearningRepository, @@ -33,6 +34,7 @@ import { CryptoRepository, DeviceInfoRepository, FilesystemProvider, + GeocodingRepository, JobRepository, MachineLearningRepository, MediaRepository, @@ -50,8 +52,9 @@ const providers: Provider[] = [ { provide: ICommunicationRepository, useClass: CommunicationRepository }, { provide: ICryptoRepository, useClass: CryptoRepository }, { provide: IDeviceInfoRepository, useClass: DeviceInfoRepository }, - { provide: IKeyRepository, useClass: APIKeyRepository }, + { provide: IGeocodingRepository, useClass: GeocodingRepository }, { provide: IJobRepository, useClass: JobRepository }, + { provide: IKeyRepository, useClass: APIKeyRepository }, { provide: IMachineLearningRepository, useClass: MachineLearningRepository }, { provide: IMediaRepository, useClass: MediaRepository }, { provide: ISearchRepository, useClass: TypesenseRepository }, diff --git a/server/libs/infra/src/repositories/geocoding.repository.ts b/server/libs/infra/src/repositories/geocoding.repository.ts new file mode 100644 index 0000000000..982fd2f1fa --- /dev/null +++ b/server/libs/infra/src/repositories/geocoding.repository.ts @@ -0,0 +1,44 @@ +import { GeoPoint, ReverseGeocodeResult } from '@app/domain'; +import { localGeocodingConfig } from '@app/infra'; +import { Injectable, Logger } from '@nestjs/common'; +import { getName } from 'i18n-iso-countries'; +import geocoder, { AddressObject, InitOptions } from 'local-reverse-geocoder'; +import { promisify } from 'util'; + +export interface AdminCode { + name: string; + asciiName: string; + geoNameId: string; +} + +export type GeoData = AddressObject & { + admin1Code?: AdminCode | string; + admin2Code?: AdminCode | string; +}; + +const init = (options: InitOptions): Promise => new Promise((resolve) => geocoder.init(options, resolve)); +const lookup = promisify(geocoder.lookUp).bind(geocoder); + +@Injectable() +export class GeocodingRepository { + private logger = new Logger(GeocodingRepository.name); + + async init(): Promise { + await init(localGeocodingConfig); + } + + async reverseGeocode(point: GeoPoint): Promise { + this.logger.debug(`Request: ${point.latitude},${point.longitude}`); + + const [address] = await lookup([point], 1); + this.logger.verbose(`Raw: ${JSON.stringify(address, null, 2)}`); + + const { countryCode, name: city, admin1Code, admin2Code } = address[0] as GeoData; + const country = getName(countryCode, 'en'); + const stateParts = [(admin2Code as AdminCode)?.name, (admin1Code as AdminCode)?.name].filter((name) => !!name); + const state = stateParts.length > 0 ? stateParts.join(', ') : null; + this.logger.debug(`Normalized: ${JSON.stringify({ country, state, city })}`); + + return { country, state, city }; + } +} diff --git a/server/libs/infra/src/repositories/index.ts b/server/libs/infra/src/repositories/index.ts index dca8e811bb..6b144b2b81 100644 --- a/server/libs/infra/src/repositories/index.ts +++ b/server/libs/infra/src/repositories/index.ts @@ -5,6 +5,7 @@ export * from './communication.repository'; export * from './crypto.repository'; export * from './device-info.repository'; export * from './filesystem.provider'; +export * from './geocoding.repository'; export * from './job.repository'; export * from './machine-learning.repository'; export * from './media.repository'; diff --git a/server/libs/infra/src/repositories/job.repository.ts b/server/libs/infra/src/repositories/job.repository.ts index 8619a048d1..f129ade23b 100644 --- a/server/libs/infra/src/repositories/job.repository.ts +++ b/server/libs/infra/src/repositories/job.repository.ts @@ -1,8 +1,8 @@ import { IAssetJob, + IAssetUploadedJob, IBaseJob, IJobRepository, - IMetadataExtractionJob, JobCounts, JobItem, JobName, @@ -30,7 +30,7 @@ export class JobRepository implements IJobRepository { @InjectQueue(QueueName.BACKGROUND_TASK) private backgroundTask: Queue, @InjectQueue(QueueName.OBJECT_TAGGING) private objectTagging: Queue, @InjectQueue(QueueName.CLIP_ENCODING) private clipEmbedding: Queue, - @InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue, + @InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue, @InjectQueue(QueueName.STORAGE_TEMPLATE_MIGRATION) private storageTemplateMigration: Queue, @InjectQueue(QueueName.THUMBNAIL_GENERATION) private generateThumbnail: Queue, @InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue, @@ -88,7 +88,6 @@ export class JobRepository implements IJobRepository { case JobName.QUEUE_METADATA_EXTRACTION: case JobName.EXIF_EXTRACTION: case JobName.EXTRACT_VIDEO_METADATA: - case JobName.REVERSE_GEOCODING: await this.metadataExtraction.add(item.name, item.data); break;