From 4f89195702c623953deef2f2ec2398c33948351c Mon Sep 17 00:00:00 2001 From: pokjay <31060527+pokjay@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:27:07 +0300 Subject: [PATCH] feat(server): country geocoding for remote locations (#10950) Co-authored-by: Zack Pollard Co-authored-by: Daniel Dietzler --- e2e/src/api/specs/asset.e2e-spec.ts | 16 ++++ server/src/constants.ts | 1 + server/src/entities/index.ts | 2 + .../natural-earth-countries.entity.ts | 19 ++++ .../1720375641148-natural-earth-countries.ts | 14 +++ server/src/repositories/map.repository.ts | 87 +++++++++++++++++-- .../asset-viewer/detail-panel-location.svelte | 6 +- 7 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 server/src/entities/natural-earth-countries.entity.ts create mode 100644 server/src/migrations/1720375641148-natural-earth-countries.ts diff --git a/e2e/src/api/specs/asset.e2e-spec.ts b/e2e/src/api/specs/asset.e2e-spec.ts index d0cf0c492c..694114aed5 100644 --- a/e2e/src/api/specs/asset.e2e-spec.ts +++ b/e2e/src/api/specs/asset.e2e-spec.ts @@ -507,6 +507,22 @@ describe('/asset', () => { expect(status).toEqual(200); }); + it('should geocode country from gps data in the middle of nowhere', async () => { + const { status } = await request(app) + .put(`/assets/${user1Assets[0].id}`) + .set('Authorization', `Bearer ${user1.accessToken}`) + .send({ latitude: 42, longitude: 69 }); + expect(status).toEqual(200); + + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + const asset = await getAssetInfo({ id: user1Assets[0].id }, { headers: asBearerAuth(user1.accessToken) }); + expect(asset).toMatchObject({ + id: user1Assets[0].id, + exifInfo: expect.objectContaining({ city: null, country: 'Kazakhstan' }), + }); + }); + it('should set the description', async () => { const { status, body } = await request(app) .put(`/assets/${user1Assets[0].id}`) diff --git a/server/src/constants.ts b/server/src/constants.ts index cd418e9234..0d1d992992 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -43,6 +43,7 @@ export const resourcePaths = { admin1: join(folders.geodata, 'admin1CodesASCII.txt'), admin2: join(folders.geodata, 'admin2Codes.txt'), cities500: join(folders.geodata, citiesFile), + naturalEarthCountriesPath: join(folders.geodata, 'ne_10m_admin_0_countries.geojson'), }, web: { root: folders.web, diff --git a/server/src/entities/index.ts b/server/src/entities/index.ts index 6090b8f918..148e264095 100644 --- a/server/src/entities/index.ts +++ b/server/src/entities/index.ts @@ -12,6 +12,7 @@ import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity'; import { LibraryEntity } from 'src/entities/library.entity'; import { MemoryEntity } from 'src/entities/memory.entity'; import { MoveEntity } from 'src/entities/move.entity'; +import { NaturalEarthCountriesEntity } from 'src/entities/natural-earth-countries.entity'; import { PartnerEntity } from 'src/entities/partner.entity'; import { PersonEntity } from 'src/entities/person.entity'; import { SessionEntity } from 'src/entities/session.entity'; @@ -36,6 +37,7 @@ export const entities = [ ExifEntity, FaceSearchEntity, GeodataPlacesEntity, + NaturalEarthCountriesEntity, MemoryEntity, MoveEntity, PartnerEntity, diff --git a/server/src/entities/natural-earth-countries.entity.ts b/server/src/entities/natural-earth-countries.entity.ts new file mode 100644 index 0000000000..19a12fa07b --- /dev/null +++ b/server/src/entities/natural-earth-countries.entity.ts @@ -0,0 +1,19 @@ +import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; + +@Entity('naturalearth_countries', { synchronize: false }) +export class NaturalEarthCountriesEntity { + @PrimaryGeneratedColumn() + id!: number; + + @Column({ type: 'varchar', length: 50 }) + admin!: string; + + @Column({ type: 'varchar', length: 3 }) + admin_a3!: string; + + @Column({ type: 'varchar', length: 50 }) + type!: string; + + @Column({ type: 'polygon' }) + coordinates!: string; +} diff --git a/server/src/migrations/1720375641148-natural-earth-countries.ts b/server/src/migrations/1720375641148-natural-earth-countries.ts new file mode 100644 index 0000000000..8c58321dca --- /dev/null +++ b/server/src/migrations/1720375641148-natural-earth-countries.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class NaturalEarthCountries1720375641148 implements MigrationInterface { + name = 'NaturalEarthCountries1720375641148' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "naturalearth_countries" ("id" SERIAL NOT NULL, "admin" character varying(50) NOT NULL, "admin_a3" character varying(3) NOT NULL, "type" character varying(50) NOT NULL, "coordinates" polygon NOT NULL, CONSTRAINT "PK_21a6d86d1ab5d841648212e5353" PRIMARY KEY ("id"))`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "naturalearth_countries"`); + } + +} diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.ts index fbba3b6c53..a1a958c517 100644 --- a/server/src/repositories/map.repository.ts +++ b/server/src/repositories/map.repository.ts @@ -7,6 +7,7 @@ import readLine from 'node:readline'; import { citiesFile, resourcePaths } from 'src/constants'; import { AssetEntity } from 'src/entities/asset.entity'; import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity'; +import { NaturalEarthCountriesEntity } from 'src/entities/natural-earth-countries.entity'; import { SystemMetadataKey } from 'src/entities/system-metadata.entity'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { @@ -28,6 +29,8 @@ export class MapRepository implements IMapRepository { constructor( @InjectRepository(AssetEntity) private assetRepository: Repository, @InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository, + @InjectRepository(NaturalEarthCountriesEntity) + private naturalEarthCountriesRepository: Repository, @InjectDataSource() private dataSource: DataSource, @Inject(ISystemMetadataRepository) private metadataRepository: ISystemMetadataRepository, @Inject(ILoggerRepository) private logger: ILoggerRepository, @@ -46,6 +49,7 @@ export class MapRepository implements IMapRepository { } await this.importGeodata(); + await this.importNaturalEarthCountries(); await this.metadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, { lastUpdate: geodataDate, @@ -130,22 +134,93 @@ export class MapRepository implements IMapRepository { .limit(1) .getOne(); - if (!response) { + if (response) { + this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`); + + const { countryCode, name: city, admin1Name } = response; + const country = getName(countryCode, 'en') ?? null; + const state = admin1Name; + + return { country, state, city }; + } + + this.logger.warn( + `Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`, + ); + + const ne_response = await this.naturalEarthCountriesRepository + .createQueryBuilder('naturalearth_countries') + .where('coordinates @> point (:longitude, :latitude)', point) + .limit(1) + .getOne(); + + if (!ne_response) { this.logger.warn( - `Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`, + `Response from database for natural earth reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`, ); + return null; } - this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`); + this.logger.verbose(`Raw: ${JSON.stringify(ne_response, ['id', 'admin', 'admin_a3', 'type'], 2)}`); - const { countryCode, name: city, admin1Name } = response; - const country = getName(countryCode, 'en') ?? null; - const state = admin1Name; + const { admin_a3 } = ne_response; + const country = getName(admin_a3, 'en') ?? null; + const state = null; + const city = null; return { country, state, city }; } + private transformCoordinatesToPolygon(coordinates: number[][][]): string { + const pointsString = coordinates.map((point) => `(${point[0]},${point[1]})`).join(', '); + return `(${pointsString})`; + } + + private async importNaturalEarthCountries() { + const queryRunner = this.dataSource.createQueryRunner(); + await queryRunner.connect(); + + try { + await queryRunner.startTransaction(); + await queryRunner.manager.clear(NaturalEarthCountriesEntity); + + const fileContent = await readFile(resourcePaths.geodata.naturalEarthCountriesPath, 'utf8'); + const geoJSONData = JSON.parse(fileContent); + + if (geoJSONData.type !== 'FeatureCollection' || !Array.isArray(geoJSONData.features)) { + this.logger.fatal('Invalid GeoJSON FeatureCollection'); + return; + } + + for await (const feature of geoJSONData.features) { + for (const polygon of feature.geometry.coordinates) { + const featureRecord = new NaturalEarthCountriesEntity(); + featureRecord.admin = feature.properties.ADMIN; + featureRecord.admin_a3 = feature.properties.ADM0_A3; + featureRecord.type = feature.properties.TYPE; + + if (feature.geometry.type === 'MultiPolygon') { + featureRecord.coordinates = this.transformCoordinatesToPolygon(polygon[0]); + await queryRunner.manager.save(featureRecord); + } else if (feature.geometry.type === 'Polygon') { + featureRecord.coordinates = this.transformCoordinatesToPolygon(polygon); + await queryRunner.manager.save(featureRecord); + break; + } + } + } + + await queryRunner.commitTransaction(); + } catch (error) { + this.logger.fatal('Error importing natural earth country data', error); + await queryRunner.rollbackTransaction(); + throw error; + } finally { + await queryRunner.release(); + } + } + private async importGeodata() { const queryRunner = this.dataSource.createQueryRunner(); await queryRunner.connect(); diff --git a/web/src/lib/components/asset-viewer/detail-panel-location.svelte b/web/src/lib/components/asset-viewer/detail-panel-location.svelte index 81ed06be0c..a93c90d0d4 100644 --- a/web/src/lib/components/asset-viewer/detail-panel-location.svelte +++ b/web/src/lib/components/asset-viewer/detail-panel-location.svelte @@ -26,7 +26,7 @@ } -{#if asset.exifInfo?.city} +{#if asset.exifInfo?.country}