self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-11-28 09:33:27 +02:00
Code Issues Releases Activity

refactor(server)!: move markers and style to dedicated map endpoint/controller (#9832)

Browse Source 
* move markers and style to dedicated map endpoint

* chore: open api

* chore: clean up repos

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
This commit is contained in:
Daniel Dietzler 2024-05-29 17:51:01 +02:00 committed by GitHub
parent 5ef144bf79
commit 5463660746
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
38 changed files with 814 additions and 692 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
e2e/src/api/specs/asset.e2e-spec.ts
View File

@ -641,95 +641,6 @@ describe('/asset', () => {
});
});
describe('GET /asset/map-marker', () => {
beforeAll(async () => {
const files = [
'formats/avif/8bit-sRGB.avif',
'formats/jpg/el_torcal_rocks.jpg',
'formats/jxl/8bit-sRGB.jxl',
'formats/heic/IMG_2682.heic',
'formats/png/density_plot.png',
'formats/raw/Nikon/D80/glarus.nef',
'formats/raw/Nikon/D700/philadelphia.nef',
'formats/raw/Panasonic/DMC-GH4/4_3.rw2',
'formats/raw/Sony/ILCE-6300/12bit-compressed-(3_2).arw',
'formats/raw/Sony/ILCE-7M2/14bit-uncompressed-(3_2).arw',
];
utils.resetEvents();
const uploadFile = async (input: string) => {
const filepath = join(testAssetDir, input);
const { id } = await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(filepath), filename: basename(filepath) },
});
await utils.waitForWebsocketEvent({ event: 'assetUpload', id });
};
const uploads = files.map((f) => uploadFile(f));
await Promise.all(uploads);
}, 30_000);
it('should require authentication', async () => {
const { status, body } = await request(app).get('/asset/map-marker');
expect(status).toBe(401);
expect(body).toEqual(errorDto.unauthorized);
});
// TODO archive one of these assets
it('should get map markers for all non-archived assets', async () => {
const { status, body } = await request(app)
.get('/asset/map-marker')
.query({ isArchived: false })
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(status).toBe(200);
expect(body).toHaveLength(2);
expect(body).toEqual([
{
city: 'Palisade',
country: 'United States of America',
id: expect.any(String),
lat: expect.closeTo(39.115),
lon: expect.closeTo(-108.400_968),
state: 'Colorado',
},
{
city: 'Ralston',
country: 'United States of America',
id: expect.any(String),
lat: expect.closeTo(41.2203),
lon: expect.closeTo(-96.071_625),
state: 'Nebraska',
},
]);
});
// TODO archive one of these assets
it('should get all map markers', async () => {
const { status, body } = await request(app)
.get('/asset/map-marker')
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(status).toBe(200);
expect(body).toEqual([
{
city: 'Palisade',
country: 'United States of America',
id: expect.any(String),
lat: expect.closeTo(39.115),
lon: expect.closeTo(-108.400_968),
state: 'Colorado',
},
{
city: 'Ralston',
country: 'United States of America',
id: expect.any(String),
lat: expect.closeTo(41.2203),
lon: expect.closeTo(-96.071_625),
state: 'Nebraska',
},
]);
});
});
describe('PUT /asset', () => {
it('should require authentication', async () => {
const { status, body } = await request(app).put('/asset');

162
e2e/src/api/specs/map.e2e-spec.ts Normal file
View File

@ -0,0 +1,162 @@
import { AssetFileUploadResponseDto, LoginResponseDto, SharedLinkType } from '@immich/sdk';
import { readFile } from 'node:fs/promises';
import { basename, join } from 'node:path';
import { Socket } from 'socket.io-client';
import { createUserDto } from 'src/fixtures';
import { errorDto } from 'src/responses';
import { app, testAssetDir, utils } from 'src/utils';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
describe('/map', () => {
let websocket: Socket;
let admin: LoginResponseDto;
let nonAdmin: LoginResponseDto;
let asset: AssetFileUploadResponseDto;
beforeAll(async () => {
await utils.resetDatabase();
admin = await utils.adminSetup({ onboarding: false });
nonAdmin = await utils.userSetup(admin.accessToken, createUserDto.user1);
websocket = await utils.connectWebsocket(admin.accessToken);
asset = await utils.createAsset(admin.accessToken);
const files = ['formats/heic/IMG_2682.heic', 'metadata/gps-position/thompson-springs.jpg'];
utils.resetEvents();
const uploadFile = async (input: string) => {
const filepath = join(testAssetDir, input);
const { id } = await utils.createAsset(admin.accessToken, {
assetData: { bytes: await readFile(filepath), filename: basename(filepath) },
});
await utils.waitForWebsocketEvent({ event: 'assetUpload', id });
};
await Promise.all(files.map((f) => uploadFile(f)));
});
afterAll(() => {
utils.disconnectWebsocket(websocket);
});
describe('GET /map/markers', () => {
it('should require authentication', async () => {
const { status, body } = await request(app).get('/map/markers');
expect(status).toBe(401);
expect(body).toEqual(errorDto.unauthorized);
});
// TODO archive one of these assets
it('should get map markers for all non-archived assets', async () => {
const { status, body } = await request(app)
.get('/map/markers')
.query({ isArchived: false })
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(status).toBe(200);
expect(body).toHaveLength(2);
expect(body).toEqual([
{
city: 'Palisade',
country: 'United States of America',
id: expect.any(String),
lat: expect.closeTo(39.115),
lon: expect.closeTo(-108.400_968),
state: 'Colorado',
},
{
city: 'Ralston',
country: 'United States of America',
id: expect.any(String),
lat: expect.closeTo(41.2203),
lon: expect.closeTo(-96.071_625),
state: 'Nebraska',
},
]);
});
// TODO archive one of these assets
it('should get all map markers', async () => {
const { status, body } = await request(app)
.get('/map/markers')
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(status).toBe(200);
expect(body).toEqual([
{
city: 'Palisade',
country: 'United States of America',
id: expect.any(String),
lat: expect.closeTo(39.115),
lon: expect.closeTo(-108.400_968),
state: 'Colorado',
},
{
city: 'Ralston',
country: 'United States of America',
id: expect.any(String),
lat: expect.closeTo(41.2203),
lon: expect.closeTo(-96.071_625),
state: 'Nebraska',
},
]);
});
});
describe('GET /map/style.json', () => {
it('should require authentication', async () => {
const { status, body } = await request(app).get('/map/style.json');
expect(status).toBe(401);
expect(body).toEqual(errorDto.unauthorized);
});
it('should allow shared link access', async () => {
const sharedLink = await utils.createSharedLink(admin.accessToken, {
type: SharedLinkType.Individual,
assetIds: [asset.id],
});
const { status, body } = await request(app).get(`/map/style.json?key=${sharedLink.key}`).query({ theme: 'dark' });
expect(status).toBe(200);
expect(body).toEqual(expect.objectContaining({ id: 'immich-map-dark' }));
});
it('should throw an error if a theme is not light or dark', async () => {
for (const theme of ['dark1', true, 123, '', null, undefined]) {
const { status, body } = await request(app)
.get('/map/style.json')
.query({ theme })
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(status).toBe(400);
expect(body).toEqual(errorDto.badRequest(['theme must be one of the following values: light, dark']));
}
});
it('should return the light style.json', async () => {
const { status, body } = await request(app)
.get('/map/style.json')
.query({ theme: 'light' })
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(status).toBe(200);
expect(body).toEqual(expect.objectContaining({ id: 'immich-map-light' }));
});
it('should return the dark style.json', async () => {
const { status, body } = await request(app)
.get('/map/style.json')
.query({ theme: 'dark' })
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(status).toBe(200);
expect(body).toEqual(expect.objectContaining({ id: 'immich-map-dark' }));
});
it('should not require admin authentication', async () => {
const { status, body } = await request(app)
.get('/map/style.json')
.query({ theme: 'dark' })
.set('Authorization', `Bearer ${nonAdmin.accessToken}`);
expect(status).toBe(200);
expect(body).toEqual(expect.objectContaining({ id: 'immich-map-dark' }));
});
});
});

67
e2e/src/api/specs/system-config.e2e-spec.ts
View File

@ -1,5 +1,4 @@
import { AssetFileUploadResponseDto, LoginResponseDto, SharedLinkType, getConfig } from '@immich/sdk';
import { createUserDto } from 'src/fixtures';
import { LoginResponseDto, getConfig } from '@immich/sdk';
import { errorDto } from 'src/responses';
import { app, asBearerAuth, utils } from 'src/utils';
import request from 'supertest';
@ -9,74 +8,10 @@ const getSystemConfig = (accessToken: string) => getConfig({ headers: asBearerAu
describe('/system-config', () => {
let admin: LoginResponseDto;
let nonAdmin: LoginResponseDto;
let asset: AssetFileUploadResponseDto;
beforeAll(async () => {
await utils.resetDatabase();
admin = await utils.adminSetup();
nonAdmin = await utils.userSetup(admin.accessToken, createUserDto.user1);
asset = await utils.createAsset(admin.accessToken);
});
describe('GET /system-config/map/style.json', () => {
it('should require authentication', async () => {
const { status, body } = await request(app).get('/system-config/map/style.json');
expect(status).toBe(401);
expect(body).toEqual(errorDto.unauthorized);
});
it('should allow shared link access', async () => {
const sharedLink = await utils.createSharedLink(admin.accessToken, {
type: SharedLinkType.Individual,
assetIds: [asset.id],
});
const { status, body } = await request(app)
.get(`/system-config/map/style.json?key=${sharedLink.key}`)
.query({ theme: 'dark' });
expect(status).toBe(200);
expect(body).toEqual(expect.objectContaining({ id: 'immich-map-dark' }));
});
it('should throw an error if a theme is not light or dark', async () => {
for (const theme of ['dark1', true, 123, '', null, undefined]) {
const { status, body } = await request(app)
.get('/system-config/map/style.json')
.query({ theme })
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(status).toBe(400);
expect(body).toEqual(errorDto.badRequest(['theme must be one of the following values: light, dark']));
}
});
it('should return the light style.json', async () => {
const { status, body } = await request(app)
.get('/system-config/map/style.json')
.query({ theme: 'light' })
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(status).toBe(200);
expect(body).toEqual(expect.objectContaining({ id: 'immich-map-light' }));
});
it('should return the dark style.json', async () => {
const { status, body } = await request(app)
.get('/system-config/map/style.json')
.query({ theme: 'dark' })
.set('Authorization', `Bearer ${admin.accessToken}`);
expect(status).toBe(200);
expect(body).toEqual(expect.objectContaining({ id: 'immich-map-dark' }));
});
it('should not require admin authentication', async () => {
const { status, body } = await request(app)
.get('/system-config/map/style.json')
.query({ theme: 'dark' })
.set('Authorization', `Bearer ${nonAdmin.accessToken}`);
expect(status).toBe(200);
expect(body).toEqual(expect.objectContaining({ id: 'immich-map-dark' }));
});
});
describe('PUT /system-config', () => {

4
mobile/lib/providers/map/map_state.provider.dart
View File

@ -46,7 +46,7 @@ class MapStateNotifier extends _$MapStateNotifier {
// Fetch and save light theme
final lightResponse = await ref
.read(apiServiceProvider)
.systemConfigApi
.mapApi
.getMapStyleWithHttpInfo(MapTheme.light);
if (lightResponse.statusCode >= HttpStatus.badRequest) {
@ -74,7 +74,7 @@ class MapStateNotifier extends _$MapStateNotifier {
// Fetch and save dark theme
final darkResponse = await ref
.read(apiServiceProvider)
.systemConfigApi
.mapApi
.getMapStyleWithHttpInfo(MapTheme.dark);
if (darkResponse.statusCode >= HttpStatus.badRequest) {

2
mobile/lib/services/api.service.dart
View File

@ -19,6 +19,7 @@ class ApiService {
late AssetApi assetApi;
late SearchApi searchApi;
late ServerInfoApi serverInfoApi;
late MapApi mapApi;
late PartnerApi partnerApi;
late PersonApi personApi;
late AuditApi auditApi;
@ -50,6 +51,7 @@ class ApiService {
assetApi = AssetApi(_apiClient);
serverInfoApi = ServerInfoApi(_apiClient);
searchApi = SearchApi(_apiClient);
mapApi = MapApi(_apiClient);
partnerApi = PartnerApi(_apiClient);
personApi = PersonApi(_apiClient);
auditApi = AuditApi(_apiClient);

2
mobile/lib/services/map.service.dart
View File

@ -19,7 +19,7 @@ class MapSerivce with ErrorLoggerMixin {
}) async {
return logError(
() async {
final markers = await _apiService.assetApi.getMapMarkers(
final markers = await _apiService.mapApi.getMapMarkers(
isFavorite: isFavorite,
isArchived: withArchived,
withPartners: withPartners,

BIN
mobile/openapi/README.md generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/api.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/api/asset_api.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/api/map_api.dart generated Normal file
View File

Binary file not shown.

BIN
mobile/openapi/lib/api/system_config_api.dart generated
View File

Binary file not shown.

270
open-api/immich-openapi-specs.json
View File

@ -1598,92 +1598,6 @@
]
}
},
"/asset/map-marker": {
"get": {
"operationId": "getMapMarkers",
"parameters": [
{
"name": "fileCreatedAfter",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "fileCreatedBefore",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "isArchived",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isFavorite",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withPartners",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withSharedAlbums",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"items": {
"$ref": "#/components/schemas/MapMarkerResponseDto"
},
"type": "array"
}
}
},
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Asset"
]
}
},
"/asset/memory-lane": {
"get": {
"operationId": "getMemoryLane",
@ -3131,6 +3045,141 @@
]
}
},
"/map/markers": {
"get": {
"operationId": "getMapMarkers",
"parameters": [
{
"name": "fileCreatedAfter",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "fileCreatedBefore",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "isArchived",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "isFavorite",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withPartners",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
},
{
"name": "withSharedAlbums",
"required": false,
"in": "query",
"schema": {
"type": "boolean"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"items": {
"$ref": "#/components/schemas/MapMarkerResponseDto"
},
"type": "array"
}
}
},
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Map"
]
}
},
"/map/style.json": {
"get": {
"operationId": "getMapStyle",
"parameters": [
{
"name": "key",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "theme",
"required": true,
"in": "query",
"schema": {
"$ref": "#/components/schemas/MapTheme"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
},
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Map"
]
}
},
"/memories": {
"get": {
"operationId": "searchMemories",
@ -5512,55 +5561,6 @@
]
}
},
"/system-config/map/style.json": {
"get": {
"operationId": "getMapStyle",
"parameters": [
{
"name": "key",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "theme",
"required": true,
"in": "query",
"schema": {
"$ref": "#/components/schemas/MapTheme"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
},
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"System Config"
]
}
},
"/system-config/storage-template-options": {
"get": {
"operationId": "getStorageTemplateOptions",

96
open-api/typescript-sdk/src/fetch-client.ts
View File

@ -303,14 +303,6 @@ export type AssetJobsDto = {
assetIds: string[];
name: AssetJobName;
};
export type MapMarkerResponseDto = {
city: string | null;
country: string | null;
id: string;
lat: number;
lon: number;
state: string | null;
};
export type MemoryLaneResponseDto = {
assets: AssetResponseDto[];
yearsAgo: number;
@ -516,6 +508,14 @@ export type ValidateLibraryImportPathResponseDto = {
export type ValidateLibraryResponseDto = {
importPaths?: ValidateLibraryImportPathResponseDto[];
};
export type MapMarkerResponseDto = {
city: string | null;
country: string | null;
id: string;
lat: number;
lon: number;
state: string | null;
};
export type OnThisDayDto = {
year: number;
};
@ -1518,28 +1518,6 @@ export function runAssetJobs({ assetJobsDto }: {
body: assetJobsDto
})));
}
export function getMapMarkers({ fileCreatedAfter, fileCreatedBefore, isArchived, isFavorite, withPartners, withSharedAlbums }: {
fileCreatedAfter?: string;
fileCreatedBefore?: string;
isArchived?: boolean;
isFavorite?: boolean;
withPartners?: boolean;
withSharedAlbums?: boolean;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: MapMarkerResponseDto[];
}>(`/asset/map-marker${QS.query(QS.explode({
fileCreatedAfter,
fileCreatedBefore,
isArchived,
isFavorite,
withPartners,
withSharedAlbums
}))}`, {
...opts
}));
}
export function getMemoryLane({ day, month }: {
day: number;
month: number;
@ -1930,6 +1908,42 @@ export function validate({ id, validateLibraryDto }: {
body: validateLibraryDto
})));
}
export function getMapMarkers({ fileCreatedAfter, fileCreatedBefore, isArchived, isFavorite, withPartners, withSharedAlbums }: {
fileCreatedAfter?: string;
fileCreatedBefore?: string;
isArchived?: boolean;
isFavorite?: boolean;
withPartners?: boolean;
withSharedAlbums?: boolean;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: MapMarkerResponseDto[];
}>(`/map/markers${QS.query(QS.explode({
fileCreatedAfter,
fileCreatedBefore,
isArchived,
isFavorite,
withPartners,
withSharedAlbums
}))}`, {
...opts
}));
}
export function getMapStyle({ key, theme }: {
key?: string;
theme: MapTheme;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: object;
}>(`/map/style.json${QS.query(QS.explode({
key,
theme
}))}`, {
...opts
}));
}
export function searchMemories(opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
@ -2568,20 +2582,6 @@ export function getConfigDefaults(opts?: Oazapfts.RequestOpts) {
...opts
}));
}
export function getMapStyle({ key, theme }: {
key?: string;
theme: MapTheme;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: object;
}>(`/system-config/map/style.json${QS.query(QS.explode({
key,
theme
}))}`, {
...opts
}));
}
export function getStorageTemplateOptions(opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
@ -2977,6 +2977,10 @@ export enum JobCommand {
Empty = "empty",
ClearFailed = "clear-failed"
}
export enum MapTheme {
Light = "light",
Dark = "dark"
}
export enum Type2 {
OnThisDay = "on_this_day"
}
@ -3073,10 +3077,6 @@ export enum ModelType {
FacialRecognition = "facial-recognition",
Clip = "clip"
}
export enum MapTheme {
Light = "light",
Dark = "dark"
}
export enum TimeBucketSize {
Day = "DAY",
Month = "MONTH"

8
server/src/controllers/asset.controller.ts
View File

@ -12,7 +12,7 @@ import {
UpdateAssetDto,
} from 'src/dtos/asset.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { MapMarkerDto, MapMarkerResponseDto, MemoryLaneDto } from 'src/dtos/search.dto';
import { MemoryLaneDto } from 'src/dtos/search.dto';
import { UpdateStackParentDto } from 'src/dtos/stack.dto';
import { Auth, Authenticated } from 'src/middleware/auth.guard';
import { Route } from 'src/middleware/file-upload.interceptor';
@ -24,12 +24,6 @@ import { UUIDParamDto } from 'src/validation';
export class AssetController {
constructor(private service: AssetService) {}
@Get('map-marker')
@Authenticated()
getMapMarkers(@Auth() auth: AuthDto, @Query() options: MapMarkerDto): Promise<MapMarkerResponseDto[]> {
return this.service.getMapMarkers(auth, options);
}
@Get('memory-lane')
@Authenticated()
getMemoryLane(@Auth() auth: AuthDto, @Query() dto: MemoryLaneDto): Promise<MemoryLaneResponseDto[]> {

2
server/src/controllers/index.ts
View File

@ -13,6 +13,7 @@ import { FaceController } from 'src/controllers/face.controller';
import { ReportController } from 'src/controllers/file-report.controller';
import { JobController } from 'src/controllers/job.controller';
import { LibraryController } from 'src/controllers/library.controller';
import { MapController } from 'src/controllers/map.controller';
import { MemoryController } from 'src/controllers/memory.controller';
import { OAuthController } from 'src/controllers/oauth.controller';
import { PartnerController } from 'src/controllers/partner.controller';
@ -45,6 +46,7 @@ export const controllers = [
FaceController,
JobController,
LibraryController,
MapController,
MemoryController,
OAuthController,
PartnerController,

25
server/src/controllers/map.controller.ts Normal file
View File

@ -0,0 +1,25 @@
import { Controller, Get, Query } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { AuthDto } from 'src/dtos/auth.dto';
import { MapMarkerDto, MapMarkerResponseDto } from 'src/dtos/search.dto';
import { MapThemeDto } from 'src/dtos/system-config.dto';
import { Auth, Authenticated } from 'src/middleware/auth.guard';
import { MapService } from 'src/services/map.service';
@ApiTags('Map')
@Controller('map')
export class MapController {
constructor(private service: MapService) {}
@Get('markers')
@Authenticated()
getMapMarkers(@Auth() auth: AuthDto, @Query() options: MapMarkerDto): Promise<MapMarkerResponseDto[]> {
return this.service.getMapMarkers(auth, options);
}
@Authenticated({ sharedLink: true })
@Get('style.json')
getMapStyle(@Query() dto: MapThemeDto) {
return this.service.getMapStyle(dto.theme);
}
}

10
server/src/controllers/system-config.controller.ts
View File

@ -1,6 +1,6 @@
import { Body, Controller, Get, Put, Query } from '@nestjs/common';
import { Body, Controller, Get, Put } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { MapThemeDto, SystemConfigDto, SystemConfigTemplateStorageOptionDto } from 'src/dtos/system-config.dto';
import { SystemConfigDto, SystemConfigTemplateStorageOptionDto } from 'src/dtos/system-config.dto';
import { Authenticated } from 'src/middleware/auth.guard';
import { SystemConfigService } from 'src/services/system-config.service';
@ -32,10 +32,4 @@ export class SystemConfigController {
getStorageTemplateOptions(): SystemConfigTemplateStorageOptionDto {
return this.service.getStorageTemplateOptions();
}
@Authenticated({ sharedLink: true })
@Get('map/style.json')
getMapStyle(@Query() dto: MapThemeDto) {
return this.service.getMapStyle(dto.theme);
}
}

15
server/src/interfaces/asset.interface.ts
View File

@ -2,7 +2,6 @@ import { AssetOrder } from 'src/entities/album.entity';
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
import { AssetEntity, AssetType } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { ReverseGeocodeResult } from 'src/interfaces/metadata.interface';
import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface';
import { Paginated, PaginationOptions } from 'src/utils/pagination';
import { FindOptionsOrder, FindOptionsRelations, FindOptionsSelect } from 'typeorm';
@ -22,19 +21,6 @@ export interface LivePhotoSearchOptions {
type: AssetType;
}
export interface MapMarkerSearchOptions {
isArchived?: boolean;
isFavorite?: boolean;
fileCreatedBefore?: Date;
fileCreatedAfter?: Date;
}
export interface MapMarker extends ReverseGeocodeResult {
id: string;
lat: number;
lon: number;
}
export enum WithoutProperty {
THUMBNAIL = 'thumbnail',
ENCODED_VIDEO = 'encoded-video',
@ -195,7 +181,6 @@ export interface IAssetRepository {
softDeleteAll(ids: string[]): Promise<void>;
restoreAll(ids: string[]): Promise<void>;
findLivePhotoMatch(options: LivePhotoSearchOptions): Promise<AssetEntity | null>;
getMapMarkers(ownerIds: string[], albumIds: string[], options?: MapMarkerSearchOptions): Promise<MapMarker[]>;
getStatistics(ownerId: string, options: AssetStatsOptions): Promise<AssetStats>;
getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;
getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>;

32
server/src/interfaces/map.interface.ts Normal file
View File

@ -0,0 +1,32 @@
export const IMapRepository = 'IMapRepository';
export interface MapMarkerSearchOptions {
isArchived?: boolean;
isFavorite?: boolean;
fileCreatedBefore?: Date;
fileCreatedAfter?: Date;
}
export interface GeoPoint {
latitude: number;
longitude: number;
}
export interface ReverseGeocodeResult {
country: string | null;
state: string | null;
city: string | null;
}
export interface MapMarker extends ReverseGeocodeResult {
id: string;
lat: number;
lon: number;
}
export interface IMapRepository {
init(): Promise<void>;
reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null>;
getMapMarkers(ownerIds: string[], albumIds: string[], options?: MapMarkerSearchOptions): Promise<MapMarker[]>;
fetchStyle(url: string): Promise<any>;
}

13
server/src/interfaces/metadata.interface.ts
View File

@ -2,17 +2,6 @@ import { BinaryField, Tags } from 'exiftool-vendored';
export const IMetadataRepository = 'IMetadataRepository';
export interface GeoPoint {
latitude: number;
longitude: number;
}
export interface ReverseGeocodeResult {
country: string | null;
state: string | null;
city: string | null;
}
export interface ExifDuration {
Value: number;
Scale?: number;
@ -33,9 +22,7 @@ export interface ImmichTags extends Omit<Tags, 'FocalLength' | 'Duration'> {
}
export interface IMetadataRepository {
init(): Promise<void>;
teardown(): Promise<void>;
reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null>;
readTags(path: string): Promise<ImmichTags | null>;
writeTags(path: string, tags: Partial<Tags>): Promise<void>;
extractBinaryTag(tagName: string, path: string): Promise<Buffer>;

1
server/src/interfaces/system-metadata.interface.ts
View File

@ -5,6 +5,5 @@ export const ISystemMetadataRepository = 'ISystemMetadataRepository';
export interface ISystemMetadataRepository {
get<T extends keyof SystemMetadata>(key: T): Promise<SystemMetadata[T] | null>;
set<T extends keyof SystemMetadata>(key: T, value: SystemMetadata[T]): Promise<void>;
fetchStyle(url: string): Promise<any>;
readFile(filename: string): Promise<string>;
}

55
server/src/repositories/asset.repository.ts
View File

@ -21,8 +21,6 @@ import {
AssetUpdateOptions,
IAssetRepository,
LivePhotoSearchOptions,
MapMarker,
MapMarkerSearchOptions,
MonthDay,
TimeBucketItem,
TimeBucketOptions,
@ -31,7 +29,7 @@ import {
WithoutProperty,
} from 'src/interfaces/asset.interface';
import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface';
import { OptionalBetween, searchAssetBuilder } from 'src/utils/database';
import { searchAssetBuilder } from 'src/utils/database';
import { Instrumentation } from 'src/utils/instrumentation';
import { Paginated, PaginationMode, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination';
import {
@ -547,57 +545,6 @@ export class AssetRepository implements IAssetRepository {
});
}
async getMapMarkers(
ownerIds: string[],
albumIds: string[],
options: MapMarkerSearchOptions = {},
): Promise<MapMarker[]> {
const { isArchived, isFavorite, fileCreatedAfter, fileCreatedBefore } = options;
const where = {
isVisible: true,
isArchived,
exifInfo: {
latitude: Not(IsNull()),
longitude: Not(IsNull()),
},
isFavorite,
fileCreatedAt: OptionalBetween(fileCreatedAfter, fileCreatedBefore),
};
const assets = await this.repository.find({
select: {
id: true,
exifInfo: {
city: true,
state: true,
country: true,
latitude: true,
longitude: true,
},
},
where: [
{ ...where, ownerId: In([...ownerIds]) },
{ ...where, albums: { id: In([...albumIds]) } },
],
relations: {
exifInfo: true,
},
order: {
fileCreatedAt: 'DESC',
},
});
return assets.map((asset) => ({
id: asset.id,
lat: asset.exifInfo!.latitude!,
lon: asset.exifInfo!.longitude!,
city: asset.exifInfo!.city,
state: asset.exifInfo!.state,
country: asset.exifInfo!.country,
}));
}
async getStatistics(ownerId: string, options: AssetStatsOptions): Promise<AssetStats> {
const builder = this.repository
.createQueryBuilder('asset')

15
server/src/repositories/index.ts
View File

@ -14,6 +14,7 @@ import { IJobRepository } from 'src/interfaces/job.interface';
import { ILibraryRepository } from 'src/interfaces/library.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IMachineLearningRepository } from 'src/interfaces/machine-learning.interface';
import { IMapRepository } from 'src/interfaces/map.interface';
import { IMediaRepository } from 'src/interfaces/media.interface';
import { IMemoryRepository } from 'src/interfaces/memory.interface';
import { IMetadataRepository } from 'src/interfaces/metadata.interface';
@ -46,6 +47,7 @@ import { JobRepository } from 'src/repositories/job.repository';
import { LibraryRepository } from 'src/repositories/library.repository';
import { LoggerRepository } from 'src/repositories/logger.repository';
import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';
import { MapRepository } from 'src/repositories/map.repository';
import { MediaRepository } from 'src/repositories/media.repository';
import { MemoryRepository } from 'src/repositories/memory.repository';
import { MetadataRepository } from 'src/repositories/metadata.repository';
@ -64,8 +66,8 @@ import { TagRepository } from 'src/repositories/tag.repository';
import { UserRepository } from 'src/repositories/user.repository';
export const repositories = [
{ provide: IActivityRepository, useClass: ActivityRepository },
{ provide: IAccessRepository, useClass: AccessRepository },
{ provide: IActivityRepository, useClass: ActivityRepository },
{ provide: IAlbumRepository, useClass: AlbumRepository },
{ provide: IAlbumUserRepository, useClass: AlbumUserRepository },
{ provide: IAssetRepository, useClass: AssetRepository },
@ -76,10 +78,12 @@ export const repositories = [
{ provide: IDatabaseRepository, useClass: DatabaseRepository },
{ provide: IEventRepository, useClass: EventRepository },
{ provide: IJobRepository, useClass: JobRepository },
{ provide: ILoggerRepository, useClass: LoggerRepository },
{ provide: ILibraryRepository, useClass: LibraryRepository },
{ provide: IKeyRepository, useClass: ApiKeyRepository },
{ provide: ILibraryRepository, useClass: LibraryRepository },
{ provide: ILoggerRepository, useClass: LoggerRepository },
{ provide: IMachineLearningRepository, useClass: MachineLearningRepository },
{ provide: IMapRepository, useClass: MapRepository },
{ provide: IMediaRepository, useClass: MediaRepository },
{ provide: IMemoryRepository, useClass: MemoryRepository },
{ provide: IMetadataRepository, useClass: MetadataRepository },
{ provide: IMetricRepository, useClass: MetricRepository },
@ -87,13 +91,12 @@ export const repositories = [
{ provide: INotificationRepository, useClass: NotificationRepository },
{ provide: IPartnerRepository, useClass: PartnerRepository },
{ provide: IPersonRepository, useClass: PersonRepository },
{ provide: IServerInfoRepository, useClass: ServerInfoRepository },
{ provide: ISharedLinkRepository, useClass: SharedLinkRepository },
{ provide: ISearchRepository, useClass: SearchRepository },
{ provide: IServerInfoRepository, useClass: ServerInfoRepository },
{ provide: ISessionRepository, useClass: SessionRepository },
{ provide: ISharedLinkRepository, useClass: SharedLinkRepository },
{ provide: IStorageRepository, useClass: StorageRepository },
{ provide: ISystemMetadataRepository, useClass: SystemMetadataRepository },
{ provide: ITagRepository, useClass: TagRepository },
{ provide: IMediaRepository, useClass: MediaRepository },
{ provide: IUserRepository, useClass: UserRepository },
];

246
server/src/repositories/map.repository.ts Normal file
View File

@ -0,0 +1,246 @@
import { Inject, Injectable } from '@nestjs/common';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { getName } from 'i18n-iso-countries';
import { createReadStream, existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import readLine from 'node:readline';
import { citiesFile, geodataAdmin1Path, geodataAdmin2Path, geodataCities500Path, geodataDatePath } from 'src/constants';
import { AssetEntity } from 'src/entities/asset.entity';
import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
import { SystemMetadataKey } from 'src/entities/system-metadata.entity';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import {
GeoPoint,
IMapRepository,
MapMarker,
MapMarkerSearchOptions,
ReverseGeocodeResult,
} from 'src/interfaces/map.interface';
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
import { OptionalBetween } from 'src/utils/database';
import { Instrumentation } from 'src/utils/instrumentation';
import { DataSource, In, IsNull, Not, QueryRunner, Repository } from 'typeorm';
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
@Instrumentation()
@Injectable()
export class MapRepository implements IMapRepository {
constructor(
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
@InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,
@InjectDataSource() private dataSource: DataSource,
@Inject(ISystemMetadataRepository) private metadataRepository: ISystemMetadataRepository,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
) {
this.logger.setContext(MapRepository.name);
}
async init(): Promise<void> {
this.logger.log('Initializing metadata repository');
const geodataDate = await readFile(geodataDatePath, 'utf8');
// TODO move to service init
const geocodingMetadata = await this.metadataRepository.get(SystemMetadataKey.REVERSE_GEOCODING_STATE);
if (geocodingMetadata?.lastUpdate === geodataDate) {
return;
}
await this.importGeodata();
await this.metadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, {
lastUpdate: geodataDate,
lastImportFileName: citiesFile,
});
this.logger.log('Geodata import completed');
}
async getMapMarkers(
ownerIds: string[],
albumIds: string[],
options: MapMarkerSearchOptions = {},
): Promise<MapMarker[]> {
const { isArchived, isFavorite, fileCreatedAfter, fileCreatedBefore } = options;
const where = {
isVisible: true,
isArchived,
exifInfo: {
latitude: Not(IsNull()),
longitude: Not(IsNull()),
},
isFavorite,
fileCreatedAt: OptionalBetween(fileCreatedAfter, fileCreatedBefore),
};
const assets = await this.assetRepository.find({
select: {
id: true,
exifInfo: {
city: true,
state: true,
country: true,
latitude: true,
longitude: true,
},
},
where: [
{ ...where, ownerId: In([...ownerIds]) },
{ ...where, albums: { id: In([...albumIds]) } },
],
relations: {
exifInfo: true,
},
order: {
fileCreatedAt: 'DESC',
},
});
return assets.map((asset) => ({
id: asset.id,
lat: asset.exifInfo!.latitude!,
lon: asset.exifInfo!.longitude!,
city: asset.exifInfo!.city,
state: asset.exifInfo!.state,
country: asset.exifInfo!.country,
}));
}
async fetchStyle(url: string) {
try {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`Failed to fetch data from ${url} with status ${response.status}: ${await response.text()}`);
}
return response.json();
} catch (error) {
throw new Error(`Failed to fetch data from ${url}: ${error}`);
}
}
async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null> {
this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
const response = await this.geodataPlacesRepository
.createQueryBuilder('geoplaces')
.where('earth_box(ll_to_earth(:latitude, :longitude), 25000) @> "earthCoord"', point)
.orderBy('earth_distance(ll_to_earth(:latitude, :longitude), "earthCoord")')
.limit(1)
.getOne();
if (!response) {
this.logger.warn(
`Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`,
);
return null;
}
this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`);
const { countryCode, name: city, admin1Name } = response;
const country = getName(countryCode, 'en') ?? null;
const state = admin1Name;
return { country, state, city };
}
private async importGeodata() {
const queryRunner = this.dataSource.createQueryRunner();
await queryRunner.connect();
const admin1 = await this.loadAdmin(geodataAdmin1Path);
const admin2 = await this.loadAdmin(geodataAdmin2Path);
try {
await queryRunner.startTransaction();
await queryRunner.manager.clear(GeodataPlacesEntity);
await this.loadCities500(queryRunner, admin1, admin2);
await queryRunner.commitTransaction();
} catch (error) {
this.logger.fatal('Error importing geodata', error);
await queryRunner.rollbackTransaction();
throw error;
} finally {
await queryRunner.release();
}
}
private async loadGeodataToTableFromFile(
queryRunner: QueryRunner,
lineToEntityMapper: (lineSplit: string[]) => GeodataPlacesEntity,
filePath: string,
options?: { entityFilter?: (linesplit: string[]) => boolean },
) {
const _entityFilter = options?.entityFilter ?? (() => true);
if (!existsSync(filePath)) {
this.logger.error(`Geodata file ${filePath} not found`);
throw new Error(`Geodata file ${filePath} not found`);
}
const input = createReadStream(filePath);
let bufferGeodata: QueryDeepPartialEntity<GeodataPlacesEntity>[] = [];
const lineReader = readLine.createInterface({ input });
for await (const line of lineReader) {
const lineSplit = line.split('\t');
if (!_entityFilter(lineSplit)) {
continue;
}
const geoData = lineToEntityMapper(lineSplit);
bufferGeodata.push(geoData);
if (bufferGeodata.length > 1000) {
await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
bufferGeodata = [];
}
}
await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
}
private async loadCities500(
queryRunner: QueryRunner,
admin1Map: Map<string, string>,
admin2Map: Map<string, string>,
) {
await this.loadGeodataToTableFromFile(
queryRunner,
(lineSplit: string[]) =>
this.geodataPlacesRepository.create({
id: Number.parseInt(lineSplit[0]),
name: lineSplit[1],
alternateNames: lineSplit[3],
latitude: Number.parseFloat(lineSplit[4]),
longitude: Number.parseFloat(lineSplit[5]),
countryCode: lineSplit[8],
admin1Code: lineSplit[10],
admin2Code: lineSplit[11],
modificationDate: lineSplit[18],
admin1Name: admin1Map.get(`${lineSplit[8]}.${lineSplit[10]}`),
admin2Name: admin2Map.get(`${lineSplit[8]}.${lineSplit[10]}.${lineSplit[11]}`),
}),
geodataCities500Path,
{ entityFilter: (lineSplit) => lineSplit[7] != 'PPLX' },
);
}
private async loadAdmin(filePath: string) {
if (!existsSync(filePath)) {
this.logger.error(`Geodata file ${filePath} not found`);
throw new Error(`Geodata file ${filePath} not found`);
}
const input = createReadStream(filePath);
const lineReader = readLine.createInterface({ input: input });
const adminMap = new Map<string, string>();
for await (const line of lineReader) {
const lineSplit = line.split('\t');
adminMap.set(lineSplit[0], lineSplit[1]);
}
return adminMap;
}
}

158
server/src/repositories/metadata.repository.ts
View File

@ -2,21 +2,13 @@ import { Inject, Injectable } from '@nestjs/common';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { DefaultReadTaskOptions, Tags, exiftool } from 'exiftool-vendored';
import geotz from 'geo-tz';
import { getName } from 'i18n-iso-countries';
import { createReadStream, existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import readLine from 'node:readline';
import { citiesFile, geodataAdmin1Path, geodataAdmin2Path, geodataCities500Path, geodataDatePath } from 'src/constants';
import { DummyValue, GenerateSql } from 'src/decorators';
import { ExifEntity } from 'src/entities/exif.entity';
import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
import { SystemMetadataKey } from 'src/entities/system-metadata.entity';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { GeoPoint, IMetadataRepository, ImmichTags, ReverseGeocodeResult } from 'src/interfaces/metadata.interface';
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
import { IMetadataRepository, ImmichTags } from 'src/interfaces/metadata.interface';
import { Instrumentation } from 'src/utils/instrumentation';
import { DataSource, QueryRunner, Repository } from 'typeorm';
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
import { DataSource, Repository } from 'typeorm';
@Instrumentation()
@Injectable()
@ -24,162 +16,16 @@ export class MetadataRepository implements IMetadataRepository {
constructor(
@InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
@InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,
@Inject(ISystemMetadataRepository)
private systemMetadataRepository: ISystemMetadataRepository,
@InjectDataSource() private dataSource: DataSource,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
) {
this.logger.setContext(MetadataRepository.name);
}
async init(): Promise<void> {
this.logger.log('Initializing metadata repository');
const geodataDate = await readFile(geodataDatePath, 'utf8');
// TODO move to metadata service init
const geocodingMetadata = await this.systemMetadataRepository.get(SystemMetadataKey.REVERSE_GEOCODING_STATE);
if (geocodingMetadata?.lastUpdate === geodataDate) {
return;
}
await this.importGeodata();
await this.systemMetadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, {
lastUpdate: geodataDate,
lastImportFileName: citiesFile,
});
this.logger.log('Geodata import completed');
}
private async importGeodata() {
const queryRunner = this.dataSource.createQueryRunner();
await queryRunner.connect();
const admin1 = await this.loadAdmin(geodataAdmin1Path);
const admin2 = await this.loadAdmin(geodataAdmin2Path);
try {
await queryRunner.startTransaction();
await queryRunner.manager.clear(GeodataPlacesEntity);
await this.loadCities500(queryRunner, admin1, admin2);
await queryRunner.commitTransaction();
} catch (error) {
this.logger.fatal('Error importing geodata', error);
await queryRunner.rollbackTransaction();
throw error;
} finally {
await queryRunner.release();
}
}
private async loadGeodataToTableFromFile(
queryRunner: QueryRunner,
lineToEntityMapper: (lineSplit: string[]) => GeodataPlacesEntity,
filePath: string,
options?: { entityFilter?: (linesplit: string[]) => boolean },
) {
const _entityFilter = options?.entityFilter ?? (() => true);
if (!existsSync(filePath)) {
this.logger.error(`Geodata file ${filePath} not found`);
throw new Error(`Geodata file ${filePath} not found`);
}
const input = createReadStream(filePath);
let bufferGeodata: QueryDeepPartialEntity<GeodataPlacesEntity>[] = [];
const lineReader = readLine.createInterface({ input });
for await (const line of lineReader) {
const lineSplit = line.split('\t');
if (!_entityFilter(lineSplit)) {
continue;
}
const geoData = lineToEntityMapper(lineSplit);
bufferGeodata.push(geoData);
if (bufferGeodata.length > 1000) {
await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
bufferGeodata = [];
}
}
await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
}
private async loadCities500(
queryRunner: QueryRunner,
admin1Map: Map<string, string>,
admin2Map: Map<string, string>,
) {
await this.loadGeodataToTableFromFile(
queryRunner,
(lineSplit: string[]) =>
this.geodataPlacesRepository.create({
id: Number.parseInt(lineSplit[0]),
name: lineSplit[1],
alternateNames: lineSplit[3],
latitude: Number.parseFloat(lineSplit[4]),
longitude: Number.parseFloat(lineSplit[5]),
countryCode: lineSplit[8],
admin1Code: lineSplit[10],
admin2Code: lineSplit[11],
modificationDate: lineSplit[18],
admin1Name: admin1Map.get(`${lineSplit[8]}.${lineSplit[10]}`),
admin2Name: admin2Map.get(`${lineSplit[8]}.${lineSplit[10]}.${lineSplit[11]}`),
}),
geodataCities500Path,
{ entityFilter: (lineSplit) => lineSplit[7] != 'PPLX' },
);
}
private async loadAdmin(filePath: string) {
if (!existsSync(filePath)) {
this.logger.error(`Geodata file ${filePath} not found`);
throw new Error(`Geodata file ${filePath} not found`);
}
const input = createReadStream(filePath);
const lineReader = readLine.createInterface({ input: input });
const adminMap = new Map<string, string>();
for await (const line of lineReader) {
const lineSplit = line.split('\t');
adminMap.set(lineSplit[0], lineSplit[1]);
}
return adminMap;
}
async teardown() {
await exiftool.end();
}
async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null> {
this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
const response = await this.geodataPlacesRepository
.createQueryBuilder('geoplaces')
.where('earth_box(ll_to_earth(:latitude, :longitude), 25000) @> "earthCoord"', point)
.orderBy('earth_distance(ll_to_earth(:latitude, :longitude), "earthCoord")')
.limit(1)
.getOne();
if (!response) {
this.logger.warn(
`Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`,
);
return null;
}
this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`);
const { countryCode, name: city, admin1Name } = response;
const country = getName(countryCode, 'en') ?? null;
const state = admin1Name;
return { country, state, city };
}
readTags(path: string): Promise<ImmichTags | null> {
return exiftool
.read(path, undefined, {

14
server/src/repositories/system-metadata.repository.ts
View File

@ -26,20 +26,6 @@ export class SystemMetadataRepository implements ISystemMetadataRepository {
await this.repository.upsert({ key, value }, { conflictPaths: { key: true } });
}
async fetchStyle(url: string) {
try {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`Failed to fetch data from ${url} with status ${response.status}: ${await response.text()}`);
}
return response.json();
} catch (error) {
throw new Error(`Failed to fetch data from ${url}: ${error}`);
}
}
readFile(filename: string): Promise<string> {
return readFile(filename, { encoding: 'utf8' });
}

26
server/src/services/asset.service.spec.ts
View File

@ -2,7 +2,6 @@ import { BadRequestException, UnauthorizedException } from '@nestjs/common';
import { mapAsset } from 'src/dtos/asset-response.dto';
import { AssetJobName, AssetStatsResponseDto, UploadFieldName } from 'src/dtos/asset.dto';
import { AssetEntity, AssetType } from 'src/entities/asset.entity';
import { IAlbumRepository } from 'src/interfaces/album.interface';
import { IAssetStackRepository } from 'src/interfaces/asset-stack.interface';
import { AssetStats, IAssetRepository } from 'src/interfaces/asset.interface';
import { ClientEvent, IEventRepository } from 'src/interfaces/event.interface';
@ -19,7 +18,6 @@ import { faceStub } from 'test/fixtures/face.stub';
import { partnerStub } from 'test/fixtures/partner.stub';
import { userStub } from 'test/fixtures/user.stub';
import { IAccessRepositoryMock, newAccessRepositoryMock } from 'test/repositories/access.repository.mock';
import { newAlbumRepositoryMock } from 'test/repositories/album.repository.mock';
import { newAssetStackRepositoryMock } from 'test/repositories/asset-stack.repository.mock';
import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
import { newEventRepositoryMock } from 'test/repositories/event.repository.mock';
@ -162,7 +160,6 @@ describe(AssetService.name, () => {
let systemMock: Mocked<ISystemMetadataRepository>;
let partnerMock: Mocked<IPartnerRepository>;
let assetStackMock: Mocked<IAssetStackRepository>;
let albumMock: Mocked<IAlbumRepository>;
let loggerMock: Mocked<ILoggerRepository>;
it('should work', () => {
@ -185,7 +182,6 @@ describe(AssetService.name, () => {
systemMock = newSystemMetadataRepositoryMock();
partnerMock = newPartnerRepositoryMock();
assetStackMock = newAssetStackRepositoryMock();
albumMock = newAlbumRepositoryMock();
loggerMock = newLoggerRepositoryMock();
sut = new AssetService(
@ -198,7 +194,6 @@ describe(AssetService.name, () => {
eventMock,
partnerMock,
assetStackMock,
albumMock,
loggerMock,
);
@ -314,27 +309,6 @@ describe(AssetService.name, () => {
});
});
describe('getMapMarkers', () => {
it('should get geo information of assets', async () => {
const asset = assetStub.withLocation;
const marker = {
id: asset.id,
lat: asset.exifInfo!.latitude!,
lon: asset.exifInfo!.longitude!,
city: asset.exifInfo!.city,
state: asset.exifInfo!.state,
country: asset.exifInfo!.country,
};
partnerMock.getAll.mockResolvedValue([]);
assetMock.getMapMarkers.mockResolvedValue([marker]);
const markers = await sut.getMapMarkers(authStub.user1, {});
expect(markers).toHaveLength(1);
expect(markers[0]).toEqual(marker);
});
});
describe('getMemoryLane', () => {
beforeAll(() => {
vitest.useFakeTimers();

28
server/src/services/asset.service.ts
View File

@ -24,11 +24,10 @@ import {
mapStats,
} from 'src/dtos/asset.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { MapMarkerDto, MapMarkerResponseDto, MemoryLaneDto } from 'src/dtos/search.dto';
import { MemoryLaneDto } from 'src/dtos/search.dto';
import { UpdateStackParentDto } from 'src/dtos/stack.dto';
import { AssetEntity } from 'src/entities/asset.entity';
import { IAccessRepository } from 'src/interfaces/access.interface';
import { IAlbumRepository } from 'src/interfaces/album.interface';
import { IAssetStackRepository } from 'src/interfaces/asset-stack.interface';
import { IAssetRepository } from 'src/interfaces/asset.interface';
import { ClientEvent, IEventRepository } from 'src/interfaces/event.interface';
@ -65,7 +64,6 @@ export class AssetService {
@Inject(IEventRepository) private eventRepository: IEventRepository,
@Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
@Inject(IAssetStackRepository) private assetStackRepository: IAssetStackRepository,
@Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
) {
this.logger.setContext(AssetService.name);
@ -153,30 +151,6 @@ export class AssetService {
return folder;
}
async getMapMarkers(auth: AuthDto, options: MapMarkerDto): Promise<MapMarkerResponseDto[]> {
const userIds: string[] = [auth.user.id];
// TODO convert to SQL join
if (options.withPartners) {
const partners = await this.partnerRepository.getAll(auth.user.id);
const partnersIds = partners
.filter((partner) => partner.sharedBy && partner.sharedWith && partner.sharedById != auth.user.id)
.map((partner) => partner.sharedById);
userIds.push(...partnersIds);
}
// TODO convert to SQL join
const albumIds: string[] = [];
if (options.withSharedAlbums) {
const [ownedAlbums, sharedAlbums] = await Promise.all([
this.albumRepository.getOwned(auth.user.id),
this.albumRepository.getShared(auth.user.id),
]);
albumIds.push(...ownedAlbums.map((album) => album.id), ...sharedAlbums.map((album) => album.id));
}
return this.assetRepository.getMapMarkers(userIds, albumIds, options);
}
async getMemoryLane(auth: AuthDto, dto: MemoryLaneDto): Promise<MemoryLaneResponseDto[]> {
const currentYear = new Date().getFullYear();

8
server/src/services/index.ts
View File

@ -13,6 +13,7 @@ import { DownloadService } from 'src/services/download.service';
import { DuplicateService } from 'src/services/duplicate.service';
import { JobService } from 'src/services/job.service';
import { LibraryService } from 'src/services/library.service';
import { MapService } from 'src/services/map.service';
import { MediaService } from 'src/services/media.service';
import { MemoryService } from 'src/services/memory.service';
import { MetadataService } from 'src/services/metadata.service';
@ -38,11 +39,10 @@ import { UserService } from 'src/services/user.service';
import { VersionService } from 'src/services/version.service';
export const services = [
ApiService,
MicroservicesService,
APIKeyService,
ActivityService,
AlbumService,
ApiService,
AssetMediaService,
AssetService,
AssetServiceV1,
@ -54,9 +54,11 @@ export const services = [
DuplicateService,
JobService,
LibraryService,
MapService,
MediaService,
MemoryService,
MetadataService,
MicroservicesService,
NotificationService,
PartnerService,
PersonService,
@ -73,7 +75,7 @@ export const services = [
TagService,
TimelineService,
TrashService,
UserService,
UserAdminService,
UserService,
VersionService,
];

54
server/src/services/map.service.spec.ts Normal file
View File

@ -0,0 +1,54 @@
import { IAlbumRepository } from 'src/interfaces/album.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IMapRepository } from 'src/interfaces/map.interface';
import { IPartnerRepository } from 'src/interfaces/partner.interface';
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
import { MapService } from 'src/services/map.service';
import { assetStub } from 'test/fixtures/asset.stub';
import { authStub } from 'test/fixtures/auth.stub';
import { newAlbumRepositoryMock } from 'test/repositories/album.repository.mock';
import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock';
import { newMapRepositoryMock } from 'test/repositories/map.repository.mock';
import { newPartnerRepositoryMock } from 'test/repositories/partner.repository.mock';
import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock';
import { Mocked } from 'vitest';
describe(MapService.name, () => {
let sut: MapService;
let albumMock: Mocked<IAlbumRepository>;
let loggerMock: Mocked<ILoggerRepository>;
let partnerMock: Mocked<IPartnerRepository>;
let mapMock: Mocked<IMapRepository>;
let systemMetadataMock: Mocked<ISystemMetadataRepository>;
beforeEach(() => {
albumMock = newAlbumRepositoryMock();
loggerMock = newLoggerRepositoryMock();
partnerMock = newPartnerRepositoryMock();
mapMock = newMapRepositoryMock();
systemMetadataMock = newSystemMetadataRepositoryMock();
sut = new MapService(albumMock, loggerMock, partnerMock, mapMock, systemMetadataMock);
});
describe('getMapMarkers', () => {
it('should get geo information of assets', async () => {
const asset = assetStub.withLocation;
const marker = {
id: asset.id,
lat: asset.exifInfo!.latitude!,
lon: asset.exifInfo!.longitude!,
city: asset.exifInfo!.city,
state: asset.exifInfo!.state,
country: asset.exifInfo!.country,
};
partnerMock.getAll.mockResolvedValue([]);
mapMock.getMapMarkers.mockResolvedValue([marker]);
const markers = await sut.getMapMarkers(authStub.user1, {});
expect(markers).toHaveLength(1);
expect(markers[0]).toEqual(marker);
});
});
});

59
server/src/services/map.service.ts Normal file
View File

@ -0,0 +1,59 @@
import { Inject } from '@nestjs/common';
import { SystemConfigCore } from 'src/cores/system-config.core';
import { AuthDto } from 'src/dtos/auth.dto';
import { MapMarkerDto, MapMarkerResponseDto } from 'src/dtos/search.dto';
import { IAlbumRepository } from 'src/interfaces/album.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IMapRepository } from 'src/interfaces/map.interface';
import { IPartnerRepository } from 'src/interfaces/partner.interface';
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
export class MapService {
private configCore: SystemConfigCore;
constructor(
@Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
@Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
@Inject(IMapRepository) private mapRepository: IMapRepository,
@Inject(ISystemMetadataRepository) private systemMetadataRepository: ISystemMetadataRepository,
) {
this.logger.setContext(MapService.name);
this.configCore = SystemConfigCore.create(systemMetadataRepository, logger);
}
async getMapMarkers(auth: AuthDto, options: MapMarkerDto): Promise<MapMarkerResponseDto[]> {
const userIds: string[] = [auth.user.id];
// TODO convert to SQL join
if (options.withPartners) {
const partners = await this.partnerRepository.getAll(auth.user.id);
const partnersIds = partners
.filter((partner) => partner.sharedBy && partner.sharedWith && partner.sharedById != auth.user.id)
.map((partner) => partner.sharedById);
userIds.push(...partnersIds);
}
// TODO convert to SQL join
const albumIds: string[] = [];
if (options.withSharedAlbums) {
const [ownedAlbums, sharedAlbums] = await Promise.all([
this.albumRepository.getOwned(auth.user.id),
this.albumRepository.getShared(auth.user.id),
]);
albumIds.push(...ownedAlbums.map((album) => album.id), ...sharedAlbums.map((album) => album.id));
}
return this.mapRepository.getMapMarkers(userIds, albumIds, options);
}
async getMapStyle(theme: 'light' | 'dark') {
const { map } = await this.configCore.getConfig();
const styleUrl = theme === 'dark' ? map.darkStyle : map.lightStyle;
if (styleUrl) {
return this.mapRepository.fetchStyle(styleUrl);
}
return JSON.parse(await this.systemMetadataRepository.readFile(`./resources/style-${theme}.json`));
}
}

11
server/src/services/metadata.service.spec.ts
View File

@ -11,6 +11,7 @@ import { IDatabaseRepository } from 'src/interfaces/database.interface';
import { ClientEvent, IEventRepository } from 'src/interfaces/event.interface';
import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IMapRepository } from 'src/interfaces/map.interface';
import { IMediaRepository } from 'src/interfaces/media.interface';
import { IMetadataRepository, ImmichTags } from 'src/interfaces/metadata.interface';
import { IMoveRepository } from 'src/interfaces/move.interface';
@ -29,6 +30,7 @@ import { newDatabaseRepositoryMock } from 'test/repositories/database.repository
import { newEventRepositoryMock } from 'test/repositories/event.repository.mock';
import { newJobRepositoryMock } from 'test/repositories/job.repository.mock';
import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock';
import { newMapRepositoryMock } from 'test/repositories/map.repository.mock';
import { newMediaRepositoryMock } from 'test/repositories/media.repository.mock';
import { newMetadataRepositoryMock } from 'test/repositories/metadata.repository.mock';
import { newMoveRepositoryMock } from 'test/repositories/move.repository.mock';
@ -44,6 +46,7 @@ describe(MetadataService.name, () => {
let systemMock: Mocked<ISystemMetadataRepository>;
let cryptoRepository: Mocked<ICryptoRepository>;
let jobMock: Mocked<IJobRepository>;
let mapMock: Mocked<IMapRepository>;
let metadataMock: Mocked<IMetadataRepository>;
let moveMock: Mocked<IMoveRepository>;
let mediaMock: Mocked<IMediaRepository>;
@ -60,6 +63,7 @@ describe(MetadataService.name, () => {
assetMock = newAssetRepositoryMock();
cryptoRepository = newCryptoRepositoryMock();
jobMock = newJobRepositoryMock();
mapMock = newMapRepositoryMock();
metadataMock = newMetadataRepositoryMock();
moveMock = newMoveRepositoryMock();
personMock = newPersonRepositoryMock();
@ -78,6 +82,7 @@ describe(MetadataService.name, () => {
cryptoRepository,
databaseMock,
jobMock,
mapMock,
mediaMock,
metadataMock,
moveMock,
@ -102,7 +107,7 @@ describe(MetadataService.name, () => {
await sut.init();
expect(jobMock.pause).toHaveBeenCalledTimes(1);
expect(metadataMock.init).toHaveBeenCalledTimes(1);
expect(mapMock.init).toHaveBeenCalledTimes(1);
expect(jobMock.resume).toHaveBeenCalledTimes(1);
});
@ -112,7 +117,7 @@ describe(MetadataService.name, () => {
await sut.init();
expect(jobMock.pause).not.toHaveBeenCalled();
expect(metadataMock.init).not.toHaveBeenCalled();
expect(mapMock.init).not.toHaveBeenCalled();
expect(jobMock.resume).not.toHaveBeenCalled();
});
});
@ -297,7 +302,7 @@ describe(MetadataService.name, () => {
it('should apply reverse geocoding', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.withLocation]);
systemMock.get.mockResolvedValue({ reverseGeocoding: { enabled: true } });
metadataMock.reverseGeocode.mockResolvedValue({ city: 'City', state: 'State', country: 'Country' });
mapMock.reverseGeocode.mockResolvedValue({ city: 'City', state: 'State', country: 'Country' });
metadataMock.readTags.mockResolvedValue({
GPSLatitude: assetStub.withLocation.exifInfo!.latitude!,
GPSLongitude: assetStub.withLocation.exifInfo!.longitude!,

6
server/src/services/metadata.service.ts
View File

@ -26,6 +26,7 @@ import {
QueueName,
} from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IMapRepository } from 'src/interfaces/map.interface';
import { IMediaRepository } from 'src/interfaces/media.interface';
import { IMetadataRepository, ImmichTags } from 'src/interfaces/metadata.interface';
import { IMoveRepository } from 'src/interfaces/move.interface';
@ -108,6 +109,7 @@ export class MetadataService {
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
@Inject(IDatabaseRepository) private databaseRepository: IDatabaseRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IMapRepository) private mapRepository: IMapRepository,
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
@Inject(IMetadataRepository) private repository: IMetadataRepository,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@ -144,7 +146,7 @@ export class MetadataService {
try {
await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
await this.databaseRepository.withLock(DatabaseLock.GeodataImport, () => this.repository.init());
await this.databaseRepository.withLock(DatabaseLock.GeodataImport, () => this.mapRepository.init());
await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
this.logger.log(`Initialized local reverse geocoder`);
@ -337,7 +339,7 @@ export class MetadataService {
}
try {
const reverseGeocode = await this.repository.reverseGeocode({ latitude, longitude });
const reverseGeocode = await this.mapRepository.reverseGeocode({ latitude, longitude });
if (!reverseGeocode) {
return;
}

13
server/src/services/system-config.service.ts
View File

@ -31,7 +31,7 @@ export class SystemConfigService {
private core: SystemConfigCore;
constructor(
@Inject(ISystemMetadataRepository) private repository: ISystemMetadataRepository,
@Inject(ISystemMetadataRepository) repository: ISystemMetadataRepository,
@Inject(IEventRepository) private eventRepository: IEventRepository,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
@Inject(ISearchRepository) private smartInfoRepository: ISearchRepository,
@ -109,17 +109,6 @@ export class SystemConfigService {
return options;
}
async getMapStyle(theme: 'light' | 'dark') {
const { map } = await this.getConfig();
const styleUrl = theme === 'dark' ? map.darkStyle : map.lightStyle;
if (styleUrl) {
return this.repository.fetchStyle(styleUrl);
}
return JSON.parse(await this.repository.readFile(`./resources/style-${theme}.json`));
}
async getCustomCss(): Promise<string> {
const { theme } = await this.core.getConfig();
return theme.customCss;

1
server/test/repositories/asset.repository.mock.ts
View File

@ -31,7 +31,6 @@ export const newAssetRepositoryMock = (): Mocked<IAssetRepository> => {
update: vitest.fn(),
remove: vitest.fn(),
findLivePhotoMatch: vitest.fn(),
getMapMarkers: vitest.fn(),
getStatistics: vitest.fn(),
getTimeBucket: vitest.fn(),
getTimeBuckets: vitest.fn(),

11
server/test/repositories/map.repository.mock.ts Normal file
View File

@ -0,0 +1,11 @@
import { IMapRepository } from 'src/interfaces/map.interface';
import { Mocked } from 'vitest';
export const newMapRepositoryMock = (): Mocked<IMapRepository> => {
return {
init: vitest.fn(),
reverseGeocode: vitest.fn(),
getMapMarkers: vitest.fn(),
fetchStyle: vitest.fn(),
};
};

2
server/test/repositories/metadata.repository.mock.ts
View File

@ -3,9 +3,7 @@ import { Mocked, vitest } from 'vitest';
export const newMetadataRepositoryMock = (): Mocked<IMetadataRepository> => {
return {
init: vitest.fn(),
teardown: vitest.fn(),
reverseGeocode: vitest.fn(),
readTags: vitest.fn(),
writeTags: vitest.fn(),
extractBinaryTag: vitest.fn(),

1
server/test/repositories/system-metadata.repository.mock.ts
View File

@ -11,6 +11,5 @@ export const newSystemMetadataRepositoryMock = (reset = true): Mocked<ISystemMet
get: vitest.fn() as any,
set: vitest.fn(),
readFile: vitest.fn(),
fetchStyle: vitest.fn(),
};
};