From 103cb60a579ed5496cb6fb4c1e47af529206c85f Mon Sep 17 00:00:00 2001 From: Fynn Petersen-Frey <10599762+fyfrey@users.noreply.github.com> Date: Tue, 16 Apr 2024 07:26:37 +0200 Subject: [PATCH] feat(server): efficient full app sync (#8755) * feat(server): efficient full app sync * add SQL, fix test compile issues * fix linter warning * new sync controller+service, add tests * enable new sync controller+service * Update server/src/services/sync.service.ts Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> --------- Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> --- mobile/openapi/.openapi-generator/FILES | 6 + mobile/openapi/README.md | 3 + .../openapi/doc/AssetDeltaSyncResponseDto.md | 17 ++ mobile/openapi/doc/SyncApi.md | 135 +++++++++++++++ mobile/openapi/lib/api.dart | 2 + mobile/openapi/lib/api/sync_api.dart | 150 +++++++++++++++++ mobile/openapi/lib/api_client.dart | 2 + .../model/asset_delta_sync_response_dto.dart | 116 +++++++++++++ .../asset_delta_sync_response_dto_test.dart | 37 ++++ mobile/openapi/test/sync_api_test.dart | 31 ++++ open-api/immich-openapi-specs.json | 159 ++++++++++++++++++ open-api/typescript-sdk/src/fetch-client.ts | 39 +++++ server/src/controllers/index.ts | 2 + server/src/controllers/sync.controller.ts | 24 +++ server/src/dtos/sync.dto.ts | 38 +++++ server/src/interfaces/asset.interface.ts | 16 ++ server/src/interfaces/audit.interface.ts | 6 +- server/src/queries/asset.repository.sql | 148 ++++++++++++++++ server/src/repositories/asset.repository.ts | 59 ++++++- server/src/repositories/audit.repository.ts | 9 +- server/src/services/audit.service.spec.ts | 6 +- server/src/services/audit.service.ts | 4 +- server/src/services/index.ts | 2 + server/src/services/sync.service.spec.ts | 101 +++++++++++ server/src/services/sync.service.ts | 77 +++++++++ .../repositories/asset.repository.mock.ts | 2 + 26 files changed, 1178 insertions(+), 13 deletions(-) create mode 100644 mobile/openapi/doc/AssetDeltaSyncResponseDto.md create mode 100644 mobile/openapi/doc/SyncApi.md create mode 100644 mobile/openapi/lib/api/sync_api.dart create mode 100644 mobile/openapi/lib/model/asset_delta_sync_response_dto.dart create mode 100644 mobile/openapi/test/asset_delta_sync_response_dto_test.dart create mode 100644 mobile/openapi/test/sync_api_test.dart create mode 100644 server/src/controllers/sync.controller.ts create mode 100644 server/src/dtos/sync.dto.ts create mode 100644 server/src/services/sync.service.spec.ts create mode 100644 server/src/services/sync.service.ts diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 263687549f..b296bbcb55 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -24,6 +24,7 @@ doc/AssetBulkUploadCheckDto.md doc/AssetBulkUploadCheckItem.md doc/AssetBulkUploadCheckResponseDto.md doc/AssetBulkUploadCheckResult.md +doc/AssetDeltaSyncResponseDto.md doc/AssetFaceResponseDto.md doc/AssetFaceUpdateDto.md doc/AssetFaceUpdateItem.md @@ -149,6 +150,7 @@ doc/SharedLinkType.md doc/SignUpDto.md doc/SmartInfoResponseDto.md doc/SmartSearchDto.md +doc/SyncApi.md doc/SystemConfigApi.md doc/SystemConfigDto.md doc/SystemConfigFFmpegDto.md @@ -218,6 +220,7 @@ lib/api/person_api.dart lib/api/search_api.dart lib/api/server_info_api.dart lib/api/shared_link_api.dart +lib/api/sync_api.dart lib/api/system_config_api.dart lib/api/tag_api.dart lib/api/timeline_api.dart @@ -248,6 +251,7 @@ lib/model/asset_bulk_upload_check_dto.dart lib/model/asset_bulk_upload_check_item.dart lib/model/asset_bulk_upload_check_response_dto.dart lib/model/asset_bulk_upload_check_result.dart +lib/model/asset_delta_sync_response_dto.dart lib/model/asset_face_response_dto.dart lib/model/asset_face_update_dto.dart lib/model/asset_face_update_item.dart @@ -427,6 +431,7 @@ test/asset_bulk_upload_check_dto_test.dart test/asset_bulk_upload_check_item_test.dart test/asset_bulk_upload_check_response_dto_test.dart test/asset_bulk_upload_check_result_test.dart +test/asset_delta_sync_response_dto_test.dart test/asset_face_response_dto_test.dart test/asset_face_update_dto_test.dart test/asset_face_update_item_test.dart @@ -552,6 +557,7 @@ test/shared_link_type_test.dart test/sign_up_dto_test.dart test/smart_info_response_dto_test.dart test/smart_search_dto_test.dart +test/sync_api_test.dart test/system_config_api_test.dart test/system_config_dto_test.dart test/system_config_f_fmpeg_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 9a03fbd61c..730307b9bf 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -191,6 +191,8 @@ Class | Method | HTTP request | Description *SharedLinkApi* | [**removeSharedLink**](doc//SharedLinkApi.md#removesharedlink) | **DELETE** /shared-link/{id} | *SharedLinkApi* | [**removeSharedLinkAssets**](doc//SharedLinkApi.md#removesharedlinkassets) | **DELETE** /shared-link/{id}/assets | *SharedLinkApi* | [**updateSharedLink**](doc//SharedLinkApi.md#updatesharedlink) | **PATCH** /shared-link/{id} | +*SyncApi* | [**getAllForUserFullSync**](doc//SyncApi.md#getallforuserfullsync) | **GET** /sync/full-sync | +*SyncApi* | [**getDeltaSync**](doc//SyncApi.md#getdeltasync) | **GET** /sync/delta-sync | *SystemConfigApi* | [**getConfig**](doc//SystemConfigApi.md#getconfig) | **GET** /system-config | *SystemConfigApi* | [**getConfigDefaults**](doc//SystemConfigApi.md#getconfigdefaults) | **GET** /system-config/defaults | *SystemConfigApi* | [**getMapStyle**](doc//SystemConfigApi.md#getmapstyle) | **GET** /system-config/map/style.json | @@ -240,6 +242,7 @@ Class | Method | HTTP request | Description - [AssetBulkUploadCheckItem](doc//AssetBulkUploadCheckItem.md) - [AssetBulkUploadCheckResponseDto](doc//AssetBulkUploadCheckResponseDto.md) - [AssetBulkUploadCheckResult](doc//AssetBulkUploadCheckResult.md) + - [AssetDeltaSyncResponseDto](doc//AssetDeltaSyncResponseDto.md) - [AssetFaceResponseDto](doc//AssetFaceResponseDto.md) - [AssetFaceUpdateDto](doc//AssetFaceUpdateDto.md) - [AssetFaceUpdateItem](doc//AssetFaceUpdateItem.md) diff --git a/mobile/openapi/doc/AssetDeltaSyncResponseDto.md b/mobile/openapi/doc/AssetDeltaSyncResponseDto.md new file mode 100644 index 0000000000..527203a4b8 --- /dev/null +++ b/mobile/openapi/doc/AssetDeltaSyncResponseDto.md @@ -0,0 +1,17 @@ +# openapi.model.AssetDeltaSyncResponseDto + +## Load the model package +```dart +import 'package:openapi/api.dart'; +``` + +## Properties +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**deleted** | **List** | | [default to const []] +**needsFullSync** | **bool** | | +**upserted** | [**List**](AssetResponseDto.md) | | [default to const []] + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/mobile/openapi/doc/SyncApi.md b/mobile/openapi/doc/SyncApi.md new file mode 100644 index 0000000000..1b28e10c8c --- /dev/null +++ b/mobile/openapi/doc/SyncApi.md @@ -0,0 +1,135 @@ +# openapi.api.SyncApi + +## Load the API package +```dart +import 'package:openapi/api.dart'; +``` + +All URIs are relative to */api* + +Method | HTTP request | Description +------------- | ------------- | ------------- +[**getAllForUserFullSync**](SyncApi.md#getallforuserfullsync) | **GET** /sync/full-sync | +[**getDeltaSync**](SyncApi.md#getdeltasync) | **GET** /sync/delta-sync | + + +# **getAllForUserFullSync** +> List getAllForUserFullSync(limit, updatedUntil, lastCreationDate, lastId, userId) + + + +### Example +```dart +import 'package:openapi/api.dart'; +// TODO Configure API key authorization: cookie +//defaultApiClient.getAuthentication('cookie').apiKey = 'YOUR_API_KEY'; +// uncomment below to setup prefix (e.g. Bearer) for API key, if needed +//defaultApiClient.getAuthentication('cookie').apiKeyPrefix = 'Bearer'; +// TODO Configure API key authorization: api_key +//defaultApiClient.getAuthentication('api_key').apiKey = 'YOUR_API_KEY'; +// uncomment below to setup prefix (e.g. Bearer) for API key, if needed +//defaultApiClient.getAuthentication('api_key').apiKeyPrefix = 'Bearer'; +// TODO Configure HTTP Bearer authorization: bearer +// Case 1. Use String Token +//defaultApiClient.getAuthentication('bearer').setAccessToken('YOUR_ACCESS_TOKEN'); +// Case 2. Use Function which generate token. +// String yourTokenGeneratorFunction() { ... } +//defaultApiClient.getAuthentication('bearer').setAccessToken(yourTokenGeneratorFunction); + +final api_instance = SyncApi(); +final limit = 56; // int | +final updatedUntil = 2013-10-20T19:20:30+01:00; // DateTime | +final lastCreationDate = 2013-10-20T19:20:30+01:00; // DateTime | +final lastId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String | +final userId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String | + +try { + final result = api_instance.getAllForUserFullSync(limit, updatedUntil, lastCreationDate, lastId, userId); + print(result); +} catch (e) { + print('Exception when calling SyncApi->getAllForUserFullSync: $e\n'); +} +``` + +### Parameters + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + **limit** | **int**| | + **updatedUntil** | **DateTime**| | + **lastCreationDate** | **DateTime**| | [optional] + **lastId** | **String**| | [optional] + **userId** | **String**| | [optional] + +### Return type + +[**List**](AssetResponseDto.md) + +### Authorization + +[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer) + +### HTTP request headers + + - **Content-Type**: Not defined + - **Accept**: application/json + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) + +# **getDeltaSync** +> AssetDeltaSyncResponseDto getDeltaSync(updatedAfter, userIds) + + + +### Example +```dart +import 'package:openapi/api.dart'; +// TODO Configure API key authorization: cookie +//defaultApiClient.getAuthentication('cookie').apiKey = 'YOUR_API_KEY'; +// uncomment below to setup prefix (e.g. Bearer) for API key, if needed +//defaultApiClient.getAuthentication('cookie').apiKeyPrefix = 'Bearer'; +// TODO Configure API key authorization: api_key +//defaultApiClient.getAuthentication('api_key').apiKey = 'YOUR_API_KEY'; +// uncomment below to setup prefix (e.g. Bearer) for API key, if needed +//defaultApiClient.getAuthentication('api_key').apiKeyPrefix = 'Bearer'; +// TODO Configure HTTP Bearer authorization: bearer +// Case 1. Use String Token +//defaultApiClient.getAuthentication('bearer').setAccessToken('YOUR_ACCESS_TOKEN'); +// Case 2. Use Function which generate token. +// String yourTokenGeneratorFunction() { ... } +//defaultApiClient.getAuthentication('bearer').setAccessToken(yourTokenGeneratorFunction); + +final api_instance = SyncApi(); +final updatedAfter = 2013-10-20T19:20:30+01:00; // DateTime | +final userIds = []; // List | + +try { + final result = api_instance.getDeltaSync(updatedAfter, userIds); + print(result); +} catch (e) { + print('Exception when calling SyncApi->getDeltaSync: $e\n'); +} +``` + +### Parameters + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + **updatedAfter** | **DateTime**| | + **userIds** | [**List**](String.md)| | [default to const []] + +### Return type + +[**AssetDeltaSyncResponseDto**](AssetDeltaSyncResponseDto.md) + +### Authorization + +[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer) + +### HTTP request headers + + - **Content-Type**: Not defined + - **Accept**: application/json + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) + diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index ae458f5de1..e7320d5bb2 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -46,6 +46,7 @@ part 'api/person_api.dart'; part 'api/search_api.dart'; part 'api/server_info_api.dart'; part 'api/shared_link_api.dart'; +part 'api/sync_api.dart'; part 'api/system_config_api.dart'; part 'api/tag_api.dart'; part 'api/timeline_api.dart'; @@ -69,6 +70,7 @@ part 'model/asset_bulk_upload_check_dto.dart'; part 'model/asset_bulk_upload_check_item.dart'; part 'model/asset_bulk_upload_check_response_dto.dart'; part 'model/asset_bulk_upload_check_result.dart'; +part 'model/asset_delta_sync_response_dto.dart'; part 'model/asset_face_response_dto.dart'; part 'model/asset_face_update_dto.dart'; part 'model/asset_face_update_item.dart'; diff --git a/mobile/openapi/lib/api/sync_api.dart b/mobile/openapi/lib/api/sync_api.dart new file mode 100644 index 0000000000..fdfd8b9ac7 --- /dev/null +++ b/mobile/openapi/lib/api/sync_api.dart @@ -0,0 +1,150 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + + +class SyncApi { + SyncApi([ApiClient? apiClient]) : apiClient = apiClient ?? defaultApiClient; + + final ApiClient apiClient; + + /// Performs an HTTP 'GET /sync/full-sync' operation and returns the [Response]. + /// Parameters: + /// + /// * [int] limit (required): + /// + /// * [DateTime] updatedUntil (required): + /// + /// * [DateTime] lastCreationDate: + /// + /// * [String] lastId: + /// + /// * [String] userId: + Future getAllForUserFullSyncWithHttpInfo(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async { + // ignore: prefer_const_declarations + final path = r'/sync/full-sync'; + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + if (lastCreationDate != null) { + queryParams.addAll(_queryParams('', 'lastCreationDate', lastCreationDate)); + } + if (lastId != null) { + queryParams.addAll(_queryParams('', 'lastId', lastId)); + } + queryParams.addAll(_queryParams('', 'limit', limit)); + queryParams.addAll(_queryParams('', 'updatedUntil', updatedUntil)); + if (userId != null) { + queryParams.addAll(_queryParams('', 'userId', userId)); + } + + const contentTypes = []; + + + return apiClient.invokeAPI( + path, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Parameters: + /// + /// * [int] limit (required): + /// + /// * [DateTime] updatedUntil (required): + /// + /// * [DateTime] lastCreationDate: + /// + /// * [String] lastId: + /// + /// * [String] userId: + Future?> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async { + final response = await getAllForUserFullSyncWithHttpInfo(limit, updatedUntil, lastCreationDate: lastCreationDate, lastId: lastId, userId: userId, ); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + final responseBody = await _decodeBodyBytes(response); + return (await apiClient.deserializeAsync(responseBody, 'List') as List) + .cast() + .toList(growable: false); + + } + return null; + } + + /// Performs an HTTP 'GET /sync/delta-sync' operation and returns the [Response]. + /// Parameters: + /// + /// * [DateTime] updatedAfter (required): + /// + /// * [List] userIds (required): + Future getDeltaSyncWithHttpInfo(DateTime updatedAfter, List userIds,) async { + // ignore: prefer_const_declarations + final path = r'/sync/delta-sync'; + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + queryParams.addAll(_queryParams('', 'updatedAfter', updatedAfter)); + queryParams.addAll(_queryParams('multi', 'userIds', userIds)); + + const contentTypes = []; + + + return apiClient.invokeAPI( + path, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Parameters: + /// + /// * [DateTime] updatedAfter (required): + /// + /// * [List] userIds (required): + Future getDeltaSync(DateTime updatedAfter, List userIds,) async { + final response = await getDeltaSyncWithHttpInfo(updatedAfter, userIds,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'AssetDeltaSyncResponseDto',) as AssetDeltaSyncResponseDto; + + } + return null; + } +} diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 757f475683..4bbae89285 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -216,6 +216,8 @@ class ApiClient { return AssetBulkUploadCheckResponseDto.fromJson(value); case 'AssetBulkUploadCheckResult': return AssetBulkUploadCheckResult.fromJson(value); + case 'AssetDeltaSyncResponseDto': + return AssetDeltaSyncResponseDto.fromJson(value); case 'AssetFaceResponseDto': return AssetFaceResponseDto.fromJson(value); case 'AssetFaceUpdateDto': diff --git a/mobile/openapi/lib/model/asset_delta_sync_response_dto.dart b/mobile/openapi/lib/model/asset_delta_sync_response_dto.dart new file mode 100644 index 0000000000..5d7679e734 --- /dev/null +++ b/mobile/openapi/lib/model/asset_delta_sync_response_dto.dart @@ -0,0 +1,116 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class AssetDeltaSyncResponseDto { + /// Returns a new [AssetDeltaSyncResponseDto] instance. + AssetDeltaSyncResponseDto({ + this.deleted = const [], + required this.needsFullSync, + this.upserted = const [], + }); + + List deleted; + + bool needsFullSync; + + List upserted; + + @override + bool operator ==(Object other) => identical(this, other) || other is AssetDeltaSyncResponseDto && + _deepEquality.equals(other.deleted, deleted) && + other.needsFullSync == needsFullSync && + _deepEquality.equals(other.upserted, upserted); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (deleted.hashCode) + + (needsFullSync.hashCode) + + (upserted.hashCode); + + @override + String toString() => 'AssetDeltaSyncResponseDto[deleted=$deleted, needsFullSync=$needsFullSync, upserted=$upserted]'; + + Map toJson() { + final json = {}; + json[r'deleted'] = this.deleted; + json[r'needsFullSync'] = this.needsFullSync; + json[r'upserted'] = this.upserted; + return json; + } + + /// Returns a new [AssetDeltaSyncResponseDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static AssetDeltaSyncResponseDto? fromJson(dynamic value) { + if (value is Map) { + final json = value.cast(); + + return AssetDeltaSyncResponseDto( + deleted: json[r'deleted'] is Iterable + ? (json[r'deleted'] as Iterable).cast().toList(growable: false) + : const [], + needsFullSync: mapValueOfType(json, r'needsFullSync')!, + upserted: AssetResponseDto.listFromJson(json[r'upserted']), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = AssetDeltaSyncResponseDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = AssetDeltaSyncResponseDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of AssetDeltaSyncResponseDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = AssetDeltaSyncResponseDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'deleted', + 'needsFullSync', + 'upserted', + }; +} + diff --git a/mobile/openapi/test/asset_delta_sync_response_dto_test.dart b/mobile/openapi/test/asset_delta_sync_response_dto_test.dart new file mode 100644 index 0000000000..20104c08c6 --- /dev/null +++ b/mobile/openapi/test/asset_delta_sync_response_dto_test.dart @@ -0,0 +1,37 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +import 'package:openapi/api.dart'; +import 'package:test/test.dart'; + +// tests for AssetDeltaSyncResponseDto +void main() { + // final instance = AssetDeltaSyncResponseDto(); + + group('test AssetDeltaSyncResponseDto', () { + // List deleted (default value: const []) + test('to test the property `deleted`', () async { + // TODO + }); + + // bool needsFullSync + test('to test the property `needsFullSync`', () async { + // TODO + }); + + // List upserted (default value: const []) + test('to test the property `upserted`', () async { + // TODO + }); + + + }); + +} diff --git a/mobile/openapi/test/sync_api_test.dart b/mobile/openapi/test/sync_api_test.dart new file mode 100644 index 0000000000..ad9ef0f92f --- /dev/null +++ b/mobile/openapi/test/sync_api_test.dart @@ -0,0 +1,31 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +import 'package:openapi/api.dart'; +import 'package:test/test.dart'; + + +/// tests for SyncApi +void main() { + // final instance = SyncApi(); + + group('tests for SyncApi', () { + //Future> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime lastCreationDate, String lastId, String userId }) async + test('test getAllForUserFullSync', () async { + // TODO + }); + + //Future getDeltaSync(DateTime updatedAfter, List userIds) async + test('test getDeltaSync', () async { + // TODO + }); + + }); +} diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index fbedfd359d..8f53f838b0 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -5566,6 +5566,140 @@ ] } }, + "/sync/delta-sync": { + "get": { + "operationId": "getDeltaSync", + "parameters": [ + { + "name": "updatedAfter", + "required": true, + "in": "query", + "schema": { + "format": "date-time", + "type": "string" + } + }, + { + "name": "userIds", + "required": true, + "in": "query", + "schema": { + "format": "uuid", + "type": "array", + "items": { + "type": "string" + } + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssetDeltaSyncResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + } + }, + "/sync/full-sync": { + "get": { + "operationId": "getAllForUserFullSync", + "parameters": [ + { + "name": "lastCreationDate", + "required": false, + "in": "query", + "schema": { + "format": "date-time", + "type": "string" + } + }, + { + "name": "lastId", + "required": false, + "in": "query", + "schema": { + "format": "uuid", + "type": "string" + } + }, + { + "name": "limit", + "required": true, + "in": "query", + "schema": { + "type": "integer" + } + }, + { + "name": "updatedUntil", + "required": true, + "in": "query", + "schema": { + "format": "date-time", + "type": "string" + } + }, + { + "name": "userId", + "required": false, + "in": "query", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/AssetResponseDto" + }, + "type": "array" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + } + }, "/system-config": { "get": { "operationId": "getConfig", @@ -7335,6 +7469,31 @@ ], "type": "object" }, + "AssetDeltaSyncResponseDto": { + "properties": { + "deleted": { + "items": { + "type": "string" + }, + "type": "array" + }, + "needsFullSync": { + "type": "boolean" + }, + "upserted": { + "items": { + "$ref": "#/components/schemas/AssetResponseDto" + }, + "type": "array" + } + }, + "required": [ + "deleted", + "needsFullSync", + "upserted" + ], + "type": "object" + }, "AssetFaceResponseDto": { "properties": { "boundingBoxX1": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 43f766dfe2..96b071f1f9 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -835,6 +835,11 @@ export type AssetIdsResponseDto = { error?: Error2; success: boolean; }; +export type AssetDeltaSyncResponseDto = { + deleted: string[]; + needsFullSync: boolean; + upserted: AssetResponseDto[]; +}; export type SystemConfigFFmpegDto = { accel: TranscodeHWAccel; acceptedAudioCodecs: AudioCodec[]; @@ -2507,6 +2512,40 @@ export function addSharedLinkAssets({ id, key, assetIdsDto }: { body: assetIdsDto }))); } +export function getDeltaSync({ updatedAfter, userIds }: { + updatedAfter: string; + userIds: string[]; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: AssetDeltaSyncResponseDto; + }>(`/sync/delta-sync${QS.query(QS.explode({ + updatedAfter, + userIds + }))}`, { + ...opts + })); +} +export function getAllForUserFullSync({ lastCreationDate, lastId, limit, updatedUntil, userId }: { + lastCreationDate?: string; + lastId?: string; + limit: number; + updatedUntil: string; + userId?: string; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: AssetResponseDto[]; + }>(`/sync/full-sync${QS.query(QS.explode({ + lastCreationDate, + lastId, + limit, + updatedUntil, + userId + }))}`, { + ...opts + })); +} export function getConfig(opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ status: 200; diff --git a/server/src/controllers/index.ts b/server/src/controllers/index.ts index ce51aa4c01..d136a52b04 100644 --- a/server/src/controllers/index.ts +++ b/server/src/controllers/index.ts @@ -17,6 +17,7 @@ import { PersonController } from 'src/controllers/person.controller'; import { SearchController } from 'src/controllers/search.controller'; import { ServerInfoController } from 'src/controllers/server-info.controller'; import { SharedLinkController } from 'src/controllers/shared-link.controller'; +import { SyncController } from 'src/controllers/sync.controller'; import { SystemConfigController } from 'src/controllers/system-config.controller'; import { TagController } from 'src/controllers/tag.controller'; import { TimelineController } from 'src/controllers/timeline.controller'; @@ -43,6 +44,7 @@ export const controllers = [ SearchController, ServerInfoController, SharedLinkController, + SyncController, SystemConfigController, TagController, TimelineController, diff --git a/server/src/controllers/sync.controller.ts b/server/src/controllers/sync.controller.ts new file mode 100644 index 0000000000..c12d42df23 --- /dev/null +++ b/server/src/controllers/sync.controller.ts @@ -0,0 +1,24 @@ +import { Controller, Get, Query } from '@nestjs/common'; +import { ApiTags } from '@nestjs/swagger'; +import { AssetResponseDto } from 'src/dtos/asset-response.dto'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto'; +import { Auth, Authenticated } from 'src/middleware/auth.guard'; +import { SyncService } from 'src/services/sync.service'; + +@ApiTags('Sync') +@Controller('sync') +@Authenticated() +export class SyncController { + constructor(private service: SyncService) {} + + @Get('full-sync') + getAllForUserFullSync(@Auth() auth: AuthDto, @Query() dto: AssetFullSyncDto): Promise { + return this.service.getAllAssetsForUserFullSync(auth, dto); + } + + @Get('delta-sync') + getDeltaSync(@Auth() auth: AuthDto, @Query() dto: AssetDeltaSyncDto): Promise { + return this.service.getChangesForDeltaSync(auth, dto); + } +} diff --git a/server/src/dtos/sync.dto.ts b/server/src/dtos/sync.dto.ts new file mode 100644 index 0000000000..a69062ec2d --- /dev/null +++ b/server/src/dtos/sync.dto.ts @@ -0,0 +1,38 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { Type } from 'class-transformer'; +import { IsInt, IsPositive } from 'class-validator'; +import { AssetResponseDto } from 'src/dtos/asset-response.dto'; +import { ValidateDate, ValidateUUID } from 'src/validation'; + +export class AssetFullSyncDto { + @ValidateUUID({ optional: true }) + lastId?: string; + + @ValidateDate({ optional: true }) + lastCreationDate?: Date; + + @ValidateDate() + updatedUntil!: Date; + + @IsInt() + @IsPositive() + @Type(() => Number) + @ApiProperty({ type: 'integer' }) + limit!: number; + + @ValidateUUID({ optional: true }) + userId?: string; +} + +export class AssetDeltaSyncDto { + @ValidateDate() + updatedAfter!: Date; + @ValidateUUID({ each: true }) + userIds!: string[]; +} + +export class AssetDeltaSyncResponseDto { + needsFullSync!: boolean; + upserted!: AssetResponseDto[]; + deleted!: string[]; +} diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index c5905f3b55..9c2ebe3e73 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -133,6 +133,20 @@ export interface MetadataSearchOptions { numResults: number; } +export interface AssetFullSyncOptions { + ownerId: string; + lastCreationDate?: Date; + lastId?: string; + updatedUntil: Date; + limit: number; +} + +export interface AssetDeltaSyncOptions { + userIds: string[]; + updatedAfter: Date; + limit: number; +} + export type AssetPathEntity = Pick; export const IAssetRepository = 'IAssetRepository'; @@ -175,4 +189,6 @@ export interface IAssetRepository { getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise>; getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise>; searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise; + getAllForUserFullSync(options: AssetFullSyncOptions): Promise; + getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise; } diff --git a/server/src/interfaces/audit.interface.ts b/server/src/interfaces/audit.interface.ts index 767a4bc2f6..b023d00d56 100644 --- a/server/src/interfaces/audit.interface.ts +++ b/server/src/interfaces/audit.interface.ts @@ -1,14 +1,14 @@ -import { AuditEntity, DatabaseAction, EntityType } from 'src/entities/audit.entity'; +import { DatabaseAction, EntityType } from 'src/entities/audit.entity'; export const IAuditRepository = 'IAuditRepository'; export interface AuditSearch { action?: DatabaseAction; entityType?: EntityType; - ownerId?: string; + userIds: string[]; } export interface IAuditRepository { - getAfter(since: Date, options: AuditSearch): Promise; + getAfter(since: Date, options: AuditSearch): Promise; removeBefore(before: Date): Promise; } diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index e223539dc0..86e9796faa 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -768,3 +768,151 @@ ORDER BY "asset"."fileCreatedAt" DESC LIMIT 250 + +-- AssetRepository.getAllForUserFullSync +SELECT + "asset"."id" AS "asset_id", + "asset"."deviceAssetId" AS "asset_deviceAssetId", + "asset"."ownerId" AS "asset_ownerId", + "asset"."libraryId" AS "asset_libraryId", + "asset"."deviceId" AS "asset_deviceId", + "asset"."type" AS "asset_type", + "asset"."originalPath" AS "asset_originalPath", + "asset"."previewPath" AS "asset_previewPath", + "asset"."thumbnailPath" AS "asset_thumbnailPath", + "asset"."thumbhash" AS "asset_thumbhash", + "asset"."encodedVideoPath" AS "asset_encodedVideoPath", + "asset"."createdAt" AS "asset_createdAt", + "asset"."updatedAt" AS "asset_updatedAt", + "asset"."deletedAt" AS "asset_deletedAt", + "asset"."fileCreatedAt" AS "asset_fileCreatedAt", + "asset"."localDateTime" AS "asset_localDateTime", + "asset"."fileModifiedAt" AS "asset_fileModifiedAt", + "asset"."isFavorite" AS "asset_isFavorite", + "asset"."isArchived" AS "asset_isArchived", + "asset"."isExternal" AS "asset_isExternal", + "asset"."isReadOnly" AS "asset_isReadOnly", + "asset"."isOffline" AS "asset_isOffline", + "asset"."checksum" AS "asset_checksum", + "asset"."duration" AS "asset_duration", + "asset"."isVisible" AS "asset_isVisible", + "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", + "asset"."originalFileName" AS "asset_originalFileName", + "asset"."sidecarPath" AS "asset_sidecarPath", + "asset"."stackId" AS "asset_stackId", + "exifInfo"."assetId" AS "exifInfo_assetId", + "exifInfo"."description" AS "exifInfo_description", + "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth", + "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight", + "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte", + "exifInfo"."orientation" AS "exifInfo_orientation", + "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal", + "exifInfo"."modifyDate" AS "exifInfo_modifyDate", + "exifInfo"."timeZone" AS "exifInfo_timeZone", + "exifInfo"."latitude" AS "exifInfo_latitude", + "exifInfo"."longitude" AS "exifInfo_longitude", + "exifInfo"."projectionType" AS "exifInfo_projectionType", + "exifInfo"."city" AS "exifInfo_city", + "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID", + "exifInfo"."autoStackId" AS "exifInfo_autoStackId", + "exifInfo"."state" AS "exifInfo_state", + "exifInfo"."country" AS "exifInfo_country", + "exifInfo"."make" AS "exifInfo_make", + "exifInfo"."model" AS "exifInfo_model", + "exifInfo"."lensModel" AS "exifInfo_lensModel", + "exifInfo"."fNumber" AS "exifInfo_fNumber", + "exifInfo"."focalLength" AS "exifInfo_focalLength", + "exifInfo"."iso" AS "exifInfo_iso", + "exifInfo"."exposureTime" AS "exifInfo_exposureTime", + "exifInfo"."profileDescription" AS "exifInfo_profileDescription", + "exifInfo"."colorspace" AS "exifInfo_colorspace", + "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", + "exifInfo"."fps" AS "exifInfo_fps", + "stack"."id" AS "stack_id", + "stack"."primaryAssetId" AS "stack_primaryAssetId" +FROM + "assets" "asset" + LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" + LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" +WHERE + "asset"."ownerId" = $1 + AND ("asset"."fileCreatedAt", "asset"."id") < ($2, $3) + AND "asset"."updatedAt" <= $4 + AND "asset"."isVisible" = true +ORDER BY + "asset"."fileCreatedAt" DESC, + "asset"."id" DESC +LIMIT + 10 + +-- AssetRepository.getChangedDeltaSync +SELECT + "AssetEntity"."id" AS "AssetEntity_id", + "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", + "AssetEntity"."ownerId" AS "AssetEntity_ownerId", + "AssetEntity"."libraryId" AS "AssetEntity_libraryId", + "AssetEntity"."deviceId" AS "AssetEntity_deviceId", + "AssetEntity"."type" AS "AssetEntity_type", + "AssetEntity"."originalPath" AS "AssetEntity_originalPath", + "AssetEntity"."previewPath" AS "AssetEntity_previewPath", + "AssetEntity"."thumbnailPath" AS "AssetEntity_thumbnailPath", + "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash", + "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath", + "AssetEntity"."createdAt" AS "AssetEntity_createdAt", + "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt", + "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt", + "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt", + "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime", + "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt", + "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite", + "AssetEntity"."isArchived" AS "AssetEntity_isArchived", + "AssetEntity"."isExternal" AS "AssetEntity_isExternal", + "AssetEntity"."isReadOnly" AS "AssetEntity_isReadOnly", + "AssetEntity"."isOffline" AS "AssetEntity_isOffline", + "AssetEntity"."checksum" AS "AssetEntity_checksum", + "AssetEntity"."duration" AS "AssetEntity_duration", + "AssetEntity"."isVisible" AS "AssetEntity_isVisible", + "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId", + "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName", + "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath", + "AssetEntity"."stackId" AS "AssetEntity_stackId", + "AssetEntity__AssetEntity_exifInfo"."assetId" AS "AssetEntity__AssetEntity_exifInfo_assetId", + "AssetEntity__AssetEntity_exifInfo"."description" AS "AssetEntity__AssetEntity_exifInfo_description", + "AssetEntity__AssetEntity_exifInfo"."exifImageWidth" AS "AssetEntity__AssetEntity_exifInfo_exifImageWidth", + "AssetEntity__AssetEntity_exifInfo"."exifImageHeight" AS "AssetEntity__AssetEntity_exifInfo_exifImageHeight", + "AssetEntity__AssetEntity_exifInfo"."fileSizeInByte" AS "AssetEntity__AssetEntity_exifInfo_fileSizeInByte", + "AssetEntity__AssetEntity_exifInfo"."orientation" AS "AssetEntity__AssetEntity_exifInfo_orientation", + "AssetEntity__AssetEntity_exifInfo"."dateTimeOriginal" AS "AssetEntity__AssetEntity_exifInfo_dateTimeOriginal", + "AssetEntity__AssetEntity_exifInfo"."modifyDate" AS "AssetEntity__AssetEntity_exifInfo_modifyDate", + "AssetEntity__AssetEntity_exifInfo"."timeZone" AS "AssetEntity__AssetEntity_exifInfo_timeZone", + "AssetEntity__AssetEntity_exifInfo"."latitude" AS "AssetEntity__AssetEntity_exifInfo_latitude", + "AssetEntity__AssetEntity_exifInfo"."longitude" AS "AssetEntity__AssetEntity_exifInfo_longitude", + "AssetEntity__AssetEntity_exifInfo"."projectionType" AS "AssetEntity__AssetEntity_exifInfo_projectionType", + "AssetEntity__AssetEntity_exifInfo"."city" AS "AssetEntity__AssetEntity_exifInfo_city", + "AssetEntity__AssetEntity_exifInfo"."livePhotoCID" AS "AssetEntity__AssetEntity_exifInfo_livePhotoCID", + "AssetEntity__AssetEntity_exifInfo"."autoStackId" AS "AssetEntity__AssetEntity_exifInfo_autoStackId", + "AssetEntity__AssetEntity_exifInfo"."state" AS "AssetEntity__AssetEntity_exifInfo_state", + "AssetEntity__AssetEntity_exifInfo"."country" AS "AssetEntity__AssetEntity_exifInfo_country", + "AssetEntity__AssetEntity_exifInfo"."make" AS "AssetEntity__AssetEntity_exifInfo_make", + "AssetEntity__AssetEntity_exifInfo"."model" AS "AssetEntity__AssetEntity_exifInfo_model", + "AssetEntity__AssetEntity_exifInfo"."lensModel" AS "AssetEntity__AssetEntity_exifInfo_lensModel", + "AssetEntity__AssetEntity_exifInfo"."fNumber" AS "AssetEntity__AssetEntity_exifInfo_fNumber", + "AssetEntity__AssetEntity_exifInfo"."focalLength" AS "AssetEntity__AssetEntity_exifInfo_focalLength", + "AssetEntity__AssetEntity_exifInfo"."iso" AS "AssetEntity__AssetEntity_exifInfo_iso", + "AssetEntity__AssetEntity_exifInfo"."exposureTime" AS "AssetEntity__AssetEntity_exifInfo_exposureTime", + "AssetEntity__AssetEntity_exifInfo"."profileDescription" AS "AssetEntity__AssetEntity_exifInfo_profileDescription", + "AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace", + "AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample", + "AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps", + "AssetEntity__AssetEntity_stack"."id" AS "AssetEntity__AssetEntity_stack_id", + "AssetEntity__AssetEntity_stack"."primaryAssetId" AS "AssetEntity__AssetEntity_stack_primaryAssetId" +FROM + "assets" "AssetEntity" + LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id" + LEFT JOIN "asset_stack" "AssetEntity__AssetEntity_stack" ON "AssetEntity__AssetEntity_stack"."id" = "AssetEntity"."stackId" +WHERE + ( + ("AssetEntity"."ownerId" IN ($1)) + AND ("AssetEntity"."isVisible" = $2) + AND ("AssetEntity"."updatedAt" > $3) + ) diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 724c1b0c0f..ddc666edd3 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -2,15 +2,18 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import path from 'node:path'; import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; -import { AssetOrder } from 'src/entities/album.entity'; +import { AlbumEntity, AssetOrder } from 'src/entities/album.entity'; import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; import { AssetEntity, AssetType } from 'src/entities/asset.entity'; import { ExifEntity } from 'src/entities/exif.entity'; +import { PartnerEntity } from 'src/entities/partner.entity'; import { SmartInfoEntity } from 'src/entities/smart-info.entity'; import { AssetBuilderOptions, AssetCreate, + AssetDeltaSyncOptions, AssetExploreFieldOptions, + AssetFullSyncOptions, AssetPathEntity, AssetStats, AssetStatsOptions, @@ -39,6 +42,7 @@ import { FindOptionsWhere, In, IsNull, + MoreThan, Not, Repository, } from 'typeorm'; @@ -61,6 +65,8 @@ export class AssetRepository implements IAssetRepository { @InjectRepository(ExifEntity) private exifRepository: Repository, @InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository, @InjectRepository(SmartInfoEntity) private smartInfoRepository: Repository, + @InjectRepository(PartnerEntity) private partnerRepository: Repository, + @InjectRepository(AlbumEntity) private albumRepository: Repository, ) {} async upsertExif(exif: Partial): Promise { @@ -781,4 +787,55 @@ export class AssetRepository implements IAssetRepository { }) as AssetEntity, ); } + + @GenerateSql({ + params: [ + { + ownerId: DummyValue.UUID, + lastCreationDate: DummyValue.DATE, + lastId: DummyValue.STRING, + updatedUntil: DummyValue.DATE, + limit: 10, + }, + ], + }) + getAllForUserFullSync(options: AssetFullSyncOptions): Promise { + const { ownerId, lastCreationDate, lastId, updatedUntil, limit } = options; + let builder = this.repository + .createQueryBuilder('asset') + .leftJoinAndSelect('asset.exifInfo', 'exifInfo') + .leftJoinAndSelect('asset.stack', 'stack') + .where('asset.ownerId = :ownerId', { ownerId }); + if (lastCreationDate !== undefined && lastId !== undefined) { + builder = builder.andWhere('(asset.fileCreatedAt, asset.id) < (:lastCreationDate, :lastId)', { + lastCreationDate, + lastId, + }); + } + return builder + .andWhere('asset.updatedAt <= :updatedUntil', { updatedUntil }) + .andWhere('asset.isVisible = true') + .orderBy('asset.fileCreatedAt', 'DESC') + .addOrderBy('asset.id', 'DESC') + .limit(limit) + .withDeleted() + .getMany(); + } + + @GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE }] }) + getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise { + return this.repository.find({ + where: { + ownerId: In(options.userIds), + isVisible: true, + updatedAt: MoreThan(options.updatedAfter), + }, + relations: { + exifInfo: true, + stack: true, + }, + take: options.limit, + withDeleted: true, + }); + } } diff --git a/server/src/repositories/audit.repository.ts b/server/src/repositories/audit.repository.ts index 50f5631f3a..3332b06fe1 100644 --- a/server/src/repositories/audit.repository.ts +++ b/server/src/repositories/audit.repository.ts @@ -2,24 +2,25 @@ import { InjectRepository } from '@nestjs/typeorm'; import { AuditEntity } from 'src/entities/audit.entity'; import { AuditSearch, IAuditRepository } from 'src/interfaces/audit.interface'; import { Instrumentation } from 'src/utils/instrumentation'; -import { LessThan, MoreThan, Repository } from 'typeorm'; +import { In, LessThan, MoreThan, Repository } from 'typeorm'; @Instrumentation() export class AuditRepository implements IAuditRepository { constructor(@InjectRepository(AuditEntity) private repository: Repository) {} - getAfter(since: Date, options: AuditSearch): Promise { + getAfter(since: Date, options: AuditSearch): Promise { return this.repository .createQueryBuilder('audit') .where({ createdAt: MoreThan(since), action: options.action, entityType: options.entityType, - ownerId: options.ownerId, + ownerId: In(options.userIds), }) .distinctOn(['audit.entityId', 'audit.entityType']) .orderBy('audit.entityId, audit.entityType, audit.createdAt', 'DESC') - .getMany(); + .select('audit.entityId') + .getRawMany(); } async removeBefore(before: Date): Promise { diff --git a/server/src/services/audit.service.spec.ts b/server/src/services/audit.service.spec.ts index 4af5c1f94d..8d6a3ea2e8 100644 --- a/server/src/services/audit.service.spec.ts +++ b/server/src/services/audit.service.spec.ts @@ -61,13 +61,13 @@ describe(AuditService.name, () => { expect(auditMock.getAfter).toHaveBeenCalledWith(date, { action: DatabaseAction.DELETE, - ownerId: authStub.admin.user.id, + userIds: [authStub.admin.user.id], entityType: EntityType.ASSET, }); }); it('should get any new or updated assets and deleted ids', async () => { - auditMock.getAfter.mockResolvedValue([auditStub.delete]); + auditMock.getAfter.mockResolvedValue([auditStub.delete.entityId]); const date = new Date(); await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({ @@ -77,7 +77,7 @@ describe(AuditService.name, () => { expect(auditMock.getAfter).toHaveBeenCalledWith(date, { action: DatabaseAction.DELETE, - ownerId: authStub.admin.user.id, + userIds: [authStub.admin.user.id], entityType: EntityType.ASSET, }); }); diff --git a/server/src/services/audit.service.ts b/server/src/services/audit.service.ts index d40167429f..b15ee9240a 100644 --- a/server/src/services/audit.service.ts +++ b/server/src/services/audit.service.ts @@ -53,7 +53,7 @@ export class AuditService { await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId); const audits = await this.repository.getAfter(dto.after, { - ownerId: userId, + userIds: [userId], entityType: dto.entityType, action: DatabaseAction.DELETE, }); @@ -62,7 +62,7 @@ export class AuditService { return { needsFullSync: duration > AUDIT_LOG_MAX_DURATION, - ids: audits.map(({ entityId }) => entityId), + ids: audits, }; } diff --git a/server/src/services/index.ts b/server/src/services/index.ts index 3c903c927d..6c40f8420a 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -22,6 +22,7 @@ import { SharedLinkService } from 'src/services/shared-link.service'; import { SmartInfoService } from 'src/services/smart-info.service'; import { StorageTemplateService } from 'src/services/storage-template.service'; import { StorageService } from 'src/services/storage.service'; +import { SyncService } from 'src/services/sync.service'; import { SystemConfigService } from 'src/services/system-config.service'; import { TagService } from 'src/services/tag.service'; import { TimelineService } from 'src/services/timeline.service'; @@ -53,6 +54,7 @@ export const services = [ SmartInfoService, StorageService, StorageTemplateService, + SyncService, SystemConfigService, TagService, TimelineService, diff --git a/server/src/services/sync.service.spec.ts b/server/src/services/sync.service.spec.ts new file mode 100644 index 0000000000..35ef4a6302 --- /dev/null +++ b/server/src/services/sync.service.spec.ts @@ -0,0 +1,101 @@ +import { mapAsset } from 'src/dtos/asset-response.dto'; +import { AssetEntity } from 'src/entities/asset.entity'; +import { IAccessRepository } from 'src/interfaces/access.interface'; +import { IAssetRepository } from 'src/interfaces/asset.interface'; +import { IAuditRepository } from 'src/interfaces/audit.interface'; +import { IPartnerRepository } from 'src/interfaces/partner.interface'; +import { SyncService } from 'src/services/sync.service'; +import { assetStub } from 'test/fixtures/asset.stub'; +import { authStub } from 'test/fixtures/auth.stub'; +import { partnerStub } from 'test/fixtures/partner.stub'; +import { newAccessRepositoryMock } from 'test/repositories/access.repository.mock'; +import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock'; +import { newAuditRepositoryMock } from 'test/repositories/audit.repository.mock'; +import { newPartnerRepositoryMock } from 'test/repositories/partner.repository.mock'; + +const untilDate = new Date(2024); +const mapAssetOpts = { auth: authStub.user1, stripMetadata: false, withStack: true }; + +describe(SyncService.name, () => { + let sut: SyncService; + let accessMock: jest.Mocked; + let assetMock: jest.Mocked; + let partnerMock: jest.Mocked; + let auditMock: jest.Mocked; + + beforeEach(() => { + partnerMock = newPartnerRepositoryMock(); + assetMock = newAssetRepositoryMock(); + accessMock = newAccessRepositoryMock(); + auditMock = newAuditRepositoryMock(); + sut = new SyncService(accessMock, assetMock, partnerMock, auditMock); + }); + + it('should exist', () => { + expect(sut).toBeDefined(); + }); + + describe('getAllAssetsForUserFullSync', () => { + it('should return a list of all assets owned by the user', async () => { + assetMock.getAllForUserFullSync.mockResolvedValue([assetStub.external, assetStub.hasEncodedVideo]); + await expect( + sut.getAllAssetsForUserFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate }), + ).resolves.toEqual([ + mapAsset(assetStub.external, mapAssetOpts), + mapAsset(assetStub.hasEncodedVideo, mapAssetOpts), + ]); + expect(assetMock.getAllForUserFullSync).toHaveBeenCalledWith({ + ownerId: authStub.user1.user.id, + updatedUntil: untilDate, + limit: 2, + }); + }); + }); + + describe('getChangesForDeltaSync', () => { + it('should return a response requiring a full sync when partners are out of sync', async () => { + partnerMock.getAll.mockResolvedValue([partnerStub.adminToUser1]); + await expect( + sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), + ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] }); + expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0); + expect(auditMock.getAfter).toHaveBeenCalledTimes(0); + }); + + it('should return a response requiring a full sync when last sync was too long ago', async () => { + partnerMock.getAll.mockResolvedValue([]); + await expect( + sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }), + ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] }); + expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0); + expect(auditMock.getAfter).toHaveBeenCalledTimes(0); + }); + + it('should return a response requiring a full sync when there are too many changes', async () => { + partnerMock.getAll.mockResolvedValue([]); + assetMock.getChangedDeltaSync.mockResolvedValue( + Array.from({ length: 10_000 }).fill(assetStub.image), + ); + await expect( + sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), + ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] }); + expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1); + expect(auditMock.getAfter).toHaveBeenCalledTimes(0); + }); + + it('should return a response with changes and deletions', async () => { + partnerMock.getAll.mockResolvedValue([]); + assetMock.getChangedDeltaSync.mockResolvedValue([assetStub.image1]); + auditMock.getAfter.mockResolvedValue([assetStub.external.id]); + await expect( + sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), + ).resolves.toEqual({ + needsFullSync: false, + upserted: [mapAsset(assetStub.image1, mapAssetOpts)], + deleted: [assetStub.external.id], + }); + expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1); + expect(auditMock.getAfter).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts new file mode 100644 index 0000000000..be11d36fa0 --- /dev/null +++ b/server/src/services/sync.service.ts @@ -0,0 +1,77 @@ +import { Inject } from '@nestjs/common'; +import _ from 'lodash'; +import { DateTime } from 'luxon'; +import { AUDIT_LOG_MAX_DURATION } from 'src/constants'; +import { AccessCore, Permission } from 'src/cores/access.core'; +import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto'; +import { DatabaseAction, EntityType } from 'src/entities/audit.entity'; +import { IAccessRepository } from 'src/interfaces/access.interface'; +import { IAssetRepository } from 'src/interfaces/asset.interface'; +import { IAuditRepository } from 'src/interfaces/audit.interface'; +import { IPartnerRepository } from 'src/interfaces/partner.interface'; + +export class SyncService { + private access: AccessCore; + + constructor( + @Inject(IAccessRepository) accessRepository: IAccessRepository, + @Inject(IAssetRepository) private assetRepository: IAssetRepository, + @Inject(IPartnerRepository) private partnerRepository: IPartnerRepository, + @Inject(IAuditRepository) private auditRepository: IAuditRepository, + ) { + this.access = AccessCore.create(accessRepository); + } + + async getAllAssetsForUserFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise { + const userId = dto.userId || auth.user.id; + await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId); + const assets = await this.assetRepository.getAllForUserFullSync({ + ownerId: userId, + lastCreationDate: dto.lastCreationDate, + updatedUntil: dto.updatedUntil, + lastId: dto.lastId, + limit: dto.limit, + }); + const options = { auth, stripMetadata: false, withStack: true }; + return assets.map((a) => mapAsset(a, options)); + } + + async getChangesForDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise { + await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds); + const partner = await this.partnerRepository.getAll(auth.user.id); + const userIds = [auth.user.id, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)]; + userIds.sort(); + dto.userIds.sort(); + const duration = DateTime.now().diff(DateTime.fromJSDate(dto.updatedAfter)); + + if (!_.isEqual(userIds, dto.userIds) || duration > AUDIT_LOG_MAX_DURATION) { + // app does not have the correct partners synced + // or app has not synced in the last 100 days + return { needsFullSync: true, deleted: [], upserted: [] }; + } + + const limit = 10_000; + const upserted = await this.assetRepository.getChangedDeltaSync({ limit, updatedAfter: dto.updatedAfter, userIds }); + + if (upserted.length === limit) { + // too many changes -> do a full sync (paginated) instead + return { needsFullSync: true, deleted: [], upserted: [] }; + } + + const deleted = await this.auditRepository.getAfter(dto.updatedAfter, { + userIds: userIds, + entityType: EntityType.ASSET, + action: DatabaseAction.DELETE, + }); + + const options = { auth, stripMetadata: false, withStack: true }; + const result = { + needsFullSync: false, + upserted: upserted.map((a) => mapAsset(a, options)), + deleted, + }; + return result; + } +} diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index 4133880d6f..0bd6d04e07 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -35,5 +35,7 @@ export const newAssetRepositoryMock = (): jest.Mocked => { getAssetIdByCity: jest.fn(), getAssetIdByTag: jest.fn(), searchMetadata: jest.fn(), + getAllForUserFullSync: jest.fn(), + getChangedDeltaSync: jest.fn(), }; };