feat(server): efficient full app sync (#8755)

* feat(server): efficient full app sync

* add SQL, fix test compile issues

* fix linter warning

* new sync controller+service, add tests

* enable new sync controller+service

* Update server/src/services/sync.service.ts

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

mobile/openapi/lib/api.dart

@@ -46,6 +46,7 @@ part 'api/person_api.dart';
 part 'api/search_api.dart';
 part 'api/server_info_api.dart';
 part 'api/shared_link_api.dart';
+part 'api/sync_api.dart';
 part 'api/system_config_api.dart';
 part 'api/tag_api.dart';
 part 'api/timeline_api.dart';
@@ -69,6 +70,7 @@ part 'model/asset_bulk_upload_check_dto.dart';
 part 'model/asset_bulk_upload_check_item.dart';
 part 'model/asset_bulk_upload_check_response_dto.dart';
 part 'model/asset_bulk_upload_check_result.dart';
+part 'model/asset_delta_sync_response_dto.dart';
 part 'model/asset_face_response_dto.dart';
 part 'model/asset_face_update_dto.dart';
 part 'model/asset_face_update_item.dart';

mobile/openapi/lib/api/sync_api.dart

@@ -0,0 +1,150 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+
+class SyncApi {
+  SyncApi([ApiClient? apiClient]) : apiClient = apiClient ?? defaultApiClient;
+
+  final ApiClient apiClient;
+
+  /// Performs an HTTP 'GET /sync/full-sync' operation and returns the [Response].
+  /// Parameters:
+  ///
+  /// * [int] limit (required):
+  ///
+  /// * [DateTime] updatedUntil (required):
+  ///
+  /// * [DateTime] lastCreationDate:
+  ///
+  /// * [String] lastId:
+  ///
+  /// * [String] userId:
+  Future<Response> getAllForUserFullSyncWithHttpInfo(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async {
+    // ignore: prefer_const_declarations
+    final path = r'/sync/full-sync';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+    if (lastCreationDate != null) {
+      queryParams.addAll(_queryParams('', 'lastCreationDate', lastCreationDate));
+    }
+    if (lastId != null) {
+      queryParams.addAll(_queryParams('', 'lastId', lastId));
+    }
+      queryParams.addAll(_queryParams('', 'limit', limit));
+      queryParams.addAll(_queryParams('', 'updatedUntil', updatedUntil));
+    if (userId != null) {
+      queryParams.addAll(_queryParams('', 'userId', userId));
+    }
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      path,
+      'GET',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  /// Parameters:
+  ///
+  /// * [int] limit (required):
+  ///
+  /// * [DateTime] updatedUntil (required):
+  ///
+  /// * [DateTime] lastCreationDate:
+  ///
+  /// * [String] lastId:
+  ///
+  /// * [String] userId:
+  Future<List<AssetResponseDto>?> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async {
+    final response = await getAllForUserFullSyncWithHttpInfo(limit, updatedUntil,  lastCreationDate: lastCreationDate, lastId: lastId, userId: userId, );
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+    // When a remote server returns no body with a status of 204, we shall not decode it.
+    // At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
+    // FormatException when trying to decode an empty string.
+    if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
+      final responseBody = await _decodeBodyBytes(response);
+      return (await apiClient.deserializeAsync(responseBody, 'List<AssetResponseDto>') as List)
+        .cast<AssetResponseDto>()
+        .toList(growable: false);
+
+    }
+    return null;
+  }
+
+  /// Performs an HTTP 'GET /sync/delta-sync' operation and returns the [Response].
+  /// Parameters:
+  ///
+  /// * [DateTime] updatedAfter (required):
+  ///
+  /// * [List<String>] userIds (required):
+  Future<Response> getDeltaSyncWithHttpInfo(DateTime updatedAfter, List<String> userIds,) async {
+    // ignore: prefer_const_declarations
+    final path = r'/sync/delta-sync';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+      queryParams.addAll(_queryParams('', 'updatedAfter', updatedAfter));
+      queryParams.addAll(_queryParams('multi', 'userIds', userIds));
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      path,
+      'GET',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  /// Parameters:
+  ///
+  /// * [DateTime] updatedAfter (required):
+  ///
+  /// * [List<String>] userIds (required):
+  Future<AssetDeltaSyncResponseDto?> getDeltaSync(DateTime updatedAfter, List<String> userIds,) async {
+    final response = await getDeltaSyncWithHttpInfo(updatedAfter, userIds,);
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+    // When a remote server returns no body with a status of 204, we shall not decode it.
+    // At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
+    // FormatException when trying to decode an empty string.
+    if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
+      return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'AssetDeltaSyncResponseDto',) as AssetDeltaSyncResponseDto;
+    
+    }
+    return null;
+  }
+}

mobile/openapi/lib/api_client.dart

@@ -216,6 +216,8 @@ class ApiClient {
           return AssetBulkUploadCheckResponseDto.fromJson(value);
         case 'AssetBulkUploadCheckResult':
           return AssetBulkUploadCheckResult.fromJson(value);
+        case 'AssetDeltaSyncResponseDto':
+          return AssetDeltaSyncResponseDto.fromJson(value);
         case 'AssetFaceResponseDto':
           return AssetFaceResponseDto.fromJson(value);
         case 'AssetFaceUpdateDto':

mobile/openapi/lib/model/asset_delta_sync_response_dto.dart

@@ -0,0 +1,116 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class AssetDeltaSyncResponseDto {
+  /// Returns a new [AssetDeltaSyncResponseDto] instance.
+  AssetDeltaSyncResponseDto({
+    this.deleted = const [],
+    required this.needsFullSync,
+    this.upserted = const [],
+  });
+
+  List<String> deleted;
+
+  bool needsFullSync;
+
+  List<AssetResponseDto> upserted;
+
+  @override
+  bool operator ==(Object other) => identical(this, other) || other is AssetDeltaSyncResponseDto &&
+    _deepEquality.equals(other.deleted, deleted) &&
+    other.needsFullSync == needsFullSync &&
+    _deepEquality.equals(other.upserted, upserted);
+
+  @override
+  int get hashCode =>
+    // ignore: unnecessary_parenthesis
+    (deleted.hashCode) +
+    (needsFullSync.hashCode) +
+    (upserted.hashCode);
+
+  @override
+  String toString() => 'AssetDeltaSyncResponseDto[deleted=$deleted, needsFullSync=$needsFullSync, upserted=$upserted]';
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+      json[r'deleted'] = this.deleted;
+      json[r'needsFullSync'] = this.needsFullSync;
+      json[r'upserted'] = this.upserted;
+    return json;
+  }
+
+  /// Returns a new [AssetDeltaSyncResponseDto] instance and imports its values from
+  /// [value] if it's a [Map], null otherwise.
+  // ignore: prefer_constructors_over_static_methods
+  static AssetDeltaSyncResponseDto? fromJson(dynamic value) {
+    if (value is Map) {
+      final json = value.cast<String, dynamic>();
+
+      return AssetDeltaSyncResponseDto(
+        deleted: json[r'deleted'] is Iterable
+            ? (json[r'deleted'] as Iterable).cast<String>().toList(growable: false)
+            : const [],
+        needsFullSync: mapValueOfType<bool>(json, r'needsFullSync')!,
+        upserted: AssetResponseDto.listFromJson(json[r'upserted']),
+      );
+    }
+    return null;
+  }
+
+  static List<AssetDeltaSyncResponseDto> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <AssetDeltaSyncResponseDto>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = AssetDeltaSyncResponseDto.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+
+  static Map<String, AssetDeltaSyncResponseDto> mapFromJson(dynamic json) {
+    final map = <String, AssetDeltaSyncResponseDto>{};
+    if (json is Map && json.isNotEmpty) {
+      json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+      for (final entry in json.entries) {
+        final value = AssetDeltaSyncResponseDto.fromJson(entry.value);
+        if (value != null) {
+          map[entry.key] = value;
+        }
+      }
+    }
+    return map;
+  }
+
+  // maps a json object with a list of AssetDeltaSyncResponseDto-objects as value to a dart map
+  static Map<String, List<AssetDeltaSyncResponseDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
+    final map = <String, List<AssetDeltaSyncResponseDto>>{};
+    if (json is Map && json.isNotEmpty) {
+      // ignore: parameter_assignments
+      json = json.cast<String, dynamic>();
+      for (final entry in json.entries) {
+        map[entry.key] = AssetDeltaSyncResponseDto.listFromJson(entry.value, growable: growable,);
+      }
+    }
+    return map;
+  }
+
+  /// The list of required keys that must be present in a JSON.
+  static const requiredKeys = <String>{
+    'deleted',
+    'needsFullSync',
+    'upserted',
+  };
+}
+