You've already forked immich
mirror of
https://github.com/immich-app/immich.git
synced 2025-08-09 23:17:29 +02:00
feat(mobile): assets + exif stream sync placeholder (#17677)
* feat(mobile): assets + exif stream sync placeholder * feat(mobile): assets + exif stream sync placeholder * refactor * fix: test * fix:test * refactor(mobile): sync stream service (#17687) * refactor: sync stream to use callbacks * pr feedback * pr feedback * pr feedback * fix: test --------- Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com> --------- Co-authored-by: shenlong <139912620+shenlong-tanwen@users.noreply.github.com> Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com>
This commit is contained in:
Alex
GitHub
@@ -2,25 +2,11 @@
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart';
|
||||
import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart';
|
||||
import 'package:immich_mobile/domain/models/sync_event.model.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:openapi/api.dart';
|
||||
import 'package:worker_manager/worker_manager.dart';
|
||||
|
||||
const _kSyncTypeOrder = [
|
||||
SyncEntityType.userDeleteV1,
|
||||
SyncEntityType.userV1,
|
||||
SyncEntityType.partnerDeleteV1,
|
||||
SyncEntityType.partnerV1,
|
||||
SyncEntityType.assetDeleteV1,
|
||||
SyncEntityType.assetV1,
|
||||
SyncEntityType.assetExifV1,
|
||||
SyncEntityType.partnerAssetDeleteV1,
|
||||
SyncEntityType.partnerAssetV1,
|
||||
SyncEntityType.partnerAssetExifV1,
|
||||
];
|
||||
|
||||
class SyncStreamService {
|
||||
final Logger _logger = Logger('SyncStreamService');
|
||||
@@ -37,164 +23,70 @@ class SyncStreamService {
|
||||
_syncStreamRepository = syncStreamRepository,
|
||||
_cancelChecker = cancelChecker;
|
||||
|
||||
Future<bool> _handleSyncData(
|
||||
bool get isCancelled => _cancelChecker?.call() ?? false;
|
||||
|
||||
Future<void> sync() => _syncApiRepository.streamChanges(_handleEvents);
|
||||
|
||||
Future<void> _handleEvents(List<SyncEvent> events, Function() abort) async {
|
||||
List<SyncEvent> items = [];
|
||||
for (final event in events) {
|
||||
if (isCancelled) {
|
||||
_logger.warning("Sync stream cancelled");
|
||||
abort();
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.type != items.firstOrNull?.type) {
|
||||
await _processBatch(items);
|
||||
}
|
||||
|
||||
items.add(event);
|
||||
}
|
||||
|
||||
await _processBatch(items);
|
||||
}
|
||||
|
||||
Future<void> _processBatch(List<SyncEvent> batch) async {
|
||||
if (batch.isEmpty) {
|
||||
return;
|
||||
}
|
||||
|
||||
final type = batch.first.type;
|
||||
await _handleSyncData(type, batch.map((e) => e.data));
|
||||
await _syncApiRepository.ack([batch.last.ack]);
|
||||
batch.clear();
|
||||
}
|
||||
|
||||
Future<void> _handleSyncData(
|
||||
SyncEntityType type,
|
||||
// ignore: avoid-dynamic
|
||||
Iterable<dynamic> data,
|
||||
) async {
|
||||
if (data.isEmpty) {
|
||||
_logger.warning("Received empty sync data for $type");
|
||||
return false;
|
||||
}
|
||||
|
||||
_logger.fine("Processing sync data for $type of length ${data.length}");
|
||||
|
||||
try {
|
||||
if (type == SyncEntityType.partnerV1) {
|
||||
return await _syncStreamRepository.updatePartnerV1(data.cast());
|
||||
}
|
||||
|
||||
if (type == SyncEntityType.partnerDeleteV1) {
|
||||
return await _syncStreamRepository.deletePartnerV1(data.cast());
|
||||
}
|
||||
|
||||
if (type == SyncEntityType.userV1) {
|
||||
return await _syncStreamRepository.updateUsersV1(data.cast());
|
||||
}
|
||||
|
||||
if (type == SyncEntityType.userDeleteV1) {
|
||||
return await _syncStreamRepository.deleteUsersV1(data.cast());
|
||||
}
|
||||
} catch (error, stack) {
|
||||
_logger.severe("Error processing sync data for $type", error, stack);
|
||||
return false;
|
||||
// ignore: prefer-switch-expression
|
||||
switch (type) {
|
||||
case SyncEntityType.userV1:
|
||||
return _syncStreamRepository.updateUsersV1(data.cast());
|
||||
case SyncEntityType.userDeleteV1:
|
||||
return _syncStreamRepository.deleteUsersV1(data.cast());
|
||||
case SyncEntityType.partnerV1:
|
||||
return _syncStreamRepository.updatePartnerV1(data.cast());
|
||||
case SyncEntityType.partnerDeleteV1:
|
||||
return _syncStreamRepository.deletePartnerV1(data.cast());
|
||||
case SyncEntityType.assetV1:
|
||||
return _syncStreamRepository.updateAssetsV1(data.cast());
|
||||
case SyncEntityType.assetDeleteV1:
|
||||
return _syncStreamRepository.deleteAssetsV1(data.cast());
|
||||
case SyncEntityType.assetExifV1:
|
||||
return _syncStreamRepository.updateAssetsExifV1(data.cast());
|
||||
case SyncEntityType.partnerAssetV1:
|
||||
return _syncStreamRepository.updatePartnerAssetsV1(data.cast());
|
||||
case SyncEntityType.partnerAssetDeleteV1:
|
||||
return _syncStreamRepository.deletePartnerAssetsV1(data.cast());
|
||||
case SyncEntityType.partnerAssetExifV1:
|
||||
return _syncStreamRepository.updatePartnerAssetsExifV1(data.cast());
|
||||
default:
|
||||
_logger.warning("Unknown sync data type: $type");
|
||||
}
|
||||
|
||||
_logger.warning("Unknown sync data type: $type");
|
||||
return false;
|
||||
}
|
||||
|
||||
Future<void> _syncEvent(List<SyncRequestType> types) {
|
||||
_logger.info("Syncing Events: $types");
|
||||
final streamCompleter = Completer();
|
||||
bool shouldComplete = false;
|
||||
// the onDone callback might fire before the events are processed
|
||||
// the following flag ensures that the onDone callback is not called
|
||||
// before the events are processed and also that events are processed sequentially
|
||||
Completer? mutex;
|
||||
StreamSubscription? subscription;
|
||||
try {
|
||||
subscription = _syncApiRepository.getSyncEvents(types).listen(
|
||||
(events) async {
|
||||
if (events.isEmpty) {
|
||||
_logger.warning("Received empty sync events");
|
||||
return;
|
||||
}
|
||||
|
||||
// If previous events are still being processed, wait for them to finish
|
||||
if (mutex != null) {
|
||||
await mutex!.future;
|
||||
}
|
||||
|
||||
if (_cancelChecker?.call() ?? false) {
|
||||
_logger.info("Sync cancelled, stopping stream");
|
||||
subscription?.cancel();
|
||||
if (!streamCompleter.isCompleted) {
|
||||
streamCompleter.completeError(
|
||||
CanceledError(),
|
||||
StackTrace.current,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Take control of the mutex and process the events
|
||||
mutex = Completer();
|
||||
|
||||
try {
|
||||
final eventsMap = events.groupListsBy((event) => event.type);
|
||||
final Map<SyncEntityType, String> acks = {};
|
||||
|
||||
for (final type in _kSyncTypeOrder) {
|
||||
final data = eventsMap[type];
|
||||
if (data == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (_cancelChecker?.call() ?? false) {
|
||||
_logger.info("Sync cancelled, stopping stream");
|
||||
mutex?.complete();
|
||||
mutex = null;
|
||||
if (!streamCompleter.isCompleted) {
|
||||
streamCompleter.completeError(
|
||||
CanceledError(),
|
||||
StackTrace.current,
|
||||
);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.isEmpty) {
|
||||
_logger.warning("Received empty sync events for $type");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (await _handleSyncData(type, data.map((e) => e.data))) {
|
||||
// ignore: avoid-unsafe-collection-methods
|
||||
acks[type] = data.last.ack;
|
||||
} else {
|
||||
_logger.warning("Failed to handle sync events for $type");
|
||||
}
|
||||
}
|
||||
|
||||
if (acks.isNotEmpty) {
|
||||
await _syncApiRepository.ack(acks.values.toList());
|
||||
}
|
||||
_logger.info("$types events processed");
|
||||
} catch (error, stack) {
|
||||
_logger.warning("Error handling sync events", error, stack);
|
||||
} finally {
|
||||
mutex?.complete();
|
||||
mutex = null;
|
||||
}
|
||||
|
||||
if (shouldComplete) {
|
||||
_logger.info("Sync done, completing stream");
|
||||
if (!streamCompleter.isCompleted) streamCompleter.complete();
|
||||
}
|
||||
},
|
||||
onError: (error, stack) {
|
||||
_logger.warning("Error in sync stream for $types", error, stack);
|
||||
// Do not proceed if the stream errors
|
||||
if (!streamCompleter.isCompleted) {
|
||||
// ignore: avoid-missing-completer-stack-trace
|
||||
streamCompleter.completeError(error, stack);
|
||||
}
|
||||
},
|
||||
onDone: () {
|
||||
_logger.info("$types stream done");
|
||||
if (mutex == null && !streamCompleter.isCompleted) {
|
||||
streamCompleter.complete();
|
||||
} else {
|
||||
// Marks the stream as done but does not complete the completer
|
||||
// until the events are processed
|
||||
shouldComplete = true;
|
||||
}
|
||||
},
|
||||
);
|
||||
} catch (error, stack) {
|
||||
_logger.severe("Error starting sync stream", error, stack);
|
||||
if (!streamCompleter.isCompleted) {
|
||||
streamCompleter.completeError(error, stack);
|
||||
}
|
||||
}
|
||||
return streamCompleter.future.whenComplete(() {
|
||||
_logger.info("Sync stream completed");
|
||||
return subscription?.cancel();
|
||||
});
|
||||
}
|
||||
|
||||
Future<void> syncUsers() =>
|
||||
_syncEvent([SyncRequestType.usersV1, SyncRequestType.partnersV1]);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user