self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-08-08 23:07:06 +02:00
Code Issues Releases Activity

feat: user sync stream (#16862)

Browse Source 
* refactor: user entity

* chore: rebase fixes

* refactor: remove int user Id

* refactor: migrate store userId from int to string

* refactor: rename uid to id

* feat: drift

* pr feedback

* refactor: move common overrides to mixin

* refactor: remove int user Id

* refactor: migrate store userId from int to string

* refactor: rename uid to id

* feat: user & partner sync stream

* pr changes

* refactor: sync service and add tests

* chore: remove generated change

* chore: move sync model

* rebase: convert string ids to byte uuids

* rebase

* add processing logs

* batch db calls

* rewrite isolate manager

* rewrite with worker_manager

* misc fixes

* add sync order test

---------

Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com>
Co-authored-by: Alex <alex.tran1502@gmail.com>
This commit is contained in:
shenlong
2025-04-17 20:55:27 +05:30
committed by GitHub
parent 067338b0ed
commit 81ed54aa61
28 changed files with 1065 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
mobile/lib/infrastructure/repositories/sync_api.repository.dart
View File

@ -1,37 +1,36 @@
import 'dart:async';
import 'dart:convert';
import 'package:flutter/foundation.dart';
import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart';
import 'package:immich_mobile/domain/models/sync/sync_event.model.dart';
import 'package:immich_mobile/services/api.service.dart';
import 'package:openapi/api.dart';
import 'package:http/http.dart' as http;
import 'package:immich_mobile/constants/constants.dart';
import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart';
import 'package:immich_mobile/domain/models/sync_event.model.dart';
import 'package:immich_mobile/services/api.service.dart';
import 'package:logging/logging.dart';
import 'package:openapi/api.dart';
class SyncApiRepository implements ISyncApiRepository {
final Logger _logger = Logger('SyncApiRepository');
final ApiService _api;
const SyncApiRepository(this._api);
final int _batchSize;
SyncApiRepository(this._api, {int batchSize = kSyncEventBatchSize})
: _batchSize = batchSize;
@override
Stream<List<SyncEvent>> watchUserSyncEvent() {
return _getSyncStream(
SyncStreamDto(types: [SyncRequestType.usersV1]),
);
Stream<List<SyncEvent>> getSyncEvents(List<SyncRequestType> type) {
return _getSyncStream(SyncStreamDto(types: type));
}
@override
Future<void> ack(String data) {
return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: [data]));
Future<void> ack(List<String> data) {
return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: data));
}
Stream<List<SyncEvent>> _getSyncStream(
SyncStreamDto dto, {
int batchSize = 5000,
}) async* {
Stream<List<SyncEvent>> _getSyncStream(SyncStreamDto dto) async* {
final client = http.Client();
final endpoint = "${_api.apiClient.basePath}/sync/stream";
final headers = <String, String>{
final headers = {
'Content-Type': 'application/json',
'Accept': 'application/jsonlines+json',
};
@ -61,52 +60,54 @@ class SyncApiRepository implements ISyncApiRepository {
await for (final chunk in response.stream.transform(utf8.decoder)) {
previousChunk += chunk;
final parts = previousChunk.split('\n');
final parts = previousChunk.toString().split('\n');
previousChunk = parts.removeLast();
lines.addAll(parts);
if (lines.length < batchSize) {
if (lines.length < _batchSize) {
continue;
}
yield await compute(_parseSyncResponse, lines);
yield _parseSyncResponse(lines);
lines.clear();
}
} finally {
if (lines.isNotEmpty) {
yield await compute(_parseSyncResponse, lines);
yield _parseSyncResponse(lines);
}
client.close();
}
}
List<SyncEvent> _parseSyncResponse(List<String> lines) {
final List<SyncEvent> data = [];
for (final line in lines) {
try {
final jsonData = jsonDecode(line);
final type = SyncEntityType.fromJson(jsonData['type'])!;
final dataJson = jsonData['data'];
final ack = jsonData['ack'];
final converter = _kResponseMap[type];
if (converter == null) {
_logger.warning("[_parseSyncResponse] Unknown type $type");
continue;
}
data.add(SyncEvent(type: type, data: converter(dataJson), ack: ack));
} catch (error, stack) {
_logger.severe("[_parseSyncResponse] Error parsing json", error, stack);
}
}
return data;
}
}
// ignore: avoid-dynamic
const _kResponseMap = <SyncEntityType, Function(dynamic)>{
SyncEntityType.userV1: SyncUserV1.fromJson,
SyncEntityType.userDeleteV1: SyncUserDeleteV1.fromJson,
SyncEntityType.partnerV1: SyncPartnerV1.fromJson,
SyncEntityType.partnerDeleteV1: SyncPartnerDeleteV1.fromJson,
};
// Need to be outside of the class to be able to use compute
List<SyncEvent> _parseSyncResponse(List<String> lines) {
final List<SyncEvent> data = [];
for (var line in lines) {
try {
final jsonData = jsonDecode(line);
final type = SyncEntityType.fromJson(jsonData['type'])!;
final dataJson = jsonData['data'];
final ack = jsonData['ack'];
final converter = _kResponseMap[type];
if (converter == null) {
debugPrint("[_parseSyncReponse] Unknown type $type");
continue;
}
data.add(SyncEvent(data: converter(dataJson), ack: ack));
} catch (error, stack) {
debugPrint("[_parseSyncReponse] Error parsing json $error $stack");
}
}
return data;
}