self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-10-31 00:18:28 +02:00
Code Activity

feat(mobile): efficient asset sync (#3945)

Browse Source 
* feat(mobile): efficient asset sync
This commit is contained in:
Fynn Petersen-Frey
2023-09-10 14:51:18 +02:00
committed by GitHub
parent 4b11e925d9
commit 5d1011b482
15 changed files with 379 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
mobile/lib/modules/partner/views/partner_detail_page.dart
View File

@@ -1,4 +1,5 @@
import 'package:flutter/material.dart';
import 'package:flutter_hooks/flutter_hooks.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/modules/home/ui/asset_grid/immich_asset_grid.dart';
import 'package:immich_mobile/shared/models/user.dart';
@@ -14,6 +15,14 @@ class PartnerDetailPage extends HookConsumerWidget {
Widget build(BuildContext context, WidgetRef ref) {
final assets = ref.watch(assetsProvider(partner.isarId));
useEffect(
() {
ref.read(assetProvider.notifier).getPartnerAssets(partner);
return null;
},
[],
);
return Scaffold(
appBar: AppBar(
title: Text("${partner.firstName} ${partner.lastName}"),
@@ -30,7 +39,8 @@ class PartnerDetailPage extends HookConsumerWidget {
)
: ImmichAssetGrid(
renderList: renderList,
onRefresh: () => ref.read(assetProvider.notifier).getAllAsset(),
onRefresh: () =>
ref.read(assetProvider.notifier).getPartnerAssets(partner),
),
error: (e, _) => Text("Error loading partners:\n$e"),
loading: () => const Center(child: ImmichLoadingIndicator()),

1
mobile/lib/routing/router.gr.dart
View File

@@ -1351,6 +1351,7 @@ class MemoryRouteArgs {
}
}
/// generated route for
/// [MapPage]
class MapRoute extends PageRouteInfo<void> {
const MapRoute()

3
mobile/lib/routing/tab_navigation_observer.dart
View File

@@ -10,6 +10,7 @@ import 'package:immich_mobile/modules/album/providers/shared_album.provider.dart
import 'package:immich_mobile/shared/models/store.dart';
import 'package:immich_mobile/shared/models/user.dart';
import 'package:immich_mobile/shared/providers/api.provider.dart';
import 'package:immich_mobile/shared/providers/asset.provider.dart';
import 'package:immich_mobile/shared/providers/server_info.provider.dart';
class TabNavigationObserver extends AutoRouterObserver {
@@ -42,6 +43,7 @@ class TabNavigationObserver extends AutoRouterObserver {
if (route.name == 'SharingRoute') {
ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums();
ref.read(assetProvider.notifier).getPartnerAssets();
}
if (route.name == 'LibraryRoute') {
@@ -50,6 +52,7 @@ class TabNavigationObserver extends AutoRouterObserver {
if (route.name == 'HomeRoute') {
ref.invalidate(memoryFutureProvider);
Future(() => ref.read(assetProvider.notifier).getAllAsset());
// Update user info
try {

12
mobile/lib/shared/models/asset.dart
View File

@@ -417,17 +417,17 @@ enum AssetState {
extension AssetsHelper on IsarCollection<Asset> {
Future<int> deleteAllByRemoteId(Iterable<String> ids) =>
ids.isEmpty ? Future.value(0) : _remote(ids).deleteAll();
ids.isEmpty ? Future.value(0) : remote(ids).deleteAll();
Future<int> deleteAllByLocalId(Iterable<String> ids) =>
ids.isEmpty ? Future.value(0) : _local(ids).deleteAll();
ids.isEmpty ? Future.value(0) : local(ids).deleteAll();
Future<List<Asset>> getAllByRemoteId(Iterable<String> ids) =>
ids.isEmpty ? Future.value([]) : _remote(ids).findAll();
ids.isEmpty ? Future.value([]) : remote(ids).findAll();
Future<List<Asset>> getAllByLocalId(Iterable<String> ids) =>
ids.isEmpty ? Future.value([]) : _local(ids).findAll();
ids.isEmpty ? Future.value([]) : local(ids).findAll();
QueryBuilder<Asset, Asset, QAfterWhereClause> _remote(Iterable<String> ids) =>
QueryBuilder<Asset, Asset, QAfterWhereClause> remote(Iterable<String> ids) =>
where().anyOf(ids, (q, String e) => q.remoteIdEqualTo(e));
QueryBuilder<Asset, Asset, QAfterWhereClause> _local(Iterable<String> ids) {
QueryBuilder<Asset, Asset, QAfterWhereClause> local(Iterable<String> ids) {
return where().anyOf(ids, (q, String e) => q.localIdEqualTo(e));
}
}

5
mobile/lib/shared/models/etag.dart
View File

@@ -5,9 +5,10 @@ part 'etag.g.dart';
@Collection(inheritance: false)
class ETag {
ETag({required this.id, this.value});
ETag({required this.id, this.assetCount, this.time});
Id get isarId => fastHash(id);
@Index(unique: true, replace: true, type: IndexType.hash)
String id;
String? value;
int? assetCount;
DateTime? time;
}

275
mobile/lib/shared/models/etag.g.dart
View File

@@ -17,15 +17,20 @@ const ETagSchema = CollectionSchema(
name: r'ETag',
id: -644290296585643859,
properties: {
r'id': PropertySchema(
r'assetCount': PropertySchema(
id: 0,
name: r'assetCount',
type: IsarType.long,
),
r'id': PropertySchema(
id: 1,
name: r'id',
type: IsarType.string,
),
r'value': PropertySchema(
id: 1,
name: r'value',
type: IsarType.string,
r'time': PropertySchema(
id: 2,
name: r'time',
type: IsarType.dateTime,
)
},
estimateSize: _eTagEstimateSize,
@@ -63,12 +68,6 @@ int _eTagEstimateSize(
) {
var bytesCount = offsets.last;
bytesCount += 3 + object.id.length * 3;
{
final value = object.value;
if (value != null) {
bytesCount += 3 + value.length * 3;
}
}
return bytesCount;
}
@@ -78,8 +77,9 @@ void _eTagSerialize(
List<int> offsets,
Map<Type, List<int>> allOffsets,
) {
writer.writeString(offsets[0], object.id);
writer.writeString(offsets[1], object.value);
writer.writeLong(offsets[0], object.assetCount);
writer.writeString(offsets[1], object.id);
writer.writeDateTime(offsets[2], object.time);
}
ETag _eTagDeserialize(
@@ -89,8 +89,9 @@ ETag _eTagDeserialize(
Map<Type, List<int>> allOffsets,
) {
final object = ETag(
id: reader.readString(offsets[0]),
value: reader.readStringOrNull(offsets[1]),
assetCount: reader.readLongOrNull(offsets[0]),
id: reader.readString(offsets[1]),
time: reader.readDateTimeOrNull(offsets[2]),
);
return object;
}
@@ -103,9 +104,11 @@ P _eTagDeserializeProp<P>(
) {
switch (propertyId) {
case 0:
return (reader.readString(offset)) as P;
return (reader.readLongOrNull(offset)) as P;
case 1:
return (reader.readStringOrNull(offset)) as P;
return (reader.readString(offset)) as P;
case 2:
return (reader.readDateTimeOrNull(offset)) as P;
default:
throw IsarError('Unknown property with id $propertyId');
}
@@ -294,6 +297,75 @@ extension ETagQueryWhere on QueryBuilder<ETag, ETag, QWhereClause> {
}
extension ETagQueryFilter on QueryBuilder<ETag, ETag, QFilterCondition> {
QueryBuilder<ETag, ETag, QAfterFilterCondition> assetCountIsNull() {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(const FilterCondition.isNull(
property: r'assetCount',
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> assetCountIsNotNull() {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(const FilterCondition.isNotNull(
property: r'assetCount',
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> assetCountEqualTo(
int? value) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.equalTo(
property: r'assetCount',
value: value,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> assetCountGreaterThan(
int? value, {
bool include = false,
}) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.greaterThan(
include: include,
property: r'assetCount',
value: value,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> assetCountLessThan(
int? value, {
bool include = false,
}) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.lessThan(
include: include,
property: r'assetCount',
value: value,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> assetCountBetween(
int? lower,
int? upper, {
bool includeLower = true,
bool includeUpper = true,
}) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.between(
property: r'assetCount',
lower: lower,
includeLower: includeLower,
upper: upper,
includeUpper: includeUpper,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> idEqualTo(
String value, {
bool caseSensitive = true,
@@ -474,146 +546,70 @@ extension ETagQueryFilter on QueryBuilder<ETag, ETag, QFilterCondition> {
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueIsNull() {
QueryBuilder<ETag, ETag, QAfterFilterCondition> timeIsNull() {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(const FilterCondition.isNull(
property: r'value',
property: r'time',
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueIsNotNull() {
QueryBuilder<ETag, ETag, QAfterFilterCondition> timeIsNotNull() {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(const FilterCondition.isNotNull(
property: r'value',
property: r'time',
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueEqualTo(
String? value, {
bool caseSensitive = true,
}) {
QueryBuilder<ETag, ETag, QAfterFilterCondition> timeEqualTo(DateTime? value) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.equalTo(
property: r'value',
property: r'time',
value: value,
caseSensitive: caseSensitive,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueGreaterThan(
String? value, {
QueryBuilder<ETag, ETag, QAfterFilterCondition> timeGreaterThan(
DateTime? value, {
bool include = false,
bool caseSensitive = true,
}) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.greaterThan(
include: include,
property: r'value',
property: r'time',
value: value,
caseSensitive: caseSensitive,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueLessThan(
String? value, {
QueryBuilder<ETag, ETag, QAfterFilterCondition> timeLessThan(
DateTime? value, {
bool include = false,
bool caseSensitive = true,
}) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.lessThan(
include: include,
property: r'value',
property: r'time',
value: value,
caseSensitive: caseSensitive,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueBetween(
String? lower,
String? upper, {
QueryBuilder<ETag, ETag, QAfterFilterCondition> timeBetween(
DateTime? lower,
DateTime? upper, {
bool includeLower = true,
bool includeUpper = true,
bool caseSensitive = true,
}) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.between(
property: r'value',
property: r'time',
lower: lower,
includeLower: includeLower,
upper: upper,
includeUpper: includeUpper,
caseSensitive: caseSensitive,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueStartsWith(
String value, {
bool caseSensitive = true,
}) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.startsWith(
property: r'value',
value: value,
caseSensitive: caseSensitive,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueEndsWith(
String value, {
bool caseSensitive = true,
}) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.endsWith(
property: r'value',
value: value,
caseSensitive: caseSensitive,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueContains(String value,
{bool caseSensitive = true}) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.contains(
property: r'value',
value: value,
caseSensitive: caseSensitive,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueMatches(String pattern,
{bool caseSensitive = true}) {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.matches(
property: r'value',
wildcard: pattern,
caseSensitive: caseSensitive,
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueIsEmpty() {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.equalTo(
property: r'value',
value: '',
));
});
}
QueryBuilder<ETag, ETag, QAfterFilterCondition> valueIsNotEmpty() {
return QueryBuilder.apply(this, (query) {
return query.addFilterCondition(FilterCondition.greaterThan(
property: r'value',
value: '',
));
});
}
@@ -624,6 +620,18 @@ extension ETagQueryObject on QueryBuilder<ETag, ETag, QFilterCondition> {}
extension ETagQueryLinks on QueryBuilder<ETag, ETag, QFilterCondition> {}
extension ETagQuerySortBy on QueryBuilder<ETag, ETag, QSortBy> {
QueryBuilder<ETag, ETag, QAfterSortBy> sortByAssetCount() {
return QueryBuilder.apply(this, (query) {
return query.addSortBy(r'assetCount', Sort.asc);
});
}
QueryBuilder<ETag, ETag, QAfterSortBy> sortByAssetCountDesc() {
return QueryBuilder.apply(this, (query) {
return query.addSortBy(r'assetCount', Sort.desc);
});
}
QueryBuilder<ETag, ETag, QAfterSortBy> sortById() {
return QueryBuilder.apply(this, (query) {
return query.addSortBy(r'id', Sort.asc);
@@ -636,20 +644,32 @@ extension ETagQuerySortBy on QueryBuilder<ETag, ETag, QSortBy> {
});
}
QueryBuilder<ETag, ETag, QAfterSortBy> sortByValue() {
QueryBuilder<ETag, ETag, QAfterSortBy> sortByTime() {
return QueryBuilder.apply(this, (query) {
return query.addSortBy(r'value', Sort.asc);
return query.addSortBy(r'time', Sort.asc);
});
}
QueryBuilder<ETag, ETag, QAfterSortBy> sortByValueDesc() {
QueryBuilder<ETag, ETag, QAfterSortBy> sortByTimeDesc() {
return QueryBuilder.apply(this, (query) {
return query.addSortBy(r'value', Sort.desc);
return query.addSortBy(r'time', Sort.desc);
});
}
}
extension ETagQuerySortThenBy on QueryBuilder<ETag, ETag, QSortThenBy> {
QueryBuilder<ETag, ETag, QAfterSortBy> thenByAssetCount() {
return QueryBuilder.apply(this, (query) {
return query.addSortBy(r'assetCount', Sort.asc);
});
}
QueryBuilder<ETag, ETag, QAfterSortBy> thenByAssetCountDesc() {
return QueryBuilder.apply(this, (query) {
return query.addSortBy(r'assetCount', Sort.desc);
});
}
QueryBuilder<ETag, ETag, QAfterSortBy> thenById() {
return QueryBuilder.apply(this, (query) {
return query.addSortBy(r'id', Sort.asc);
@@ -674,20 +694,26 @@ extension ETagQuerySortThenBy on QueryBuilder<ETag, ETag, QSortThenBy> {
});
}
QueryBuilder<ETag, ETag, QAfterSortBy> thenByValue() {
QueryBuilder<ETag, ETag, QAfterSortBy> thenByTime() {
return QueryBuilder.apply(this, (query) {
return query.addSortBy(r'value', Sort.asc);
return query.addSortBy(r'time', Sort.asc);
});
}
QueryBuilder<ETag, ETag, QAfterSortBy> thenByValueDesc() {
QueryBuilder<ETag, ETag, QAfterSortBy> thenByTimeDesc() {
return QueryBuilder.apply(this, (query) {
return query.addSortBy(r'value', Sort.desc);
return query.addSortBy(r'time', Sort.desc);
});
}
}
extension ETagQueryWhereDistinct on QueryBuilder<ETag, ETag, QDistinct> {
QueryBuilder<ETag, ETag, QDistinct> distinctByAssetCount() {
return QueryBuilder.apply(this, (query) {
return query.addDistinctBy(r'assetCount');
});
}
QueryBuilder<ETag, ETag, QDistinct> distinctById(
{bool caseSensitive = true}) {
return QueryBuilder.apply(this, (query) {
@@ -695,10 +721,9 @@ extension ETagQueryWhereDistinct on QueryBuilder<ETag, ETag, QDistinct> {
});
}
QueryBuilder<ETag, ETag, QDistinct> distinctByValue(
{bool caseSensitive = true}) {
QueryBuilder<ETag, ETag, QDistinct> distinctByTime() {
return QueryBuilder.apply(this, (query) {
return query.addDistinctBy(r'value', caseSensitive: caseSensitive);
return query.addDistinctBy(r'time');
});
}
}
@@ -710,15 +735,21 @@ extension ETagQueryProperty on QueryBuilder<ETag, ETag, QQueryProperty> {
});
}
QueryBuilder<ETag, int?, QQueryOperations> assetCountProperty() {
return QueryBuilder.apply(this, (query) {
return query.addPropertyName(r'assetCount');
});
}
QueryBuilder<ETag, String, QQueryOperations> idProperty() {
return QueryBuilder.apply(this, (query) {
return query.addPropertyName(r'id');
});
}
QueryBuilder<ETag, String?, QQueryOperations> valueProperty() {
QueryBuilder<ETag, DateTime?, QQueryOperations> timeProperty() {
return QueryBuilder.apply(this, (query) {
return query.addPropertyName(r'value');
return query.addPropertyName(r'time');
});
}
}

17
mobile/lib/shared/providers/app_state.provider.dart
View File

@@ -1,4 +1,6 @@
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/modules/album/providers/album.provider.dart';
import 'package:immich_mobile/modules/album/providers/shared_album.provider.dart';
import 'package:immich_mobile/modules/backup/background_service/background.service.dart';
import 'package:immich_mobile/modules/backup/models/backup_state.model.dart';
import 'package:immich_mobile/modules/backup/providers/backup.provider.dart';
@@ -11,6 +13,7 @@ import 'package:immich_mobile/modules/settings/providers/notification_permission
import 'package:immich_mobile/shared/providers/asset.provider.dart';
import 'package:immich_mobile/shared/providers/release_info.provider.dart';
import 'package:immich_mobile/shared/providers/server_info.provider.dart';
import 'package:immich_mobile/shared/providers/tab.provider.dart';
import 'package:immich_mobile/shared/providers/websocket.provider.dart';
import 'package:immich_mobile/shared/services/immich_logger.service.dart';
import 'package:permission_handler/permission_handler.dart';
@@ -47,8 +50,18 @@ class AppStateNotiifer extends StateNotifier<AppStateEnum> {
if (isAuthenticated && (permission.isGranted || permission.isLimited)) {
ref.read(backupProvider.notifier).resumeBackup();
ref.read(backgroundServiceProvider).resumeServiceIfEnabled();
ref.watch(assetProvider.notifier).getAllAsset();
ref.watch(serverInfoProvider.notifier).getServerVersion();
ref.read(serverInfoProvider.notifier).getServerVersion();
switch (ref.read(tabProvider)) {
case TabEnum.home:
ref.read(assetProvider.notifier).getAllAsset();
case TabEnum.search:
// nothing to do
case TabEnum.sharing:
ref.read(assetProvider.notifier).getPartnerAssets();
ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums();
case TabEnum.library:
ref.read(albumProvider.notifier).getAllAlbums();
}
}
ref.watch(websocketProvider.notifier).connect();

29
mobile/lib/shared/providers/asset.provider.dart
View File

@@ -27,6 +27,7 @@ class AssetNotifier extends StateNotifier<bool> {
final log = Logger('AssetNotifier');
bool _getAllAssetInProgress = false;
bool _deleteInProgress = false;
bool _getPartnerAssetsInProgress = false;
AssetNotifier(
this._assetService,
@@ -49,15 +50,10 @@ class AssetNotifier extends StateNotifier<bool> {
await clearAssetsAndAlbums(_db);
log.info("Manual refresh requested, cleared assets and albums from db");
}
await _userService.refreshUsers();
final bool newRemote = await _assetService.refreshRemoteAssets();
final bool newLocal = await _albumService.refreshDeviceAlbums();
debugPrint("newRemote: $newRemote, newLocal: $newLocal");
final List<User> partners =
await _db.users.filter().isPartnerSharedWithEqualTo(true).findAll();
for (User u in partners) {
await _assetService.refreshRemoteAssets(u);
}
log.info("Load assets: ${stopwatch.elapsedMilliseconds}ms");
} finally {
_getAllAssetInProgress = false;
@@ -65,6 +61,27 @@ class AssetNotifier extends StateNotifier<bool> {
}
}
Future<void> getPartnerAssets([User? partner]) async {
if (_getPartnerAssetsInProgress) return;
try {
final stopwatch = Stopwatch()..start();
_getPartnerAssetsInProgress = true;
if (partner == null) {
await _userService.refreshUsers();
final List<User> partners =
await _db.users.filter().isPartnerSharedWithEqualTo(true).findAll();
for (User u in partners) {
await _assetService.refreshRemoteAssets(u);
}
} else {
await _assetService.refreshRemoteAssets(partner);
}
log.info("Load partner assets: ${stopwatch.elapsedMilliseconds}ms");
} finally {
_getPartnerAssetsInProgress = false;
}
}
Future<void> clearAllAsset() {
return clearAssetsAndAlbums(_db);
}

13
mobile/lib/shared/providers/tab.provider.dart Normal file
View File

@@ -0,0 +1,13 @@
import 'package:hooks_riverpod/hooks_riverpod.dart';
enum TabEnum {
home,
search,
sharing,
library,
}
/// Provides the currently active tab
final tabProvider = StateProvider<TabEnum>(
(ref) => TabEnum.home,
);

2
mobile/lib/shared/services/api.service.dart
View File

@@ -20,6 +20,7 @@ class ApiService {
late ServerInfoApi serverInfoApi;
late PartnerApi partnerApi;
late PersonApi personApi;
late AuditApi auditApi;
ApiService() {
final endpoint = Store.tryGet(StoreKey.serverEndpoint);
@@ -43,6 +44,7 @@ class ApiService {
searchApi = SearchApi(_apiClient);
partnerApi = PartnerApi(_apiClient);
personApi = PersonApi(_apiClient);
auditApi = AuditApi(_apiClient);
}
Future<String> resolveAndSetEndpoint(String serverUrl) async {

43
mobile/lib/shared/services/asset.service.dart
View File

@@ -3,7 +3,6 @@ import 'dart:async';
import 'package:flutter/material.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/shared/models/asset.dart';
import 'package:immich_mobile/shared/models/etag.dart';
import 'package:immich_mobile/shared/models/exif_info.dart';
import 'package:immich_mobile/shared/models/store.dart';
import 'package:immich_mobile/shared/models/user.dart';
@@ -11,7 +10,6 @@ import 'package:immich_mobile/shared/providers/api.provider.dart';
import 'package:immich_mobile/shared/providers/db.provider.dart';
import 'package:immich_mobile/shared/services/api.service.dart';
import 'package:immich_mobile/shared/services/sync.service.dart';
import 'package:immich_mobile/utils/openapi_extensions.dart';
import 'package:isar/isar.dart';
import 'package:logging/logging.dart';
import 'package:openapi/api.dart';
@@ -39,37 +37,34 @@ class AssetService {
/// Checks the server for updated assets and updates the local database if
/// required. Returns `true` if there were any changes.
Future<bool> refreshRemoteAssets([User? user]) async {
user ??= Store.get(StoreKey.currentUser);
user ??= Store.get<User>(StoreKey.currentUser);
final Stopwatch sw = Stopwatch()..start();
final int numOwnedRemoteAssets = await _db.assets
.where()
.remoteIdIsNotNull()
.filter()
.ownerIdEqualTo(user!.isarId)
.count();
final bool changes = await _syncService.syncRemoteAssetsToDb(
user,
() async => (await _getRemoteAssets(
hasCache: numOwnedRemoteAssets > 0,
user: user!,
)),
_getRemoteAssetChanges,
_getRemoteAssets,
);
debugPrint("refreshRemoteAssets full took ${sw.elapsedMilliseconds}ms");
return changes;
}
/// Returns `(null, null)` if changes are invalid -> requires full sync
Future<(List<Asset>? toUpsert, List<String>? toDelete)>
_getRemoteAssetChanges(User user, DateTime since) async {
final deleted = await _apiService.auditApi
.getAuditDeletes(EntityType.ASSET, since, userId: user.id);
if (deleted == null || deleted.needsFullSync) return (null, null);
final assetDto = await _apiService.assetApi
.getAllAssets(userId: user.id, updatedAfter: since);
if (assetDto == null) return (null, null);
return (assetDto.map(Asset.remote).toList(), deleted.ids);
}
/// Returns `null` if the server state did not change, else list of assets
Future<List<Asset>?> _getRemoteAssets({
required bool hasCache,
required User user,
}) async {
Future<List<Asset>?> _getRemoteAssets(User user) async {
try {
final etag = hasCache ? _db.eTags.getByIdSync(user.id)?.value : null;
final (List<AssetResponseDto>? assets, String? newETag) =
await _apiService.assetApi.getAllAssetsWithETag(
eTag: etag,
userId: user.id,
);
final List<AssetResponseDto>? assets =
await _apiService.assetApi.getAllAssets(userId: user.id);
if (assets == null) {
return null;
} else if (assets.isNotEmpty && assets.first.ownerId != user.id) {
@@ -77,8 +72,6 @@ class AssetService {
" The server returned assets for user ${assets.first.ownerId}"
" while requesting assets of user ${user.id}");
return null;
} else if (newETag != etag) {
_db.writeTxn(() => _db.eTags.put(ETag(id: user.id, value: newETag)));
}
return assets.map(Asset.remote).toList();
} catch (error, stack) {

92
mobile/lib/shared/services/sync.service.dart
View File

@@ -69,9 +69,17 @@ class SyncService {
/// Returns `true` if there were any changes
Future<bool> syncRemoteAssetsToDb(
User user,
FutureOr<List<Asset>?> Function() loadAssets,
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
User user,
DateTime since,
) getChangedAssets,
FutureOr<List<Asset>?> Function(User user) loadAssets,
) =>
_lock.run(() => _syncRemoteAssetsToDb(user, loadAssets));
_lock.run(
() async =>
await _syncRemoteAssetChanges(user, getChangedAssets) ??
await _syncRemoteAssetsFull(user, loadAssets),
);
/// Syncs remote albums to the database
/// returns `true` if there were any changes
@@ -130,13 +138,59 @@ class SyncService {
return true;
}
/// Syncs remote assets to the databas
/// returns `true` if there were any changes
Future<bool> _syncRemoteAssetsToDb(
/// Efficiently syncs assets via changes. Returns `null` when a full sync is required.
Future<bool?> _syncRemoteAssetChanges(
User user,
FutureOr<List<Asset>?> Function() loadAssets,
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
User user,
DateTime since,
) getChangedAssets,
) async {
final List<Asset>? remote = await loadAssets();
final DateTime? since = _db.eTags.getByIdSync(user.id)?.time?.toUtc();
if (since == null) return null;
final DateTime now = DateTime.now();
final (toUpsert, toDelete) = await getChangedAssets(user, since);
if (toUpsert == null || toDelete == null) return null;
try {
if (toDelete.isNotEmpty) {
await _handleRemoteAssetRemoval(toDelete);
}
if (toUpsert.isNotEmpty) {
final (_, updated) = await _linkWithExistingFromDb(toUpsert);
await upsertAssetsWithExif(updated);
}
if (toUpsert.isNotEmpty || toDelete.isNotEmpty) {
await _updateUserAssetsETag(user, now);
return true;
}
return false;
} on IsarError catch (e) {
_log.severe("Failed to sync remote assets to db: $e");
}
return null;
}
/// Deletes remote-only assets, updates merged assets to be local-only
Future<void> _handleRemoteAssetRemoval(List<String> idsToDelete) {
return _db.writeTxn(() async {
await _db.assets.remote(idsToDelete).filter().localIdIsNull().deleteAll();
final onlyLocal = await _db.assets.remote(idsToDelete).findAll();
if (onlyLocal.isNotEmpty) {
for (final Asset a in onlyLocal) {
a.remoteId = null;
}
await _db.assets.putAll(onlyLocal);
}
});
}
/// Syncs assets by loading and comparing all assets from the server.
Future<bool> _syncRemoteAssetsFull(
User user,
FutureOr<List<Asset>?> Function(User user) loadAssets,
) async {
final DateTime now = DateTime.now();
final List<Asset>? remote = await loadAssets(user);
if (remote == null) {
return false;
}
@@ -150,6 +204,7 @@ class SyncService {
remote.sort(Asset.compareByChecksum);
final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true);
if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) {
await _updateUserAssetsETag(user, now);
return false;
}
final idsToDelete = toRemove.map((e) => e.id).toList();
@@ -159,9 +214,13 @@ class SyncService {
} on IsarError catch (e) {
_log.severe("Failed to sync remote assets to db: $e");
}
await _updateUserAssetsETag(user, now);
return true;
}
Future<void> _updateUserAssetsETag(User user, DateTime time) =>
_db.writeTxn(() => _db.eTags.put(ETag(id: user.id, time: time)));
/// Syncs remote albums to the database
/// returns `true` if there were any changes
Future<bool> _syncRemoteAlbumsToDb(
@@ -450,6 +509,14 @@ class SyncService {
_log.fine(
"Only excluded assets in local album ${ape.name} changed. Stopping sync.",
);
if (assetCountOnDevice !=
_db.eTags.getByIdSync(ape.eTagKeyAssetCount)?.assetCount) {
await _db.writeTxn(
() => _db.eTags.put(
ETag(id: ape.eTagKeyAssetCount, assetCount: assetCountOnDevice),
),
);
}
return false;
}
_log.fine(
@@ -477,7 +544,7 @@ class SyncService {
album.thumbnail.value ??= await album.assets.filter().findFirst();
await album.thumbnail.save();
await _db.eTags.put(
ETag(id: ape.eTagKeyAssetCount, value: assetCountOnDevice.toString()),
ETag(id: ape.eTagKeyAssetCount, assetCount: assetCountOnDevice),
);
});
_log.info("Synced changes of local album ${ape.name} to DB");
@@ -496,7 +563,7 @@ class SyncService {
}
final int totalOnDevice = await ape.assetCountAsync;
final int lastKnownTotal =
(await _db.eTags.getById(ape.eTagKeyAssetCount))?.value?.toInt() ?? 0;
(await _db.eTags.getById(ape.eTagKeyAssetCount))?.assetCount ?? 0;
final AssetPathEntity? modified = totalOnDevice > lastKnownTotal
? await ape.fetchPathProperties(
filterOptionGroup: FilterOptionGroup(
@@ -523,9 +590,8 @@ class SyncService {
await _db.assets.putAll(updated);
await album.assets.update(link: existingInDb + updated);
await _db.albums.put(album);
await _db.eTags.put(
ETag(id: ape.eTagKeyAssetCount, value: totalOnDevice.toString()),
);
await _db.eTags
.put(ETag(id: ape.eTagKeyAssetCount, assetCount: totalOnDevice));
});
_log.info("Fast synced local album ${ape.name} to DB");
} on IsarError catch (e) {
@@ -667,7 +733,7 @@ class SyncService {
a.lastModified == null ||
!a.lastModified!.isAtSameMomentAs(b.modifiedAt) ||
await a.assetCountAsync !=
(await _db.eTags.getById(a.eTagKeyAssetCount))?.value?.toInt();
(await _db.eTags.getById(a.eTagKeyAssetCount))?.assetCount;
}
}

3
mobile/lib/shared/views/tab_controller_page.dart
View File

@@ -7,6 +7,7 @@ import 'package:immich_mobile/modules/asset_viewer/providers/scroll_notifier.pro
import 'package:immich_mobile/modules/home/providers/multiselect.provider.dart';
import 'package:immich_mobile/routing/router.dart';
import 'package:immich_mobile/shared/providers/asset.provider.dart';
import 'package:immich_mobile/shared/providers/tab.provider.dart';
class TabControllerPage extends HookConsumerWidget {
const TabControllerPage({Key? key}) : super(key: key);
@@ -51,6 +52,7 @@ class TabControllerPage extends HookConsumerWidget {
}
HapticFeedback.selectionClick();
tabsRouter.setActiveIndex(index);
ref.read(tabProvider.notifier).state = TabEnum.values[index];
},
selectedIconTheme: IconThemeData(
color: Theme.of(context).primaryColor,
@@ -103,6 +105,7 @@ class TabControllerPage extends HookConsumerWidget {
}
HapticFeedback.selectionClick();
tabsRouter.setActiveIndex(index);
ref.read(tabProvider.notifier).state = TabEnum.values[index];
},
destinations: [
NavigationDestination(

59
mobile/lib/utils/openapi_extensions.dart
View File

@@ -1,59 +0,0 @@
import 'dart:convert';
import 'dart:io';
import 'package:http/http.dart';
import 'package:openapi/api.dart';
/// Extension methods to retrieve ETag together with the API call
extension WithETag on AssetApi {
/// Get all AssetEntity belong to the user
///
/// Parameters:
///
/// * [String] eTag:
/// ETag of data already cached on the client
Future<(List<AssetResponseDto>? assets, String? eTag)> getAllAssetsWithETag({
String? eTag,
String? userId,
bool? isFavorite,
bool? isArchived,
}) async {
final response = await getAllAssetsWithHttpInfo(
ifNoneMatch: eTag,
userId: userId,
isFavorite: isFavorite,
isArchived: isArchived,
);
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}
// When a remote server returns no body with a status of 204, we shall not decode it.
// At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
// FormatException when trying to decode an empty string.
if (response.body.isNotEmpty &&
response.statusCode != HttpStatus.noContent) {
final responseBody = await _decodeBodyBytes(response);
final etag = response.headers[HttpHeaders.etagHeader];
final data = (await apiClient.deserializeAsync(
responseBody,
'List<AssetResponseDto>',
) as List)
.cast<AssetResponseDto>()
.toList();
return (data, etag);
}
return (null, null);
}
}
/// Returns the decoded body as UTF-8 if the given headers indicate an 'application/json'
/// content type. Otherwise, returns the decoded body as decoded by dart:http package.
Future<String> _decodeBodyBytes(Response response) async {
final contentType = response.headers['content-type'];
return contentType != null &&
contentType.toLowerCase().startsWith('application/json')
? response.bodyBytes.isEmpty
? ''
: utf8.decode(response.bodyBytes)
: response.body;
}

70
mobile/test/sync_service_test.dart
View File

@@ -2,6 +2,7 @@ import 'package:flutter/widgets.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:immich_mobile/shared/models/album.dart';
import 'package:immich_mobile/shared/models/asset.dart';
import 'package:immich_mobile/shared/models/etag.dart';
import 'package:immich_mobile/shared/models/exif_info.dart';
import 'package:immich_mobile/shared/models/logger_message.model.dart';
import 'package:immich_mobile/shared/models/store.dart';
@@ -17,7 +18,6 @@ void main() {
required String checksum,
String? localId,
String? remoteId,
int deviceId = 1,
int ownerId = 590700560494856554, // hash of "1"
}) {
final DateTime date = DateTime(2000);
@@ -46,6 +46,7 @@ void main() {
UserSchema,
StoreValueSchema,
LoggerMessageSchema,
ETagSchema,
],
maxSizeMiB: 256,
directory: ".",
@@ -73,8 +74,8 @@ void main() {
await Store.put(StoreKey.currentUser, owner);
});
final List<Asset> initialAssets = [
makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2),
makeAsset(checksum: "a", remoteId: "0-1"),
makeAsset(checksum: "b", remoteId: "2-1"),
makeAsset(checksum: "c", localId: "1", remoteId: "1-1"),
makeAsset(checksum: "d", localId: "2"),
makeAsset(checksum: "e", localId: "3"),
@@ -88,12 +89,13 @@ void main() {
test('test inserting existing assets', () async {
SyncService s = SyncService(db, hs);
final List<Asset> remoteAssets = [
makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2),
makeAsset(checksum: "a", remoteId: "0-1"),
makeAsset(checksum: "b", remoteId: "2-1"),
makeAsset(checksum: "c", remoteId: "1-1"),
];
expect(db.assets.countSync(), 5);
final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
final bool c1 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
expect(c1, false);
expect(db.assets.countSync(), 5);
});
@@ -101,15 +103,16 @@ void main() {
test('test inserting new assets', () async {
SyncService s = SyncService(db, hs);
final List<Asset> remoteAssets = [
makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2),
makeAsset(checksum: "a", remoteId: "0-1"),
makeAsset(checksum: "b", remoteId: "2-1"),
makeAsset(checksum: "c", remoteId: "1-1"),
makeAsset(checksum: "d", remoteId: "1-2"),
makeAsset(checksum: "f", remoteId: "1-4"),
makeAsset(checksum: "g", remoteId: "3-1", deviceId: 3),
makeAsset(checksum: "g", remoteId: "3-1"),
];
expect(db.assets.countSync(), 5);
final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
final bool c1 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
expect(c1, true);
expect(db.assets.countSync(), 7);
});
@@ -117,31 +120,56 @@ void main() {
test('test syncing duplicate assets', () async {
SyncService s = SyncService(db, hs);
final List<Asset> remoteAssets = [
makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
makeAsset(checksum: "a", remoteId: "0-1"),
makeAsset(checksum: "b", remoteId: "1-1"),
makeAsset(checksum: "c", remoteId: "2-1", deviceId: 2),
makeAsset(checksum: "h", remoteId: "2-1b", deviceId: 2),
makeAsset(checksum: "i", remoteId: "2-1c", deviceId: 2),
makeAsset(checksum: "j", remoteId: "2-1d", deviceId: 2),
makeAsset(checksum: "c", remoteId: "2-1"),
makeAsset(checksum: "h", remoteId: "2-1b"),
makeAsset(checksum: "i", remoteId: "2-1c"),
makeAsset(checksum: "j", remoteId: "2-1d"),
];
expect(db.assets.countSync(), 5);
final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
final bool c1 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
expect(c1, true);
expect(db.assets.countSync(), 8);
final bool c2 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
final bool c2 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
expect(c2, false);
expect(db.assets.countSync(), 8);
remoteAssets.removeAt(4);
final bool c3 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
final bool c3 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
expect(c3, true);
expect(db.assets.countSync(), 7);
remoteAssets.add(makeAsset(checksum: "k", remoteId: "2-1e", deviceId: 2));
remoteAssets.add(makeAsset(checksum: "l", remoteId: "2-2", deviceId: 2));
final bool c4 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
remoteAssets.add(makeAsset(checksum: "k", remoteId: "2-1e"));
remoteAssets.add(makeAsset(checksum: "l", remoteId: "2-2"));
final bool c4 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
expect(c4, true);
expect(db.assets.countSync(), 9);
});
test('test efficient sync', () async {
SyncService s = SyncService(db, hs);
final List<Asset> toUpsert = [
makeAsset(checksum: "a", remoteId: "0-1"), // changed
makeAsset(checksum: "f", remoteId: "0-2"), // new
makeAsset(checksum: "g", remoteId: "0-3"), // new
];
toUpsert[0].isFavorite = true;
final List<String> toDelete = ["2-1", "1-1"];
final bool c = await s.syncRemoteAssetsToDb(
owner,
(user, since) async => (toUpsert, toDelete),
(user) => throw Exception(),
);
expect(c, true);
expect(db.assets.countSync(), 6);
});
});
}
Future<(List<Asset>?, List<String>?)> _failDiff(User user, DateTime time) =>
Future.value((null, null));
class MockHashService extends Mock implements HashService {}