1
0
mirror of https://github.com/immich-app/immich.git synced 2024-12-25 10:43:13 +02:00

feature(mobile): Hardening synchronization mechanism + Pull to refresh (#2085)

* fix(mobile): allow syncing duplicate local IDs

* enable to run isar unit tests on CI

* serialize sync operations, add pull to refresh on timeline

---------

Co-authored-by: Fynn Petersen-Frey <zoodyy@users.noreply.github.com>
This commit is contained in:
Fynn Petersen-Frey 2023-03-27 04:35:52 +02:00 committed by GitHub
parent 1a94530935
commit cae37657e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 531 additions and 153 deletions

View File

@ -50,6 +50,7 @@ void main() async {
await initApp();
await migrateHiveToStoreIfNecessary();
await migrateJsonCacheIfNecessary();
await migrateDatabaseIfNeeded(db);
runApp(getMainWidget(db));
}

View File

@ -53,7 +53,7 @@ class AlbumThumbnailCard extends StatelessWidget {
// Add the owner name to the subtitle
String? owner;
if (showOwner) {
if (album.ownerId == Store.get(StoreKey.userRemoteId)) {
if (album.ownerId == Store.get(StoreKey.currentUser).id) {
owner = 'album_thumbnail_owned'.tr();
} else if (album.ownerName != null) {
owner = 'album_thumbnail_shared_by'.tr(args: [album.ownerName!]);

View File

@ -17,7 +17,7 @@ class SharingPage extends HookConsumerWidget {
@override
Widget build(BuildContext context, WidgetRef ref) {
final List<Album> sharedAlbums = ref.watch(sharedAlbumProvider);
final userId = store.Store.get(store.StoreKey.userRemoteId);
final userId = store.Store.get(store.StoreKey.currentUser).id;
var isDarkMode = Theme.of(context).brightness == Brightness.dark;
useEffect(

View File

@ -17,10 +17,12 @@ class ImmichAssetGrid extends HookConsumerWidget {
final bool selectionActive;
final List<Asset> assets;
final RenderList? renderList;
final Future<void> Function()? onRefresh;
const ImmichAssetGrid({
super.key,
required this.assets,
this.onRefresh,
this.renderList,
this.assetsPerRow,
this.showStorageIndicator,
@ -62,11 +64,12 @@ class ImmichAssetGrid extends HookConsumerWidget {
enabled: enableHeroAnimations.value,
child: ImmichAssetGridView(
allAssets: assets,
assetsPerRow: assetsPerRow
?? settings.getSetting(AppSettingsEnum.tilesPerRow),
onRefresh: onRefresh,
assetsPerRow: assetsPerRow ??
settings.getSetting(AppSettingsEnum.tilesPerRow),
listener: listener,
showStorageIndicator: showStorageIndicator
?? settings.getSetting(AppSettingsEnum.storageIndicator),
showStorageIndicator: showStorageIndicator ??
settings.getSetting(AppSettingsEnum.storageIndicator),
renderList: renderList!,
margin: margin,
selectionActive: selectionActive,
@ -76,26 +79,25 @@ class ImmichAssetGrid extends HookConsumerWidget {
}
return renderListFuture.when(
data: (renderList) =>
WillPopScope(
onWillPop: onWillPop,
child: HeroMode(
enabled: enableHeroAnimations.value,
child: ImmichAssetGridView(
allAssets: assets,
assetsPerRow: assetsPerRow
?? settings.getSetting(AppSettingsEnum.tilesPerRow),
listener: listener,
showStorageIndicator: showStorageIndicator
?? settings.getSetting(AppSettingsEnum.storageIndicator),
renderList: renderList,
margin: margin,
selectionActive: selectionActive,
),
data: (renderList) => WillPopScope(
onWillPop: onWillPop,
child: HeroMode(
enabled: enableHeroAnimations.value,
child: ImmichAssetGridView(
allAssets: assets,
onRefresh: onRefresh,
assetsPerRow: assetsPerRow ??
settings.getSetting(AppSettingsEnum.tilesPerRow),
listener: listener,
showStorageIndicator: showStorageIndicator ??
settings.getSetting(AppSettingsEnum.storageIndicator),
renderList: renderList,
margin: margin,
selectionActive: selectionActive,
),
),
error: (err, stack) =>
Center(child: Text("$err")),
),
error: (err, stack) => Center(child: Text("$err")),
loading: () => const Center(
child: ImmichLoadingIndicator(),
),

View File

@ -199,21 +199,23 @@ class ImmichAssetGridViewState extends State<ImmichAssetGridView> {
addRepaintBoundaries: true,
);
if (!useDragScrolling) {
return listWidget;
}
final child = useDragScrolling
? DraggableScrollbar.semicircle(
scrollStateListener: dragScrolling,
itemPositionsListener: _itemPositionsListener,
controller: _itemScrollController,
backgroundColor: Theme.of(context).hintColor,
labelTextBuilder: _labelBuilder,
labelConstraints: const BoxConstraints(maxHeight: 28),
scrollbarAnimationDuration: const Duration(seconds: 1),
scrollbarTimeToFade: const Duration(seconds: 4),
child: listWidget,
)
: listWidget;
return DraggableScrollbar.semicircle(
scrollStateListener: dragScrolling,
itemPositionsListener: _itemPositionsListener,
controller: _itemScrollController,
backgroundColor: Theme.of(context).hintColor,
labelTextBuilder: _labelBuilder,
labelConstraints: const BoxConstraints(maxHeight: 28),
scrollbarAnimationDuration: const Duration(seconds: 1),
scrollbarTimeToFade: const Duration(seconds: 4),
child: listWidget,
);
return widget.onRefresh == null
? child
: RefreshIndicator(onRefresh: widget.onRefresh!, child: child);
}
@override
@ -248,7 +250,7 @@ class ImmichAssetGridViewState extends State<ImmichAssetGridView> {
}
void _scrollToTop() {
// for some reason, this is necessary as well in order
// for some reason, this is necessary as well in order
// to correctly reposition the drag thumb scroll bar
_itemScrollController.jumpTo(
index: 0,
@ -281,6 +283,7 @@ class ImmichAssetGridView extends StatefulWidget {
final ImmichAssetGridSelectionListener? listener;
final bool selectionActive;
final List<Asset> allAssets;
final Future<void> Function()? onRefresh;
const ImmichAssetGridView({
super.key,
@ -291,6 +294,7 @@ class ImmichAssetGridView extends StatefulWidget {
this.listener,
this.margin = 5.0,
this.selectionActive = false,
this.onRefresh,
});
@override

View File

@ -43,6 +43,7 @@ class HomePage extends HookConsumerWidget {
final albumService = ref.watch(albumServiceProvider);
final tipOneOpacity = useState(0.0);
final refreshCount = useState(0);
useEffect(
() {
@ -182,6 +183,22 @@ class HomePage extends HookConsumerWidget {
}
}
Future<void> refreshAssets() async {
debugPrint("refreshCount.value ${refreshCount.value}");
final fullRefresh = refreshCount.value > 0;
await ref.read(assetProvider.notifier).getAllAsset(clear: fullRefresh);
if (fullRefresh) {
// refresh was forced: user requested another refresh within 2 seconds
refreshCount.value = 0;
} else {
refreshCount.value++;
// set counter back to 0 if user does not request refresh again
Timer(const Duration(seconds: 2), () {
refreshCount.value = 0;
});
}
}
buildLoadingIndicator() {
Timer(const Duration(seconds: 2), () {
tipOneOpacity.value = 1;
@ -241,6 +258,7 @@ class HomePage extends HookConsumerWidget {
.getSetting(AppSettingsEnum.storageIndicator),
listener: selectionListener,
selectionActive: selectionEnabledHook.value,
onRefresh: refreshAssets,
),
if (selectionEnabledHook.value)
SafeArea(

View File

@ -78,7 +78,6 @@ class AuthenticationNotifier extends StateNotifier<AuthenticationState> {
await Future.wait([
_apiService.authenticationApi.logout(),
Store.delete(StoreKey.assetETag),
Store.delete(StoreKey.userRemoteId),
Store.delete(StoreKey.currentUser),
Store.delete(StoreKey.accessToken),
]);
@ -133,7 +132,6 @@ class AuthenticationNotifier extends StateNotifier<AuthenticationState> {
var deviceInfo = await _deviceInfoService.getDeviceInfo();
Store.put(StoreKey.deviceId, deviceInfo["deviceId"]);
Store.put(StoreKey.deviceIdHash, fastHash(deviceInfo["deviceId"]));
Store.put(StoreKey.userRemoteId, userResponseDto.id);
Store.put(StoreKey.currentUser, User.fromDto(userResponseDto));
Store.put(StoreKey.serverUrl, serverUrl);
Store.put(StoreKey.accessToken, accessToken);

View File

@ -15,11 +15,11 @@ class Asset {
Asset.remote(AssetResponseDto remote)
: remoteId = remote.id,
isLocal = false,
fileCreatedAt = DateTime.parse(remote.fileCreatedAt).toUtc(),
fileModifiedAt = DateTime.parse(remote.fileModifiedAt).toUtc(),
updatedAt = DateTime.parse(remote.updatedAt).toUtc(),
// use -1 as fallback duration (to not mix it up with non-video assets correctly having duration=0)
durationInSeconds = remote.duration.toDuration()?.inSeconds ?? -1,
fileCreatedAt = DateTime.parse(remote.fileCreatedAt),
fileModifiedAt = DateTime.parse(remote.fileModifiedAt),
updatedAt = DateTime.parse(remote.updatedAt),
durationInSeconds = remote.duration.toDuration()?.inSeconds ?? 0,
type = remote.type.toAssetType(),
fileName = p.basename(remote.originalPath),
height = remote.exifInfo?.exifImageHeight?.toInt(),
width = remote.exifInfo?.exifImageWidth?.toInt(),
@ -35,15 +35,16 @@ class Asset {
: localId = local.id,
isLocal = true,
durationInSeconds = local.duration,
type = AssetType.values[local.typeInt],
height = local.height,
width = local.width,
fileName = local.title!,
deviceId = Store.get(StoreKey.deviceIdHash),
ownerId = Store.get(StoreKey.currentUser).isarId,
fileModifiedAt = local.modifiedDateTime.toUtc(),
updatedAt = local.modifiedDateTime.toUtc(),
fileModifiedAt = local.modifiedDateTime,
updatedAt = local.modifiedDateTime,
isFavorite = local.isFavorite,
fileCreatedAt = local.createDateTime.toUtc() {
fileCreatedAt = local.createDateTime {
if (fileCreatedAt.year == 1970) {
fileCreatedAt = fileModifiedAt;
}
@ -61,6 +62,7 @@ class Asset {
required this.fileModifiedAt,
required this.updatedAt,
required this.durationInSeconds,
required this.type,
this.width,
this.height,
required this.fileName,
@ -77,10 +79,10 @@ class Asset {
AssetEntity? get local {
if (isLocal && _local == null) {
_local = AssetEntity(
id: localId.toString(),
id: localId,
typeInt: isImage ? 1 : 2,
width: width!,
height: height!,
width: width ?? 0,
height: height ?? 0,
duration: durationInSeconds,
createDateSecond: fileCreatedAt.millisecondsSinceEpoch ~/ 1000,
modifiedDateSecond: fileModifiedAt.millisecondsSinceEpoch ~/ 1000,
@ -96,7 +98,7 @@ class Asset {
String? remoteId;
@Index(
unique: true,
unique: false,
replace: false,
type: IndexType.hash,
composite: [CompositeIndex('deviceId')],
@ -115,6 +117,9 @@ class Asset {
int durationInSeconds;
@Enumerated(EnumType.ordinal)
AssetType type;
short? width;
short? height;
@ -140,7 +145,7 @@ class Asset {
bool get isRemote => remoteId != null;
@ignore
bool get isImage => durationInSeconds == 0;
bool get isImage => type == AssetType.image;
@ignore
Duration get duration => Duration(seconds: durationInSeconds);
@ -148,12 +153,43 @@ class Asset {
@override
bool operator ==(other) {
if (other is! Asset) return false;
return id == other.id;
return id == other.id &&
remoteId == other.remoteId &&
localId == other.localId &&
deviceId == other.deviceId &&
ownerId == other.ownerId &&
fileCreatedAt == other.fileCreatedAt &&
fileModifiedAt == other.fileModifiedAt &&
updatedAt == other.updatedAt &&
durationInSeconds == other.durationInSeconds &&
type == other.type &&
width == other.width &&
height == other.height &&
fileName == other.fileName &&
livePhotoVideoId == other.livePhotoVideoId &&
isFavorite == other.isFavorite &&
isLocal == other.isLocal;
}
@override
@ignore
int get hashCode => id.hashCode;
int get hashCode =>
id.hashCode ^
remoteId.hashCode ^
localId.hashCode ^
deviceId.hashCode ^
ownerId.hashCode ^
fileCreatedAt.hashCode ^
fileModifiedAt.hashCode ^
updatedAt.hashCode ^
durationInSeconds.hashCode ^
type.hashCode ^
width.hashCode ^
height.hashCode ^
fileName.hashCode ^
livePhotoVideoId.hashCode ^
isFavorite.hashCode ^
isLocal.hashCode;
bool updateFromAssetEntity(AssetEntity ae) {
// TODO check more fields;
@ -192,9 +228,24 @@ class Asset {
}
}
static int compareByDeviceIdLocalId(Asset a, Asset b) {
final int order = a.deviceId.compareTo(b.deviceId);
return order == 0 ? a.localId.compareTo(b.localId) : order;
/// compares assets by [ownerId], [deviceId], [localId]
static int compareByOwnerDeviceLocalId(Asset a, Asset b) {
final int ownerIdOrder = a.ownerId.compareTo(b.ownerId);
if (ownerIdOrder != 0) {
return ownerIdOrder;
}
final int deviceIdOrder = a.deviceId.compareTo(b.deviceId);
if (deviceIdOrder != 0) {
return deviceIdOrder;
}
final int localIdOrder = a.localId.compareTo(b.localId);
return localIdOrder;
}
/// compares assets by [ownerId], [deviceId], [localId], [fileModifiedAt]
static int compareByOwnerDeviceLocalIdModified(Asset a, Asset b) {
final int order = compareByOwnerDeviceLocalId(a, b);
return order != 0 ? order : a.fileModifiedAt.compareTo(b.fileModifiedAt);
}
static int compareById(Asset a, Asset b) => a.id.compareTo(b.id);
@ -203,6 +254,30 @@ class Asset {
a.localId.compareTo(b.localId);
}
enum AssetType {
// do not change this order!
other,
image,
video,
audio,
}
extension AssetTypeEnumHelper on AssetTypeEnum {
AssetType toAssetType() {
switch (this) {
case AssetTypeEnum.IMAGE:
return AssetType.image;
case AssetTypeEnum.VIDEO:
return AssetType.video;
case AssetTypeEnum.AUDIO:
return AssetType.audio;
case AssetTypeEnum.OTHER:
return AssetType.other;
}
throw Exception();
}
}
extension AssetsHelper on IsarCollection<Asset> {
Future<int> deleteAllByRemoteId(Iterable<String> ids) =>
ids.isEmpty ? Future.value(0) : _remote(ids).deleteAll();

View File

@ -138,7 +138,7 @@ class StoreKeyNotFoundException implements Exception {
/// Key for each possible value in the `Store`.
/// Defines the data type for each value
enum StoreKey<T> {
userRemoteId<String>(0, type: String),
version<int>(0, type: int),
assetETag<String>(1, type: String),
currentUser<User>(2, type: User, fromDb: _getUser, toDb: _toUser),
deviceIdHash<int>(3, type: int),

View File

@ -1,7 +1,5 @@
import 'package:flutter/foundation.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/modules/album/services/album.service.dart';
import 'package:immich_mobile/shared/models/album.dart';
import 'package:immich_mobile/shared/models/exif_info.dart';
import 'package:immich_mobile/shared/models/store.dart';
import 'package:immich_mobile/shared/models/user.dart';
@ -12,6 +10,9 @@ import 'package:immich_mobile/modules/settings/providers/app_settings.provider.d
import 'package:immich_mobile/modules/settings/services/app_settings.service.dart';
import 'package:immich_mobile/shared/models/asset.dart';
import 'package:collection/collection.dart';
import 'package:immich_mobile/shared/services/sync.service.dart';
import 'package:immich_mobile/utils/async_mutex.dart';
import 'package:immich_mobile/utils/db.dart';
import 'package:intl/intl.dart';
import 'package:isar/isar.dart';
import 'package:logging/logging.dart';
@ -53,15 +54,18 @@ class AssetNotifier extends StateNotifier<AssetsState> {
final AssetService _assetService;
final AppSettingsService _settingsService;
final AlbumService _albumService;
final SyncService _syncService;
final Isar _db;
final log = Logger('AssetNotifier');
bool _getAllAssetInProgress = false;
bool _deleteInProgress = false;
final AsyncMutex _stateUpdateLock = AsyncMutex();
AssetNotifier(
this._assetService,
this._settingsService,
this._albumService,
this._syncService,
this._db,
) : super(AssetsState.fromAssetList([]));
@ -81,24 +85,30 @@ class AssetNotifier extends StateNotifier<AssetsState> {
await _updateAssetsState(state.allAssets);
}
getAllAsset() async {
Future<void> getAllAsset({bool clear = false}) async {
if (_getAllAssetInProgress || _deleteInProgress) {
// guard against multiple calls to this method while it's still working
return;
}
final stopwatch = Stopwatch();
final stopwatch = Stopwatch()..start();
try {
_getAllAssetInProgress = true;
final User me = Store.get(StoreKey.currentUser);
final int cachedCount =
await _db.assets.filter().ownerIdEqualTo(me.isarId).count();
stopwatch.start();
if (cachedCount > 0 && cachedCount != state.allAssets.length) {
await _updateAssetsState(await _getUserAssets(me.isarId));
log.info(
"Reading assets ${state.allAssets.length} from DB: ${stopwatch.elapsedMilliseconds}ms",
);
stopwatch.reset();
if (clear) {
await clearAssetsAndAlbums(_db);
log.info("Manual refresh requested, cleared assets and albums from db");
} else if (_stateUpdateLock.enqueued <= 1) {
final int cachedCount =
await _db.assets.filter().ownerIdEqualTo(me.isarId).count();
if (cachedCount > 0 && cachedCount != state.allAssets.length) {
await _stateUpdateLock.run(
() async => _updateAssetsState(await _getUserAssets(me.isarId)),
);
log.info(
"Reading assets ${state.allAssets.length} from DB: ${stopwatch.elapsedMilliseconds}ms",
);
stopwatch.reset();
}
}
final bool newRemote = await _assetService.refreshRemoteAssets();
final bool newLocal = await _albumService.refreshDeviceAlbums();
@ -112,10 +122,14 @@ class AssetNotifier extends StateNotifier<AssetsState> {
return;
}
stopwatch.reset();
final assets = await _getUserAssets(me.isarId);
if (!const ListEquality().equals(assets, state.allAssets)) {
log.info("setting new asset state");
await _updateAssetsState(assets);
if (_stateUpdateLock.enqueued <= 1) {
_stateUpdateLock.run(() async {
final assets = await _getUserAssets(me.isarId);
if (!const ListEquality().equals(assets, state.allAssets)) {
log.info("setting new asset state");
await _updateAssetsState(assets);
}
});
}
} finally {
_getAllAssetInProgress = false;
@ -130,47 +144,18 @@ class AssetNotifier extends StateNotifier<AssetsState> {
Future<void> clearAllAsset() {
state = AssetsState.empty();
return _db.writeTxn(() async {
await _db.assets.clear();
await _db.exifInfos.clear();
await _db.albums.clear();
});
return clearAssetsAndAlbums(_db);
}
Future<void> onNewAssetUploaded(Asset newAsset) async {
final int i = state.allAssets.indexWhere(
(a) =>
a.isRemote ||
(a.localId == newAsset.localId && a.deviceId == newAsset.deviceId),
);
if (i == -1 ||
state.allAssets[i].localId != newAsset.localId ||
state.allAssets[i].deviceId != newAsset.deviceId) {
await _updateAssetsState([...state.allAssets, newAsset]);
} else {
// unify local/remote assets by replacing the
// local-only asset in the DB with a local&remote asset
final Asset? inDb = await _db.assets
.where()
.localIdDeviceIdEqualTo(newAsset.localId, newAsset.deviceId)
.findFirst();
if (inDb != null) {
newAsset.id = inDb.id;
newAsset.isLocal = inDb.isLocal;
}
// order is important to keep all local-only assets at the beginning!
await _updateAssetsState([
...state.allAssets.slice(0, i),
...state.allAssets.slice(i + 1),
newAsset,
]);
}
try {
await _db.writeTxn(() => newAsset.put(_db));
} on IsarError catch (e) {
debugPrint(e.toString());
final bool ok = await _syncService.syncNewAssetToDb(newAsset);
if (ok && _stateUpdateLock.enqueued <= 1) {
// run this sequentially if there is at most 1 other task waiting
await _stateUpdateLock.run(() async {
final userId = Store.get(StoreKey.currentUser).isarId;
final assets = await _getUserAssets(userId);
await _updateAssetsState(assets);
});
}
}
@ -253,6 +238,7 @@ final assetProvider = StateNotifierProvider<AssetNotifier, AssetsState>((ref) {
ref.watch(assetServiceProvider),
ref.watch(appSettingsServiceProvider),
ref.watch(albumServiceProvider),
ref.watch(syncServiceProvider),
ref.watch(dbProvider),
);
});

View File

@ -45,14 +45,11 @@ class AssetService {
.filter()
.ownerIdEqualTo(Store.get(StoreKey.currentUser).isarId)
.count();
final List<AssetResponseDto>? dtos =
await _getRemoteAssets(hasCache: numOwnedRemoteAssets > 0);
if (dtos == null) {
debugPrint("refreshRemoteAssets fast took ${sw.elapsedMilliseconds}ms");
return false;
}
final bool changes = await _syncService
.syncRemoteAssetsToDb(dtos.map(Asset.remote).toList());
final bool changes = await _syncService.syncRemoteAssetsToDb(
() async => (await _getRemoteAssets(hasCache: numOwnedRemoteAssets > 0))
?.map(Asset.remote)
.toList(),
);
debugPrint("refreshRemoteAssets full took ${sw.elapsedMilliseconds}ms");
return changes;
}

View File

@ -20,7 +20,7 @@ class ImmichLogger {
static final ImmichLogger _instance = ImmichLogger._internal();
final maxLogEntries = 200;
final Isar _db = Isar.getInstance()!;
final List<LoggerMessage> _msgBuffer = [];
List<LoggerMessage> _msgBuffer = [];
Timer? _timer;
factory ImmichLogger() => _instance;
@ -41,7 +41,12 @@ class ImmichLogger {
final msgCount = _db.loggerMessages.countSync();
if (msgCount > maxLogEntries) {
final numberOfEntryToBeDeleted = msgCount - maxLogEntries;
_db.loggerMessages.where().limit(numberOfEntryToBeDeleted).deleteAll();
_db.writeTxn(
() => _db.loggerMessages
.where()
.limit(numberOfEntryToBeDeleted)
.deleteAll(),
);
}
}
@ -63,8 +68,9 @@ class ImmichLogger {
void _flushBufferToDatabase() {
_timer = null;
_db.writeTxnSync(() => _db.loggerMessages.putAllSync(_msgBuffer));
_msgBuffer.clear();
final buffer = _msgBuffer;
_msgBuffer = [];
_db.writeTxn(() => _db.loggerMessages.putAll(buffer));
}
void clearLogs() {
@ -111,7 +117,7 @@ class ImmichLogger {
void flush() {
if (_timer != null) {
_timer!.cancel();
_flushBufferToDatabase();
_db.writeTxnSync(() => _db.loggerMessages.putAllSync(_msgBuffer));
}
}
}

View File

@ -1,7 +1,6 @@
import 'dart:async';
import 'package:collection/collection.dart';
import 'package:flutter/foundation.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/shared/models/album.dart';
import 'package:immich_mobile/shared/models/asset.dart';
@ -61,8 +60,10 @@ class SyncService {
/// Syncs remote assets owned by the logged-in user to the DB
/// Returns `true` if there were any changes
Future<bool> syncRemoteAssetsToDb(List<Asset> remote) =>
_lock.run(() => _syncRemoteAssetsToDb(remote));
Future<bool> syncRemoteAssetsToDb(
FutureOr<List<Asset>?> Function() loadAssets,
) =>
_lock.run(() => _syncRemoteAssetsToDb(loadAssets));
/// Syncs remote albums to the database
/// returns `true` if there were any changes
@ -97,19 +98,72 @@ class SyncService {
.toList();
}
/// Syncs a new asset to the db. Returns `true` if successful
Future<bool> syncNewAssetToDb(Asset newAsset) =>
_lock.run(() => _syncNewAssetToDb(newAsset));
// private methods:
/// Syncs a new asset to the db. Returns `true` if successful
Future<bool> _syncNewAssetToDb(Asset newAsset) async {
final List<Asset> inDb = await _db.assets
.where()
.localIdDeviceIdEqualTo(newAsset.localId, newAsset.deviceId)
.findAll();
Asset? match;
if (inDb.length == 1) {
// exactly one match: trivial case
match = inDb.first;
} else if (inDb.length > 1) {
// TODO instead of this heuristics: match by checksum once available
for (Asset a in inDb) {
if (a.ownerId == newAsset.ownerId &&
a.fileModifiedAt == newAsset.fileModifiedAt) {
assert(match == null);
match = a;
}
}
if (match == null) {
for (Asset a in inDb) {
if (a.ownerId == newAsset.ownerId) {
assert(match == null);
match = a;
}
}
}
}
if (match != null) {
// unify local/remote assets by replacing the
// local-only asset in the DB with a local&remote asset
newAsset.updateFromDb(match);
}
try {
await _db.writeTxn(() => newAsset.put(_db));
} on IsarError catch (e) {
_log.severe("Failed to put new asset into db: $e");
return false;
}
return true;
}
/// Syncs remote assets to the databas
/// returns `true` if there were any changes
Future<bool> _syncRemoteAssetsToDb(List<Asset> remote) async {
Future<bool> _syncRemoteAssetsToDb(
FutureOr<List<Asset>?> Function() loadAssets,
) async {
final List<Asset>? remote = await loadAssets();
if (remote == null) {
return false;
}
final User user = Store.get(StoreKey.currentUser);
final List<Asset> inDb = await _db.assets
.filter()
.ownerIdEqualTo(user.isarId)
.sortByDeviceId()
.thenByLocalId()
.thenByFileModifiedAt()
.findAll();
remote.sort(Asset.compareByDeviceIdLocalId);
remote.sort(Asset.compareByOwnerDeviceLocalIdModified);
final diff = _diffAssets(remote, inDb, remote: true);
if (diff.first.isEmpty && diff.second.isEmpty && diff.third.isEmpty) {
return false;
@ -119,7 +173,7 @@ class SyncService {
await _db.writeTxn(() => _db.assets.deleteAll(idsToDelete));
await _upsertAssetsWithExif(diff.first + diff.second);
} on IsarError catch (e) {
debugPrint(e.toString());
_log.severe("Failed to sync remote assets to db: $e");
}
return true;
}
@ -188,10 +242,15 @@ class SyncService {
if (dto.assetCount != dto.assets.length) {
return false;
}
final assetsInDb =
await album.assets.filter().sortByDeviceId().thenByLocalId().findAll();
final assetsInDb = await album.assets
.filter()
.sortByOwnerId()
.thenByDeviceId()
.thenByLocalId()
.thenByFileModifiedAt()
.findAll();
final List<Asset> assetsOnRemote = dto.getAssets();
assetsOnRemote.sort(Asset.compareByDeviceIdLocalId);
assetsOnRemote.sort(Asset.compareByOwnerDeviceLocalIdModified);
final d = _diffAssets(assetsOnRemote, assetsInDb);
final List<Asset> toAdd = d.first, toUpdate = d.second, toUnlink = d.third;
@ -237,7 +296,7 @@ class SyncService {
await _db.albums.put(album);
});
} on IsarError catch (e) {
debugPrint(e.toString());
_log.severe("Failed to sync remote album to database $e");
}
if (album.shared || dto.shared) {
@ -300,7 +359,7 @@ class SyncService {
assert(ok);
_log.info("Removed local album $album from DB");
} catch (e) {
_log.warning("Failed to remove local album $album from DB");
_log.severe("Failed to remove local album $album from DB");
}
}
@ -331,7 +390,7 @@ class SyncService {
_addAlbumFromDevice(ape, existing, excludedAssets),
onlySecond: (Album a) => _removeAlbumFromDb(a, deleteCandidates),
);
final pair = _handleAssetRemoval(deleteCandidates, existing);
final pair = _handleAssetRemoval(deleteCandidates, existing, remote: false);
if (pair.first.isNotEmpty || pair.second.isNotEmpty) {
await _db.writeTxn(() async {
await _db.assets.deleteAll(pair.first);
@ -366,7 +425,12 @@ class SyncService {
}
// general case, e.g. some assets have been deleted or there are excluded albums on iOS
final inDb = await album.assets.filter().sortByLocalId().findAll();
final inDb = await album.assets
.filter()
.ownerIdEqualTo(Store.get(StoreKey.currentUser).isarId)
.deviceIdEqualTo(Store.get(StoreKey.deviceIdHash))
.sortByLocalId()
.findAll();
final List<Asset> onDevice =
await ape.getAssets(excludedAssets: excludedAssets);
onDevice.sort(Asset.compareByLocalId);
@ -401,7 +465,7 @@ class SyncService {
});
_log.info("Synced changes of local album $ape to DB");
} on IsarError catch (e) {
_log.warning("Failed to update synced album $ape in DB: $e");
_log.severe("Failed to update synced album $ape in DB: $e");
}
return true;
@ -438,7 +502,7 @@ class SyncService {
});
_log.info("Fast synced local album $ape to DB");
} on IsarError catch (e) {
_log.warning("Failed to fast sync local album $ape to DB: $e");
_log.severe("Failed to fast sync local album $ape to DB: $e");
return false;
}
@ -470,7 +534,7 @@ class SyncService {
await _db.writeTxn(() => _db.albums.store(a));
_log.info("Added a new local album to DB: $ape");
} on IsarError catch (e) {
_log.warning("Failed to add new local album $ape to DB: $e");
_log.severe("Failed to add new local album $ape to DB: $e");
}
}
@ -487,15 +551,19 @@ class SyncService {
assets,
(q, Asset e) => q.localIdDeviceIdEqualTo(e.localId, e.deviceId),
)
.sortByDeviceId()
.sortByOwnerId()
.thenByDeviceId()
.thenByLocalId()
.thenByFileModifiedAt()
.findAll();
assets.sort(Asset.compareByDeviceIdLocalId);
assets.sort(Asset.compareByOwnerDeviceLocalIdModified);
final List<Asset> existing = [], toUpsert = [];
diffSortedListsSync(
inDb,
assets,
compare: Asset.compareByDeviceIdLocalId,
// do not compare by modified date because for some assets dates differ on
// client and server, thus never reaching "both" case below
compare: Asset.compareByOwnerDeviceLocalId,
both: (Asset a, Asset b) {
if ((a.isLocal || !b.isLocal) &&
(a.isRemote || !b.isRemote) &&
@ -541,7 +609,7 @@ Triple<List<Asset>, List<Asset>, List<Asset>> _diffAssets(
List<Asset> assets,
List<Asset> inDb, {
bool? remote,
int Function(Asset, Asset) compare = Asset.compareByDeviceIdLocalId,
int Function(Asset, Asset) compare = Asset.compareByOwnerDeviceLocalId,
}) {
final List<Asset> toAdd = [];
final List<Asset> toUpdate = [];
@ -582,15 +650,20 @@ Triple<List<Asset>, List<Asset>, List<Asset>> _diffAssets(
/// returns a tuple (toDelete toUpdate) when assets are to be deleted
Pair<List<int>, List<Asset>> _handleAssetRemoval(
List<Asset> deleteCandidates,
List<Asset> existing,
) {
List<Asset> existing, {
bool? remote,
}) {
if (deleteCandidates.isEmpty) {
return const Pair([], []);
}
deleteCandidates.sort(Asset.compareById);
existing.sort(Asset.compareById);
final triple =
_diffAssets(existing, deleteCandidates, compare: Asset.compareById);
final triple = _diffAssets(
existing,
deleteCandidates,
compare: Asset.compareById,
remote: remote,
);
return Pair(triple.third.map((e) => e.id).toList(), triple.second);
}

View File

@ -3,12 +3,17 @@ import 'dart:async';
/// Async mutex to guarantee actions are performed sequentially and do not interleave
class AsyncMutex {
Future _running = Future.value(null);
int _enqueued = 0;
get enqueued => _enqueued;
/// Execute [operation] exclusively, after any currently running operations.
/// Returns a [Future] with the result of the [operation].
Future<T> run<T>(Future<T> Function() operation) {
final completer = Completer<T>();
_enqueued++;
_running.whenComplete(() {
_enqueued--;
completer.complete(Future<T>.sync(operation));
});
return _running = completer.future;

14
mobile/lib/utils/db.dart Normal file
View File

@ -0,0 +1,14 @@
import 'package:immich_mobile/shared/models/album.dart';
import 'package:immich_mobile/shared/models/asset.dart';
import 'package:immich_mobile/shared/models/exif_info.dart';
import 'package:immich_mobile/shared/models/store.dart';
import 'package:isar/isar.dart';
Future<void> clearAssetsAndAlbums(Isar db) async {
await Store.delete(StoreKey.assetETag);
await db.writeTxn(() async {
await db.assets.clear();
await db.exifInfos.clear();
await db.albums.clear();
});
}

View File

@ -15,6 +15,7 @@ import 'package:immich_mobile/modules/settings/services/app_settings.service.dar
import 'package:immich_mobile/shared/models/immich_logger_message.model.dart';
import 'package:immich_mobile/shared/models/store.dart';
import 'package:immich_mobile/shared/services/asset_cache.service.dart';
import 'package:immich_mobile/utils/db.dart';
import 'package:isar/isar.dart';
Future<void> migrateHiveToStoreIfNecessary() async {
@ -53,7 +54,6 @@ Future<void> _migrateLoginInfoBox(Box<HiveSavedLoginInfo> box) async {
}
Future<void> _migrateHiveUserInfoBox(Box box) async {
await _migrateKey(box, userIdKey, StoreKey.userRemoteId);
await _migrateKey(box, assetEtagKey, StoreKey.assetETag);
if (Store.tryGet(StoreKey.deviceId) == null) {
await _migrateKey(box, deviceIdKey, StoreKey.deviceId);
@ -143,3 +143,16 @@ Future<void> migrateJsonCacheIfNecessary() async {
await SharedAlbumCacheService().invalidate();
await AssetCacheService().invalidate();
}
Future<void> migrateDatabaseIfNeeded(Isar db) async {
final int version = Store.get(StoreKey.version, 1);
switch (version) {
case 1:
await _migrateV1ToV2(db);
}
}
Future<void> _migrateV1ToV2(Isar db) async {
await clearAssetsAndAlbums(db);
await Store.put(StoreKey.version, 2);
}

View File

@ -20,6 +20,7 @@ void main() {
fileModifiedAt: date,
updatedAt: date,
durationInSeconds: 0,
type: AssetType.image,
fileName: '',
isFavorite: false,
isLocal: false,

View File

@ -0,0 +1,41 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:immich_mobile/utils/async_mutex.dart';
void main() {
group('Test AsyncMutex grouped', () {
test('test ordered execution', () async {
AsyncMutex lock = AsyncMutex();
List<int> events = [];
expect(0, lock.enqueued);
lock.run(
() => Future.delayed(
const Duration(milliseconds: 10),
() => events.add(1),
),
);
expect(1, lock.enqueued);
lock.run(
() => Future.delayed(
const Duration(milliseconds: 3),
() => events.add(2),
),
);
expect(2, lock.enqueued);
lock.run(
() => Future.delayed(
const Duration(milliseconds: 1),
() => events.add(3),
),
);
expect(3, lock.enqueued);
await lock.run(
() => Future.delayed(
const Duration(milliseconds: 10),
() => events.add(4),
),
);
expect(0, lock.enqueued);
expect(events, [1, 2, 3, 4]);
});
});
}

View File

@ -23,6 +23,7 @@ Asset _getTestAsset(int id, bool favorite) {
updatedAt: DateTime.now(),
isLocal: false,
durationInSeconds: 0,
type: AssetType.image,
fileName: '',
isFavorite: favorite,
);

View File

@ -0,0 +1,143 @@
import 'package:flutter/widgets.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:immich_mobile/shared/models/album.dart';
import 'package:immich_mobile/shared/models/asset.dart';
import 'package:immich_mobile/shared/models/exif_info.dart';
import 'package:immich_mobile/shared/models/logger_message.model.dart';
import 'package:immich_mobile/shared/models/store.dart';
import 'package:immich_mobile/shared/models/user.dart';
import 'package:immich_mobile/shared/services/immich_logger.service.dart';
import 'package:immich_mobile/shared/services/sync.service.dart';
import 'package:isar/isar.dart';
void main() {
Asset makeAsset({
required String localId,
String? remoteId,
int deviceId = 1,
int ownerId = 590700560494856554, // hash of "1"
bool isLocal = false,
}) {
final DateTime date = DateTime(2000);
return Asset(
localId: localId,
remoteId: remoteId,
deviceId: deviceId,
ownerId: ownerId,
fileCreatedAt: date,
fileModifiedAt: date,
updatedAt: date,
durationInSeconds: 0,
type: AssetType.image,
fileName: localId,
isFavorite: false,
isLocal: isLocal,
);
}
Isar loadDb() {
return Isar.openSync(
[
ExifInfoSchema,
AssetSchema,
AlbumSchema,
UserSchema,
StoreValueSchema,
LoggerMessageSchema
],
maxSizeMiB: 256,
);
}
group('Test SyncService grouped', () {
late final Isar db;
setUpAll(() async {
WidgetsFlutterBinding.ensureInitialized();
await Isar.initializeIsarCore(download: true);
db = loadDb();
ImmichLogger();
db.writeTxnSync(() => db.clearSync());
Store.init(db);
await Store.put(
StoreKey.currentUser,
User(
id: "1",
updatedAt: DateTime.now(),
email: "a@b.c",
firstName: "first",
lastName: "last",
isAdmin: false,
),
);
});
final List<Asset> initialAssets = [
makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
makeAsset(localId: "1", remoteId: "1-1", isLocal: true),
makeAsset(localId: "2", isLocal: true),
makeAsset(localId: "3", isLocal: true),
];
setUp(() {
db.writeTxnSync(() {
db.assets.clearSync();
db.assets.putAllSync(initialAssets);
});
});
test('test inserting existing assets', () async {
SyncService s = SyncService(db);
final List<Asset> remoteAssets = [
makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
makeAsset(localId: "1", remoteId: "1-1"),
];
expect(db.assets.countSync(), 5);
final bool c1 = await s.syncRemoteAssetsToDb(() => remoteAssets);
expect(c1, false);
expect(db.assets.countSync(), 5);
});
test('test inserting new assets', () async {
SyncService s = SyncService(db);
final List<Asset> remoteAssets = [
makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
makeAsset(localId: "1", remoteId: "1-1"),
makeAsset(localId: "2", remoteId: "1-2"),
makeAsset(localId: "4", remoteId: "1-4"),
makeAsset(localId: "1", remoteId: "3-1", deviceId: 3),
];
expect(db.assets.countSync(), 5);
final bool c1 = await s.syncRemoteAssetsToDb(() => remoteAssets);
expect(c1, true);
expect(db.assets.countSync(), 7);
});
test('test syncing duplicate assets', () async {
SyncService s = SyncService(db);
final List<Asset> remoteAssets = [
makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
makeAsset(localId: "1", remoteId: "1-1"),
makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
makeAsset(localId: "1", remoteId: "2-1b", deviceId: 2),
makeAsset(localId: "1", remoteId: "2-1c", deviceId: 2),
makeAsset(localId: "1", remoteId: "2-1d", deviceId: 2),
];
expect(db.assets.countSync(), 5);
final bool c1 = await s.syncRemoteAssetsToDb(() => remoteAssets);
expect(c1, true);
expect(db.assets.countSync(), 8);
final bool c2 = await s.syncRemoteAssetsToDb(() => remoteAssets);
expect(c2, false);
expect(db.assets.countSync(), 8);
remoteAssets.removeAt(4);
final bool c3 = await s.syncRemoteAssetsToDb(() => remoteAssets);
expect(c3, true);
expect(db.assets.countSync(), 7);
remoteAssets.add(makeAsset(localId: "1", remoteId: "2-1e", deviceId: 2));
remoteAssets.add(makeAsset(localId: "2", remoteId: "2-2", deviceId: 2));
final bool c4 = await s.syncRemoteAssetsToDb(() => remoteAssets);
expect(c4, true);
expect(db.assets.countSync(), 9);
});
});
}