feature(mobile): hash assets & sync via checksum (#2592)

* compare different sha1 implementations

* remove openssl sha1

* sync via checksum

* hash assets in batches

* hash in background, show spinner in tab

* undo tmp changes

* migrate by clearing assets

* ignore duplicate assets

* error handling

* trigger sync/merge after download and update view

* review feedback improvements

* hash in background isolate on iOS

* rework linking assets with existing from DB

* fine-grained errors on unique index violation

* hash lenth validation

* revert compute in background on iOS

* ignore duplicate assets on device

* fix bug with batching based on accumulated size

---------

Co-authored-by: Fynn Petersen-Frey <zoodyy@users.noreply.github.com>

mobile/android/app/build.gradle

@@ -84,6 +84,7 @@ flutter {
 
 dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
+    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$kotlin_coroutines_version"
     implementation "androidx.work:work-runtime-ktx:$work_version"
     implementation "androidx.concurrent:concurrent-futures:$concurrent_version"
     implementation "com.google.guava:guava:$guava_version"

mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt

@@ -1,10 +1,15 @@
 package app.alextran.immich
 
 import android.content.Context
+import android.util.Log
 import io.flutter.embedding.engine.plugins.FlutterPlugin
 import io.flutter.plugin.common.BinaryMessenger
 import io.flutter.plugin.common.MethodCall
 import io.flutter.plugin.common.MethodChannel
+import java.security.MessageDigest
+import java.io.File
+import java.io.FileInputStream
+import kotlinx.coroutines.*
 
 /**
  * Android plugin for Dart `BackgroundService`
@@ -16,6 +21,7 @@ class BackgroundServicePlugin : FlutterPlugin, MethodChannel.MethodCallHandler {
 
     private var methodChannel: MethodChannel? = null
     private var context: Context? = null
+    private val sha1: MessageDigest = MessageDigest.getInstance("SHA-1")
 
     override fun onAttachedToEngine(binding: FlutterPlugin.FlutterPluginBinding) {
         onAttachedToEngine(binding.applicationContext, binding.binaryMessenger)
@@ -70,9 +76,40 @@ class BackgroundServicePlugin : FlutterPlugin, MethodChannel.MethodCallHandler {
             "isIgnoringBatteryOptimizations" -> {
                 result.success(BackupWorker.isIgnoringBatteryOptimizations(ctx))
             }
+            "digestFiles" -> {
+                val args = call.arguments<ArrayList<String>>()!!
+                GlobalScope.launch(Dispatchers.IO) {
+                    val buf = ByteArray(BUFSIZE)
+                    val digest: MessageDigest = MessageDigest.getInstance("SHA-1")
+                    val hashes = arrayOfNulls<ByteArray>(args.size)
+                    for (i in args.indices) {
+                        val path = args[i]
+                        var len = 0
+                        try {
+                            val file = FileInputStream(path)
+                            try {
+                                while (true) {
+                                    len = file.read(buf)
+                                    if (len != BUFSIZE) break
+                                    digest.update(buf)
+                                }
+                            } finally {
+                                file.close()
+                            }
+                            digest.update(buf, 0, len)
+                            hashes[i] = digest.digest()
+                        } catch (e: Exception) {
+                            // skip this file
+                            Log.w(TAG, "Failed to hash file ${args[i]}: $e")
+                        }
+                    }
+                    result.success(hashes.asList())
+                }
+            }
             else -> result.notImplemented()
         }
     }
 }
 
 private const val TAG = "BackgroundServicePlugin"
+private const val BUFSIZE = 2*1024*1024;

mobile/android/build.gradle

@@ -1,5 +1,6 @@
 buildscript {
     ext.kotlin_version = '1.8.20'
+    ext.kotlin_coroutines_version = '1.7.1'
     ext.work_version = '2.7.1'
     ext.concurrent_version = '1.1.0'
     ext.guava_version = '31.0.1-android'

mobile/lib/main.dart

@@ -19,9 +19,11 @@ import 'package:immich_mobile/modules/settings/providers/notification_permission
 import 'package:immich_mobile/routing/router.dart';
 import 'package:immich_mobile/routing/tab_navigation_observer.dart';
 import 'package:immich_mobile/shared/models/album.dart';
+import 'package:immich_mobile/shared/models/android_device_asset.dart';
 import 'package:immich_mobile/shared/models/asset.dart';
 import 'package:immich_mobile/shared/models/etag.dart';
 import 'package:immich_mobile/shared/models/exif_info.dart';
+import 'package:immich_mobile/shared/models/ios_device_asset.dart';
 import 'package:immich_mobile/shared/models/logger_message.model.dart';
 import 'package:immich_mobile/shared/models/store.dart';
 import 'package:immich_mobile/shared/models/user.dart';
@@ -91,6 +93,7 @@ Future<Isar> loadDb() async {
       DuplicatedAssetSchema,
       LoggerMessageSchema,
       ETagSchema,
+      Platform.isAndroid ? AndroidDeviceAssetSchema : IOSDeviceAssetSchema,
     ],
     directory: dir.path,
     maxSizeMiB: 256,

mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart

@@ -2,6 +2,7 @@ import 'package:easy_localization/easy_localization.dart';
 import 'package:flutter/material.dart';
 import 'package:fluttertoast/fluttertoast.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/modules/album/services/album.service.dart';
 import 'package:immich_mobile/modules/asset_viewer/models/image_viewer_page_state.model.dart';
 import 'package:immich_mobile/modules/asset_viewer/services/image_viewer.service.dart';
 import 'package:immich_mobile/shared/models/asset.dart';
@@ -12,9 +13,13 @@ import 'package:immich_mobile/shared/ui/share_dialog.dart';
 class ImageViewerStateNotifier extends StateNotifier<ImageViewerPageState> {
   final ImageViewerService _imageViewerService;
   final ShareService _shareService;
+  final AlbumService _albumService;
 
-  ImageViewerStateNotifier(this._imageViewerService, this._shareService)
+  ImageViewerStateNotifier(
-      : super(
+    this._imageViewerService,
+    this._shareService,
+    this._albumService,
+  ) : super(
           ImageViewerPageState(
             downloadAssetStatus: DownloadAssetStatus.idle,
           ),
@@ -34,6 +39,7 @@ class ImageViewerStateNotifier extends StateNotifier<ImageViewerPageState> {
         toastType: ToastType.success,
         gravity: ToastGravity.BOTTOM,
       );
+      _albumService.refreshDeviceAlbums();
     } else {
       state = state.copyWith(downloadAssetStatus: DownloadAssetStatus.error);
       ImmichToast.show(
@@ -66,5 +72,6 @@ final imageViewerStateProvider =
   ((ref) => ImageViewerStateNotifier(
         ref.watch(imageViewerServiceProvider),
         ref.watch(shareServiceProvider),
+        ref.watch(albumServiceProvider),
       )),
 );

mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart

@@ -72,15 +72,7 @@ class TopControlAppBar extends HookConsumerWidget {
                     color: Colors.grey[200],
                   ),
           ),
-        if (!asset.isLocal)
+        if (asset.storage == AssetState.remote)
-          IconButton(
-            onPressed: onDownloadPressed,
-            icon: Icon(
-              Icons.cloud_download_outlined,
-              color: Colors.grey[200],
-            ),
-          ),
-        if (asset.storage == AssetState.merged)
           IconButton(
             onPressed: onDownloadPressed,
             icon: Icon(

mobile/lib/modules/asset_viewer/views/gallery_viewer.dart

@@ -287,7 +287,7 @@ class GalleryViewerPage extends HookConsumerWidget {
             isFavorite: asset().isFavorite,
             onMoreInfoPressed: showInfo,
             onFavorite: asset().isRemote ? () => toggleFavorite(asset()) : null,
-            onDownloadPressed: asset().storage == AssetState.local
+            onDownloadPressed: asset().isLocal
                 ? null
                 : () =>
                     ref.watch(imageViewerStateProvider.notifier).downloadAsset(

mobile/lib/modules/backup/background_service/background.service.dart

@@ -132,6 +132,17 @@ class BackgroundService {
     }
   }
 
+  Future<Uint8List?> digestFile(String path) {
+    return _foregroundChannel.invokeMethod<Uint8List>("digestFile", [path]);
+  }
+
+  Future<List<Uint8List?>?> digestFiles(List<String> paths) {
+    return _foregroundChannel.invokeListMethod<Uint8List?>(
+      "digestFiles",
+      paths,
+    );
+  }
+
   /// Updates the notification shown by the background service
   Future<bool?> _updateNotification({
     String? title,

mobile/lib/modules/home/views/home_page.dart

@@ -47,11 +47,11 @@ class HomePage extends HookConsumerWidget {
 
     useEffect(
       () {
-        ref.watch(websocketProvider.notifier).connect();
+        ref.read(websocketProvider.notifier).connect();
-        ref.watch(assetProvider.notifier).getAllAsset();
+        Future(() => ref.read(assetProvider.notifier).getAllAsset());
-        ref.watch(albumProvider.notifier).getAllAlbums();
+        ref.read(albumProvider.notifier).getAllAlbums();
-        ref.watch(sharedAlbumProvider.notifier).getAllSharedAlbums();
+        ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums();
-        ref.watch(serverInfoProvider.notifier).getServerVersion();
+        ref.read(serverInfoProvider.notifier).getServerVersion();
 
         selectionEnabledHook.addListener(() {
           multiselectEnabled.state = selectionEnabledHook.value;
@@ -144,7 +144,7 @@ class HomePage extends HookConsumerWidget {
           );
           if (remoteAssets.isNotEmpty) {
             await ref
-                .watch(assetProvider.notifier)
+                .read(assetProvider.notifier)
                 .toggleArchive(remoteAssets, true);
 
             final assetOrAssets = remoteAssets.length > 1 ? 'assets' : 'asset';
@@ -163,7 +163,7 @@ class HomePage extends HookConsumerWidget {
       void onDelete() async {
         processing.value = true;
         try {
-          await ref.watch(assetProvider.notifier).deleteAssets(selection.value);
+          await ref.read(assetProvider.notifier).deleteAssets(selection.value);
           selectionEnabledHook.value = false;
         } finally {
           processing.value = false;

mobile/lib/shared/models/album.dart

@@ -166,23 +166,10 @@ extension AssetsHelper on IsarCollection<Album> {
   }
 }
 
-extension AssetPathEntityHelper on AssetPathEntity {
-  Future<List<Asset>> getAssets({
-    int start = 0,
-    int end = 0x7fffffffffffffff,
-    Set<String>? excludedAssets,
-  }) async {
-    final assetEntities = await getAssetListRange(start: start, end: end);
-    if (excludedAssets != null) {
-      return assetEntities
-          .where((e) => !excludedAssets.contains(e.id))
-          .map(Asset.local)
-          .toList();
-    }
-    return assetEntities.map(Asset.local).toList();
-  }
-}
-
 extension AlbumResponseDtoHelper on AlbumResponseDto {
   List<Asset> getAssets() => assets.map(Asset.remote).toList();
 }
+
+extension AssetPathEntityHelper on AssetPathEntity {
+  String get eTagKeyAssetCount => "device-album-$id-asset-count";
+}

mobile/lib/shared/models/android_device_asset.dart

@@ -0,0 +1,10 @@
+import 'package:immich_mobile/shared/models/device_asset.dart';
+import 'package:isar/isar.dart';
+
+part 'android_device_asset.g.dart';
+
+@Collection()
+class AndroidDeviceAsset extends DeviceAsset {
+  AndroidDeviceAsset({required this.id, required super.hash});
+  Id id;
+}

mobile/lib/shared/models/asset.dart

@@ -1,3 +1,5 @@
+import 'dart:convert';
+
 import 'package:immich_mobile/shared/models/exif_info.dart';
 import 'package:immich_mobile/shared/models/store.dart';
 import 'package:immich_mobile/utils/hash.dart';
@@ -14,7 +16,7 @@ part 'asset.g.dart';
 class Asset {
   Asset.remote(AssetResponseDto remote)
       : remoteId = remote.id,
-        isLocal = false,
+        checksum = remote.checksum,
         fileCreatedAt = remote.fileCreatedAt,
         fileModifiedAt = remote.fileModifiedAt,
         updatedAt = remote.updatedAt,
@@ -24,23 +26,20 @@ class Asset {
         height = remote.exifInfo?.exifImageHeight?.toInt(),
         width = remote.exifInfo?.exifImageWidth?.toInt(),
         livePhotoVideoId = remote.livePhotoVideoId,
-        localId = remote.deviceAssetId,
-        deviceId = fastHash(remote.deviceId),
         ownerId = fastHash(remote.ownerId),
         exifInfo =
             remote.exifInfo != null ? ExifInfo.fromDto(remote.exifInfo!) : null,
         isFavorite = remote.isFavorite,
         isArchived = remote.isArchived;
 
-  Asset.local(AssetEntity local)
+  Asset.local(AssetEntity local, List<int> hash)
       : localId = local.id,
-        isLocal = true,
+        checksum = base64.encode(hash),
         durationInSeconds = local.duration,
         type = AssetType.values[local.typeInt],
         height = local.height,
         width = local.width,
         fileName = local.title!,
-        deviceId = Store.get(StoreKey.deviceIdHash),
         ownerId = Store.get(StoreKey.currentUser).isarId,
         fileModifiedAt = local.modifiedDateTime,
         updatedAt = local.modifiedDateTime,
@@ -53,13 +52,15 @@ class Asset {
     if (local.latitude != null) {
       exifInfo = ExifInfo(lat: local.latitude, long: local.longitude);
     }
+    _local = local;
+    assert(hash.length == 20, "invalid SHA1 hash");
   }
 
   Asset({
     this.id = Isar.autoIncrement,
+    required this.checksum,
     this.remoteId,
     required this.localId,
-    required this.deviceId,
     required this.ownerId,
     required this.fileCreatedAt,
     required this.fileModifiedAt,
@@ -72,7 +73,6 @@ class Asset {
     this.livePhotoVideoId,
     this.exifInfo,
     required this.isFavorite,
-    required this.isLocal,
     required this.isArchived,
   });
 
@@ -83,7 +83,7 @@ class Asset {
   AssetEntity? get local {
     if (isLocal && _local == null) {
       _local = AssetEntity(
-        id: localId,
+        id: localId!,
         typeInt: isImage ? 1 : 2,
         width: width ?? 0,
         height: height ?? 0,
@@ -98,18 +98,21 @@ class Asset {
 
   Id id = Isar.autoIncrement;
 
+  /// stores the raw SHA1 bytes as a base64 String
+  /// because Isar cannot sort lists of byte arrays
+  @Index(
+    unique: true,
+    replace: false,
+    type: IndexType.hash,
+    composite: [CompositeIndex("ownerId")],
+  )
+  String checksum;
+
   @Index(unique: false, replace: false, type: IndexType.hash)
   String? remoteId;
 
-  @Index(
+  @Index(unique: false, replace: false, type: IndexType.hash)
-    unique: false,
+  String? localId;
-    replace: false,
-    type: IndexType.hash,
-    composite: [CompositeIndex('deviceId')],
-  )
-  String localId;
-
-  int deviceId;
 
   int ownerId;
 
@@ -134,14 +137,15 @@ class Asset {
 
   bool isFavorite;
 
-  /// `true` if this [Asset] is present on the device
-  bool isLocal;
-
   bool isArchived;
 
   @ignore
   ExifInfo? exifInfo;
 
+  /// `true` if this [Asset] is present on the device
+  @ignore
+  bool get isLocal => localId != null;
+
   @ignore
   bool get isInDb => id != Isar.autoIncrement;
 
@@ -175,9 +179,9 @@ class Asset {
   bool operator ==(other) {
     if (other is! Asset) return false;
     return id == other.id &&
+        checksum == other.checksum &&
         remoteId == other.remoteId &&
         localId == other.localId &&
-        deviceId == other.deviceId &&
         ownerId == other.ownerId &&
         fileCreatedAt.isAtSameMomentAs(other.fileCreatedAt) &&
         fileModifiedAt.isAtSameMomentAs(other.fileModifiedAt) &&
@@ -197,9 +201,9 @@ class Asset {
   @ignore
   int get hashCode =>
       id.hashCode ^
+      checksum.hashCode ^
       remoteId.hashCode ^
       localId.hashCode ^
-      deviceId.hashCode ^
       ownerId.hashCode ^
       fileCreatedAt.hashCode ^
       fileModifiedAt.hashCode ^
@@ -217,8 +221,7 @@ class Asset {
   /// Returns `true` if this [Asset] can updated with values from parameter [a]
   bool canUpdate(Asset a) {
     assert(isInDb);
-    assert(localId == a.localId);
+    assert(checksum == a.checksum);
-    assert(deviceId == a.deviceId);
     assert(a.storage != AssetState.merged);
     return a.updatedAt.isAfter(updatedAt) ||
         a.isRemote && !isRemote ||
@@ -239,11 +242,18 @@ class Asset {
       if (a.isRemote) {
         return a._copyWith(
           id: id,
-          isLocal: isLocal,
+          localId: localId,
           width: a.width ?? width,
           height: a.height ?? height,
           exifInfo: a.exifInfo?.copyWith(id: id) ?? exifInfo,
         );
+      } else if (isRemote) {
+        return _copyWith(
+          localId: localId ?? a.localId,
+          width: width ?? a.width,
+          height: height ?? a.height,
+          exifInfo: exifInfo ?? a.exifInfo?.copyWith(id: id),
+        );
       } else {
         return a._copyWith(
           id: id,
@@ -270,7 +280,7 @@ class Asset {
       } else {
         // add only missing values (and set isLocal to true)
         return _copyWith(
-          isLocal: true,
+          localId: localId ?? a.localId,
           width: width ?? a.width,
           height: height ?? a.height,
           exifInfo: exifInfo ?? a.exifInfo?.copyWith(id: id),
@@ -281,9 +291,9 @@ class Asset {
 
   Asset _copyWith({
     Id? id,
+    String? checksum,
     String? remoteId,
     String? localId,
-    int? deviceId,
     int? ownerId,
     DateTime? fileCreatedAt,
     DateTime? fileModifiedAt,
@@ -295,15 +305,14 @@ class Asset {
     String? fileName,
     String? livePhotoVideoId,
     bool? isFavorite,
-    bool? isLocal,
     bool? isArchived,
     ExifInfo? exifInfo,
   }) =>
       Asset(
         id: id ?? this.id,
+        checksum: checksum ?? this.checksum,
         remoteId: remoteId ?? this.remoteId,
         localId: localId ?? this.localId,
-        deviceId: deviceId ?? this.deviceId,
         ownerId: ownerId ?? this.ownerId,
         fileCreatedAt: fileCreatedAt ?? this.fileCreatedAt,
         fileModifiedAt: fileModifiedAt ?? this.fileModifiedAt,
@@ -315,7 +324,6 @@ class Asset {
         fileName: fileName ?? this.fileName,
         livePhotoVideoId: livePhotoVideoId ?? this.livePhotoVideoId,
         isFavorite: isFavorite ?? this.isFavorite,
-        isLocal: isLocal ?? this.isLocal,
         isArchived: isArchived ?? this.isArchived,
         exifInfo: exifInfo ?? this.exifInfo,
       );
@@ -328,39 +336,36 @@ class Asset {
     }
   }
 
-  /// compares assets by [ownerId], [deviceId], [localId]
-  static int compareByOwnerDeviceLocalId(Asset a, Asset b) {
-    final int ownerIdOrder = a.ownerId.compareTo(b.ownerId);
-    if (ownerIdOrder != 0) {
-      return ownerIdOrder;
-    }
-    final int deviceIdOrder = a.deviceId.compareTo(b.deviceId);
-    if (deviceIdOrder != 0) {
-      return deviceIdOrder;
-    }
-    final int localIdOrder = a.localId.compareTo(b.localId);
-    return localIdOrder;
-  }
-
-  /// compares assets by [ownerId], [deviceId], [localId], [fileModifiedAt]
-  static int compareByOwnerDeviceLocalIdModified(Asset a, Asset b) {
-    final int order = compareByOwnerDeviceLocalId(a, b);
-    return order != 0 ? order : a.fileModifiedAt.compareTo(b.fileModifiedAt);
-  }
-
   static int compareById(Asset a, Asset b) => a.id.compareTo(b.id);
 
-  static int compareByLocalId(Asset a, Asset b) =>
+  static int compareByChecksum(Asset a, Asset b) =>
-      a.localId.compareTo(b.localId);
+      a.checksum.compareTo(b.checksum);
+
+  static int compareByOwnerChecksum(Asset a, Asset b) {
+    final int ownerIdOrder = a.ownerId.compareTo(b.ownerId);
+    if (ownerIdOrder != 0) return ownerIdOrder;
+    return compareByChecksum(a, b);
+  }
+
+  static int compareByOwnerChecksumCreatedModified(Asset a, Asset b) {
+    final int ownerIdOrder = a.ownerId.compareTo(b.ownerId);
+    if (ownerIdOrder != 0) return ownerIdOrder;
+    final int checksumOrder = compareByChecksum(a, b);
+    if (checksumOrder != 0) return checksumOrder;
+    final int createdOrder = a.fileCreatedAt.compareTo(b.fileCreatedAt);
+    if (createdOrder != 0) return createdOrder;
+    return a.fileModifiedAt.compareTo(b.fileModifiedAt);
+  }
 
   @override
   String toString() {
     return """
 {
+  "id": ${id == Isar.autoIncrement ? '"N/A"' : id},
   "remoteId": "${remoteId ?? "N/A"}",
-  "localId": "$localId", 
+  "localId": "${localId ?? "N/A"}",
-  "deviceId": "$deviceId", 
+  "checksum": "$checksum",
-  "ownerId": "$ownerId", 
+  "ownerId": $ownerId, 
   "livePhotoVideoId": "${livePhotoVideoId ?? "N/A"}",
   "fileCreatedAt": "$fileCreatedAt",
   "fileModifiedAt": "$fileModifiedAt", 
@@ -369,9 +374,8 @@ class Asset {
   "type": "$type",
   "fileName": "$fileName", 
   "isFavorite": $isFavorite, 
-  "isLocal": $isLocal,
   "isRemote: $isRemote,
-  "storage": $storage,
+  "storage": "$storage",
   "width": ${width ?? "N/A"},
   "height": ${height ?? "N/A"},
   "isArchived": $isArchived
@@ -424,10 +428,6 @@ extension AssetsHelper on IsarCollection<Asset> {
   QueryBuilder<Asset, Asset, QAfterWhereClause> _remote(Iterable<String> ids) =>
       where().anyOf(ids, (q, String e) => q.remoteIdEqualTo(e));
   QueryBuilder<Asset, Asset, QAfterWhereClause> _local(Iterable<String> ids) {
-    return where().anyOf(
+    return where().anyOf(ids, (q, String e) => q.localIdEqualTo(e));
-      ids,
-      (q, String e) =>
-          q.localIdDeviceIdEqualTo(e, Store.get(StoreKey.deviceIdHash)),
-    );
   }
 }

mobile/lib/shared/models/device_asset.dart

@@ -0,0 +1,8 @@
+import 'package:isar/isar.dart';
+
+class DeviceAsset {
+  DeviceAsset({required this.hash});
+
+  @Index(unique: false, type: IndexType.hash)
+  List<byte> hash;
+}

mobile/lib/shared/models/ios_device_asset.dart

@@ -0,0 +1,14 @@
+import 'package:immich_mobile/shared/models/device_asset.dart';
+import 'package:immich_mobile/utils/hash.dart';
+import 'package:isar/isar.dart';
+
+part 'ios_device_asset.g.dart';
+
+@Collection()
+class IOSDeviceAsset extends DeviceAsset {
+  IOSDeviceAsset({required this.id, required super.hash});
+
+  @Index(replace: true, unique: true, type: IndexType.hash)
+  String id;
+  Id get isarId => fastHash(id);
+}

mobile/lib/shared/providers/asset.provider.dart

@@ -18,11 +18,7 @@ import 'package:logging/logging.dart';
 import 'package:openapi/api.dart';
 import 'package:photo_manager/photo_manager.dart';
 
-/// State does not contain archived assets.
+class AssetNotifier extends StateNotifier<bool> {
-/// Use database provider if you want to access the isArchived assets
-class AssetsState {}
-
-class AssetNotifier extends StateNotifier<AssetsState> {
   final AssetService _assetService;
   final AlbumService _albumService;
   final UserService _userService;
@@ -38,7 +34,7 @@ class AssetNotifier extends StateNotifier<AssetsState> {
     this._userService,
     this._syncService,
     this._db,
-  ) : super(AssetsState());
+  ) : super(false);
 
   Future<void> getAllAsset({bool clear = false}) async {
     if (_getAllAssetInProgress || _deleteInProgress) {
@@ -48,14 +44,15 @@ class AssetNotifier extends StateNotifier<AssetsState> {
     final stopwatch = Stopwatch()..start();
     try {
       _getAllAssetInProgress = true;
+      state = true;
       if (clear) {
         await clearAssetsAndAlbums(_db);
         log.info("Manual refresh requested, cleared assets and albums from db");
       }
+      await _userService.refreshUsers();
       final bool newRemote = await _assetService.refreshRemoteAssets();
       final bool newLocal = await _albumService.refreshDeviceAlbums();
       debugPrint("newRemote: $newRemote, newLocal: $newLocal");
-      await _userService.refreshUsers();
       final List<User> partners =
           await _db.users.filter().isPartnerSharedWithEqualTo(true).findAll();
       for (User u in partners) {
@@ -64,6 +61,7 @@ class AssetNotifier extends StateNotifier<AssetsState> {
       log.info("Load assets: ${stopwatch.elapsedMilliseconds}ms");
     } finally {
       _getAllAssetInProgress = false;
+      state = false;
     }
   }
 
@@ -79,6 +77,7 @@ class AssetNotifier extends StateNotifier<AssetsState> {
 
   Future<void> deleteAssets(Set<Asset> deleteAssets) async {
     _deleteInProgress = true;
+    state = true;
     try {
       final localDeleted = await _deleteLocalAssets(deleteAssets);
       final remoteDeleted = await _deleteRemoteAssets(deleteAssets);
@@ -91,24 +90,14 @@ class AssetNotifier extends StateNotifier<AssetsState> {
       }
     } finally {
       _deleteInProgress = false;
+      state = false;
     }
   }
 
   Future<List<String>> _deleteLocalAssets(Set<Asset> assetsToDelete) async {
-    final int deviceId = Store.get(StoreKey.deviceIdHash);
+    final List<String> local =
-    final List<String> local = [];
+        assetsToDelete.where((a) => a.isLocal).map((a) => a.localId!).toList();
     // Delete asset from device
-    for (final Asset asset in assetsToDelete) {
-      if (asset.isLocal) {
-        local.add(asset.localId);
-      } else if (asset.deviceId == deviceId) {
-        // Delete asset on device if it is still present
-        var localAsset = await AssetEntity.fromId(asset.localId);
-        if (localAsset != null) {
-          local.add(localAsset.id);
-        }
-      }
-    }
     if (local.isNotEmpty) {
       try {
         return await PhotoManager.editor.deleteWithIds(local);
@@ -153,7 +142,7 @@ class AssetNotifier extends StateNotifier<AssetsState> {
   }
 }
 
-final assetProvider = StateNotifierProvider<AssetNotifier, AssetsState>((ref) {
+final assetProvider = StateNotifierProvider<AssetNotifier, bool>((ref) {
   return AssetNotifier(
     ref.watch(assetServiceProvider),
     ref.watch(albumServiceProvider),

mobile/lib/shared/services/hash.service.dart

@@ -0,0 +1,175 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:crypto/crypto.dart';
+import 'package:flutter/foundation.dart';
+import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/modules/backup/background_service/background.service.dart';
+import 'package:immich_mobile/shared/models/android_device_asset.dart';
+import 'package:immich_mobile/shared/models/asset.dart';
+import 'package:immich_mobile/shared/models/device_asset.dart';
+import 'package:immich_mobile/shared/models/ios_device_asset.dart';
+import 'package:immich_mobile/shared/providers/db.provider.dart';
+import 'package:immich_mobile/utils/builtin_extensions.dart';
+import 'package:isar/isar.dart';
+import 'package:logging/logging.dart';
+import 'package:photo_manager/photo_manager.dart';
+
+class HashService {
+  HashService(this._db, this._backgroundService);
+  final Isar _db;
+  final BackgroundService _backgroundService;
+  final _log = Logger('HashService');
+
+  /// Returns all assets that were successfully hashed
+  Future<List<Asset>> getHashedAssets(
+    AssetPathEntity album, {
+    int start = 0,
+    int end = 0x7fffffffffffffff,
+    Set<String>? excludedAssets,
+  }) async {
+    final entities = await album.getAssetListRange(start: start, end: end);
+    final filtered = excludedAssets == null
+        ? entities
+        : entities.where((e) => !excludedAssets.contains(e.id)).toList();
+    return _hashAssets(filtered);
+  }
+
+  /// Converts a list of [AssetEntity]s to [Asset]s including only those
+  /// that were successfully hashed. Hashes are looked up in a DB table
+  /// [AndroidDeviceAsset] / [IOSDeviceAsset] by local id. Only missing
+  /// entries are newly hashed and added to the DB table.
+  Future<List<Asset>> _hashAssets(List<AssetEntity> assetEntities) async {
+    const int batchFileCount = 128;
+    const int batchDataSize = 1024 * 1024 * 1024; // 1GB
+
+    final ids = assetEntities
+        .map(Platform.isAndroid ? (a) => a.id.toInt() : (a) => a.id)
+        .toList();
+    final List<DeviceAsset?> hashes = await _lookupHashes(ids);
+    final List<DeviceAsset> toAdd = [];
+    final List<String> toHash = [];
+
+    int bytes = 0;
+
+    for (int i = 0; i < assetEntities.length; i++) {
+      if (hashes[i] != null) {
+        continue;
+      }
+      final file = await assetEntities[i].originFile;
+      if (file == null) {
+        _log.warning(
+          "Failed to get file for asset ${assetEntities[i].id}, skipping",
+        );
+        continue;
+      }
+      bytes += await file.length();
+      toHash.add(file.path);
+      final deviceAsset = Platform.isAndroid
+          ? AndroidDeviceAsset(id: ids[i] as int, hash: const [])
+          : IOSDeviceAsset(id: ids[i] as String, hash: const []);
+      toAdd.add(deviceAsset);
+      hashes[i] = deviceAsset;
+      if (toHash.length == batchFileCount || bytes >= batchDataSize) {
+        await _processBatch(toHash, toAdd);
+        toAdd.clear();
+        toHash.clear();
+        bytes = 0;
+      }
+    }
+    if (toHash.isNotEmpty) {
+      await _processBatch(toHash, toAdd);
+    }
+    return _mapAllHashedAssets(assetEntities, hashes);
+  }
+
+  /// Lookup hashes of assets by their local ID
+  Future<List<DeviceAsset?>> _lookupHashes(List<Object> ids) =>
+      Platform.isAndroid
+          ? _db.androidDeviceAssets.getAll(ids.cast())
+          : _db.iOSDeviceAssets.getAllById(ids.cast());
+
+  /// Processes a batch of files and saves any successfully hashed
+  /// values to the DB table.
+  Future<void> _processBatch(
+    final List<String> toHash,
+    final List<DeviceAsset> toAdd,
+  ) async {
+    final hashes = await _hashFiles(toHash);
+    bool anyNull = false;
+    for (int j = 0; j < hashes.length; j++) {
+      if (hashes[j]?.length == 20) {
+        toAdd[j].hash = hashes[j]!;
+      } else {
+        _log.warning("Failed to hash file ${toHash[j]}, skipping");
+        anyNull = true;
+      }
+    }
+    final validHashes = anyNull
+        ? toAdd.where((e) => e.hash.length == 20).toList(growable: false)
+        : toAdd;
+    await _db.writeTxn(
+      () => Platform.isAndroid
+          ? _db.androidDeviceAssets.putAll(validHashes.cast())
+          : _db.iOSDeviceAssets.putAll(validHashes.cast()),
+    );
+    _log.fine("Hashed ${validHashes.length}/${toHash.length} assets");
+  }
+
+  /// Hashes the given files and returns a list of the same length
+  /// files that could not be hashed have a `null` value
+  Future<List<Uint8List?>> _hashFiles(List<String> paths) async {
+    if (Platform.isAndroid) {
+      final List<Uint8List?>? hashes =
+          await _backgroundService.digestFiles(paths);
+      if (hashes == null) {
+        throw Exception("Hashing ${paths.length} files failed");
+      }
+      return hashes;
+    } else if (Platform.isIOS) {
+      final List<Uint8List?> result = List.filled(paths.length, null);
+      for (int i = 0; i < paths.length; i++) {
+        result[i] = await _hashAssetDart(File(paths[i]));
+      }
+      return result;
+    } else {
+      throw Exception("_hashFiles implementation missing");
+    }
+  }
+
+  /// Hashes a single file using Dart's crypto package
+  Future<Uint8List?> _hashAssetDart(File f) async {
+    late Digest output;
+    final sink = sha1.startChunkedConversion(
+      ChunkedConversionSink<Digest>.withCallback((accumulated) {
+        output = accumulated.first;
+      }),
+    );
+    await for (final chunk in f.openRead()) {
+      sink.add(chunk);
+    }
+    sink.close();
+    return Uint8List.fromList(output.bytes);
+  }
+
+  /// Converts [AssetEntity]s that were successfully hashed to [Asset]s
+  List<Asset> _mapAllHashedAssets(
+    List<AssetEntity> assets,
+    List<DeviceAsset?> hashes,
+  ) {
+    final List<Asset> result = [];
+    for (int i = 0; i < assets.length; i++) {
+      if (hashes[i] != null && hashes[i]!.hash.isNotEmpty) {
+        result.add(Asset.local(assets[i], hashes[i]!.hash));
+      }
+    }
+    return result;
+  }
+}
+
+final hashServiceProvider = Provider(
+  (ref) => HashService(
+    ref.watch(dbProvider),
+    ref.watch(backgroundServiceProvider),
+  ),
+);

mobile/lib/shared/services/sync.service.dart

@@ -4,10 +4,12 @@ import 'package:collection/collection.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/shared/models/album.dart';
 import 'package:immich_mobile/shared/models/asset.dart';
+import 'package:immich_mobile/shared/models/etag.dart';
 import 'package:immich_mobile/shared/models/exif_info.dart';
 import 'package:immich_mobile/shared/models/store.dart';
 import 'package:immich_mobile/shared/models/user.dart';
 import 'package:immich_mobile/shared/providers/db.provider.dart';
+import 'package:immich_mobile/shared/services/hash.service.dart';
 import 'package:immich_mobile/utils/async_mutex.dart';
 import 'package:immich_mobile/utils/builtin_extensions.dart';
 import 'package:immich_mobile/utils/diff.dart';
@@ -16,15 +18,17 @@ import 'package:logging/logging.dart';
 import 'package:openapi/api.dart';
 import 'package:photo_manager/photo_manager.dart';
 
-final syncServiceProvider =
+final syncServiceProvider = Provider(
-    Provider((ref) => SyncService(ref.watch(dbProvider)));
+  (ref) => SyncService(ref.watch(dbProvider), ref.watch(hashServiceProvider)),
+);
 
 class SyncService {
   final Isar _db;
+  final HashService _hashService;
   final AsyncMutex _lock = AsyncMutex();
   final Logger _log = Logger('SyncService');
 
-  SyncService(this._db);
+  SyncService(this._db, this._hashService);
 
   // public methods:
 
@@ -33,6 +37,7 @@ class SyncService {
   Future<bool> syncUsersFromServer(List<User> users) async {
     users.sortBy((u) => u.id);
     final dbUsers = await _db.users.where().sortById().findAll();
+    assert(dbUsers.isSortedBy((u) => u.id), "dbUsers not sorted!");
     final List<int> toDelete = [];
     final List<User> toUpsert = [];
     final changes = diffSortedListsSync(
@@ -108,40 +113,16 @@ class SyncService {
   // private methods:
 
   /// Syncs a new asset to the db. Returns `true` if successful
-  Future<bool> _syncNewAssetToDb(Asset newAsset) async {
+  Future<bool> _syncNewAssetToDb(Asset a) async {
-    final List<Asset> inDb = await _db.assets
+    final Asset? inDb =
-        .where()
+        await _db.assets.getByChecksumOwnerId(a.checksum, a.ownerId);
-        .localIdDeviceIdEqualTo(newAsset.localId, newAsset.deviceId)
+    if (inDb != null) {
-        .findAll();
-    Asset? match;
-    if (inDb.length == 1) {
-      // exactly one match: trivial case
-      match = inDb.first;
-    } else if (inDb.length > 1) {
-      // TODO instead of this heuristics: match by checksum once available
-      for (Asset a in inDb) {
-        if (a.ownerId == newAsset.ownerId &&
-            a.fileModifiedAt.isAtSameMomentAs(newAsset.fileModifiedAt)) {
-          assert(match == null);
-          match = a;
-        }
-      }
-      if (match == null) {
-        for (Asset a in inDb) {
-          if (a.ownerId == newAsset.ownerId) {
-            assert(match == null);
-            match = a;
-          }
-        }
-      }
-    }
-    if (match != null) {
       // unify local/remote assets by replacing the
       // local-only asset in the DB with a local&remote asset
-      newAsset = match.updatedCopy(newAsset);
+      a = inDb.updatedCopy(a);
     }
     try {
-      await _db.writeTxn(() => newAsset.put(_db));
+      await _db.writeTxn(() => a.put(_db));
     } on IsarError catch (e) {
       _log.severe("Failed to put new asset into db: $e");
       return false;
@@ -162,11 +143,11 @@ class SyncService {
     final List<Asset> inDb = await _db.assets
         .filter()
         .ownerIdEqualTo(user.isarId)
-        .sortByDeviceId()
+        .sortByChecksum()
-        .thenByLocalId()
-        .thenByFileModifiedAt()
         .findAll();
-    remote.sort(Asset.compareByOwnerDeviceLocalIdModified);
+    assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!");
+
+    remote.sort(Asset.compareByChecksum);
     final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true);
     if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) {
       return false;
@@ -199,6 +180,7 @@ class SyncService {
       query = baseQuery.owner((q) => q.isarIdEqualTo(me.isarId));
     }
     final List<Album> dbAlbums = await query.sortByRemoteId().findAll();
+    assert(dbAlbums.isSortedBy((e) => e.remoteId!), "dbAlbums not sorted!");
 
     final List<Asset> toDelete = [];
     final List<Asset> existing = [];
@@ -245,16 +227,16 @@ class SyncService {
     if (dto.assetCount != dto.assets.length) {
       return false;
     }
-    final assetsInDb = await album.assets
+    final assetsInDb =
-        .filter()
+        await album.assets.filter().sortByOwnerId().thenByChecksum().findAll();
-        .sortByOwnerId()
+    assert(assetsInDb.isSorted(Asset.compareByOwnerChecksum), "inDb unsorted!");
-        .thenByDeviceId()
-        .thenByLocalId()
-        .thenByFileModifiedAt()
-        .findAll();
     final List<Asset> assetsOnRemote = dto.getAssets();
-    assetsOnRemote.sort(Asset.compareByOwnerDeviceLocalIdModified);
+    assetsOnRemote.sort(Asset.compareByOwnerChecksum);
-    final (toAdd, toUpdate, toUnlink) = _diffAssets(assetsOnRemote, assetsInDb);
+    final (toAdd, toUpdate, toUnlink) = _diffAssets(
+      assetsOnRemote,
+      assetsInDb,
+      compare: Asset.compareByOwnerChecksum,
+    );
 
     // update shared users
     final List<User> sharedUsers = album.sharedUsers.toList(growable: false);
@@ -297,6 +279,7 @@ class SyncService {
         await album.assets.update(link: assetsToLink, unlink: toUnlink.cast());
         await _db.albums.put(album);
       });
+      _log.info("Synced changes of remote album ${album.name} to DB");
     } on IsarError catch (e) {
       _log.severe("Failed to sync remote album to database $e");
     }
@@ -382,10 +365,11 @@ class SyncService {
     Set<String>? excludedAssets,
   ]) async {
     onDevice.sort((a, b) => a.id.compareTo(b.id));
-    final List<Album> inDb =
+    final inDb =
         await _db.albums.where().localIdIsNotNull().sortByLocalId().findAll();
     final List<Asset> deleteCandidates = [];
     final List<Asset> existing = [];
+    assert(inDb.isSorted((a, b) => a.localId!.compareTo(b.localId!)), "sort!");
     final bool anyChanges = await diffSortedLists(
       onDevice,
       inDb,
@@ -447,14 +431,15 @@ class SyncService {
     final inDb = await album.assets
         .filter()
         .ownerIdEqualTo(Store.get(StoreKey.currentUser).isarId)
-        .deviceIdEqualTo(Store.get(StoreKey.deviceIdHash))
+        .sortByChecksum()
-        .sortByLocalId()
         .findAll();
+    assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!");
+    final int assetCountOnDevice = await ape.assetCountAsync;
     final List<Asset> onDevice =
-        await ape.getAssets(excludedAssets: excludedAssets);
+        await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets);
-    onDevice.sort(Asset.compareByLocalId);
+    _removeDuplicates(onDevice);
-    final (toAdd, toUpdate, toDelete) =
+    // _removeDuplicates sorts `onDevice` by checksum
-        _diffAssets(onDevice, inDb, compare: Asset.compareByLocalId);
+    final (toAdd, toUpdate, toDelete) = _diffAssets(onDevice, inDb);
     if (toAdd.isEmpty &&
         toUpdate.isEmpty &&
         toDelete.isEmpty &&
@@ -491,6 +476,9 @@ class SyncService {
         await _db.albums.put(album);
         album.thumbnail.value ??= await album.assets.filter().findFirst();
         await album.thumbnail.save();
+        await _db.eTags.put(
+          ETag(id: ape.eTagKeyAssetCount, value: assetCountOnDevice.toString()),
+        );
       });
       _log.info("Synced changes of local album ${ape.name} to DB");
     } on IsarError catch (e) {
@@ -503,8 +491,13 @@ class SyncService {
   /// fast path for common case: only new assets were added to device album
   /// returns `true` if successfull, else `false`
   Future<bool> _syncDeviceAlbumFast(AssetPathEntity ape, Album album) async {
+    if (!(ape.lastModified ?? DateTime.now()).isAfter(album.modifiedAt)) {
+      return false;
+    }
     final int totalOnDevice = await ape.assetCountAsync;
-    final AssetPathEntity? modified = totalOnDevice > album.assetCount
+    final int lastKnownTotal =
+        (await _db.eTags.getById(ape.eTagKeyAssetCount))?.value?.toInt() ?? 0;
+    final AssetPathEntity? modified = totalOnDevice > lastKnownTotal
         ? await ape.fetchPathProperties(
             filterOptionGroup: FilterOptionGroup(
               updateTimeCond: DateTimeCond(
@@ -517,17 +510,22 @@ class SyncService {
     if (modified == null) {
       return false;
     }
-    final List<Asset> newAssets = await modified.getAssets();
+    final List<Asset> newAssets = await _hashService.getHashedAssets(modified);
-    if (totalOnDevice != album.assets.length + newAssets.length) {
+
+    if (totalOnDevice != lastKnownTotal + newAssets.length) {
       return false;
     }
     album.modifiedAt = ape.lastModified ?? DateTime.now();
+    _removeDuplicates(newAssets);
     final (existingInDb, updated) = await _linkWithExistingFromDb(newAssets);
     try {
       await _db.writeTxn(() async {
         await _db.assets.putAll(updated);
         await album.assets.update(link: existingInDb + updated);
         await _db.albums.put(album);
+        await _db.eTags.put(
+          ETag(id: ape.eTagKeyAssetCount, value: totalOnDevice.toString()),
+        );
       });
       _log.info("Fast synced local album ${ape.name} to DB");
     } on IsarError catch (e) {
@@ -547,7 +545,9 @@ class SyncService {
   ]) async {
     _log.info("Syncing a new local album to DB: ${ape.name}");
     final Album a = Album.local(ape);
-    final assets = await ape.getAssets(excludedAssets: excludedAssets);
+    final assets =
+        await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets);
+    _removeDuplicates(assets);
     final (existingInDb, updated) = await _linkWithExistingFromDb(assets);
     _log.info(
       "${existingInDb.length} assets already existed in DB, to upsert ${updated.length}",
@@ -570,44 +570,29 @@ class SyncService {
   Future<(List<Asset> existing, List<Asset> updated)> _linkWithExistingFromDb(
     List<Asset> assets,
   ) async {
-    if (assets.isEmpty) {
+    if (assets.isEmpty) return ([].cast<Asset>(), [].cast<Asset>());
-      return ([].cast<Asset>(), [].cast<Asset>());
+
-    }
+    final List<Asset?> inDb = await _db.assets.getAllByChecksumOwnerId(
-    final List<Asset> inDb = await _db.assets
+      assets.map((a) => a.checksum).toList(growable: false),
-        .where()
+      assets.map((a) => a.ownerId).toInt64List(),
-        .anyOf(
-          assets,
-          (q, Asset e) => q.localIdDeviceIdEqualTo(e.localId, e.deviceId),
-        )
-        .sortByOwnerId()
-        .thenByDeviceId()
-        .thenByLocalId()
-        .thenByFileModifiedAt()
-        .findAll();
-    assets.sort(Asset.compareByOwnerDeviceLocalIdModified);
-    final List<Asset> existing = [], toUpsert = [];
-    diffSortedListsSync(
-      inDb,
-      assets,
-      // do not compare by modified date because for some assets dates differ on
-      // client and server, thus never reaching "both" case below
-      compare: Asset.compareByOwnerDeviceLocalId,
-      both: (Asset a, Asset b) {
-        if (a.canUpdate(b)) {
-          toUpsert.add(a.updatedCopy(b));
-          return true;
-        } else {
-          existing.add(a);
-          return false;
-        }
-      },
-      onlyFirst: (Asset a) => _log.finer(
-        "_linkWithExistingFromDb encountered asset only in DB: $a",
-        null,
-        StackTrace.current,
-      ),
-      onlySecond: (Asset b) => toUpsert.add(b),
     );
+    assert(inDb.length == assets.length);
+    final List<Asset> existing = [], toUpsert = [];
+    for (int i = 0; i < assets.length; i++) {
+      final Asset? b = inDb[i];
+      if (b == null) {
+        toUpsert.add(assets[i]);
+        continue;
+      }
+      if (b.canUpdate(assets[i])) {
+        final updated = b.updatedCopy(assets[i]);
+        assert(updated.id != Isar.autoIncrement);
+        toUpsert.add(updated);
+      } else {
+        existing.add(b);
+      }
+    }
+    assert(existing.length + toUpsert.length == assets.length);
     return (existing, toUpsert);
   }
 
@@ -627,10 +612,62 @@ class SyncService {
       });
       _log.info("Upserted ${assets.length} assets into the DB");
     } on IsarError catch (e) {
-      _log.warning(
+      _log.severe(
         "Failed to upsert ${assets.length} assets into the DB: ${e.toString()}",
       );
+      // give details on the errors
+      assets.sort(Asset.compareByOwnerChecksum);
+      final inDb = await _db.assets.getAllByChecksumOwnerId(
+        assets.map((e) => e.checksum).toList(growable: false),
+        assets.map((e) => e.ownerId).toInt64List(),
+      );
+      for (int i = 0; i < assets.length; i++) {
+        final Asset a = assets[i];
+        final Asset? b = inDb[i];
+        if (b == null) {
+          if (a.id != Isar.autoIncrement) {
+            _log.warning(
+              "Trying to update an asset that does not exist in DB:\n$a",
+            );
           }
+        } else if (a.id != b.id) {
+          _log.warning(
+            "Trying to insert another asset with the same checksum+owner. In DB:\n$b\nTo insert:\n$a",
+          );
+        }
+      }
+      for (int i = 1; i < assets.length; i++) {
+        if (Asset.compareByOwnerChecksum(assets[i - 1], assets[i]) == 0) {
+          _log.warning(
+            "Trying to insert duplicate assets:\n${assets[i - 1]}\n${assets[i]}",
+          );
+        }
+      }
+    }
+  }
+
+  List<Asset> _removeDuplicates(List<Asset> assets) {
+    final int before = assets.length;
+    assets.sort(Asset.compareByOwnerChecksumCreatedModified);
+    assets.uniqueConsecutive(
+      compare: Asset.compareByOwnerChecksum,
+      onDuplicate: (a, b) =>
+          _log.info("Ignoring duplicate assets on device:\n$a\n$b"),
+    );
+    final int duplicates = before - assets.length;
+    if (duplicates > 0) {
+      _log.warning("Ignored $duplicates duplicate assets on device");
+    }
+    return assets;
+  }
+
+  /// returns `true` if the albums differ on the surface
+  Future<bool> _hasAssetPathEntityChanged(AssetPathEntity a, Album b) async {
+    return a.name != b.name ||
+        a.lastModified == null ||
+        !a.lastModified!.isAtSameMomentAs(b.modifiedAt) ||
+        await a.assetCountAsync !=
+            (await _db.eTags.getById(a.eTagKeyAssetCount))?.value?.toInt();
   }
 }
 
@@ -639,7 +676,7 @@ class SyncService {
   List<Asset> assets,
   List<Asset> inDb, {
   bool? remote,
-  int Function(Asset, Asset) compare = Asset.compareByOwnerDeviceLocalId,
+  int Function(Asset, Asset) compare = Asset.compareByChecksum,
 }) {
   final List<Asset> toAdd = [];
   final List<Asset> toUpdate = [];
@@ -663,7 +700,7 @@ class SyncService {
         }
       } else if (remote == false && a.isRemote) {
         if (a.isLocal) {
-          a.isLocal = false;
+          a.localId = null;
           toUpdate.add(a);
         }
       } else {
@@ -685,9 +722,9 @@ class SyncService {
     return const ([], []);
   }
   deleteCandidates.sort(Asset.compareById);
-  deleteCandidates.uniqueConsecutive((a) => a.id);
+  deleteCandidates.uniqueConsecutive(compare: Asset.compareById);
   existing.sort(Asset.compareById);
-  existing.uniqueConsecutive((a) => a.id);
+  existing.uniqueConsecutive(compare: Asset.compareById);
   final (tooAdd, toUpdate, toRemove) = _diffAssets(
     existing,
     deleteCandidates,
@@ -698,14 +735,6 @@ class SyncService {
   return (toRemove.map((e) => e.id).toList(), toUpdate);
 }
 
-/// returns `true` if the albums differ on the surface
-Future<bool> _hasAssetPathEntityChanged(AssetPathEntity a, Album b) async {
-  return a.name != b.name ||
-      a.lastModified == null ||
-      !a.lastModified!.isAtSameMomentAs(b.modifiedAt) ||
-      await a.assetCountAsync != b.assetCount;
-}
-
 /// returns `true` if the albums differ on the surface
 bool _hasAlbumResponseDtoChanged(AlbumResponseDto dto, Album a) {
   return dto.assetCount != a.assetCount ||

mobile/lib/shared/views/tab_controller_page.dart

@@ -6,12 +6,39 @@ import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/modules/asset_viewer/providers/scroll_notifier.provider.dart';
 import 'package:immich_mobile/modules/home/providers/multiselect.provider.dart';
 import 'package:immich_mobile/routing/router.dart';
+import 'package:immich_mobile/shared/providers/asset.provider.dart';
 
-class TabControllerPage extends ConsumerWidget {
+class TabControllerPage extends HookConsumerWidget {
   const TabControllerPage({Key? key}) : super(key: key);
 
   @override
   Widget build(BuildContext context, WidgetRef ref) {
+    final refreshing = ref.watch(assetProvider);
+
+    Widget buildIcon(Widget icon) {
+      if (!refreshing) return icon;
+      return Stack(
+        alignment: Alignment.center,
+        clipBehavior: Clip.none,
+        children: [
+          icon,
+          Positioned(
+            right: -14,
+            child: SizedBox(
+              height: 12,
+              width: 12,
+              child: CircularProgressIndicator(
+                strokeWidth: 2,
+                valueColor: AlwaysStoppedAnimation<Color>(
+                  Theme.of(context).primaryColor,
+                ),
+              ),
+            ),
+          ),
+        ],
+      );
+    }
+
     navigationRail(TabsRouter tabsRouter) {
       return NavigationRail(
         labelType: NavigationRailLabelType.all,
@@ -83,11 +110,14 @@ class TabControllerPage extends ConsumerWidget {
             icon: const Icon(
               Icons.photo_library_outlined,
             ),
-            selectedIcon: Icon(
+            selectedIcon: buildIcon(
+              Icon(
+                size: 24,
                 Icons.photo_library,
                 color: Theme.of(context).primaryColor,
               ),
             ),
+          ),
           NavigationDestination(
             label: 'tab_controller_nav_search'.tr(),
             icon: const Icon(
@@ -113,10 +143,12 @@ class TabControllerPage extends ConsumerWidget {
             icon: const Icon(
               Icons.photo_album_outlined,
             ),
-            selectedIcon: Icon(
+            selectedIcon: buildIcon(
+              Icon(
                 Icons.photo_album_rounded,
                 color: Theme.of(context).primaryColor,
               ),
+            ),
           )
         ],
       );

mobile/lib/utils/builtin_extensions.dart

@@ -1,3 +1,5 @@
+import 'dart:typed_data';
+
 import 'package:collection/collection.dart';
 
 extension DurationExtension on String {
@@ -22,15 +24,20 @@ extension DurationExtension on String {
 }
 
 extension ListExtension<E> on List<E> {
-  List<E> uniqueConsecutive<T>([T Function(E element)? key]) {
+  List<E> uniqueConsecutive({
-    key ??= (E e) => e as T;
+    int Function(E a, E b)? compare,
+    void Function(E a, E b)? onDuplicate,
+  }) {
+    compare ??= (E a, E b) => a == b ? 0 : 1;
     int i = 1, j = 1;
     for (; i < length; i++) {
-      if (key(this[i]) != key(this[i - 1])) {
+      if (compare(this[i - 1], this[i]) != 0) {
         if (i != j) {
           this[j] = this[i];
         }
         j++;
+      } else if (onDuplicate != null) {
+        onDuplicate(this[i - 1], this[i]);
       }
     }
     length = length == 0 ? 0 : j;
@@ -45,3 +52,11 @@ extension ListExtension<E> on List<E> {
     return ListSlice<E>(this, start, end);
   }
 }
+
+extension IntListExtension on Iterable<int> {
+  Int64List toInt64List() {
+    final list = Int64List(length);
+    list.setAll(0, this);
+    return list;
+  }
+}

mobile/lib/utils/migration.dart

@@ -8,11 +8,13 @@ Future<void> migrateDatabaseIfNeeded(Isar db) async {
   final int version = Store.get(StoreKey.version, 1);
   switch (version) {
     case 1:
-      await _migrateV1ToV2(db);
+      await _migrateTo(db, 2);
+    case 2:
+      await _migrateTo(db, 3);
   }
 }
 
-Future<void> _migrateV1ToV2(Isar db) async {
+Future<void> _migrateTo(Isar db, int version) async {
   await clearAssetsAndAlbums(db);
-  await Store.put(StoreKey.version, 2);
+  await Store.put(StoreKey.version, version);
 }

mobile/pubspec.lock

@@ -242,13 +242,13 @@ packages:
     source: hosted
     version: "0.3.3+4"
   crypto:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: crypto
-      sha256: aa274aa7774f8964e4f4f38cc994db7b6158dd36e9187aaceaddc994b35c6c67
+      sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab
       url: "https://pub.dev"
     source: hosted
-    version: "3.0.2"
+    version: "3.0.3"
   csslib:
     dependency: transitive
     description:
@@ -333,10 +333,10 @@ packages:
     dependency: transitive
     description:
       name: ffi
-      sha256: a38574032c5f1dd06c4aee541789906c12ccaab8ba01446e800d9c5b79c4a978
+      sha256: ed5337a5660c506388a9f012be0288fb38b49020ce2b45fe1f8b8323fe429f99
       url: "https://pub.dev"
     source: hosted
-    version: "2.0.1"
+    version: "2.0.2"
   file:
     dependency: transitive
     description:

mobile/pubspec.yaml

@@ -45,6 +45,7 @@ dependencies:
   isar_flutter_libs: *isar_version # contains Isar Core
   permission_handler: ^10.2.0
   device_info_plus: ^8.1.0
+  crypto: ^3.0.3 # TODO remove once native crypto is used on iOS
 
   openapi:
     path: openapi

mobile/test/asset_grid_data_structure_test.dart

@@ -13,8 +13,8 @@ void main() {
 
     testAssets.add(
       Asset(
+        checksum: "",
         localId: '$i',
-        deviceId: 1,
         ownerId: 1,
         fileCreatedAt: date,
         fileModifiedAt: date,
@@ -23,7 +23,6 @@ void main() {
         type: AssetType.image,
         fileName: '',
         isFavorite: false,
-        isLocal: false,
         isArchived: false,
       ),
     );

mobile/test/builtin_extensions_test.dart

@@ -43,7 +43,12 @@ void main() {
 
     test('withKey', () {
       final a = ["a", "bb", "cc", "ddd"];
-      expect(a.uniqueConsecutive((s) => s.length), ["a", "bb", "ddd"]);
+      expect(
+        a.uniqueConsecutive(
+          compare: (s1, s2) => s1.length.compareTo(s2.length),
+        ),
+        ["a", "bb", "ddd"],
+      );
     });
   });
 }

mobile/test/sync_service_test.dart

@@ -6,32 +6,33 @@ import 'package:immich_mobile/shared/models/exif_info.dart';
 import 'package:immich_mobile/shared/models/logger_message.model.dart';
 import 'package:immich_mobile/shared/models/store.dart';
 import 'package:immich_mobile/shared/models/user.dart';
+import 'package:immich_mobile/shared/services/hash.service.dart';
 import 'package:immich_mobile/shared/services/immich_logger.service.dart';
 import 'package:immich_mobile/shared/services/sync.service.dart';
 import 'package:isar/isar.dart';
+import 'package:mockito/mockito.dart';
 
 void main() {
   Asset makeAsset({
-    required String localId,
+    required String checksum,
+    String? localId,
     String? remoteId,
     int deviceId = 1,
     int ownerId = 590700560494856554, // hash of "1"
-    bool isLocal = false,
   }) {
     final DateTime date = DateTime(2000);
     return Asset(
+      checksum: checksum,
       localId: localId,
       remoteId: remoteId,
-      deviceId: deviceId,
       ownerId: ownerId,
       fileCreatedAt: date,
       fileModifiedAt: date,
       updatedAt: date,
       durationInSeconds: 0,
       type: AssetType.image,
-      fileName: localId,
+      fileName: localId ?? remoteId ?? "",
       isFavorite: false,
-      isLocal: isLocal,
       isArchived: false,
     );
   }
@@ -53,6 +54,7 @@ void main() {
 
   group('Test SyncService grouped', () {
     late final Isar db;
+    final MockHashService hs = MockHashService();
     final owner = User(
       id: "1",
       updatedAt: DateTime.now(),
@@ -71,11 +73,11 @@ void main() {
       await Store.put(StoreKey.currentUser, owner);
     });
     final List<Asset> initialAssets = [
-      makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
+      makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
-      makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
+      makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2),
-      makeAsset(localId: "1", remoteId: "1-1", isLocal: true),
+      makeAsset(checksum: "c", localId: "1", remoteId: "1-1"),
-      makeAsset(localId: "2", isLocal: true),
+      makeAsset(checksum: "d", localId: "2"),
-      makeAsset(localId: "3", isLocal: true),
+      makeAsset(checksum: "e", localId: "3"),
     ];
     setUp(() {
       db.writeTxnSync(() {
@@ -84,11 +86,11 @@ void main() {
       });
     });
     test('test inserting existing assets', () async {
-      SyncService s = SyncService(db);
+      SyncService s = SyncService(db, hs);
       final List<Asset> remoteAssets = [
-        makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
+        makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
-        makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
+        makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2),
-        makeAsset(localId: "1", remoteId: "1-1"),
+        makeAsset(checksum: "c", remoteId: "1-1"),
       ];
       expect(db.assets.countSync(), 5);
       final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
@@ -97,14 +99,14 @@ void main() {
     });
 
     test('test inserting new assets', () async {
-      SyncService s = SyncService(db);
+      SyncService s = SyncService(db, hs);
       final List<Asset> remoteAssets = [
-        makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
+        makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
-        makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
+        makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2),
-        makeAsset(localId: "1", remoteId: "1-1"),
+        makeAsset(checksum: "c", remoteId: "1-1"),
-        makeAsset(localId: "2", remoteId: "1-2"),
+        makeAsset(checksum: "d", remoteId: "1-2"),
-        makeAsset(localId: "4", remoteId: "1-4"),
+        makeAsset(checksum: "f", remoteId: "1-4"),
-        makeAsset(localId: "1", remoteId: "3-1", deviceId: 3),
+        makeAsset(checksum: "g", remoteId: "3-1", deviceId: 3),
       ];
       expect(db.assets.countSync(), 5);
       final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
@@ -113,14 +115,14 @@ void main() {
     });
 
     test('test syncing duplicate assets', () async {
-      SyncService s = SyncService(db);
+      SyncService s = SyncService(db, hs);
       final List<Asset> remoteAssets = [
-        makeAsset(localId: "1", remoteId: "0-1", deviceId: 0),
+        makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0),
-        makeAsset(localId: "1", remoteId: "1-1"),
+        makeAsset(checksum: "b", remoteId: "1-1"),
-        makeAsset(localId: "1", remoteId: "2-1", deviceId: 2),
+        makeAsset(checksum: "c", remoteId: "2-1", deviceId: 2),
-        makeAsset(localId: "1", remoteId: "2-1b", deviceId: 2),
+        makeAsset(checksum: "h", remoteId: "2-1b", deviceId: 2),
-        makeAsset(localId: "1", remoteId: "2-1c", deviceId: 2),
+        makeAsset(checksum: "i", remoteId: "2-1c", deviceId: 2),
-        makeAsset(localId: "1", remoteId: "2-1d", deviceId: 2),
+        makeAsset(checksum: "j", remoteId: "2-1d", deviceId: 2),
       ];
       expect(db.assets.countSync(), 5);
       final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
@@ -133,11 +135,13 @@ void main() {
       final bool c3 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
       expect(c3, true);
       expect(db.assets.countSync(), 7);
-      remoteAssets.add(makeAsset(localId: "1", remoteId: "2-1e", deviceId: 2));
+      remoteAssets.add(makeAsset(checksum: "k", remoteId: "2-1e", deviceId: 2));
-      remoteAssets.add(makeAsset(localId: "2", remoteId: "2-2", deviceId: 2));
+      remoteAssets.add(makeAsset(checksum: "l", remoteId: "2-2", deviceId: 2));
       final bool c4 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets);
       expect(c4, true);
       expect(db.assets.countSync(), 9);
     });
   });
 }
+
+class MockHashService extends Mock implements HashService {}