feat(Android): find & delete corrupt asset backups (#2963)

* feat(mobile): find & delete corrupt asset backups

* show backup fix only for advanced troubleshooting

mobile/lib/modules/backup/services/backup_verification.service.dart

@@ -0,0 +1,232 @@
+import 'dart:async';
+import 'dart:typed_data';
+
+import 'package:collection/collection.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter/services.dart';
+import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/shared/models/asset.dart';
+import 'package:immich_mobile/shared/models/exif_info.dart';
+import 'package:immich_mobile/shared/models/store.dart';
+import 'package:immich_mobile/shared/providers/db.provider.dart';
+import 'package:immich_mobile/shared/services/api.service.dart';
+import 'package:immich_mobile/utils/diff.dart';
+import 'package:isar/isar.dart';
+import 'package:photo_manager/photo_manager.dart' show PhotoManager;
+
+/// Finds duplicates originating from missing EXIF information
+class BackupVerificationService {
+  final Isar _db;
+
+  BackupVerificationService(this._db);
+
+  /// Returns at most [limit] assets that were backed up without exif
+  Future<List<Asset>> findWronglyBackedUpAssets({int limit = 100}) async {
+    final owner = Store.get(StoreKey.currentUser).isarId;
+    final List<Asset> onlyLocal = await _db.assets
+        .where()
+        .remoteIdIsNull()
+        .filter()
+        .ownerIdEqualTo(owner)
+        .localIdIsNotNull()
+        .findAll();
+    final List<Asset> remoteMatches = await _getMatches(
+      _db.assets.where().localIdIsNull().filter().remoteIdIsNotNull(),
+      owner,
+      onlyLocal,
+      limit,
+    );
+    final List<Asset> localMatches = await _getMatches(
+      _db.assets.where().remoteIdIsNull().filter().localIdIsNotNull(),
+      owner,
+      remoteMatches,
+      limit,
+    );
+
+    final List<Asset> deleteCandidates = [], originals = [];
+
+    await diffSortedLists(
+      remoteMatches,
+      localMatches,
+      compare: (a, b) => a.fileName.compareTo(b.fileName),
+      both: (a, b) async {
+        a.exifInfo = await _db.exifInfos.get(a.id);
+        deleteCandidates.add(a);
+        originals.add(b);
+        return false;
+      },
+      onlyFirst: (a) {},
+      onlySecond: (b) {},
+    );
+    final isolateToken = ServicesBinding.rootIsolateToken!;
+    final List<Asset> toDelete;
+    if (deleteCandidates.length > 10) {
+      // performs 2 checks in parallel for a nice speedup
+      final half = deleteCandidates.length ~/ 2;
+      final lower = compute(
+        _computeSaveToDelete,
+        (
+          deleteCandidates: deleteCandidates.slice(0, half),
+          originals: originals.slice(0, half),
+          auth: Store.get(StoreKey.accessToken),
+          endpoint: Store.get(StoreKey.serverEndpoint),
+          rootIsolateToken: isolateToken,
+        ),
+      );
+      final upper = compute(
+        _computeSaveToDelete,
+        (
+          deleteCandidates: deleteCandidates.slice(half),
+          originals: originals.slice(half),
+          auth: Store.get(StoreKey.accessToken),
+          endpoint: Store.get(StoreKey.serverEndpoint),
+          rootIsolateToken: isolateToken,
+        ),
+      );
+      toDelete = await lower + await upper;
+    } else {
+      toDelete = await compute(
+        _computeSaveToDelete,
+        (
+          deleteCandidates: deleteCandidates,
+          originals: originals,
+          auth: Store.get(StoreKey.accessToken),
+          endpoint: Store.get(StoreKey.serverEndpoint),
+          rootIsolateToken: isolateToken,
+        ),
+      );
+    }
+    return toDelete;
+  }
+
+  static Future<List<Asset>> _computeSaveToDelete(
+    ({
+      List<Asset> deleteCandidates,
+      List<Asset> originals,
+      String auth,
+      String endpoint,
+      RootIsolateToken rootIsolateToken,
+    }) tuple,
+  ) async {
+    assert(tuple.deleteCandidates.length == tuple.originals.length);
+    final List<Asset> result = [];
+    BackgroundIsolateBinaryMessenger.ensureInitialized(tuple.rootIsolateToken);
+    await PhotoManager.setIgnorePermissionCheck(true);
+    final ApiService apiService = ApiService();
+    apiService.setEndpoint(tuple.endpoint);
+    apiService.setAccessToken(tuple.auth);
+    for (int i = 0; i < tuple.deleteCandidates.length; i++) {
+      if (await _compareAssets(
+        tuple.deleteCandidates[i],
+        tuple.originals[i],
+        apiService,
+      )) {
+        result.add(tuple.deleteCandidates[i]);
+      }
+    }
+    return result;
+  }
+
+  static Future<bool> _compareAssets(
+    Asset remote,
+    Asset local,
+    ApiService apiService,
+  ) async {
+    if (remote.checksum == local.checksum) return false;
+    ExifInfo? exif = remote.exifInfo;
+    if (exif != null && exif.lat != null) return false;
+    if (exif == null || exif.fileSize == null) {
+      final dto = await apiService.assetApi.getAssetById(remote.remoteId!);
+      if (dto != null && dto.exifInfo != null) {
+        exif = ExifInfo.fromDto(dto.exifInfo!);
+      }
+    }
+    final file = await local.local!.originFile;
+    if (exif != null && file != null && exif.fileSize != null) {
+      final origSize = await file.length();
+      if (exif.fileSize! == origSize || exif.fileSize! != origSize) {
+        final latLng = await local.local!.latlngAsync();
+
+        if (exif.lat == null &&
+            latLng.latitude != null &&
+            (remote.fileCreatedAt.isAtSameMomentAs(local.fileCreatedAt) ||
+                remote.fileModifiedAt.isAtSameMomentAs(local.fileModifiedAt) ||
+                _sameExceptTimeZone(
+                  remote.fileCreatedAt,
+                  local.fileCreatedAt,
+                ))) {
+          if (remote.type == AssetType.video) {
+            // it's very unlikely that a video of same length, filesize, name
+            // and date is wrong match. Cannot easily compare videos anyway
+            return true;
+          }
+
+          // for images: make sure they are pixel-wise identical
+          // (skip first few KBs containing metadata)
+          final Uint64List localImage =
+              _fakeDecodeImg(local, await file.readAsBytes());
+          final res = await apiService.assetApi
+              .downloadFileWithHttpInfo(remote.remoteId!);
+          final Uint64List remoteImage = _fakeDecodeImg(remote, res.bodyBytes);
+
+          final eq = const ListEquality().equals(remoteImage, localImage);
+          return eq;
+        }
+      }
+    }
+
+    return false;
+  }
+
+  static Uint64List _fakeDecodeImg(Asset asset, Uint8List bytes) {
+    const headerLength = 131072; // assume header is at most 128 KB
+    final start = bytes.length < headerLength * 2
+        ? (bytes.length ~/ (4 * 8)) * 8
+        : headerLength;
+    return bytes.buffer.asUint64List(start);
+  }
+
+  static Future<List<Asset>> _getMatches(
+    QueryBuilder<Asset, Asset, QAfterFilterCondition> query,
+    int ownerId,
+    List<Asset> assets,
+    int limit,
+  ) =>
+      query
+          .ownerIdEqualTo(ownerId)
+          .anyOf(
+            assets,
+            (q, Asset a) => q
+                .fileNameEqualTo(a.fileName)
+                .and()
+                .durationInSecondsEqualTo(a.durationInSeconds)
+                .and()
+                .fileCreatedAtBetween(
+                  a.fileCreatedAt.subtract(const Duration(hours: 12)),
+                  a.fileCreatedAt.add(const Duration(hours: 12)),
+                )
+                .and()
+                .not()
+                .checksumEqualTo(a.checksum),
+          )
+          .sortByFileName()
+          .thenByFileCreatedAt()
+          .thenByFileModifiedAt()
+          .limit(limit)
+          .findAll();
+
+  static bool _sameExceptTimeZone(DateTime a, DateTime b) {
+    final ms = a.isAfter(b)
+        ? a.millisecondsSinceEpoch - b.millisecondsSinceEpoch
+        : b.millisecondsSinceEpoch - a.microsecondsSinceEpoch;
+    final x = ms / (1000 * 60 * 30);
+    final y = ms ~/ (1000 * 60 * 30);
+    return y.toDouble() == x && y < 24;
+  }
+}
+
+final backupVerificationServiceProvider = Provider(
+  (ref) => BackupVerificationService(
+    ref.watch(dbProvider),
+  ),
+);

mobile/lib/modules/backup/ui/current_backup_asset_info_box.dart

@@ -138,6 +138,7 @@ class CurrentUploadingAssetInfoBox extends HookConsumerWidget {
     return FutureBuilder<Uint8List?>(
       future: buildAssetThumbnail(),
       builder: (context, thumbnail) => ListTile(
+        isThreeLine: true,
         leading: AnimatedCrossFade(
           alignment: Alignment.centerLeft,
           firstChild: GestureDetector(

mobile/lib/modules/backup/views/backup_controller_page.dart

@@ -1,6 +1,7 @@
 import 'dart:io';
 
 import 'package:auto_route/auto_route.dart';
+import 'package:connectivity_plus/connectivity_plus.dart';
 import 'package:easy_localization/easy_localization.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_hooks/flutter_hooks.dart';
@@ -8,15 +9,23 @@ import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/modules/backup/background_service/background.service.dart';
 import 'package:immich_mobile/modules/backup/providers/error_backup_list.provider.dart';
 import 'package:immich_mobile/modules/backup/providers/ios_background_settings.provider.dart';
+import 'package:immich_mobile/modules/backup/services/backup_verification.service.dart';
 import 'package:immich_mobile/modules/backup/ui/current_backup_asset_info_box.dart';
 import 'package:immich_mobile/modules/backup/ui/ios_debug_info_tile.dart';
 import 'package:immich_mobile/modules/backup/models/backup_state.model.dart';
 import 'package:immich_mobile/modules/backup/providers/backup.provider.dart';
+import 'package:immich_mobile/modules/settings/providers/app_settings.provider.dart';
+import 'package:immich_mobile/modules/settings/services/app_settings.service.dart';
 import 'package:immich_mobile/routing/router.dart';
+import 'package:immich_mobile/shared/models/asset.dart';
+import 'package:immich_mobile/shared/providers/asset.provider.dart';
 import 'package:immich_mobile/shared/providers/websocket.provider.dart';
 import 'package:immich_mobile/modules/backup/ui/backup_info_card.dart';
+import 'package:immich_mobile/shared/ui/confirm_dialog.dart';
+import 'package:immich_mobile/shared/ui/immich_toast.dart';
 import 'package:permission_handler/permission_handler.dart';
 import 'package:url_launcher/url_launcher.dart';
+import 'package:wakelock/wakelock.dart';
 
 class BackupControllerPage extends HookConsumerWidget {
   const BackupControllerPage({Key? key}) : super(key: key);
@@ -25,6 +34,9 @@ class BackupControllerPage extends HookConsumerWidget {
   Widget build(BuildContext context, WidgetRef ref) {
     BackUpState backupState = ref.watch(backupProvider);
     final settings = ref.watch(iOSBackgroundSettingsProvider.notifier).settings;
+    final settingsService = ref.watch(appSettingsServiceProvider);
+    final showBackupFix = Platform.isAndroid &&
+        settingsService.getSetting(AppSettingsEnum.advancedTroubleshooting);
 
     final appRefreshDisabled =
         Platform.isIOS && settings?.appRefreshEnabled != true;
@@ -37,6 +49,7 @@ class BackupControllerPage extends HookConsumerWidget {
         ? false
         : true;
     var isDarkMode = Theme.of(context).brightness == Brightness.dark;
+    final checkInProgress = useState(false);
 
     useEffect(
       () {
@@ -59,6 +72,104 @@ class BackupControllerPage extends HookConsumerWidget {
       [],
     );
 
+    Future<void> performDeletion(List<Asset> assets) async {
+      try {
+        checkInProgress.value = true;
+        ImmichToast.show(
+          context: context,
+          msg: "Deleting ${assets.length} assets on the server...",
+        );
+        await ref.read(assetProvider.notifier).deleteAssets(assets);
+        ImmichToast.show(
+          context: context,
+          msg: "Deleted ${assets.length} assets on the server. "
+              "You can now start a manual backup",
+          toastType: ToastType.success,
+        );
+      } finally {
+        checkInProgress.value = false;
+      }
+    }
+
+    void performBackupCheck() async {
+      try {
+        checkInProgress.value = true;
+        if (backupState.allUniqueAssets.length >
+            backupState.selectedAlbumsBackupAssetsIds.length) {
+          ImmichToast.show(
+            context: context,
+            msg: "Backup all assets before starting this check!",
+            toastType: ToastType.error,
+          );
+          return;
+        }
+        final connection = await Connectivity().checkConnectivity();
+        if (connection != ConnectivityResult.wifi) {
+          ImmichToast.show(
+            context: context,
+            msg: "Make sure to be connected to unmetered Wi-Fi",
+            toastType: ToastType.error,
+          );
+          return;
+        }
+        Wakelock.enable();
+        const limit = 100;
+        final toDelete = await ref
+            .read(backupVerificationServiceProvider)
+            .findWronglyBackedUpAssets(limit: limit);
+        if (toDelete.isEmpty) {
+          ImmichToast.show(
+            context: context,
+            msg: "Did not find any corrupt asset backups!",
+            toastType: ToastType.success,
+          );
+        } else {
+          await showDialog(
+            context: context,
+            builder: (context) => ConfirmDialog(
+              onOk: () => performDeletion(toDelete),
+              title: "Corrupt backups!",
+              ok: "Delete",
+              content:
+                  "Found ${toDelete.length} (max $limit at once) corrupt asset backups. "
+                  "Run the check again to find more.\n"
+                  "Do you want to delete the corrupt asset backups now?",
+            ),
+          );
+        }
+      } finally {
+        Wakelock.disable();
+        checkInProgress.value = false;
+      }
+    }
+
+    Widget buildCheckCorruptBackups() {
+      return ListTile(
+        leading: Icon(
+          Icons.warning_rounded,
+          color: Theme.of(context).primaryColor,
+        ),
+        title: const Text(
+          "Check for corrupt asset backups",
+          style: TextStyle(fontWeight: FontWeight.bold, fontSize: 14),
+        ),
+        isThreeLine: true,
+        subtitle: Column(
+          crossAxisAlignment: CrossAxisAlignment.start,
+          children: [
+            const Text("Run this check only over Wi-Fi and once all assets "
+                "have been backed-up. The procedure might take a few minutes."),
+            ElevatedButton(
+              onPressed: checkInProgress.value ? null : performBackupCheck,
+              child: checkInProgress.value
+                  ? const CircularProgressIndicator()
+                  : const Text("Perform check"),
+            ),
+          ],
+        ),
+      );
+    }
+
     Widget buildStorageInformation() {
       return ListTile(
         leading: Icon(
@@ -69,6 +180,7 @@ class BackupControllerPage extends HookConsumerWidget {
           "backup_controller_page_server_storage",
           style: TextStyle(fontWeight: FontWeight.bold, fontSize: 14),
         ).tr(),
+        isThreeLine: true,
         subtitle: Padding(
           padding: const EdgeInsets.only(top: 8.0),
           child: Column(
@@ -648,6 +760,8 @@ class BackupControllerPage extends HookConsumerWidget {
                       : buildBackgroundBackupController())
                   : buildBackgroundBackupController(),
             ),
+            if (showBackupFix) const Divider(),
+            if (showBackupFix) buildCheckCorruptBackups(),
             const Divider(),
             buildStorageInformation(),
             const Divider(),

mobile/lib/shared/providers/asset.provider.dart

@@ -75,7 +75,7 @@ class AssetNotifier extends StateNotifier<bool> {
     await _syncService.syncNewAssetToDb(newAsset);
   }
 
-  Future<void> deleteAssets(Set<Asset> deleteAssets) async {
+  Future<void> deleteAssets(Iterable<Asset> deleteAssets) async {
     _deleteInProgress = true;
     state = true;
     try {
@@ -94,7 +94,9 @@ class AssetNotifier extends StateNotifier<bool> {
     }
   }
 
-  Future<List<String>> _deleteLocalAssets(Set<Asset> assetsToDelete) async {
+  Future<List<String>> _deleteLocalAssets(
+    Iterable<Asset> assetsToDelete,
+  ) async {
     final List<String> local =
         assetsToDelete.where((a) => a.isLocal).map((a) => a.localId!).toList();
     // Delete asset from device
@@ -109,7 +111,7 @@ class AssetNotifier extends StateNotifier<bool> {
   }
 
   Future<Iterable<String>> _deleteRemoteAssets(
-    Set<Asset> assetsToDelete,
+    Iterable<Asset> assetsToDelete,
   ) async {
     final Iterable<Asset> remote = assetsToDelete.where((e) => e.isRemote);
     final List<DeleteAssetResponseDto> deleteAssetResult =

mobile/pubspec.lock

@@ -225,6 +225,22 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.17.1"
+  connectivity_plus:
+    dependency: "direct main"
+    description:
+      name: connectivity_plus
+      sha256: "8599ae9edca5ff96163fca3e36f8e481ea917d1e71cdad912c084b5579913f34"
+      url: "https://pub.dev"
+    source: hosted
+    version: "4.0.1"
+  connectivity_plus_platform_interface:
+    dependency: transitive
+    description:
+      name: connectivity_plus_platform_interface
+      sha256: cf1d1c28f4416f8c654d7dc3cd638ec586076255d407cef3ddbdaf178272a71a
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.2.4"
   convert:
     dependency: transitive
     description:
@@ -281,6 +297,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.1.0"
+  dbus:
+    dependency: transitive
+    description:
+      name: dbus
+      sha256: "6f07cba3f7b3448d42d015bfd3d53fe12e5b36da2423f23838efc1d5fb31a263"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.7.8"
   device_info_plus:
     dependency: "direct main"
     description:
@@ -748,6 +772,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.0.0"
+  nm:
+    dependency: transitive
+    description:
+      name: nm
+      sha256: "2c9aae4127bdc8993206464fcc063611e0e36e72018696cd9631023a31b24254"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.5.0"
   octo_image:
     dependency: transitive
     description:

mobile/pubspec.yaml

@@ -46,6 +46,7 @@ dependencies:
   isar_flutter_libs: *isar_version # contains Isar Core
   permission_handler: ^10.2.0
   device_info_plus: ^8.1.0
+  connectivity_plus: ^4.0.1
   crypto: ^3.0.3 # TODO remove once native crypto is used on iOS
   wakelock: ^0.6.2