self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-25 10:43:13 +02:00
Code Issues Releases Activity

feat(Android): find & delete corrupt asset backups (#2963)

Browse Source 
* feat(mobile): find & delete corrupt asset backups

* show backup fix only for advanced troubleshooting
This commit is contained in:
Fynn Petersen-Frey 2023-06-27 19:25:00 +02:00 committed by GitHub
parent 4d3ce0a65e
commit de42ebf3d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 385 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

232
mobile/lib/modules/backup/services/backup_verification.service.dart Normal file
View File

@ -0,0 +1,232 @@
import 'dart:async';
import 'dart:typed_data';
import 'package:collection/collection.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/services.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/shared/models/asset.dart';
import 'package:immich_mobile/shared/models/exif_info.dart';
import 'package:immich_mobile/shared/models/store.dart';
import 'package:immich_mobile/shared/providers/db.provider.dart';
import 'package:immich_mobile/shared/services/api.service.dart';
import 'package:immich_mobile/utils/diff.dart';
import 'package:isar/isar.dart';
import 'package:photo_manager/photo_manager.dart' show PhotoManager;
/// Finds duplicates originating from missing EXIF information
class BackupVerificationService {
final Isar _db;
BackupVerificationService(this._db);
/// Returns at most [limit] assets that were backed up without exif
Future<List<Asset>> findWronglyBackedUpAssets({int limit = 100}) async {
final owner = Store.get(StoreKey.currentUser).isarId;
final List<Asset> onlyLocal = await _db.assets
.where()
.remoteIdIsNull()
.filter()
.ownerIdEqualTo(owner)
.localIdIsNotNull()
.findAll();
final List<Asset> remoteMatches = await _getMatches(
_db.assets.where().localIdIsNull().filter().remoteIdIsNotNull(),
owner,
onlyLocal,
limit,
);
final List<Asset> localMatches = await _getMatches(
_db.assets.where().remoteIdIsNull().filter().localIdIsNotNull(),
owner,
remoteMatches,
limit,
);
final List<Asset> deleteCandidates = [], originals = [];
await diffSortedLists(
remoteMatches,
localMatches,
compare: (a, b) => a.fileName.compareTo(b.fileName),
both: (a, b) async {
a.exifInfo = await _db.exifInfos.get(a.id);
deleteCandidates.add(a);
originals.add(b);
return false;
},
onlyFirst: (a) {},
onlySecond: (b) {},
);
final isolateToken = ServicesBinding.rootIsolateToken!;
final List<Asset> toDelete;
if (deleteCandidates.length > 10) {
// performs 2 checks in parallel for a nice speedup
final half = deleteCandidates.length ~/ 2;
final lower = compute(
_computeSaveToDelete,
(
deleteCandidates: deleteCandidates.slice(0, half),
originals: originals.slice(0, half),
auth: Store.get(StoreKey.accessToken),
endpoint: Store.get(StoreKey.serverEndpoint),
rootIsolateToken: isolateToken,
),
);
final upper = compute(
_computeSaveToDelete,
(
deleteCandidates: deleteCandidates.slice(half),
originals: originals.slice(half),
auth: Store.get(StoreKey.accessToken),
endpoint: Store.get(StoreKey.serverEndpoint),
rootIsolateToken: isolateToken,
),
);
toDelete = await lower + await upper;
} else {
toDelete = await compute(
_computeSaveToDelete,
(
deleteCandidates: deleteCandidates,
originals: originals,
auth: Store.get(StoreKey.accessToken),
endpoint: Store.get(StoreKey.serverEndpoint),
rootIsolateToken: isolateToken,
),
);
}
return toDelete;
}
static Future<List<Asset>> _computeSaveToDelete(
({
List<Asset> deleteCandidates,
List<Asset> originals,
String auth,
String endpoint,
RootIsolateToken rootIsolateToken,
}) tuple,
) async {
assert(tuple.deleteCandidates.length == tuple.originals.length);
final List<Asset> result = [];
BackgroundIsolateBinaryMessenger.ensureInitialized(tuple.rootIsolateToken);
await PhotoManager.setIgnorePermissionCheck(true);
final ApiService apiService = ApiService();
apiService.setEndpoint(tuple.endpoint);
apiService.setAccessToken(tuple.auth);
for (int i = 0; i < tuple.deleteCandidates.length; i++) {
if (await _compareAssets(
tuple.deleteCandidates[i],
tuple.originals[i],
apiService,
)) {
result.add(tuple.deleteCandidates[i]);
}
}
return result;
}
static Future<bool> _compareAssets(
Asset remote,
Asset local,
ApiService apiService,
) async {
if (remote.checksum == local.checksum) return false;
ExifInfo? exif = remote.exifInfo;
if (exif != null && exif.lat != null) return false;
if (exif == null || exif.fileSize == null) {
final dto = await apiService.assetApi.getAssetById(remote.remoteId!);
if (dto != null && dto.exifInfo != null) {
exif = ExifInfo.fromDto(dto.exifInfo!);
}
}
final file = await local.local!.originFile;
if (exif != null && file != null && exif.fileSize != null) {
final origSize = await file.length();
if (exif.fileSize! == origSize || exif.fileSize! != origSize) {
final latLng = await local.local!.latlngAsync();
if (exif.lat == null &&
latLng.latitude != null &&
(remote.fileCreatedAt.isAtSameMomentAs(local.fileCreatedAt) ||
remote.fileModifiedAt.isAtSameMomentAs(local.fileModifiedAt) ||
_sameExceptTimeZone(
remote.fileCreatedAt,
local.fileCreatedAt,
))) {
if (remote.type == AssetType.video) {
// it's very unlikely that a video of same length, filesize, name
// and date is wrong match. Cannot easily compare videos anyway
return true;
}
// for images: make sure they are pixel-wise identical
// (skip first few KBs containing metadata)
final Uint64List localImage =
_fakeDecodeImg(local, await file.readAsBytes());
final res = await apiService.assetApi
.downloadFileWithHttpInfo(remote.remoteId!);
final Uint64List remoteImage = _fakeDecodeImg(remote, res.bodyBytes);
final eq = const ListEquality().equals(remoteImage, localImage);
return eq;
}
}
}
return false;
}
static Uint64List _fakeDecodeImg(Asset asset, Uint8List bytes) {
const headerLength = 131072; // assume header is at most 128 KB
final start = bytes.length < headerLength * 2
? (bytes.length ~/ (4 * 8)) * 8
: headerLength;
return bytes.buffer.asUint64List(start);
}
static Future<List<Asset>> _getMatches(
QueryBuilder<Asset, Asset, QAfterFilterCondition> query,
int ownerId,
List<Asset> assets,
int limit,
) =>
query
.ownerIdEqualTo(ownerId)
.anyOf(
assets,
(q, Asset a) => q
.fileNameEqualTo(a.fileName)
.and()
.durationInSecondsEqualTo(a.durationInSeconds)
.and()
.fileCreatedAtBetween(
a.fileCreatedAt.subtract(const Duration(hours: 12)),
a.fileCreatedAt.add(const Duration(hours: 12)),
)
.and()
.not()
.checksumEqualTo(a.checksum),
)
.sortByFileName()
.thenByFileCreatedAt()
.thenByFileModifiedAt()
.limit(limit)
.findAll();
static bool _sameExceptTimeZone(DateTime a, DateTime b) {
final ms = a.isAfter(b)
? a.millisecondsSinceEpoch - b.millisecondsSinceEpoch
: b.millisecondsSinceEpoch - a.microsecondsSinceEpoch;
final x = ms / (1000 * 60 * 30);
final y = ms ~/ (1000 * 60 * 30);
return y.toDouble() == x && y < 24;
}
}
final backupVerificationServiceProvider = Provider(
(ref) => BackupVerificationService(
ref.watch(dbProvider),
),
);

1
mobile/lib/modules/backup/ui/current_backup_asset_info_box.dart
View File

@ -138,6 +138,7 @@ class CurrentUploadingAssetInfoBox extends HookConsumerWidget {
return FutureBuilder<Uint8List?>(
future: buildAssetThumbnail(),
builder: (context, thumbnail) => ListTile(
isThreeLine: true,
leading: AnimatedCrossFade(
alignment: Alignment.centerLeft,
firstChild: GestureDetector(

114
mobile/lib/modules/backup/views/backup_controller_page.dart
View File

@ -1,6 +1,7 @@
import 'dart:io';
import 'package:auto_route/auto_route.dart';
import 'package:connectivity_plus/connectivity_plus.dart';
import 'package:easy_localization/easy_localization.dart';
import 'package:flutter/material.dart';
import 'package:flutter_hooks/flutter_hooks.dart';
@ -8,15 +9,23 @@ import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/modules/backup/background_service/background.service.dart';
import 'package:immich_mobile/modules/backup/providers/error_backup_list.provider.dart';
import 'package:immich_mobile/modules/backup/providers/ios_background_settings.provider.dart';
import 'package:immich_mobile/modules/backup/services/backup_verification.service.dart';
import 'package:immich_mobile/modules/backup/ui/current_backup_asset_info_box.dart';
import 'package:immich_mobile/modules/backup/ui/ios_debug_info_tile.dart';
import 'package:immich_mobile/modules/backup/models/backup_state.model.dart';
import 'package:immich_mobile/modules/backup/providers/backup.provider.dart';
import 'package:immich_mobile/modules/settings/providers/app_settings.provider.dart';
import 'package:immich_mobile/modules/settings/services/app_settings.service.dart';
import 'package:immich_mobile/routing/router.dart';
import 'package:immich_mobile/shared/models/asset.dart';
import 'package:immich_mobile/shared/providers/asset.provider.dart';
import 'package:immich_mobile/shared/providers/websocket.provider.dart';
import 'package:immich_mobile/modules/backup/ui/backup_info_card.dart';
import 'package:immich_mobile/shared/ui/confirm_dialog.dart';
import 'package:immich_mobile/shared/ui/immich_toast.dart';
import 'package:permission_handler/permission_handler.dart';
import 'package:url_launcher/url_launcher.dart';
import 'package:wakelock/wakelock.dart';
class BackupControllerPage extends HookConsumerWidget {
const BackupControllerPage({Key? key}) : super(key: key);
@ -25,6 +34,9 @@ class BackupControllerPage extends HookConsumerWidget {
Widget build(BuildContext context, WidgetRef ref) {
BackUpState backupState = ref.watch(backupProvider);
final settings = ref.watch(iOSBackgroundSettingsProvider.notifier).settings;
final settingsService = ref.watch(appSettingsServiceProvider);
final showBackupFix = Platform.isAndroid &&
settingsService.getSetting(AppSettingsEnum.advancedTroubleshooting);
final appRefreshDisabled =
Platform.isIOS && settings?.appRefreshEnabled != true;
@ -37,6 +49,7 @@ class BackupControllerPage extends HookConsumerWidget {
? false
: true;
var isDarkMode = Theme.of(context).brightness == Brightness.dark;
final checkInProgress = useState(false);
useEffect(
() {
@ -59,6 +72,104 @@ class BackupControllerPage extends HookConsumerWidget {
[],
);
Future<void> performDeletion(List<Asset> assets) async {
try {
checkInProgress.value = true;
ImmichToast.show(
context: context,
msg: "Deleting ${assets.length} assets on the server...",
);
await ref.read(assetProvider.notifier).deleteAssets(assets);
ImmichToast.show(
context: context,
msg: "Deleted ${assets.length} assets on the server. "
"You can now start a manual backup",
toastType: ToastType.success,
);
} finally {
checkInProgress.value = false;
}
}
void performBackupCheck() async {
try {
checkInProgress.value = true;
if (backupState.allUniqueAssets.length >
backupState.selectedAlbumsBackupAssetsIds.length) {
ImmichToast.show(
context: context,
msg: "Backup all assets before starting this check!",
toastType: ToastType.error,
);
return;
}
final connection = await Connectivity().checkConnectivity();
if (connection != ConnectivityResult.wifi) {
ImmichToast.show(
context: context,
msg: "Make sure to be connected to unmetered Wi-Fi",
toastType: ToastType.error,
);
return;
}
Wakelock.enable();
const limit = 100;
final toDelete = await ref
.read(backupVerificationServiceProvider)
.findWronglyBackedUpAssets(limit: limit);
if (toDelete.isEmpty) {
ImmichToast.show(
context: context,
msg: "Did not find any corrupt asset backups!",
toastType: ToastType.success,
);
} else {
await showDialog(
context: context,
builder: (context) => ConfirmDialog(
onOk: () => performDeletion(toDelete),
title: "Corrupt backups!",
ok: "Delete",
content:
"Found ${toDelete.length} (max $limit at once) corrupt asset backups. "
"Run the check again to find more.\n"
"Do you want to delete the corrupt asset backups now?",
),
);
}
} finally {
Wakelock.disable();
checkInProgress.value = false;
}
}
Widget buildCheckCorruptBackups() {
return ListTile(
leading: Icon(
Icons.warning_rounded,
color: Theme.of(context).primaryColor,
),
title: const Text(
"Check for corrupt asset backups",
style: TextStyle(fontWeight: FontWeight.bold, fontSize: 14),
),
isThreeLine: true,
subtitle: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
const Text("Run this check only over Wi-Fi and once all assets "
"have been backed-up. The procedure might take a few minutes."),
ElevatedButton(
onPressed: checkInProgress.value ? null : performBackupCheck,
child: checkInProgress.value
? const CircularProgressIndicator()
: const Text("Perform check"),
),
],
),
);
}
Widget buildStorageInformation() {
return ListTile(
leading: Icon(
@ -69,6 +180,7 @@ class BackupControllerPage extends HookConsumerWidget {
"backup_controller_page_server_storage",
style: TextStyle(fontWeight: FontWeight.bold, fontSize: 14),
).tr(),
isThreeLine: true,
subtitle: Padding(
padding: const EdgeInsets.only(top: 8.0),
child: Column(
@ -648,6 +760,8 @@ class BackupControllerPage extends HookConsumerWidget {
: buildBackgroundBackupController())
: buildBackgroundBackupController(),
),
if (showBackupFix) const Divider(),
if (showBackupFix) buildCheckCorruptBackups(),
const Divider(),
buildStorageInformation(),
const Divider(),

8
mobile/lib/shared/providers/asset.provider.dart
View File

@ -75,7 +75,7 @@ class AssetNotifier extends StateNotifier<bool> {
await _syncService.syncNewAssetToDb(newAsset);
}
Future<void> deleteAssets(Set<Asset> deleteAssets) async {
Future<void> deleteAssets(Iterable<Asset> deleteAssets) async {
_deleteInProgress = true;
state = true;
try {
@ -94,7 +94,9 @@ class AssetNotifier extends StateNotifier<bool> {
}
}
Future<List<String>> _deleteLocalAssets(Set<Asset> assetsToDelete) async {
Future<List<String>> _deleteLocalAssets(
Iterable<Asset> assetsToDelete,
) async {
final List<String> local =
assetsToDelete.where((a) => a.isLocal).map((a) => a.localId!).toList();
// Delete asset from device
@ -109,7 +111,7 @@ class AssetNotifier extends StateNotifier<bool> {
}
Future<Iterable<String>> _deleteRemoteAssets(
Set<Asset> assetsToDelete,
Iterable<Asset> assetsToDelete,
) async {
final Iterable<Asset> remote = assetsToDelete.where((e) => e.isRemote);
final List<DeleteAssetResponseDto> deleteAssetResult =

32
mobile/pubspec.lock
View File

@ -225,6 +225,22 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.17.1"
connectivity_plus:
dependency: "direct main"
description:
name: connectivity_plus
sha256: "8599ae9edca5ff96163fca3e36f8e481ea917d1e71cdad912c084b5579913f34"
url: "https://pub.dev"
source: hosted
version: "4.0.1"
connectivity_plus_platform_interface:
dependency: transitive
description:
name: connectivity_plus_platform_interface
sha256: cf1d1c28f4416f8c654d7dc3cd638ec586076255d407cef3ddbdaf178272a71a
url: "https://pub.dev"
source: hosted
version: "1.2.4"
convert:
dependency: transitive
description:
@ -281,6 +297,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.1.0"
dbus:
dependency: transitive
description:
name: dbus
sha256: "6f07cba3f7b3448d42d015bfd3d53fe12e5b36da2423f23838efc1d5fb31a263"
url: "https://pub.dev"
source: hosted
version: "0.7.8"
device_info_plus:
dependency: "direct main"
description:
@ -748,6 +772,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.0.0"
nm:
dependency: transitive
description:
name: nm
sha256: "2c9aae4127bdc8993206464fcc063611e0e36e72018696cd9631023a31b24254"
url: "https://pub.dev"
source: hosted
version: "0.5.0"
octo_image:
dependency: transitive
description:

1
mobile/pubspec.yaml
View File

@ -46,6 +46,7 @@ dependencies:
isar_flutter_libs: *isar_version # contains Isar Core
permission_handler: ^10.2.0
device_info_plus: ^8.1.0
connectivity_plus: ^4.0.1
crypto: ^3.0.3 # TODO remove once native crypto is used on iOS
wakelock: ^0.6.2