feat(server): near-duplicate detection (#8228)

* duplicate detection job, entity, config * queueing * job panel, update api * use embedding in db instead of fetching * disable concurrency * only queue visible assets * handle multiple duplicateIds * update concurrent queue check * add provider * add web placeholder, server endpoint, migration, various fixes * update sql * select embedding by default * rename variable * simplify * remove separate entity, handle re-running with different threshold, set default back to 0.02 * fix tests * add tests * add index to entity * formatting * update asset mock * fix `upsertJobStatus` signature * update sql * formatting * default to 0.03 * optimize clustering * use asset's `duplicateId` if present * update sql * update tests * expose admin setting * refactor * formatting * skip if ml is disabled * debug trash e2e * remove from web * remove from sidebar * test if ml is disabled * update sql * separate duplicate detection from clip in config, disable by default for now * fix doc * lower minimum `maxDistance` * update api * Add and Use Duplicate Detection Feature Flag (#9364) * Add Duplicate Detection Flag * Use Duplicate Detection Flag * Attempt Fixes for Failing Checks * lower minimum `maxDistance` * fix tests --------- Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> * chore: fixes and additions after rebase * chore: update api (remove new Role enum) * fix: left join smart search so getAll works without machine learning * test: trash e2e go back to checking length of assets is zero * chore: regen api after rebase * test: fix tests after rebase * redundant join --------- Co-authored-by: Nicholas Flamy <30300649+NicholasFlamy@users.noreply.github.com> Co-authored-by: Zack Pollard <zackpollard@ymail.com> Co-authored-by: Zack Pollard <zack@futo.org>
2025-07-04 05:50:38 +02:00 · 2024-05-16 13:08:37 -04:00
parent 673e97e71d
commit 64636c0618
61 changed files with 1254 additions and 61 deletions
--- a/mobile/openapi/lib/model/duplicate_detection_config.dart
+++ b/mobile/openapi/lib/model/duplicate_detection_config.dart
@ -0,0 +1,108 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.18
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class DuplicateDetectionConfig {
+  /// Returns a new [DuplicateDetectionConfig] instance.
+  DuplicateDetectionConfig({
+    required this.enabled,
+    required this.maxDistance,
+  });
+
+  bool enabled;
+
+  /// Minimum value: 0.001
+  /// Maximum value: 0.1
+  double maxDistance;
+
+  @override
+  bool operator ==(Object other) => identical(this, other) || other is DuplicateDetectionConfig &&
+    other.enabled == enabled &&
+    other.maxDistance == maxDistance;
+
+  @override
+  int get hashCode =>
+    // ignore: unnecessary_parenthesis
+    (enabled.hashCode) +
+    (maxDistance.hashCode);
+
+  @override
+  String toString() => 'DuplicateDetectionConfig[enabled=$enabled, maxDistance=$maxDistance]';
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+      json[r'enabled'] = this.enabled;
+      json[r'maxDistance'] = this.maxDistance;
+    return json;
+  }
+
+  /// Returns a new [DuplicateDetectionConfig] instance and imports its values from
+  /// [value] if it's a [Map], null otherwise.
+  // ignore: prefer_constructors_over_static_methods
+  static DuplicateDetectionConfig? fromJson(dynamic value) {
+    if (value is Map) {
+      final json = value.cast<String, dynamic>();
+
+      return DuplicateDetectionConfig(
+        enabled: mapValueOfType<bool>(json, r'enabled')!,
+        maxDistance: mapValueOfType<double>(json, r'maxDistance')!,
+      );
+    }
+    return null;
+  }
+
+  static List<DuplicateDetectionConfig> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <DuplicateDetectionConfig>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = DuplicateDetectionConfig.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+
+  static Map<String, DuplicateDetectionConfig> mapFromJson(dynamic json) {
+    final map = <String, DuplicateDetectionConfig>{};
+    if (json is Map && json.isNotEmpty) {
+      json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+      for (final entry in json.entries) {
+        final value = DuplicateDetectionConfig.fromJson(entry.value);
+        if (value != null) {
+          map[entry.key] = value;
+        }
+      }
+    }
+    return map;
+  }
+
+  // maps a json object with a list of DuplicateDetectionConfig-objects as value to a dart map
+  static Map<String, List<DuplicateDetectionConfig>> mapListFromJson(dynamic json, {bool growable = false,}) {
+    final map = <String, List<DuplicateDetectionConfig>>{};
+    if (json is Map && json.isNotEmpty) {
+      // ignore: parameter_assignments
+      json = json.cast<String, dynamic>();
+      for (final entry in json.entries) {
+        map[entry.key] = DuplicateDetectionConfig.listFromJson(entry.value, growable: growable,);
+      }
+    }
+    return map;
+  }
+
+  /// The list of required keys that must be present in a JSON.
+  static const requiredKeys = <String>{
+    'enabled',
+    'maxDistance',
+  };
+}
+