feat(server): Automatic watching of library folders (#6192)

* feat: initial watch support

* allow offline files

* chore: ignore query errors when resetting e2e db

* revert db query

* add savepoint

* guard the user query

* chore: openapi and db migration

* wip

* support multiple libraries

* fix tests

* wip

* can now cleanup chokidar watchers

* fix unit tests

* add library watch queue

* add missing init from merge

* wip

* can now filter file extensions

* remove watch api from non job client

* Fix e2e test

* watch library with updated import path and exclusion pattern

* add library watch frontend ui

* case sensitive watching extensions

* can auto watch libraries

* move watcher e2e tests to separate file

* don't watch libraries from a queue

* use event emitters

* shorten e2e test timeout

* refactor chokidar code to filesystem provider

* expose chokidar parameters to config file

* fix storage mock

* set default config for library watching

* add fs provider mocks

* cleanup

* add more unit tests for watcher

* chore: fix format + sql

* add more tests

* move unwatch feature back to library service

* add file event unit tests

* chore: formatting

* add documentation

* fix e2e tests

* chore: fix e2e tests

* fix library updating

* test cleanup

* fix typo

* cleanup

* fixing as per pr comments

* reduce library watch config file

* update storage config and mocks

* move negative event tests to unit tests

* fix library watcher e2e

* make watch configuration global

* remove the feature flag

* refactor watcher teardown

* fix microservices init

* centralize asset scan job queue

* improve docs

* add more tests

* chore: open api

* initialize app service

* fix docs

* fix library watch feature flag

* Update docs/docs/features/libraries.md

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* fix: import right app service

* don't be truthy

* fix test speling

* stricter library update tests

* move fs watcher mock to external file

* subscribe to config changes

* docker does not need polling

* make library watch() private

* feat: add configuration ui

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

mobile/openapi/.openapi-generator/FILES

@@ -143,6 +143,7 @@ doc/SystemConfigFFmpegDto.md
 doc/SystemConfigJobDto.md
 doc/SystemConfigLibraryDto.md
 doc/SystemConfigLibraryScanDto.md
+doc/SystemConfigLibraryWatchDto.md
 doc/SystemConfigLoggingDto.md
 doc/SystemConfigMachineLearningDto.md
 doc/SystemConfigMapDto.md
@@ -333,6 +334,7 @@ lib/model/system_config_f_fmpeg_dto.dart
 lib/model/system_config_job_dto.dart
 lib/model/system_config_library_dto.dart
 lib/model/system_config_library_scan_dto.dart
+lib/model/system_config_library_watch_dto.dart
 lib/model/system_config_logging_dto.dart
 lib/model/system_config_machine_learning_dto.dart
 lib/model/system_config_map_dto.dart
@@ -508,6 +510,7 @@ test/system_config_f_fmpeg_dto_test.dart
 test/system_config_job_dto_test.dart
 test/system_config_library_dto_test.dart
 test/system_config_library_scan_dto_test.dart
+test/system_config_library_watch_dto_test.dart
 test/system_config_logging_dto_test.dart
 test/system_config_machine_learning_dto_test.dart
 test/system_config_map_dto_test.dart

mobile/openapi/README.md

@@ -341,6 +341,7 @@ Class | Method | HTTP request | Description
  - [SystemConfigJobDto](doc//SystemConfigJobDto.md)
  - [SystemConfigLibraryDto](doc//SystemConfigLibraryDto.md)
  - [SystemConfigLibraryScanDto](doc//SystemConfigLibraryScanDto.md)
+ - [SystemConfigLibraryWatchDto](doc//SystemConfigLibraryWatchDto.md)
  - [SystemConfigLoggingDto](doc//SystemConfigLoggingDto.md)
  - [SystemConfigMachineLearningDto](doc//SystemConfigMachineLearningDto.md)
  - [SystemConfigMapDto](doc//SystemConfigMapDto.md)

mobile/openapi/doc/CreateLibraryDto.md

@@ -11,6 +11,7 @@ Name | Type | Description | Notes
 **exclusionPatterns** | **List<String>** |  | [optional] [default to const []]
 **importPaths** | **List<String>** |  | [optional] [default to const []]
 **isVisible** | **bool** |  | [optional] 
+**isWatched** | **bool** |  | [optional] 
 **name** | **String** |  | [optional] 
 **type** | [**LibraryType**](LibraryType.md) |  | 
 

mobile/openapi/doc/SystemConfigLibraryDto.md

@@ -9,6 +9,7 @@ import 'package:openapi/api.dart';
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **scan** | [**SystemConfigLibraryScanDto**](SystemConfigLibraryScanDto.md) |  | 
+**watch** | [**SystemConfigLibraryWatchDto**](SystemConfigLibraryWatchDto.md) |  | 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 

mobile/openapi/doc/SystemConfigLibraryWatchDto.md

@@ -0,0 +1,17 @@
+# openapi.model.SystemConfigLibraryWatchDto
+
+## Load the model package
+```dart
+import 'package:openapi/api.dart';
+```
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**enabled** | **bool** |  | 
+**interval** | **int** |  | 
+**usePolling** | **bool** |  | 
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

mobile/openapi/lib/api.dart

@@ -173,6 +173,7 @@ part 'model/system_config_f_fmpeg_dto.dart';
 part 'model/system_config_job_dto.dart';
 part 'model/system_config_library_dto.dart';
 part 'model/system_config_library_scan_dto.dart';
+part 'model/system_config_library_watch_dto.dart';
 part 'model/system_config_logging_dto.dart';
 part 'model/system_config_machine_learning_dto.dart';
 part 'model/system_config_map_dto.dart';

mobile/openapi/lib/api_client.dart

@@ -428,6 +428,8 @@ class ApiClient {
           return SystemConfigLibraryDto.fromJson(value);
         case 'SystemConfigLibraryScanDto':
           return SystemConfigLibraryScanDto.fromJson(value);
+        case 'SystemConfigLibraryWatchDto':
+          return SystemConfigLibraryWatchDto.fromJson(value);
         case 'SystemConfigLoggingDto':
           return SystemConfigLoggingDto.fromJson(value);
         case 'SystemConfigMachineLearningDto':

mobile/openapi/lib/model/create_library_dto.dart

@@ -16,6 +16,7 @@ class CreateLibraryDto {
     this.exclusionPatterns = const [],
     this.importPaths = const [],
     this.isVisible,
+    this.isWatched,
     this.name,
     required this.type,
   });
@@ -32,6 +33,14 @@ class CreateLibraryDto {
   ///
   bool? isVisible;
 
+  ///
+  /// Please note: This property should have been non-nullable! Since the specification file
+  /// does not include a default value (using the "default:" property), however, the generated
+  /// source code must fall back to having a nullable type.
+  /// Consider adding a "default:" property in the specification file to hide this note.
+  ///
+  bool? isWatched;
+
   ///
   /// Please note: This property should have been non-nullable! Since the specification file
   /// does not include a default value (using the "default:" property), however, the generated
@@ -47,6 +56,7 @@ class CreateLibraryDto {
     _deepEquality.equals(other.exclusionPatterns, exclusionPatterns) &&
     _deepEquality.equals(other.importPaths, importPaths) &&
     other.isVisible == isVisible &&
+    other.isWatched == isWatched &&
     other.name == name &&
     other.type == type;
 
@@ -56,11 +66,12 @@ class CreateLibraryDto {
     (exclusionPatterns.hashCode) +
     (importPaths.hashCode) +
     (isVisible == null ? 0 : isVisible!.hashCode) +
+    (isWatched == null ? 0 : isWatched!.hashCode) +
     (name == null ? 0 : name!.hashCode) +
     (type.hashCode);
 
   @override
-  String toString() => 'CreateLibraryDto[exclusionPatterns=$exclusionPatterns, importPaths=$importPaths, isVisible=$isVisible, name=$name, type=$type]';
+  String toString() => 'CreateLibraryDto[exclusionPatterns=$exclusionPatterns, importPaths=$importPaths, isVisible=$isVisible, isWatched=$isWatched, name=$name, type=$type]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
@@ -71,6 +82,11 @@ class CreateLibraryDto {
     } else {
     //  json[r'isVisible'] = null;
     }
+    if (this.isWatched != null) {
+      json[r'isWatched'] = this.isWatched;
+    } else {
+    //  json[r'isWatched'] = null;
+    }
     if (this.name != null) {
       json[r'name'] = this.name;
     } else {
@@ -95,6 +111,7 @@ class CreateLibraryDto {
             ? (json[r'importPaths'] as Iterable).cast<String>().toList(growable: false)
             : const [],
         isVisible: mapValueOfType<bool>(json, r'isVisible'),
+        isWatched: mapValueOfType<bool>(json, r'isWatched'),
         name: mapValueOfType<String>(json, r'name'),
         type: LibraryType.fromJson(json[r'type'])!,
       );

mobile/openapi/lib/model/system_config_library_dto.dart

@@ -14,25 +14,31 @@ class SystemConfigLibraryDto {
   /// Returns a new [SystemConfigLibraryDto] instance.
   SystemConfigLibraryDto({
     required this.scan,
+    required this.watch,
   });
 
   SystemConfigLibraryScanDto scan;
 
+  SystemConfigLibraryWatchDto watch;
+
   @override
   bool operator ==(Object other) => identical(this, other) || other is SystemConfigLibraryDto &&
-    other.scan == scan;
+    other.scan == scan &&
+    other.watch == watch;
 
   @override
   int get hashCode =>
     // ignore: unnecessary_parenthesis
-    (scan.hashCode);
+    (scan.hashCode) +
+    (watch.hashCode);
 
   @override
-  String toString() => 'SystemConfigLibraryDto[scan=$scan]';
+  String toString() => 'SystemConfigLibraryDto[scan=$scan, watch=$watch]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
       json[r'scan'] = this.scan;
+      json[r'watch'] = this.watch;
     return json;
   }
 
@@ -45,6 +51,7 @@ class SystemConfigLibraryDto {
 
       return SystemConfigLibraryDto(
         scan: SystemConfigLibraryScanDto.fromJson(json[r'scan'])!,
+        watch: SystemConfigLibraryWatchDto.fromJson(json[r'watch'])!,
       );
     }
     return null;
@@ -93,6 +100,7 @@ class SystemConfigLibraryDto {
   /// The list of required keys that must be present in a JSON.
   static const requiredKeys = <String>{
     'scan',
+    'watch',
   };
 }
 

mobile/openapi/lib/model/system_config_library_watch_dto.dart

@@ -0,0 +1,114 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class SystemConfigLibraryWatchDto {
+  /// Returns a new [SystemConfigLibraryWatchDto] instance.
+  SystemConfigLibraryWatchDto({
+    required this.enabled,
+    required this.interval,
+    required this.usePolling,
+  });
+
+  bool enabled;
+
+  int interval;
+
+  bool usePolling;
+
+  @override
+  bool operator ==(Object other) => identical(this, other) || other is SystemConfigLibraryWatchDto &&
+    other.enabled == enabled &&
+    other.interval == interval &&
+    other.usePolling == usePolling;
+
+  @override
+  int get hashCode =>
+    // ignore: unnecessary_parenthesis
+    (enabled.hashCode) +
+    (interval.hashCode) +
+    (usePolling.hashCode);
+
+  @override
+  String toString() => 'SystemConfigLibraryWatchDto[enabled=$enabled, interval=$interval, usePolling=$usePolling]';
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+      json[r'enabled'] = this.enabled;
+      json[r'interval'] = this.interval;
+      json[r'usePolling'] = this.usePolling;
+    return json;
+  }
+
+  /// Returns a new [SystemConfigLibraryWatchDto] instance and imports its values from
+  /// [value] if it's a [Map], null otherwise.
+  // ignore: prefer_constructors_over_static_methods
+  static SystemConfigLibraryWatchDto? fromJson(dynamic value) {
+    if (value is Map) {
+      final json = value.cast<String, dynamic>();
+
+      return SystemConfigLibraryWatchDto(
+        enabled: mapValueOfType<bool>(json, r'enabled')!,
+        interval: mapValueOfType<int>(json, r'interval')!,
+        usePolling: mapValueOfType<bool>(json, r'usePolling')!,
+      );
+    }
+    return null;
+  }
+
+  static List<SystemConfigLibraryWatchDto> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <SystemConfigLibraryWatchDto>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = SystemConfigLibraryWatchDto.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+
+  static Map<String, SystemConfigLibraryWatchDto> mapFromJson(dynamic json) {
+    final map = <String, SystemConfigLibraryWatchDto>{};
+    if (json is Map && json.isNotEmpty) {
+      json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+      for (final entry in json.entries) {
+        final value = SystemConfigLibraryWatchDto.fromJson(entry.value);
+        if (value != null) {
+          map[entry.key] = value;
+        }
+      }
+    }
+    return map;
+  }
+
+  // maps a json object with a list of SystemConfigLibraryWatchDto-objects as value to a dart map
+  static Map<String, List<SystemConfigLibraryWatchDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
+    final map = <String, List<SystemConfigLibraryWatchDto>>{};
+    if (json is Map && json.isNotEmpty) {
+      // ignore: parameter_assignments
+      json = json.cast<String, dynamic>();
+      for (final entry in json.entries) {
+        map[entry.key] = SystemConfigLibraryWatchDto.listFromJson(entry.value, growable: growable,);
+      }
+    }
+    return map;
+  }
+
+  /// The list of required keys that must be present in a JSON.
+  static const requiredKeys = <String>{
+    'enabled',
+    'interval',
+    'usePolling',
+  };
+}
+

mobile/openapi/test/create_library_dto_test.dart

@@ -31,6 +31,11 @@ void main() {
       // TODO
     });
 
+    // bool isWatched
+    test('to test the property `isWatched`', () async {
+      // TODO
+    });
+
     // String name
     test('to test the property `name`', () async {
       // TODO

mobile/openapi/test/system_config_library_dto_test.dart

@@ -21,6 +21,11 @@ void main() {
       // TODO
     });
 
+    // SystemConfigLibraryWatchDto watch
+    test('to test the property `watch`', () async {
+      // TODO
+    });
+
 
   });
 

mobile/openapi/test/system_config_library_watch_dto_test.dart

@@ -0,0 +1,37 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.12
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+import 'package:openapi/api.dart';
+import 'package:test/test.dart';
+
+// tests for SystemConfigLibraryWatchDto
+void main() {
+  // final instance = SystemConfigLibraryWatchDto();
+
+  group('test SystemConfigLibraryWatchDto', () {
+    // bool enabled
+    test('to test the property `enabled`', () async {
+      // TODO
+    });
+
+    // int interval
+    test('to test the property `interval`', () async {
+      // TODO
+    });
+
+    // bool usePolling
+    test('to test the property `usePolling`', () async {
+      // TODO
+    });
+
+
+  });
+
+}