From 47c0dc0d7ee5709fd67e47ee1045489ab5df68bd Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Fri, 11 Jul 2025 17:32:10 -0400 Subject: [PATCH] feat: nightly tasks (#19879) --- e2e/src/api/specs/system-config.e2e-spec.ts | 6 - i18n/en.json | 14 ++ mobile/openapi/README.md | 1 + mobile/openapi/lib/api.dart | 1 + mobile/openapi/lib/api_client.dart | 2 + .../openapi/lib/model/system_config_dto.dart | 10 +- .../system_config_nightly_tasks_dto.dart | 139 ++++++++++++++++++ open-api/immich-openapi-specs.json | 35 +++++ open-api/typescript-sdk/src/fetch-client.ts | 9 ++ server/src/config.ts | 16 ++ .../system-config.controller.spec.ts | 74 ++++++++++ server/src/dtos/system-config.dto.ts | 27 +++- server/src/enum.ts | 6 + server/src/services/api.service.ts | 7 +- server/src/services/job.service.spec.ts | 10 +- server/src/services/job.service.ts | 111 ++++++++++---- server/src/services/library.service.spec.ts | 12 +- server/src/services/library.service.ts | 6 +- .../services/system-config.service.spec.ts | 8 + .../nightly-tasks-settings.svelte | 81 ++++++++++ .../routes/admin/system-settings/+page.svelte | 23 ++- 21 files changed, 538 insertions(+), 60 deletions(-) create mode 100644 mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart create mode 100644 server/src/controllers/system-config.controller.spec.ts create mode 100644 web/src/lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte diff --git a/e2e/src/api/specs/system-config.e2e-spec.ts b/e2e/src/api/specs/system-config.e2e-spec.ts index 060163d7c9..1bd7bdc489 100644 --- a/e2e/src/api/specs/system-config.e2e-spec.ts +++ b/e2e/src/api/specs/system-config.e2e-spec.ts @@ -15,12 +15,6 @@ describe('/system-config', () => { }); describe('PUT /system-config', () => { - it('should require authentication', async () => { - const { status, body } = await request(app).put('/system-config'); - expect(status).toBe(401); - expect(body).toEqual(errorDto.unauthorized); - }); - it('should always return the new config', async () => { const config = await getSystemConfig(admin.accessToken); diff --git a/i18n/en.json b/i18n/en.json index 06b3a5d599..f547a4e48d 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -166,6 +166,20 @@ "metadata_settings_description": "Manage metadata settings", "migration_job": "Migration", "migration_job_description": "Migrate thumbnails for assets and faces to the latest folder structure", + "nightly_tasks_cluster_faces_setting_description": "Run facial recognition on newly detected faces", + "nightly_tasks_cluster_new_faces_setting": "Cluster new faces", + "nightly_tasks_database_cleanup_setting": "Database cleanup tasks", + "nightly_tasks_database_cleanup_setting_description": "Clean up old, expired data from the database", + "nightly_tasks_generate_memories_setting": "Generate memories", + "nightly_tasks_generate_memories_setting_description": "Create new memories from assets", + "nightly_tasks_missing_thumbnails_setting": "Generate missing thumbnails", + "nightly_tasks_missing_thumbnails_setting_description": "Queue assets without thumbnails for thumbnail generation", + "nightly_tasks_settings": "Nightly Tasks Settings", + "nightly_tasks_settings_description": "Manage nightly tasks", + "nightly_tasks_start_time_setting": "Start time", + "nightly_tasks_start_time_setting_description": "The time at which the server starts running the nightly tasks", + "nightly_tasks_sync_quota_usage_setting": "Sync quota usage", + "nightly_tasks_sync_quota_usage_setting_description": "Update user storage quota, based on current usage", "no_paths_added": "No paths added", "no_pattern_added": "No pattern added", "note_apply_storage_label_previous_assets": "Note: To apply the Storage Label to previously uploaded assets, run the", diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 5085052c05..28fa63ba84 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -509,6 +509,7 @@ Class | Method | HTTP request | Description - [SystemConfigMapDto](doc//SystemConfigMapDto.md) - [SystemConfigMetadataDto](doc//SystemConfigMetadataDto.md) - [SystemConfigNewVersionCheckDto](doc//SystemConfigNewVersionCheckDto.md) + - [SystemConfigNightlyTasksDto](doc//SystemConfigNightlyTasksDto.md) - [SystemConfigNotificationsDto](doc//SystemConfigNotificationsDto.md) - [SystemConfigOAuthDto](doc//SystemConfigOAuthDto.md) - [SystemConfigPasswordLoginDto](doc//SystemConfigPasswordLoginDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index f30481ecce..becafa06bf 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -291,6 +291,7 @@ part 'model/system_config_machine_learning_dto.dart'; part 'model/system_config_map_dto.dart'; part 'model/system_config_metadata_dto.dart'; part 'model/system_config_new_version_check_dto.dart'; +part 'model/system_config_nightly_tasks_dto.dart'; part 'model/system_config_notifications_dto.dart'; part 'model/system_config_o_auth_dto.dart'; part 'model/system_config_password_login_dto.dart'; diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index f1cf05f110..603163f00e 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -638,6 +638,8 @@ class ApiClient { return SystemConfigMetadataDto.fromJson(value); case 'SystemConfigNewVersionCheckDto': return SystemConfigNewVersionCheckDto.fromJson(value); + case 'SystemConfigNightlyTasksDto': + return SystemConfigNightlyTasksDto.fromJson(value); case 'SystemConfigNotificationsDto': return SystemConfigNotificationsDto.fromJson(value); case 'SystemConfigOAuthDto': diff --git a/mobile/openapi/lib/model/system_config_dto.dart b/mobile/openapi/lib/model/system_config_dto.dart index 59d5f09fc9..38dbb30f0c 100644 --- a/mobile/openapi/lib/model/system_config_dto.dart +++ b/mobile/openapi/lib/model/system_config_dto.dart @@ -23,6 +23,7 @@ class SystemConfigDto { required this.map, required this.metadata, required this.newVersionCheck, + required this.nightlyTasks, required this.notifications, required this.oauth, required this.passwordLogin, @@ -55,6 +56,8 @@ class SystemConfigDto { SystemConfigNewVersionCheckDto newVersionCheck; + SystemConfigNightlyTasksDto nightlyTasks; + SystemConfigNotificationsDto notifications; SystemConfigOAuthDto oauth; @@ -87,6 +90,7 @@ class SystemConfigDto { other.map == map && other.metadata == metadata && other.newVersionCheck == newVersionCheck && + other.nightlyTasks == nightlyTasks && other.notifications == notifications && other.oauth == oauth && other.passwordLogin == passwordLogin && @@ -111,6 +115,7 @@ class SystemConfigDto { (map.hashCode) + (metadata.hashCode) + (newVersionCheck.hashCode) + + (nightlyTasks.hashCode) + (notifications.hashCode) + (oauth.hashCode) + (passwordLogin.hashCode) + @@ -123,7 +128,7 @@ class SystemConfigDto { (user.hashCode); @override - String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]'; + String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, nightlyTasks=$nightlyTasks, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]'; Map toJson() { final json = {}; @@ -137,6 +142,7 @@ class SystemConfigDto { json[r'map'] = this.map; json[r'metadata'] = this.metadata; json[r'newVersionCheck'] = this.newVersionCheck; + json[r'nightlyTasks'] = this.nightlyTasks; json[r'notifications'] = this.notifications; json[r'oauth'] = this.oauth; json[r'passwordLogin'] = this.passwordLogin; @@ -169,6 +175,7 @@ class SystemConfigDto { map: SystemConfigMapDto.fromJson(json[r'map'])!, metadata: SystemConfigMetadataDto.fromJson(json[r'metadata'])!, newVersionCheck: SystemConfigNewVersionCheckDto.fromJson(json[r'newVersionCheck'])!, + nightlyTasks: SystemConfigNightlyTasksDto.fromJson(json[r'nightlyTasks'])!, notifications: SystemConfigNotificationsDto.fromJson(json[r'notifications'])!, oauth: SystemConfigOAuthDto.fromJson(json[r'oauth'])!, passwordLogin: SystemConfigPasswordLoginDto.fromJson(json[r'passwordLogin'])!, @@ -236,6 +243,7 @@ class SystemConfigDto { 'map', 'metadata', 'newVersionCheck', + 'nightlyTasks', 'notifications', 'oauth', 'passwordLogin', diff --git a/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart b/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart new file mode 100644 index 0000000000..ab7b4b37c2 --- /dev/null +++ b/mobile/openapi/lib/model/system_config_nightly_tasks_dto.dart @@ -0,0 +1,139 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigNightlyTasksDto { + /// Returns a new [SystemConfigNightlyTasksDto] instance. + SystemConfigNightlyTasksDto({ + required this.clusterNewFaces, + required this.databaseCleanup, + required this.generateMemories, + required this.missingThumbnails, + required this.startTime, + required this.syncQuotaUsage, + }); + + bool clusterNewFaces; + + bool databaseCleanup; + + bool generateMemories; + + bool missingThumbnails; + + String startTime; + + bool syncQuotaUsage; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigNightlyTasksDto && + other.clusterNewFaces == clusterNewFaces && + other.databaseCleanup == databaseCleanup && + other.generateMemories == generateMemories && + other.missingThumbnails == missingThumbnails && + other.startTime == startTime && + other.syncQuotaUsage == syncQuotaUsage; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (clusterNewFaces.hashCode) + + (databaseCleanup.hashCode) + + (generateMemories.hashCode) + + (missingThumbnails.hashCode) + + (startTime.hashCode) + + (syncQuotaUsage.hashCode); + + @override + String toString() => 'SystemConfigNightlyTasksDto[clusterNewFaces=$clusterNewFaces, databaseCleanup=$databaseCleanup, generateMemories=$generateMemories, missingThumbnails=$missingThumbnails, startTime=$startTime, syncQuotaUsage=$syncQuotaUsage]'; + + Map toJson() { + final json = {}; + json[r'clusterNewFaces'] = this.clusterNewFaces; + json[r'databaseCleanup'] = this.databaseCleanup; + json[r'generateMemories'] = this.generateMemories; + json[r'missingThumbnails'] = this.missingThumbnails; + json[r'startTime'] = this.startTime; + json[r'syncQuotaUsage'] = this.syncQuotaUsage; + return json; + } + + /// Returns a new [SystemConfigNightlyTasksDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigNightlyTasksDto? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigNightlyTasksDto"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigNightlyTasksDto( + clusterNewFaces: mapValueOfType(json, r'clusterNewFaces')!, + databaseCleanup: mapValueOfType(json, r'databaseCleanup')!, + generateMemories: mapValueOfType(json, r'generateMemories')!, + missingThumbnails: mapValueOfType(json, r'missingThumbnails')!, + startTime: mapValueOfType(json, r'startTime')!, + syncQuotaUsage: mapValueOfType(json, r'syncQuotaUsage')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigNightlyTasksDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigNightlyTasksDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigNightlyTasksDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigNightlyTasksDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'clusterNewFaces', + 'databaseCleanup', + 'generateMemories', + 'missingThumbnails', + 'startTime', + 'syncQuotaUsage', + }; +} + diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 18204c21f3..492d3cfec1 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -14318,6 +14318,9 @@ "newVersionCheck": { "$ref": "#/components/schemas/SystemConfigNewVersionCheckDto" }, + "nightlyTasks": { + "$ref": "#/components/schemas/SystemConfigNightlyTasksDto" + }, "notifications": { "$ref": "#/components/schemas/SystemConfigNotificationsDto" }, @@ -14360,6 +14363,7 @@ "map", "metadata", "newVersionCheck", + "nightlyTasks", "notifications", "oauth", "passwordLogin", @@ -14790,6 +14794,37 @@ ], "type": "object" }, + "SystemConfigNightlyTasksDto": { + "properties": { + "clusterNewFaces": { + "type": "boolean" + }, + "databaseCleanup": { + "type": "boolean" + }, + "generateMemories": { + "type": "boolean" + }, + "missingThumbnails": { + "type": "boolean" + }, + "startTime": { + "type": "string" + }, + "syncQuotaUsage": { + "type": "boolean" + } + }, + "required": [ + "clusterNewFaces", + "databaseCleanup", + "generateMemories", + "missingThumbnails", + "startTime", + "syncQuotaUsage" + ], + "type": "object" + }, "SystemConfigNotificationsDto": { "properties": { "smtp": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 88dee9bf0b..f60fa6dfe8 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -1389,6 +1389,14 @@ export type SystemConfigMetadataDto = { export type SystemConfigNewVersionCheckDto = { enabled: boolean; }; +export type SystemConfigNightlyTasksDto = { + clusterNewFaces: boolean; + databaseCleanup: boolean; + generateMemories: boolean; + missingThumbnails: boolean; + startTime: string; + syncQuotaUsage: boolean; +}; export type SystemConfigNotificationsDto = { smtp: SystemConfigSmtpDto; }; @@ -1457,6 +1465,7 @@ export type SystemConfigDto = { map: SystemConfigMapDto; metadata: SystemConfigMetadataDto; newVersionCheck: SystemConfigNewVersionCheckDto; + nightlyTasks: SystemConfigNightlyTasksDto; notifications: SystemConfigNotificationsDto; oauth: SystemConfigOAuthDto; passwordLogin: SystemConfigPasswordLoginDto; diff --git a/server/src/config.ts b/server/src/config.ts index 1fcc2e9782..90ca2c1529 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -121,6 +121,14 @@ export interface SystemConfig { newVersionCheck: { enabled: boolean; }; + nightlyTasks: { + startTime: string; + databaseCleanup: boolean; + missingThumbnails: boolean; + clusterNewFaces: boolean; + generateMemories: boolean; + syncQuotaUsage: boolean; + }; trash: { enabled: boolean; days: number; @@ -298,6 +306,14 @@ export const defaults = Object.freeze({ newVersionCheck: { enabled: true, }, + nightlyTasks: { + startTime: '00:00', + databaseCleanup: true, + generateMemories: true, + syncQuotaUsage: true, + missingThumbnails: true, + clusterNewFaces: true, + }, trash: { enabled: true, days: 30, diff --git a/server/src/controllers/system-config.controller.spec.ts b/server/src/controllers/system-config.controller.spec.ts new file mode 100644 index 0000000000..48b8c1bcf0 --- /dev/null +++ b/server/src/controllers/system-config.controller.spec.ts @@ -0,0 +1,74 @@ +import _ from 'lodash'; +import { defaults } from 'src/config'; +import { SystemConfigController } from 'src/controllers/system-config.controller'; +import { StorageTemplateService } from 'src/services/storage-template.service'; +import { SystemConfigService } from 'src/services/system-config.service'; +import request from 'supertest'; +import { errorDto } from 'test/medium/responses'; +import { ControllerContext, controllerSetup, mockBaseService } from 'test/utils'; + +describe(SystemConfigController.name, () => { + let ctx: ControllerContext; + const systemConfigService = mockBaseService(SystemConfigService); + const templateService = mockBaseService(StorageTemplateService); + + beforeAll(async () => { + ctx = await controllerSetup(SystemConfigController, [ + { provide: SystemConfigService, useValue: systemConfigService }, + { provide: StorageTemplateService, useValue: templateService }, + ]); + return () => ctx.close(); + }); + + beforeEach(() => { + systemConfigService.resetAllMocks(); + templateService.resetAllMocks(); + ctx.reset(); + }); + + describe('GET /system-config', () => { + it('should be an authenticated route', async () => { + await request(ctx.getHttpServer()).get('/system-config'); + expect(ctx.authenticate).toHaveBeenCalled(); + }); + }); + + describe('GET /system-config/defaults', () => { + it('should be an authenticated route', async () => { + await request(ctx.getHttpServer()).get('/system-config/defaults'); + expect(ctx.authenticate).toHaveBeenCalled(); + }); + }); + + describe('PUT /system-config', () => { + it('should be an authenticated route', async () => { + await request(ctx.getHttpServer()).put('/system-config'); + expect(ctx.authenticate).toHaveBeenCalled(); + }); + + describe('nightlyTasks', () => { + it('should validate nightly jobs start time', async () => { + const config = _.cloneDeep(defaults); + config.nightlyTasks.startTime = 'invalid'; + const { status, body } = await request(ctx.getHttpServer()).put('/system-config').send(config); + expect(status).toBe(400); + expect(body).toEqual(errorDto.badRequest(['nightlyTasks.startTime must be in HH:mm format'])); + }); + + it('should accept a valid time', async () => { + const config = _.cloneDeep(defaults); + config.nightlyTasks.startTime = '05:05'; + const { status } = await request(ctx.getHttpServer()).put('/system-config').send(config); + expect(status).toBe(200); + }); + + it('should validate a boolean field', async () => { + const config = _.cloneDeep(defaults); + (config.nightlyTasks.databaseCleanup as any) = 'invalid'; + const { status, body } = await request(ctx.getHttpServer()).put('/system-config').send(config); + expect(status).toBe(400); + expect(body).toEqual(errorDto.badRequest(['nightlyTasks.databaseCleanup must be a boolean value'])); + }); + }); + }); +}); diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index b0385984b4..49c5e5b4e7 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -34,7 +34,7 @@ import { VideoContainer, } from 'src/enum'; import { ConcurrentQueueName } from 'src/types'; -import { IsCronExpression, Optional, ValidateBoolean } from 'src/validation'; +import { IsCronExpression, IsDateStringFormat, Optional, ValidateBoolean } from 'src/validation'; const isLibraryScanEnabled = (config: SystemConfigLibraryScanDto) => config.enabled; const isOAuthEnabled = (config: SystemConfigOAuthDto) => config.enabled; @@ -329,6 +329,26 @@ class SystemConfigNewVersionCheckDto { enabled!: boolean; } +class SystemConfigNightlyTasksDto { + @IsDateStringFormat('HH:mm', { message: 'startTime must be in HH:mm format' }) + startTime!: string; + + @ValidateBoolean() + databaseCleanup!: boolean; + + @ValidateBoolean() + missingThumbnails!: boolean; + + @ValidateBoolean() + clusterNewFaces!: boolean; + + @ValidateBoolean() + generateMemories!: boolean; + + @ValidateBoolean() + syncQuotaUsage!: boolean; +} + class SystemConfigOAuthDto { @ValidateBoolean() autoLaunch!: boolean; @@ -638,6 +658,11 @@ export class SystemConfigDto implements SystemConfig { @IsObject() newVersionCheck!: SystemConfigNewVersionCheckDto; + @Type(() => SystemConfigNightlyTasksDto) + @ValidateNested() + @IsObject() + nightlyTasks!: SystemConfigNightlyTasksDto; + @Type(() => SystemConfigOAuthDto) @ValidateNested() @IsObject() diff --git a/server/src/enum.ts b/server/src/enum.ts index dca0f09555..d7c74a71c6 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -567,6 +567,7 @@ export enum DatabaseLock { VersionHistory = 500, CLIPDimSize = 512, Library = 1337, + NightlyJobs = 600, GetSystemConfig = 69, BackupDatabase = 42, MemoryCreation = 777, @@ -684,3 +685,8 @@ export enum AssetVisibility { HIDDEN = 'hidden', LOCKED = 'locked', } + +export enum CronJob { + LibraryScan = 'LibraryScan', + NightlyJobs = 'NightlyJobs', +} diff --git a/server/src/services/api.service.ts b/server/src/services/api.service.ts index 064b2e268d..27a776e867 100644 --- a/server/src/services/api.service.ts +++ b/server/src/services/api.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { Cron, CronExpression, Interval } from '@nestjs/schedule'; +import { Interval } from '@nestjs/schedule'; import { NextFunction, Request, Response } from 'express'; import { readFileSync } from 'node:fs'; import sanitizeHtml from 'sanitize-html'; @@ -54,11 +54,6 @@ export class ApiService { await this.versionService.handleQueueVersionCheck(); } - @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) - async onNightlyJob() { - await this.jobService.handleNightlyJobs(); - } - ssr(excludePaths: string[]) { const { resourcePaths } = this.configRepository.getEnv(); diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.ts index c9020ed96a..a18eccdd8b 100644 --- a/server/src/services/job.service.spec.ts +++ b/server/src/services/job.service.spec.ts @@ -41,12 +41,12 @@ describe(JobService.name, () => { { name: JobName.USER_DELETE_CHECK }, { name: JobName.PERSON_CLEANUP }, { name: JobName.MEMORIES_CLEANUP }, - { name: JobName.MEMORIES_CREATE }, - { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }, - { name: JobName.CLEAN_OLD_AUDIT_LOGS }, - { name: JobName.USER_SYNC_USAGE }, - { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } }, { name: JobName.CLEAN_OLD_SESSION_TOKENS }, + { name: JobName.CLEAN_OLD_AUDIT_LOGS }, + { name: JobName.MEMORIES_CREATE }, + { name: JobName.USER_SYNC_USAGE }, + { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }, + { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } }, ]); }); }); diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index e4b20ba37f..645b22249f 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -1,6 +1,7 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { ClassConstructor } from 'class-transformer'; import { snakeCase } from 'lodash'; +import { SystemConfig } from 'src/config'; import { OnEvent } from 'src/decorators'; import { mapAsset } from 'src/dtos/asset-response.dto'; import { AllJobStatusResponseDto, JobCommandDto, JobCreateDto, JobStatusDto } from 'src/dtos/job.dto'; @@ -8,6 +9,8 @@ import { AssetType, AssetVisibility, BootstrapEventPriority, + CronJob, + DatabaseLock, ImmichWorker, JobCommand, JobName, @@ -20,6 +23,7 @@ import { ArgOf, ArgsOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; import { ConcurrentQueueName, JobItem } from 'src/types'; import { hexOrBufferToBase64 } from 'src/utils/bytes'; +import { handlePromiseError } from 'src/utils/misc'; const asJobItem = (dto: JobCreateDto): JobItem => { switch (dto.name) { @@ -53,12 +57,59 @@ const asJobItem = (dto: JobCreateDto): JobItem => { } }; +const asNightlyTasksCron = (config: SystemConfig) => { + const [hours, minutes] = config.nightlyTasks.startTime.split(':').map(Number); + return `${minutes} ${hours} * * *`; +}; + @Injectable() export class JobService extends BaseService { private services: ClassConstructor[] = []; + private nightlyJobsLock = false; - @OnEvent({ name: 'config.init', workers: [ImmichWorker.MICROSERVICES] }) - onConfigInit({ newConfig: config }: ArgOf<'config.init'>) { + @OnEvent({ name: 'config.init' }) + async onConfigInit({ newConfig: config }: ArgOf<'config.init'>) { + if (this.worker === ImmichWorker.MICROSERVICES) { + this.updateQueueConcurrency(config); + return; + } + + this.nightlyJobsLock = await this.databaseRepository.tryLock(DatabaseLock.NightlyJobs); + if (this.nightlyJobsLock) { + const cronExpression = asNightlyTasksCron(config); + this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`); + this.cronRepository.create({ + name: CronJob.NightlyJobs, + expression: cronExpression, + start: true, + onTick: () => handlePromiseError(this.handleNightlyJobs(), this.logger), + }); + } + } + + @OnEvent({ name: 'config.update', server: true }) + onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) { + if (this.worker === ImmichWorker.MICROSERVICES) { + this.updateQueueConcurrency(config); + return; + } + + if (this.nightlyJobsLock) { + const cronExpression = asNightlyTasksCron(config); + this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`); + this.cronRepository.update({ name: CronJob.NightlyJobs, expression: cronExpression, start: true }); + } + } + + @OnEvent({ name: 'app.bootstrap', priority: BootstrapEventPriority.JobService }) + onBootstrap() { + this.jobRepository.setup(this.services); + if (this.worker === ImmichWorker.MICROSERVICES) { + this.jobRepository.startWorkers(); + } + } + + private updateQueueConcurrency(config: SystemConfig) { this.logger.debug(`Updating queue concurrency settings`); for (const queueName of Object.values(QueueName)) { let concurrency = 1; @@ -70,19 +121,6 @@ export class JobService extends BaseService { } } - @OnEvent({ name: 'config.update', server: true, workers: [ImmichWorker.MICROSERVICES] }) - onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) { - this.onConfigInit({ newConfig: config }); - } - - @OnEvent({ name: 'app.bootstrap', priority: BootstrapEventPriority.JobService }) - onBootstrap() { - this.jobRepository.setup(this.services); - if (this.worker === ImmichWorker.MICROSERVICES) { - this.jobRepository.startWorkers(); - } - } - setServices(services: ClassConstructor[]) { this.services = services; } @@ -233,18 +271,37 @@ export class JobService extends BaseService { } async handleNightlyJobs() { - await this.jobRepository.queueAll([ - { name: JobName.ASSET_DELETION_CHECK }, - { name: JobName.USER_DELETE_CHECK }, - { name: JobName.PERSON_CLEANUP }, - { name: JobName.MEMORIES_CLEANUP }, - { name: JobName.MEMORIES_CREATE }, - { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }, - { name: JobName.CLEAN_OLD_AUDIT_LOGS }, - { name: JobName.USER_SYNC_USAGE }, - { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } }, - { name: JobName.CLEAN_OLD_SESSION_TOKENS }, - ]); + const config = await this.getConfig({ withCache: false }); + const jobs: JobItem[] = []; + + if (config.nightlyTasks.databaseCleanup) { + jobs.push( + { name: JobName.ASSET_DELETION_CHECK }, + { name: JobName.USER_DELETE_CHECK }, + { name: JobName.PERSON_CLEANUP }, + { name: JobName.MEMORIES_CLEANUP }, + { name: JobName.CLEAN_OLD_SESSION_TOKENS }, + { name: JobName.CLEAN_OLD_AUDIT_LOGS }, + ); + } + + if (config.nightlyTasks.generateMemories) { + jobs.push({ name: JobName.MEMORIES_CREATE }); + } + + if (config.nightlyTasks.syncQuotaUsage) { + jobs.push({ name: JobName.USER_SYNC_USAGE }); + } + + if (config.nightlyTasks.missingThumbnails) { + jobs.push({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }); + } + + if (config.nightlyTasks.clusterNewFaces) { + jobs.push({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } }); + } + + await this.jobRepository.queueAll(jobs); } /** diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts index e92cdcf200..ab69e22b99 100644 --- a/server/src/services/library.service.spec.ts +++ b/server/src/services/library.service.spec.ts @@ -3,7 +3,7 @@ import { Stats } from 'node:fs'; import { defaults, SystemConfig } from 'src/config'; import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants'; import { mapLibrary } from 'src/dtos/library.dto'; -import { AssetType, ImmichWorker, JobName, JobStatus } from 'src/enum'; +import { AssetType, CronJob, ImmichWorker, JobName, JobStatus } from 'src/enum'; import { LibraryService } from 'src/services/library.service'; import { ILibraryBulkIdsJob, ILibraryFileJob } from 'src/types'; import { assetStub } from 'test/fixtures/asset.stub'; @@ -56,7 +56,11 @@ describe(LibraryService.name, () => { } as SystemConfig, }); - expect(mocks.cron.update).toHaveBeenCalledWith({ name: 'libraryScan', expression: '0 1 * * *', start: true }); + expect(mocks.cron.update).toHaveBeenCalledWith({ + name: CronJob.LibraryScan, + expression: '0 1 * * *', + start: true, + }); }); it('should initialize watcher for all external libraries', async () => { @@ -128,7 +132,7 @@ describe(LibraryService.name, () => { }); expect(mocks.cron.update).toHaveBeenCalledWith({ - name: 'libraryScan', + name: CronJob.LibraryScan, expression: systemConfigStub.libraryScan.library.scan.cronExpression, start: systemConfigStub.libraryScan.library.scan.enabled, }); @@ -149,7 +153,7 @@ describe(LibraryService.name, () => { }); expect(mocks.cron.update).toHaveBeenCalledWith({ - name: 'libraryScan', + name: CronJob.LibraryScan, expression: systemConfigStub.libraryScan.library.scan.cronExpression, start: systemConfigStub.libraryScan.library.scan.enabled, }); diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 8c7f79ee5c..f6bc5b2ebb 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -17,7 +17,7 @@ import { ValidateLibraryImportPathResponseDto, ValidateLibraryResponseDto, } from 'src/dtos/library.dto'; -import { AssetStatus, AssetType, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum'; +import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum'; import { ArgOf } from 'src/repositories/event.repository'; import { AssetSyncResult } from 'src/repositories/library.repository'; import { AssetTable } from 'src/schema/tables/asset.table'; @@ -45,7 +45,7 @@ export class LibraryService extends BaseService { if (this.lock) { this.cronRepository.create({ - name: 'libraryScan', + name: CronJob.LibraryScan, expression: scan.cronExpression, onTick: () => handlePromiseError(this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL }), this.logger), @@ -65,7 +65,7 @@ export class LibraryService extends BaseService { } this.cronRepository.update({ - name: 'libraryScan', + name: CronJob.LibraryScan, expression: library.scan.cronExpression, start: library.scan.enabled, }); diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index c7b98cc990..43be323459 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -103,6 +103,14 @@ const updatedConfig = Object.freeze({ lightStyle: 'https://tiles.immich.cloud/v1/style/light.json', darkStyle: 'https://tiles.immich.cloud/v1/style/dark.json', }, + nightlyTasks: { + startTime: '00:00', + databaseCleanup: true, + clusterNewFaces: true, + missingThumbnails: true, + generateMemories: true, + syncQuotaUsage: true, + }, reverseGeocoding: { enabled: true, }, diff --git a/web/src/lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte b/web/src/lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte new file mode 100644 index 0000000000..af653f2473 --- /dev/null +++ b/web/src/lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte @@ -0,0 +1,81 @@ + + +
+
+
+
+ + + + + + +
+ + onReset({ ...options, configKeys: ['nightlyTasks'] })} + onSave={() => onSave({ nightlyTasks: config.nightlyTasks })} + showResetToDefault={!isEqual(savedConfig.nightlyTasks, defaultConfig.nightlyTasks)} + {disabled} + /> + +
+
diff --git a/web/src/routes/admin/system-settings/+page.svelte b/web/src/routes/admin/system-settings/+page.svelte index e053eea179..3b1d68a49e 100644 --- a/web/src/routes/admin/system-settings/+page.svelte +++ b/web/src/routes/admin/system-settings/+page.svelte @@ -12,6 +12,7 @@ import MapSettings from '$lib/components/admin-page/settings/map-settings/map-settings.svelte'; import MetadataSettings from '$lib/components/admin-page/settings/metadata-settings/metadata-settings.svelte'; import NewVersionCheckSettings from '$lib/components/admin-page/settings/new-version-check-settings/new-version-check-settings.svelte'; + import NightlyTasksSettings from '$lib/components/admin-page/settings/nightly-tasks-settings/nightly-tasks-settings.svelte'; import NotificationSettings from '$lib/components/admin-page/settings/notification-settings/notification-settings.svelte'; import ServerSettings from '$lib/components/admin-page/settings/server/server-settings.svelte'; import StorageTemplateSettings from '$lib/components/admin-page/settings/storage-template/storage-template-settings.svelte'; @@ -33,6 +34,7 @@ mdiBackupRestore, mdiBellOutline, mdiBookshelf, + mdiClockOutline, mdiContentCopy, mdiDatabaseOutline, mdiDownload, @@ -136,13 +138,6 @@ key: 'job', icon: mdiSync, }, - { - component: MetadataSettings, - title: $t('admin.metadata_settings'), - subtitle: $t('admin.metadata_settings_description'), - key: 'metadata', - icon: mdiDatabaseOutline, - }, { component: LibrarySettings, title: $t('admin.library_settings'), @@ -171,6 +166,20 @@ key: 'location', icon: mdiMapMarkerOutline, }, + { + component: MetadataSettings, + title: $t('admin.metadata_settings'), + subtitle: $t('admin.metadata_settings_description'), + key: 'metadata', + icon: mdiDatabaseOutline, + }, + { + component: NightlyTasksSettings, + title: $t('admin.nightly_tasks_settings'), + subtitle: $t('admin.nightly_tasks_settings_description'), + key: 'nightly-tasks', + icon: mdiClockOutline, + }, { component: NotificationSettings, title: $t('admin.notification_settings'),