From ddb734e64a75ea51976873e9d8b81c8b47e214a0 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Sun, 29 Dec 2019 00:35:41 +0100 Subject: [PATCH] implementing job history saving --- benchmark/Benchmarks.ts | 3 +- docker/alpine/Dockerfile.build | 3 +- docker/debian-stretch/Dockerfile.build | 3 +- .../debian-stretch/selfcontained/Dockerfile | 3 +- .../thumbnail/PhotoConverterMWs.ts | 5 +- .../thumbnail/ThumbnailGeneratorMWs.ts | 4 +- .../model/database/interfaces/IJobManager.ts | 3 +- .../model/database/memory/UserManager.ts | 3 +- .../model/database/sql/SQLConnection.ts | 6 +- .../model/diagnostics/ConfigDiagnostics.ts | 5 +- .../model/fileprocessing/PhotoProcessing.ts | 43 +++----- .../model/fileprocessing/VideoProcessing.ts | 13 +++ src/backend/model/jobs/JobManager.ts | 47 ++++---- src/backend/model/jobs/JobProgressManager.ts | 83 ++++++++++++++ src/backend/model/jobs/jobs/DBResetJob.ts | 7 +- src/backend/model/jobs/jobs/FileJob.ts | 27 +++-- src/backend/model/jobs/jobs/IJob.ts | 12 +-- src/backend/model/jobs/jobs/IJobListener.ts | 9 ++ src/backend/model/jobs/jobs/IndexingJob.ts | 19 ++-- src/backend/model/jobs/jobs/Job.ts | 86 ++++++--------- src/backend/model/jobs/jobs/JobProgress.ts | 102 ++++++++++++++++++ .../model/jobs/jobs/PhotoConvertingJob.ts | 12 ++- .../model/jobs/jobs/TempFolderCleaningJob.ts | 26 +++-- .../model/jobs/jobs/ThumbnailGenerationJob.ts | 21 ++-- .../model/jobs/jobs/VideoConvertingJob.ts | 10 +- src/common/config/private/IPrivateConfig.ts | 17 +-- .../private/PrivateConfigDefaultsClass.ts | 8 +- src/common/entities/job/JobDTO.ts | 6 ++ src/common/entities/job/JobLastRunDTO.ts | 15 --- src/common/entities/job/JobProgressDTO.ts | 18 ++-- .../database/database.settings.component.html | 30 +++--- .../indexing/indexing.settings.component.html | 2 +- .../indexing/indexing.settings.component.ts | 8 +- .../jobs/jobs.settings.component.html | 4 +- .../settings/jobs/jobs.settings.component.ts | 10 +- .../job-progress.settings.component.html | 38 +++---- .../job-progress.settings.component.ts | 16 ++- .../photo/photo.settings.component.html | 2 +- .../photo/photo.settings.component.ts | 8 +- .../app/ui/settings/scheduled-jobs.service.ts | 19 +--- .../thumbnail.settings.component.html | 2 +- .../thumbnail/thumbnail.settings.component.ts | 8 +- .../video/video.settings.component.html | 2 +- .../video/video.settings.component.ts | 8 +- test/backend/SQLTestHelper.ts | 2 +- test/backend/integration/model/sql/typeorm.ts | 2 +- .../fileprocessing/PhotoProcessing.spec.ts | 15 +-- 47 files changed, 479 insertions(+), 316 deletions(-) create mode 100644 src/backend/model/jobs/JobProgressManager.ts create mode 100644 src/backend/model/jobs/jobs/IJobListener.ts create mode 100644 src/backend/model/jobs/jobs/JobProgress.ts delete mode 100644 src/common/entities/job/JobLastRunDTO.ts diff --git a/benchmark/Benchmarks.ts b/benchmark/Benchmarks.ts index 9b531fd0..45b96c87 100644 --- a/benchmark/Benchmarks.ts +++ b/benchmark/Benchmarks.ts @@ -5,6 +5,7 @@ import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker' import {IndexingManager} from '../src/backend/model/database/sql/IndexingManager'; import {SearchManager} from '../src/backend/model/database/sql/SearchManager'; import * as fs from 'fs'; +import * as path from 'path'; import {SearchTypes} from '../src/common/entities/AutoCompleteItem'; import {Utils} from '../src/common/Utils'; import {GalleryManager} from '../src/backend/model/database/sql/GalleryManager'; @@ -123,7 +124,7 @@ export class Benchmarks { fs.unlinkSync(this.dbPath); } Config.Server.Database.type = ServerConfig.DatabaseType.sqlite; - Config.Server.Database.sqlite.storage = this.dbPath; + Config.Server.Database.dbFolder = path.dirname(this.dbPath); await ObjectManagers.InitSQLManagers(); }; diff --git a/docker/alpine/Dockerfile.build b/docker/alpine/Dockerfile.build index c873d840..8dc3de49 100644 --- a/docker/alpine/Dockerfile.build +++ b/docker/alpine/Dockerfile.build @@ -16,8 +16,7 @@ ENTRYPOINT ["npm", "start", "--", \ # after a extensive job (like video converting), pigallery calls gc, to clean up everthing as fast as possible "--expose-gc", \ "--config-path=/app/data/config/config.json", \ - "--Server-Database-sqlite-storage=/app/data/db/sqlite.db", \ - "--Server-Database-memory-usersFile=/app/data/db/users.db", \ + "--Server-Database-dbFolder=/app/data/db", \ "--Server-Media-folder=/app/data/images", \ "--Server-Media-tempFolder=/app/data/tmp"] EXPOSE 80 diff --git a/docker/debian-stretch/Dockerfile.build b/docker/debian-stretch/Dockerfile.build index 04ec5a8f..e842051b 100644 --- a/docker/debian-stretch/Dockerfile.build +++ b/docker/debian-stretch/Dockerfile.build @@ -15,8 +15,7 @@ ENTRYPOINT ["npm", "start", "--", \ # after a extensive job (like video converting), pigallery calls gc, to clean up everthing as fast as possible "--expose-gc", \ "--config-path=/app/data/config/config.json", \ - "--Server-Database-sqlite-storage=/app/data/db/sqlite.db", \ - "--Server-Database-memory-usersFile=/app/data/db/users.db", \ + "--Server-Database-dbFolder=/app/data/db", \ "--Server-Media-folder=/app/data/images", \ "--Server-Media-tempFolder=/app/data/tmp"] EXPOSE 80 diff --git a/docker/debian-stretch/selfcontained/Dockerfile b/docker/debian-stretch/selfcontained/Dockerfile index 8ea09c77..6cd38f19 100644 --- a/docker/debian-stretch/selfcontained/Dockerfile +++ b/docker/debian-stretch/selfcontained/Dockerfile @@ -18,8 +18,7 @@ ENTRYPOINT ["npm", "start", "--", \ # after a extensive job (like video converting), pigallery calls gc, to clean up everthing as fast as possible "--expose-gc", \ "--config-path=/app/data/config/config.json", \ - "--Server-Database-sqlite-storage=/app/data/db/sqlite.db", \ - "--Server-Database-memory-usersFile=/app/data/db/users.db", \ + "--Server-Database-dbFolder=/app/data/db", \ "--Server-Media-folder=/app/data/images", \ "--Server-Media-tempFolder=/app/data/tmp"] EXPOSE 80 diff --git a/src/backend/middlewares/thumbnail/PhotoConverterMWs.ts b/src/backend/middlewares/thumbnail/PhotoConverterMWs.ts index e2e514a4..46cf5908 100644 --- a/src/backend/middlewares/thumbnail/PhotoConverterMWs.ts +++ b/src/backend/middlewares/thumbnail/PhotoConverterMWs.ts @@ -15,7 +15,7 @@ export class PhotoConverterMWs { } const fullMediaPath = req.resultPipe; - const convertedVideo = PhotoProcessing.generateConvertedFilePath(fullMediaPath); + const convertedVideo = PhotoProcessing.generateConvertedPath(fullMediaPath, Config.Server.Media.Photo.Converting.resolution); // check if transcoded video exist if (fs.existsSync(convertedVideo) === true) { @@ -24,8 +24,7 @@ export class PhotoConverterMWs { } if (Config.Server.Media.Photo.Converting.onTheFly === true) { - req.resultPipe = await PhotoProcessing.convertPhoto(fullMediaPath, - Config.Server.Media.Photo.Converting.resolution); + req.resultPipe = await PhotoProcessing.convertPhoto(fullMediaPath); return next(); } diff --git a/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts b/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts index 27505755..ef694f25 100644 --- a/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts +++ b/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts @@ -149,7 +149,7 @@ export class ThumbnailGeneratorMWs { const fullMediaPath = path.join(ProjectPath.ImageFolder, photos[i].directory.path, photos[i].directory.name, photos[i].name); for (let j = 0; j < Config.Client.Media.Thumbnail.thumbnailSizes.length; j++) { const size = Config.Client.Media.Thumbnail.thumbnailSizes[j]; - const thPath = PhotoProcessing.generateThumbnailPath(fullMediaPath, size); + const thPath = PhotoProcessing.generateConvertedPath(fullMediaPath, size); if (fs.existsSync(thPath) === true) { if (typeof photos[i].readyThumbnails === 'undefined') { photos[i].readyThumbnails = []; @@ -157,7 +157,7 @@ export class ThumbnailGeneratorMWs { photos[i].readyThumbnails.push(size); } } - const iconPath = PhotoProcessing.generateThumbnailPath(fullMediaPath, Config.Client.Media.Thumbnail.iconSize); + const iconPath = PhotoProcessing.generateConvertedPath(fullMediaPath, Config.Client.Media.Thumbnail.iconSize); if (fs.existsSync(iconPath) === true) { photos[i].readyIcon = true; } diff --git a/src/backend/model/database/interfaces/IJobManager.ts b/src/backend/model/database/interfaces/IJobManager.ts index 60290be1..792fd185 100644 --- a/src/backend/model/database/interfaces/IJobManager.ts +++ b/src/backend/model/database/interfaces/IJobManager.ts @@ -1,6 +1,5 @@ import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO'; import {JobDTO} from '../../../../common/entities/job/JobDTO'; -import {JobLastRunDTO} from '../../../../common/entities/job/JobLastRunDTO'; export interface IJobManager { @@ -17,5 +16,5 @@ export interface IJobManager { runSchedules(): void; - getJobLastRuns(): { [key: string]: { [key: string]: JobLastRunDTO } }; + getJobLastRuns(): { [key: string]: JobProgressDTO }; } diff --git a/src/backend/model/database/memory/UserManager.ts b/src/backend/model/database/memory/UserManager.ts index bb4df481..d907f59f 100644 --- a/src/backend/model/database/memory/UserManager.ts +++ b/src/backend/model/database/memory/UserManager.ts @@ -3,6 +3,7 @@ import {IUserManager} from '../interfaces/IUserManager'; import {ProjectPath} from '../../../ProjectPath'; import {Utils} from '../../../../common/Utils'; import * as fs from 'fs'; +import * as path from 'path'; import {PasswordHelper} from '../../PasswordHelper'; import {Config} from '../../../../common/config/private/Config'; @@ -13,7 +14,7 @@ export class UserManager implements IUserManager { constructor() { - this.dbPath = ProjectPath.getAbsolutePath(Config.Server.Database.memory.usersFile); + this.dbPath = path.join(ProjectPath.getAbsolutePath(Config.Server.Database.dbFolder), 'users.db'); if (fs.existsSync(this.dbPath)) { this.loadDB(); } diff --git a/src/backend/model/database/sql/SQLConnection.ts b/src/backend/model/database/sql/SQLConnection.ts index 20920351..30c58865 100644 --- a/src/backend/model/database/sql/SQLConnection.ts +++ b/src/backend/model/database/sql/SQLConnection.ts @@ -17,6 +17,7 @@ import {FileEntity} from './enitites/FileEntity'; import {FaceRegionEntry} from './enitites/FaceRegionEntry'; import {PersonEntry} from './enitites/PersonEntry'; import {Utils} from '../../../../common/Utils'; +import * as path from 'path'; import {ServerConfig} from '../../../../common/config/private/IPrivateConfig'; @@ -178,11 +179,14 @@ export class SQLConnection { } else if (config.type === ServerConfig.DatabaseType.sqlite) { driver = { type: 'sqlite', - database: ProjectPath.getAbsolutePath(config.sqlite.storage) + database: path.join(ProjectPath.getAbsolutePath(config.dbFolder), 'sqlite.db') }; } return driver; } + public static getSQLiteDB(config: ServerConfig.DataBaseConfig){ + return path.join(ProjectPath.getAbsolutePath(config.dbFolder), 'sqlite.db'); + } } diff --git a/src/backend/model/diagnostics/ConfigDiagnostics.ts b/src/backend/model/diagnostics/ConfigDiagnostics.ts index 0933b8aa..ec58ed3e 100644 --- a/src/backend/model/diagnostics/ConfigDiagnostics.ts +++ b/src/backend/model/diagnostics/ConfigDiagnostics.ts @@ -31,10 +31,9 @@ export class ConfigDiagnostics { } if (databaseConfig.type !== ServerConfig.DatabaseType.sqlite) { try { - await this.checkReadWritePermission(ProjectPath.getAbsolutePath(databaseConfig.sqlite.storage)); + await this.checkReadWritePermission(SQLConnection.getSQLiteDB(databaseConfig)); } catch (e) { - throw new Error('Cannot read or write sqlite storage file: ' + - ProjectPath.getAbsolutePath(databaseConfig.sqlite.storage)); + throw new Error('Cannot read or write sqlite storage file: ' + SQLConnection.getSQLiteDB(databaseConfig)); } } } diff --git a/src/backend/model/fileprocessing/PhotoProcessing.ts b/src/backend/model/fileprocessing/PhotoProcessing.ts index 8eb868ce..1bd40e2c 100644 --- a/src/backend/model/fileprocessing/PhotoProcessing.ts +++ b/src/backend/model/fileprocessing/PhotoProcessing.ts @@ -98,7 +98,7 @@ export class PhotoProcessing { } - public static generateThumbnailPath(mediaPath: string, size: number): string { + public static generateConvertedPath(mediaPath: string, size: number): string { const file = path.basename(mediaPath); return path.join(ProjectPath.TranscodedFolder, ProjectPath.getRelativePathToImages(path.dirname(mediaPath)), @@ -111,9 +111,6 @@ export class PhotoProcessing { .digest('hex') + '_' + size + '.jpg'); } - public static generateConvertedFilePath(photoPath: string): string { - return this.generateThumbnailPath(photoPath, Config.Server.Media.Photo.Converting.resolution); - } public static async isValidConvertedPath(convertedPath: string): Promise { const origFilePath = path.join(ProjectPath.ImageFolder, @@ -142,42 +139,35 @@ export class PhotoProcessing { } - public static async convertPhoto(mediaPath: string, size: number) { + public static async convertPhoto(mediaPath: string) { + return this.generateThumbnail(mediaPath, + Config.Server.Media.Photo.Converting.resolution, + ThumbnailSourceType.Photo, + false); + } + + + static async convertedPhotoExist(mediaPath: string, size: number) { + // generate thumbnail path - const outPath = PhotoProcessing.generateConvertedFilePath(mediaPath); + const outPath = PhotoProcessing.generateConvertedPath(mediaPath, size); // check if file already exist try { await fsp.access(outPath, fsConstants.R_OK); - return outPath; + return true; } catch (e) { } - - - // run on other thread - const input = { - type: ThumbnailSourceType.Photo, - mediaPath: mediaPath, - size: size, - outPath: outPath, - makeSquare: false, - qualityPriority: Config.Server.Media.Thumbnail.qualityPriority - }; - - const outDir = path.dirname(input.outPath); - - await fsp.mkdir(outDir, {recursive: true}); - await this.taskQue.execute(input); - return outPath; + return false; } public static async generateThumbnail(mediaPath: string, size: number, sourceType: ThumbnailSourceType, - makeSquare: boolean) { + makeSquare: boolean): Promise { // generate thumbnail path - const outPath = PhotoProcessing.generateThumbnailPath(mediaPath, size); + const outPath = PhotoProcessing.generateConvertedPath(mediaPath, size); // check if file already exist @@ -209,5 +199,6 @@ export class PhotoProcessing { const extension = path.extname(fullPath).toLowerCase(); return SupportedFormats.WithDots.Photos.indexOf(extension) !== -1; } + } diff --git a/src/backend/model/fileprocessing/VideoProcessing.ts b/src/backend/model/fileprocessing/VideoProcessing.ts index a06161d8..2dbe5c6a 100644 --- a/src/backend/model/fileprocessing/VideoProcessing.ts +++ b/src/backend/model/fileprocessing/VideoProcessing.ts @@ -40,6 +40,19 @@ export class VideoProcessing { return true; } + + static async convertedVideoExist(videoPath: string): Promise { + const outPath = this.generateConvertedFilePath(videoPath); + + try { + await fsp.access(outPath, fsConstants.R_OK); + return true; + } catch (e) { + } + + return false; + } + public static async convertVideo(videoPath: string): Promise { diff --git a/src/backend/model/jobs/JobManager.ts b/src/backend/model/jobs/JobManager.ts index cf954732..5740a2d3 100644 --- a/src/backend/model/jobs/JobManager.ts +++ b/src/backend/model/jobs/JobManager.ts @@ -1,50 +1,40 @@ import {IJobManager} from '../database/interfaces/IJobManager'; -import {JobProgressDTO} from '../../../common/entities/job/JobProgressDTO'; +import {JobProgressDTO, JobProgressStates} from '../../../common/entities/job/JobProgressDTO'; import {IJob} from './jobs/IJob'; import {JobRepository} from './JobRepository'; import {Config} from '../../../common/config/private/Config'; import {AfterJobTrigger, JobScheduleDTO, JobTriggerType} from '../../../common/entities/job/JobScheduleDTO'; import {Logger} from '../../Logger'; import {NotificationManager} from '../NotifocationManager'; -import {JobLastRunDTO, JobLastRunState} from '../../../common/entities/job/JobLastRunDTO'; +import {IJobListener} from './jobs/IJobListener'; +import {JobProgress} from './jobs/JobProgress'; +import {JobProgressManager} from './JobProgressManager'; -declare var global: NodeJS.Global; const LOG_TAG = '[JobManager]'; -export class JobManager implements IJobManager { - +export class JobManager implements IJobManager, IJobListener { protected timers: { schedule: JobScheduleDTO, timer: NodeJS.Timeout }[] = []; + protected progressManager: JobProgressManager = null; constructor() { + this.progressManager = new JobProgressManager(); this.runSchedules(); } getProgresses(): { [id: string]: JobProgressDTO } { - const m: { [id: string]: JobProgressDTO } = {}; - JobRepository.Instance.getAvailableJobs() - .filter(t => t.Progress) - .forEach(t => { - t.Progress.time.current = Date.now(); - m[t.Name] = t.Progress; - }); - return m; + return this.progressManager.Running; } - getJobLastRuns(): { [key: string]: { [key: string]: JobLastRunDTO } } { - const m: { [id: string]: { [id: string]: JobLastRunDTO } } = {}; - JobRepository.Instance.getAvailableJobs().forEach(t => { - m[t.Name] = t.LastRuns; - }); - return m; + getJobLastRuns(): { [key: string]: JobProgressDTO } { + return this.progressManager.Finished; } async run(jobName: string, config: T): Promise { const t = this.findJob(jobName); if (t) { - await t.start(config, (status: JobLastRunState) => { - this.onJobFinished(t, status); - }); + t.JobListener = this; + await t.start(config); } else { Logger.warn(LOG_TAG, 'cannot find job to start:' + jobName); } @@ -53,14 +43,19 @@ export class JobManager implements IJobManager { stop(jobName: string): void { const t = this.findJob(jobName); if (t) { - t.stop(); + t.cancel(); } else { Logger.warn(LOG_TAG, 'cannot find job to stop:' + jobName); } } - async onJobFinished(job: IJob, status: JobLastRunState): Promise { - if (status === JobLastRunState.canceled) { // if it was cancelled do not start the next one + onProgressUpdate = (progress: JobProgress): void => { + this.progressManager.onJobProgressUpdate(progress.toDTO()); + }; + + + onJobFinished = async (job: IJob, state: JobProgressStates): Promise => { + if (state !== JobProgressStates.finished) { // if it was not finished peacefully, do not start the next one return; } const sch = Config.Server.Jobs.scheduled.find(s => s.jobName === job.Name); @@ -75,7 +70,7 @@ export class JobManager implements IJobManager { } } } - } + }; getAvailableJobs(): IJob[] { return JobRepository.Instance.getAvailableJobs(); diff --git a/src/backend/model/jobs/JobProgressManager.ts b/src/backend/model/jobs/JobProgressManager.ts new file mode 100644 index 00000000..70a57fe6 --- /dev/null +++ b/src/backend/model/jobs/JobProgressManager.ts @@ -0,0 +1,83 @@ +import {promises as fsp} from 'fs'; +import * as path from 'path'; +import {ProjectPath} from '../../ProjectPath'; +import {Config} from '../../../common/config/private/Config'; +import {JobProgressDTO, JobProgressStates} from '../../../common/entities/job/JobProgressDTO'; + +export class JobProgressManager { + db: { [key: string]: { progress: JobProgressDTO, timestamp: number } } = {}; + private readonly dbPath: string; + private timer: NodeJS.Timeout = null; + + constructor() { + this.dbPath = path.join(ProjectPath.getAbsolutePath(Config.Server.Database.dbFolder), 'jobs.db'); + this.loadDB().catch(console.error); + } + + get Running(): { [key: string]: JobProgressDTO } { + const m: { [key: string]: JobProgressDTO } = {}; + for (const key of Object.keys(this.db)) { + if (this.db[key].progress.state === JobProgressStates.running) { + m[key] = this.db[key].progress; + m[key].time.end = Date.now(); + } + } + return m; + } + + get Finished(): { [key: string]: JobProgressDTO } { + const m: { [key: string]: JobProgressDTO } = {}; + for (const key of Object.keys(this.db)) { + if (this.db[key].progress.state !== JobProgressStates.running) { + m[key] = this.db[key].progress; + } + } + return m; + } + + onJobProgressUpdate(progress: JobProgressDTO) { + this.db[progress.HashName] = {progress: progress, timestamp: Date.now()}; + this.delayedSave(); + } + + private async loadDB() { + try { + await fsp.access(this.dbPath); + } catch (e) { + return; + } + const data = await fsp.readFile(this.dbPath, 'utf8'); + this.db = JSON.parse(data); + + while (Object.keys(this.db).length > Config.Server.Jobs.maxSavedProgress) { + let min: string = null; + for (const key of Object.keys(this.db)) { + if (min === null || this.db[min].timestamp > this.db[key].timestamp) { + min = key; + } + } + delete this.db[min]; + } + + for (const key of Object.keys(this.db)) { + if (this.db[key].progress.state === JobProgressStates.running) { + this.db[key].progress.state = JobProgressStates.interrupted; + } + } + } + + private async saveDB() { + await fsp.writeFile(this.dbPath, JSON.stringify(this.db)); + } + + private delayedSave() { + if (this.timer !== null) { + return; + } + this.timer = setTimeout(async () => { + this.saveDB().catch(console.error); + this.timer = null; + }, 1000); + } + +} diff --git a/src/backend/model/jobs/jobs/DBResetJob.ts b/src/backend/model/jobs/jobs/DBResetJob.ts index ed8d3d5c..346a7757 100644 --- a/src/backend/model/jobs/jobs/DBResetJob.ts +++ b/src/backend/model/jobs/jobs/DBResetJob.ts @@ -1,4 +1,3 @@ -import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO'; import {ObjectManagers} from '../../ObjectManagers'; import {Config} from '../../../../common/config/private/Config'; import {ConfigTemplateEntry, DefaultsJobs} from '../../../../common/entities/job/JobDTO'; @@ -19,9 +18,11 @@ export class DBRestJob extends Job { protected async init() { } - protected async step(): Promise { + protected async step(): Promise { + this.Progress.Left = 1; + this.Progress.Processed++; await ObjectManagers.getInstance().IndexingManager.resetDB(); - return null; + return false; } diff --git a/src/backend/model/jobs/jobs/FileJob.ts b/src/backend/model/jobs/jobs/FileJob.ts index 3bf127b0..69cb8d9e 100644 --- a/src/backend/model/jobs/jobs/FileJob.ts +++ b/src/backend/model/jobs/jobs/FileJob.ts @@ -1,4 +1,3 @@ -import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO'; import {ConfigTemplateEntry} from '../../../../common/entities/job/JobDTO'; import {Job} from './Job'; import * as path from 'path'; @@ -53,14 +52,14 @@ export abstract class FileJob; protected abstract async processFile(file: FileDTO): Promise; - protected async step(): Promise { + protected async step(): Promise { if (this.directoryQueue.length === 0 && this.fileQueue.length === 0) { - return null; + return false; } - this.progress.time.current = Date.now(); if (this.directoryQueue.length > 0) { if (this.config.indexedOnly === true && @@ -71,24 +70,30 @@ export abstract class FileJob 0) { + this.Progress.Left = this.fileQueue.length; const file = this.fileQueue.shift(); - this.progress.left = this.fileQueue.length; - this.progress.progress++; const filePath = path.join(file.directory.path, file.directory.name, file.name); - this.progress.comment = 'processing: ' + filePath; try { - await this.processFile(file); + if ((await this.shouldProcess(file)) === true) { + this.Progress.Processed++; + this.Progress.log('processing: ' + filePath); + await this.processFile(file); + } else { + this.Progress.log('skipping: ' + filePath); + this.Progress.Skipped++; + } } catch (e) { console.error(e); Logger.error(LOG_TAG, 'Error during processing file:' + filePath + ', ' + e.toString()); + this.Progress.log('Error during processing file:' + filePath + ', ' + e.toString()); } } - return this.progress; + return true; } private async loadADirectoryFromDisk() { const directory = this.directoryQueue.shift(); - this.progress.comment = 'scanning directory: ' + directory; + this.Progress.log('scanning directory: ' + directory); const scanned = await DiskManager.scanDirectory(directory, this.scanFilter); for (let i = 0; i < scanned.directories.length; i++) { this.directoryQueue.push(path.join(scanned.directories[i].path, scanned.directories[i].name)); @@ -106,7 +111,7 @@ export abstract class FileJob extends JobDTO { Name: string; Supported: boolean; - Progress: JobProgressDTO; - LastRuns: { [key: string]: JobLastRunDTO }; + Progress: JobProgress; + JobListener: IJobListener; - start(config: T, OnFinishCB: (status: JobLastRunState) => void): Promise; + start(config: T): Promise; - stop(): void; + cancel(): void; toJSON(): JobDTO; } diff --git a/src/backend/model/jobs/jobs/IJobListener.ts b/src/backend/model/jobs/jobs/IJobListener.ts new file mode 100644 index 00000000..f0e49c85 --- /dev/null +++ b/src/backend/model/jobs/jobs/IJobListener.ts @@ -0,0 +1,9 @@ +import {JobProgress} from './JobProgress'; +import {IJob} from './IJob'; +import {JobProgressStates} from '../../../../common/entities/job/JobProgressDTO'; + +export interface IJobListener { + onJobFinished(job: IJob, state: JobProgressStates): void; + + onProgressUpdate(progress: JobProgress): void; +} diff --git a/src/backend/model/jobs/jobs/IndexingJob.ts b/src/backend/model/jobs/jobs/IndexingJob.ts index 524deecd..60850b95 100644 --- a/src/backend/model/jobs/jobs/IndexingJob.ts +++ b/src/backend/model/jobs/jobs/IndexingJob.ts @@ -1,10 +1,10 @@ -import {JobProgressDTO, JobState} from '../../../../common/entities/job/JobProgressDTO'; import {ObjectManagers} from '../../ObjectManagers'; import * as path from 'path'; import {Config} from '../../../../common/config/private/Config'; import {Job} from './Job'; import {ConfigTemplateEntry, DefaultsJobs} from '../../../../common/entities/job/JobDTO'; import {ServerConfig} from '../../../../common/config/private/IPrivateConfig'; +import {JobProgressStates} from '../../../../common/entities/job/JobProgressDTO'; declare var global: NodeJS.Global; const LOG_TAG = '[IndexingJob]'; @@ -23,23 +23,22 @@ export class IndexingJob extends Job { this.directoriesToIndex.push('/'); } - protected async step(): Promise { + protected async step(): Promise { if (this.directoriesToIndex.length === 0) { - return null; + return false; } const directory = this.directoriesToIndex.shift(); - this.progress.comment = directory; - this.progress.left = this.directoriesToIndex.length; + this.Progress.log(directory); + this.Progress.Left = this.directoriesToIndex.length; const scanned = await ObjectManagers.getInstance().IndexingManager.indexDirectory(directory); - if (this.state !== JobState.running) { - return null; + if (this.Progress.State !== JobProgressStates.running) { + return false; } - this.progress.progress++; - this.progress.time.current = Date.now(); + this.Progress.Processed++; for (let i = 0; i < scanned.directories.length; i++) { this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name)); } - return this.progress; + return true; } diff --git a/src/backend/model/jobs/jobs/Job.ts b/src/backend/model/jobs/jobs/Job.ts index d0933b31..5ac0c78e 100644 --- a/src/backend/model/jobs/jobs/Job.ts +++ b/src/backend/model/jobs/jobs/Job.ts @@ -1,8 +1,9 @@ -import {JobProgressDTO, JobState} from '../../../../common/entities/job/JobProgressDTO'; import {Logger} from '../../../Logger'; import {IJob} from './IJob'; import {ConfigTemplateEntry, JobDTO} from '../../../../common/entities/job/JobDTO'; -import {JobLastRunDTO, JobLastRunState} from '../../../../common/entities/job/JobLastRunDTO'; +import {JobProgress} from './JobProgress'; +import {IJobListener} from './IJobListener'; +import {JobProgressStates} from '../../../../common/entities/job/JobProgressDTO'; declare const process: any; declare const global: any; @@ -10,13 +11,15 @@ declare const global: any; const LOG_TAG = '[JOB]'; export abstract class Job implements IJob { - - protected progress: JobProgressDTO = null; - protected state = JobState.idle; + protected progress: JobProgress = null; protected config: T; protected prResolve: () => void; protected IsInstant = false; - protected lastRuns: { [key: string]: JobLastRunDTO } = {}; + private jobListener: IJobListener; + + public set JobListener(value: IJobListener) { + this.jobListener = value; + } public abstract get Supported(): boolean; @@ -24,45 +27,25 @@ export abstract class Job implements IJob { public abstract get ConfigTemplate(): ConfigTemplateEntry[]; - public get LastRuns(): { [key: string]: JobLastRunDTO } { - return this.lastRuns; - } - public get Progress(): JobProgressDTO { + public get Progress(): JobProgress { return this.progress; } - public start(config: T, onFinishCB: (status: JobLastRunState) => void): Promise { - this.OnFinishCB = onFinishCB; - if (this.state === JobState.idle && this.Supported) { + public get CanRun() { + return this.Progress == null && this.Supported; + } + + public start(config: T): Promise { + if (this.CanRun) { Logger.info(LOG_TAG, 'Running job: ' + this.Name); this.config = config; - this.progress = { - progress: 0, - left: 0, - comment: '', - state: JobState.running, - time: { - start: Date.now(), - current: Date.now() - } - }; - this.lastRuns[JSON.stringify(this.config)] = { - all: 0, - done: 0, - comment: '', - config: this.config, - state: JobLastRunState.finished, - time: { - start: Date.now(), - end: Date.now() - } - }; + this.progress = new JobProgress(JobDTO.getHashName(this.Name, this.config)); + this.progress.OnChange = this.jobListener.onProgressUpdate; const pr = new Promise((resolve) => { this.prResolve = resolve; }); this.init().catch(console.error); - this.state = JobState.running; this.run(); if (!this.IsInstant) { // if instant, wait for execution, otherwise, return right away return Promise.resolve(); @@ -74,11 +57,9 @@ export abstract class Job implements IJob { } } - public stop(): void { + public cancel(): void { Logger.info(LOG_TAG, 'Stopping job: ' + this.Name); - this.state = JobState.stopping; - this.progress.state = JobState.stopping; - this.lastRuns[JSON.stringify(this.config)].state = JobLastRunState.canceled; + this.Progress.State = JobProgressStates.cancelling; } public toJSON(): JobDTO { @@ -88,18 +69,19 @@ export abstract class Job implements IJob { }; } - protected OnFinishCB = (status: JobLastRunState) => { - }; - protected abstract async step(): Promise; + protected abstract async step(): Promise; protected abstract async init(): Promise; private onFinish(): void { - this.lastRuns[JSON.stringify(this.config)].all = this.progress.left + this.progress.progress; - this.lastRuns[JSON.stringify(this.config)].done = this.progress.progress; - this.lastRuns[JSON.stringify(this.config)].time.end = Date.now(); - + if (this.Progress.State === JobProgressStates.running) { + this.Progress.State = JobProgressStates.finished; + } + if (this.Progress.State === JobProgressStates.cancelling) { + this.Progress.State = JobProgressStates.canceled; + } + const finishState = this.Progress.State; this.progress = null; if (global.gc) { global.gc(); @@ -108,25 +90,19 @@ export abstract class Job implements IJob { if (this.IsInstant) { this.prResolve(); } - this.OnFinishCB(this.lastRuns[JSON.stringify(this.config)].state); + this.jobListener.onJobFinished(this, finishState); } private run() { process.nextTick(async () => { try { - if (this.state === JobState.idle) { + if (this.Progress == null || this.Progress.State !== JobProgressStates.running) { return; } - let prg = null; - if (this.state === JobState.running) { - prg = await this.step(); - } - if (prg == null) { // finished - this.state = JobState.idle; + if (await this.step() === false) { // finished this.onFinish(); return; } - this.progress = prg; this.run(); } catch (e) { Logger.error(LOG_TAG, e); diff --git a/src/backend/model/jobs/jobs/JobProgress.ts b/src/backend/model/jobs/jobs/JobProgress.ts new file mode 100644 index 00000000..3f71007b --- /dev/null +++ b/src/backend/model/jobs/jobs/JobProgress.ts @@ -0,0 +1,102 @@ +import {JobProgressDTO, JobProgressStates} from '../../../../common/entities/job/JobProgressDTO'; + + +export class JobProgress { + private steps = { + all: 0, + processed: 0, + skipped: 0, + }; + private state = JobProgressStates.running; + private time = { + start: Date.now(), + end: null, + }; + private logs: string[] = []; + + + constructor(public readonly HashName: string) { + } + + set OnChange(val: (progress: JobProgress) => void) { + this.onChange = val; + } + + get Skipped(): number { + return this.steps.skipped; + } + + set Skipped(value: number) { + this.steps.skipped = value; + this.time.end = Date.now(); + this.onChange(this); + } + + get Processed(): number { + return this.steps.processed; + } + + set Processed(value: number) { + this.steps.processed = value; + this.time.end = Date.now(); + this.onChange(this); + } + + get Left(): number { + return this.steps.all - this.steps.processed - this.steps.skipped; + } + + set Left(value: number) { + this.steps.all = value + this.steps.skipped + this.steps.processed; + this.time.end = Date.now(); + this.onChange(this); + } + + get All(): number { + return this.steps.all; + } + + set All(value: number) { + this.steps.all = value; + this.time.end = Date.now(); + this.onChange(this); + } + + get State(): JobProgressStates { + return this.state; + } + + set State(value: JobProgressStates) { + this.state = value; + this.time.end = Date.now(); + this.onChange(this); + } + + get Logs(): string[] { + return this.logs; + } + + onChange = (progress: JobProgress) => { + }; + + log(log: string) { + while (this.logs.length > 10) { + this.logs.shift(); + } + this.logs.push(log); + this.onChange(this); + } + + toDTO(): JobProgressDTO { + return { + HashName: this.HashName, + state: this.state, + time: { + start: this.time.start, + end: this.time.end + }, + logs: this.logs, + steps: this.steps + }; + } +} diff --git a/src/backend/model/jobs/jobs/PhotoConvertingJob.ts b/src/backend/model/jobs/jobs/PhotoConvertingJob.ts index ebd9895e..46746b48 100644 --- a/src/backend/model/jobs/jobs/PhotoConvertingJob.ts +++ b/src/backend/model/jobs/jobs/PhotoConvertingJob.ts @@ -21,11 +21,15 @@ export class PhotoConvertingJob extends FileJob { } + protected async shouldProcess(file: FileDTO): Promise { + const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + return !(await PhotoProcessing.convertedPhotoExist(mPath, Config.Server.Media.Photo.Converting.resolution)); + } + + protected async processFile(file: FileDTO): Promise { - await PhotoProcessing.convertPhoto(path.join(ProjectPath.ImageFolder, - file.directory.path, - file.directory.name, - file.name), Config.Server.Media.Photo.Converting.resolution); + const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + await PhotoProcessing.convertPhoto(mPath); } diff --git a/src/backend/model/jobs/jobs/TempFolderCleaningJob.ts b/src/backend/model/jobs/jobs/TempFolderCleaningJob.ts index a6fafe63..1f2feab8 100644 --- a/src/backend/model/jobs/jobs/TempFolderCleaningJob.ts +++ b/src/backend/model/jobs/jobs/TempFolderCleaningJob.ts @@ -3,7 +3,6 @@ import * as path from 'path'; import * as util from 'util'; import {promises as fsp} from 'fs'; import {Job} from './Job'; -import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO'; import {ProjectPath} from '../../../ProjectPath'; import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing'; import {VideoProcessing} from '../../fileprocessing/VideoProcessing'; @@ -61,39 +60,38 @@ export class TempFolderCleaningJob extends Job { const validFiles = [ProjectPath.TranscodedFolder, ProjectPath.FacesFolder]; for (let i = 0; i < files.length; ++i) { if (validFiles.indexOf(files[i]) === -1) { + this.Progress.Processed++; if ((await fsp.stat(files[i])).isDirectory()) { await rimrafPR(files[i]); } else { await fsp.unlink(files[i]); } + } else { + this.Progress.Skipped++; } } - this.progress.time.current = Date.now(); - this.progress.comment = 'processing: ' + ProjectPath.TempFolder; + this.Progress.log('processing: ' + ProjectPath.TempFolder); - return this.progress; + return true; } protected async stepConvertedDirectory() { - - this.progress.time.current = Date.now(); - - const filePath = this.directoryQueue.shift(); const stat = await fsp.stat(filePath); - this.progress.left = this.directoryQueue.length; - this.progress.progress++; - this.progress.comment = 'processing: ' + filePath; + this.Progress.Left = this.directoryQueue.length; + this.Progress.log('processing: ' + filePath); if (stat.isDirectory()) { if (await this.isValidDirectory(filePath) === false) { + this.Progress.Processed++; await rimrafPR(filePath); } else { + this.Progress.Skipped++; this.directoryQueue = this.directoryQueue.concat(await this.readDir(filePath)); } } else { @@ -101,12 +99,12 @@ export class TempFolderCleaningJob extends Job { await fsp.unlink(filePath); } } - return this.progress; + return true; } - protected async step(): Promise { + protected async step(): Promise { if (this.directoryQueue.length === 0) { - return null; + return false; } if (this.tempRootCleaned === false) { this.tempRootCleaned = true; diff --git a/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts b/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts index 46c707d0..02706355 100644 --- a/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts +++ b/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts @@ -7,12 +7,12 @@ import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing'; import {ThumbnailSourceType} from '../../threading/PhotoWorker'; import {MediaDTO} from '../../../../common/entities/MediaDTO'; import {FileDTO} from '../../../../common/entities/FileDTO'; -import {JobLastRunState} from '../../../../common/entities/job/JobLastRunDTO'; const LOG_TAG = '[ThumbnailGenerationJob]'; export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOnly: boolean }> { + public readonly Name = DefaultsJobs[DefaultsJobs['Thumbnail Generation']]; constructor() { @@ -29,14 +29,14 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn return true; } - start(config: { sizes: number[], indexedOnly: boolean }, OnFinishCB: (status: JobLastRunState) => void): Promise { + start(config: { sizes: number[], indexedOnly: boolean }): Promise { for (let i = 0; i < config.sizes.length; ++i) { if (Config.Client.Media.Thumbnail.thumbnailSizes.indexOf(config.sizes[i]) === -1) { throw new Error('unknown thumbnails size: ' + config.sizes[i] + '. Add it to the possible thumbnail sizes.'); } } - return super.start(config, OnFinishCB); + return super.start(config); } protected async filterMediaFiles(files: FileDTO[]): Promise { @@ -47,13 +47,22 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn return undefined; } - protected async processFile(media: FileDTO): Promise { + protected async shouldProcess(file: FileDTO): Promise { + const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + for (let i = 0; i < this.config.sizes.length; ++i) { + if (!(await PhotoProcessing.convertedPhotoExist(mPath, this.config.sizes[i]))) { + return true; + } + } + } - const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name); + protected async processFile(file: FileDTO): Promise { + + const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); for (let i = 0; i < this.config.sizes.length; ++i) { await PhotoProcessing.generateThumbnail(mPath, this.config.sizes[i], - MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo, + MediaDTO.isVideo(file) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo, false); } diff --git a/src/backend/model/jobs/jobs/VideoConvertingJob.ts b/src/backend/model/jobs/jobs/VideoConvertingJob.ts index 5caf03a1..691532f7 100644 --- a/src/backend/model/jobs/jobs/VideoConvertingJob.ts +++ b/src/backend/model/jobs/jobs/VideoConvertingJob.ts @@ -20,12 +20,14 @@ export class VideoConvertingJob extends FileJob { return Config.Client.Media.Video.enabled === true; } + protected async shouldProcess(file: FileDTO): Promise { + const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + return !(await VideoProcessing.convertedVideoExist(mPath)); + } protected async processFile(file: FileDTO): Promise { - await VideoProcessing.convertVideo(path.join(ProjectPath.ImageFolder, - file.directory.path, - file.directory.name, - file.name)); + const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + await VideoProcessing.convertVideo(mPath); } diff --git a/src/common/config/private/IPrivateConfig.ts b/src/common/config/private/IPrivateConfig.ts index 465e65f4..8b9f99cc 100644 --- a/src/common/config/private/IPrivateConfig.ts +++ b/src/common/config/private/IPrivateConfig.ts @@ -27,19 +27,19 @@ export module ServerConfig { password: string; } - export interface SQLiteConfig { - storage: string; - } + /* + export interface SQLiteConfig { + } - export interface MemoryConfig { - usersFile: string; - } + export interface MemoryConfig { + }*/ export interface DataBaseConfig { type: DatabaseType; + dbFolder: string; mysql?: MySQLConfig; - sqlite?: SQLiteConfig; - memory?: MemoryConfig; + // sqlite?: SQLiteConfig; + // memory?: MemoryConfig; } export interface ThumbnailConfig { @@ -78,6 +78,7 @@ export module ServerConfig { } export interface JobConfig { + maxSavedProgress: number; scheduled: JobScheduleDTO[]; } diff --git a/src/common/config/private/PrivateConfigDefaultsClass.ts b/src/common/config/private/PrivateConfigDefaultsClass.ts index a2e3b038..b506a918 100644 --- a/src/common/config/private/PrivateConfigDefaultsClass.ts +++ b/src/common/config/private/PrivateConfigDefaultsClass.ts @@ -43,18 +43,13 @@ export class PrivateConfigDefaultsClass extends PublicConfigClass implements IPr photoMetadataSize: 512 * 1024, Database: { type: ServerConfig.DatabaseType.sqlite, + dbFolder: 'db', mysql: { host: '', username: '', password: '', database: '' - }, - sqlite: { - storage: 'sqlite.db' - }, - memory: { - usersFile: 'user.db' } }, Sharing: { @@ -75,6 +70,7 @@ export class PrivateConfigDefaultsClass extends PublicConfigClass implements IPr listingLimit: 1000 }, Jobs: { + maxSavedProgress: 10, scheduled: [ { name: DefaultsJobs[DefaultsJobs.Indexing], diff --git a/src/common/entities/job/JobDTO.ts b/src/common/entities/job/JobDTO.ts index 2ecd9757..10e9cd2f 100644 --- a/src/common/entities/job/JobDTO.ts +++ b/src/common/entities/job/JobDTO.ts @@ -21,3 +21,9 @@ export interface JobDTO { Name: string; ConfigTemplate: ConfigTemplateEntry[]; } + +export module JobDTO { + export const getHashName = (jobName: string, config: any = {}) => { + return jobName + '-' + JSON.stringify(config); + }; +} diff --git a/src/common/entities/job/JobLastRunDTO.ts b/src/common/entities/job/JobLastRunDTO.ts deleted file mode 100644 index 1e1442e5..00000000 --- a/src/common/entities/job/JobLastRunDTO.ts +++ /dev/null @@ -1,15 +0,0 @@ -export enum JobLastRunState { - finished = 1, canceled = 2 -} - -export interface JobLastRunDTO { - config: any; - done: number; - all: number; - state: JobLastRunState; - comment: string; - time: { - start: number, - end: number - }; -} diff --git a/src/common/entities/job/JobProgressDTO.ts b/src/common/entities/job/JobProgressDTO.ts index f743391a..e03fde5c 100644 --- a/src/common/entities/job/JobProgressDTO.ts +++ b/src/common/entities/job/JobProgressDTO.ts @@ -1,15 +1,19 @@ -export enum JobState { - idle = 1, running = 2, stopping = 3 +export enum JobProgressStates { + running = 1, cancelling = 2, interrupted = 3, canceled = 4, finished = 5 } export interface JobProgressDTO { - progress: number; - left: number; - state: JobState; - comment: string; + HashName: string; + steps: { + all: number, + processed: number, + skipped: number, + }; + state: JobProgressStates; + logs: string[]; time: { start: number, - current: number + end: number }; } diff --git a/src/frontend/app/ui/settings/database/database.settings.component.html b/src/frontend/app/ui/settings/database/database.settings.component.html index 108f530d..fb956f06 100644 --- a/src/frontend/app/ui/settings/database/database.settings.component.html +++ b/src/frontend/app/ui/settings/database/database.settings.component.html @@ -21,6 +21,17 @@ +
+ +
+ +
+ + All file-based data will be stored here (sqlite database, user database in case of memory db, job history data) + +
+
@@ -52,24 +63,7 @@
- -
- -
- -
-
- - -
- -
- -
-
- + @@ -129,7 +129,7 @@ diff --git a/src/frontend/app/ui/settings/jobs/jobs.settings.component.ts b/src/frontend/app/ui/settings/jobs/jobs.settings.component.ts index 77ed0135..e7a7a5d7 100644 --- a/src/frontend/app/ui/settings/jobs/jobs.settings.component.ts +++ b/src/frontend/app/ui/settings/jobs/jobs.settings.component.ts @@ -17,10 +17,10 @@ import { } from '../../../../../common/entities/job/JobScheduleDTO'; import {Utils} from '../../../../../common/Utils'; import {ServerConfig} from '../../../../../common/config/private/IPrivateConfig'; -import {ConfigTemplateEntry} from '../../../../../common/entities/job/JobDTO'; -import {JobState} from '../../../../../common/entities/job/JobProgressDTO'; +import {ConfigTemplateEntry, JobDTO} from '../../../../../common/entities/job/JobDTO'; import {Job} from '../../../../../backend/model/jobs/jobs/Job'; import {ModalDirective} from 'ngx-bootstrap/modal'; +import {JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO'; @Component({ selector: 'app-settings-jobs', @@ -38,7 +38,7 @@ export class JobsSettingsComponent extends SettingsComponent Last run: -
- +
+ {{lastRun.time.start | date:'medium'}} - {{lastRun.time.end | date:'mediumTime'}}
-
- - {{lastRun.done}}/{{lastRun.all}} +
+ + {{lastRun.steps.processed}}+{{lastRun.steps.skipped}}/{{lastRun.steps.all}}
-
- - {{JobLastRunState[lastRun.state]}} +
+ + {{JobProgressStates[lastRun.state]}}
- - +
@@ -30,22 +30,22 @@
{{TimeElapsed| duration:':'}}
{{TimeAll| duration:':'}}
-
+
- {{progress.progress}}/{{progress.progress + progress.left}} + [style.width.%]="(progress.steps.processed+progress.steps.skipped/(progress.steps.all))*100"> + {{progress.steps.processed}}+{{progress.steps.skipped}}/{{progress.steps.all}}
diff --git a/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.ts b/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.ts index ed25a50d..e8a4e351 100644 --- a/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.ts +++ b/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.ts @@ -1,7 +1,6 @@ import {Component, Input, OnChanges, OnDestroy} from '@angular/core'; -import {JobProgressDTO, JobState} from '../../../../../../common/entities/job/JobProgressDTO'; +import {JobProgressDTO, JobProgressStates} from '../../../../../../common/entities/job/JobProgressDTO'; import {Subscription, timer} from 'rxjs'; -import {JobLastRunDTO, JobLastRunState} from '../../../../../../common/entities/job/JobLastRunDTO'; @Component({ selector: 'app-settings-job-progress', @@ -11,10 +10,9 @@ import {JobLastRunDTO, JobLastRunState} from '../../../../../../common/entities/ export class JobProgressComponent implements OnDestroy, OnChanges { @Input() progress: JobProgressDTO; - @Input() lastRun: JobLastRunDTO; - JobState = JobState; + @Input() lastRun: JobProgressDTO; + JobProgressStates = JobProgressStates; timeCurrentCopy: number; - JobLastRunState = JobLastRunState; private timerSub: Subscription; constructor() { @@ -24,15 +22,15 @@ export class JobProgressComponent implements OnDestroy, OnChanges { if (!this.progress) { return 0; } - return (this.progress.time.current - this.progress.time.start) / - this.progress.progress * (this.progress.left + this.progress.progress); + return (this.progress.time.end - this.progress.time.start) / + (this.progress.steps.processed + this.progress.steps.skipped) * this.progress.steps.all; } get TimeLeft(): number { if (!this.progress) { return 0; } - return (this.progress.time.current - this.progress.time.start) / this.progress.progress * this.progress.left; + return (this.progress.time.end - this.progress.time.start) / this.progress.steps.all; } get TimeElapsed() { @@ -46,7 +44,7 @@ export class JobProgressComponent implements OnDestroy, OnChanges { if (!this.progress) { return; } - this.timeCurrentCopy = this.progress.time.current; + this.timeCurrentCopy = this.progress.time.end; if (!this.timerSub) { this.timerSub = timer(0, 1000).subscribe(() => { if (this.progress) { diff --git a/src/frontend/app/ui/settings/photo/photo.settings.component.html b/src/frontend/app/ui/settings/photo/photo.settings.component.html index 8af9b000..43e90f58 100644 --- a/src/frontend/app/ui/settings/photo/photo.settings.component.html +++ b/src/frontend/app/ui/settings/photo/photo.settings.component.html @@ -111,7 +111,7 @@ diff --git a/src/frontend/app/ui/settings/photo/photo.settings.component.ts b/src/frontend/app/ui/settings/photo/photo.settings.component.ts index 9a29b045..e8f86483 100644 --- a/src/frontend/app/ui/settings/photo/photo.settings.component.ts +++ b/src/frontend/app/ui/settings/photo/photo.settings.component.ts @@ -9,9 +9,9 @@ import {I18n} from '@ngx-translate/i18n-polyfill'; import {ScheduledJobsService} from '../scheduled-jobs.service'; import {ServerConfig} from '../../../../../common/config/private/IPrivateConfig'; import {Utils} from '../../../../../common/Utils'; -import {DefaultsJobs} from '../../../../../common/entities/job/JobDTO'; +import {DefaultsJobs, JobDTO} from '../../../../../common/entities/job/JobDTO'; import {ErrorDTO} from '../../../../../common/entities/Error'; -import {JobState} from '../../../../../common/entities/job/JobProgressDTO'; +import {JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO'; @Component({ @@ -28,7 +28,7 @@ export class PhotoSettingsComponent extends SettingsComponent<{ }> { resolutions = [720, 1080, 1440, 2160, 4320]; PhotoProcessingLib = ServerConfig.PhotoProcessingLib; - JobState = JobState; + JobProgressStates = JobProgressStates; libTypes = Utils .enumToArray(ServerConfig.PhotoProcessingLib).map((v) => { @@ -59,7 +59,7 @@ export class PhotoSettingsComponent extends SettingsComponent<{ get Progress() { - return this.jobsService.progress.value[DefaultsJobs[DefaultsJobs['Photo Converting']]]; + return this.jobsService.progress.value[JobDTO.getHashName(DefaultsJobs[DefaultsJobs['Photo Converting']])]; } async convertPhoto() { diff --git a/src/frontend/app/ui/settings/scheduled-jobs.service.ts b/src/frontend/app/ui/settings/scheduled-jobs.service.ts index 317a40ca..1922384c 100644 --- a/src/frontend/app/ui/settings/scheduled-jobs.service.ts +++ b/src/frontend/app/ui/settings/scheduled-jobs.service.ts @@ -1,15 +1,14 @@ import {EventEmitter, Injectable} from '@angular/core'; import {BehaviorSubject} from 'rxjs'; -import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO'; +import {JobProgressDTO, JobProgressStates} from '../../../../common/entities/job/JobProgressDTO'; import {NetworkService} from '../../model/network/network.service'; -import {JobLastRunDTO} from '../../../../common/entities/job/JobLastRunDTO'; @Injectable() export class ScheduledJobsService { public progress: BehaviorSubject<{ [key: string]: JobProgressDTO }>; - public lastRuns: BehaviorSubject<{ [key: string]: { [key: string]: JobLastRunDTO } }>; + public lastRuns: BehaviorSubject<{ [key: string]: { [key: string]: JobProgressStates } }>; public onJobFinish: EventEmitter = new EventEmitter(); timer: number = null; private subscribers = 0; @@ -19,17 +18,6 @@ export class ScheduledJobsService { this.lastRuns = new BehaviorSubject({}); } - public calcTimeElapsed(progress: JobProgressDTO) { - if (progress) { - return (progress.time.current - progress.time.start); - } - } - - public calcTimeLeft(progress: JobProgressDTO) { - if (progress) { - return (progress.time.current - progress.time.start) / progress.progress * progress.left; - } - } subscribeToProgress(): void { this.incSubscribers(); @@ -56,7 +44,7 @@ export class ScheduledJobsService { protected async getProgress(): Promise { const prevPrg = this.progress.value; this.progress.next(await this._networkService.getJson<{ [key: string]: JobProgressDTO }>('/admin/jobs/scheduled/progress')); - this.lastRuns.next(await this._networkService.getJson<{ [key: string]: { [key: string]: JobLastRunDTO } }>('/admin/jobs/scheduled/lastRun')); + this.lastRuns.next(await this._networkService.getJson<{ [key: string]: { [key: string]: JobProgressStates } }>('/admin/jobs/scheduled/lastRun')); for (const prg in prevPrg) { if (!this.progress.value.hasOwnProperty(prg)) { this.onJobFinish.emit(prg); @@ -64,6 +52,7 @@ export class ScheduledJobsService { } } + protected getProgressPeriodically() { if (this.timer != null || this.subscribers === 0) { return; diff --git a/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html b/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html index fd573621..83ac885b 100644 --- a/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html +++ b/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html @@ -85,7 +85,7 @@ diff --git a/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.ts b/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.ts index fedd720d..1752add9 100644 --- a/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.ts +++ b/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.ts @@ -7,10 +7,10 @@ import {ClientConfig} from '../../../../../common/config/public/ConfigClass'; import {ThumbnailSettingsService} from './thumbnail.settings.service'; import {I18n} from '@ngx-translate/i18n-polyfill'; import {ServerConfig} from '../../../../../common/config/private/IPrivateConfig'; -import {DefaultsJobs} from '../../../../../common/entities/job/JobDTO'; +import {DefaultsJobs, JobDTO} from '../../../../../common/entities/job/JobDTO'; import {ErrorDTO} from '../../../../../common/entities/Error'; import {ScheduledJobsService} from '../scheduled-jobs.service'; -import {JobState} from '../../../../../common/entities/job/JobProgressDTO'; +import {JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO'; @Component({ selector: 'app-settings-thumbnail', @@ -22,7 +22,7 @@ import {JobState} from '../../../../../common/entities/job/JobProgressDTO'; export class ThumbnailSettingsComponent extends SettingsComponent<{ server: ServerConfig.ThumbnailConfig, client: ClientConfig.ThumbnailConfig }> implements OnInit { - JobState = JobState; + JobProgressStates = JobProgressStates; constructor(_authService: AuthenticationService, _navigation: NavigationService, @@ -49,7 +49,7 @@ export class ThumbnailSettingsComponent } get Progress() { - return this.jobsService.progress.value[DefaultsJobs[DefaultsJobs['Thumbnail Generation']]]; + return this.jobsService.progress.value[JobDTO.getHashName(DefaultsJobs[DefaultsJobs['Thumbnail Generation']])]; } ngOnInit() { diff --git a/src/frontend/app/ui/settings/video/video.settings.component.html b/src/frontend/app/ui/settings/video/video.settings.component.html index 12ae71ef..f18de444 100644 --- a/src/frontend/app/ui/settings/video/video.settings.component.html +++ b/src/frontend/app/ui/settings/video/video.settings.component.html @@ -136,7 +136,7 @@ diff --git a/src/frontend/app/ui/settings/video/video.settings.component.ts b/src/frontend/app/ui/settings/video/video.settings.component.ts index 68f1aba8..ee4d7115 100644 --- a/src/frontend/app/ui/settings/video/video.settings.component.ts +++ b/src/frontend/app/ui/settings/video/video.settings.component.ts @@ -7,10 +7,10 @@ import {NotificationService} from '../../../model/notification.service'; import {ClientConfig} from '../../../../../common/config/public/ConfigClass'; import {I18n} from '@ngx-translate/i18n-polyfill'; import {ScheduledJobsService} from '../scheduled-jobs.service'; -import {DefaultsJobs} from '../../../../../common/entities/job/JobDTO'; +import {DefaultsJobs, JobDTO} from '../../../../../common/entities/job/JobDTO'; import {ErrorDTO} from '../../../../../common/entities/Error'; import {ServerConfig} from '../../../../../common/config/private/IPrivateConfig'; -import { JobState } from '../../../../../common/entities/job/JobProgressDTO'; +import {JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO'; @Component({ @@ -27,7 +27,7 @@ export class VideoSettingsComponent extends SettingsComponent<{ server: ServerCo formats: ServerConfig.formatType[] = ['mp4', 'webm']; fps = [24, 25, 30, 48, 50, 60]; - JobState = JobState; + JobProgressStates = JobProgressStates; constructor(_authService: AuthenticationService, _navigation: NavigationService, @@ -48,7 +48,7 @@ export class VideoSettingsComponent extends SettingsComponent<{ server: ServerCo get Progress() { - return this.jobsService.progress.value[DefaultsJobs[DefaultsJobs['Video Converting']]]; + return this.jobsService.progress.value[JobDTO.getHashName(DefaultsJobs[DefaultsJobs['Video Converting']])]; } get bitRate(): number { diff --git a/test/backend/SQLTestHelper.ts b/test/backend/SQLTestHelper.ts index 221327a9..3f1aa4ce 100644 --- a/test/backend/SQLTestHelper.ts +++ b/test/backend/SQLTestHelper.ts @@ -63,7 +63,7 @@ export class SQLTestHelper { await this.resetSQLite(); Config.Server.Database.type = ServerConfig.DatabaseType.sqlite; - Config.Server.Database.sqlite.storage = this.dbPath; + Config.Server.Database.dbFolder = path.dirname(this.dbPath); } private async initMySQL() { diff --git a/test/backend/integration/model/sql/typeorm.ts b/test/backend/integration/model/sql/typeorm.ts index 3c717fe9..9675b623 100644 --- a/test/backend/integration/model/sql/typeorm.ts +++ b/test/backend/integration/model/sql/typeorm.ts @@ -32,7 +32,7 @@ describe('Typeorm integration', () => { } Config.Server.Database.type = ServerConfig.DatabaseType.sqlite; - Config.Server.Database.sqlite.storage = dbPath; + Config.Server.Database.dbFolder = path.dirname(dbPath); }; diff --git a/test/backend/unit/model/fileprocessing/PhotoProcessing.spec.ts b/test/backend/unit/model/fileprocessing/PhotoProcessing.spec.ts index 6528143c..6ee86ffd 100644 --- a/test/backend/unit/model/fileprocessing/PhotoProcessing.spec.ts +++ b/test/backend/unit/model/fileprocessing/PhotoProcessing.spec.ts @@ -16,15 +16,18 @@ describe('PhotoProcessing', () => { const photoPath = path.join(ProjectPath.ImageFolder, 'test_png.png'); expect(await PhotoProcessing - .isValidConvertedPath(PhotoProcessing.generateConvertedFilePath(photoPath))) + .isValidConvertedPath(PhotoProcessing.generateConvertedPath(photoPath, + Config.Server.Media.Photo.Converting.resolution))) .to.be.true; expect(await PhotoProcessing - .isValidConvertedPath(PhotoProcessing.generateConvertedFilePath(photoPath + 'noPath'))) + .isValidConvertedPath(PhotoProcessing.generateConvertedPath(photoPath + 'noPath', + Config.Server.Media.Photo.Converting.resolution))) .to.be.false; { - const convertedPath = PhotoProcessing.generateConvertedFilePath(photoPath); + const convertedPath = PhotoProcessing.generateConvertedPath(photoPath, + Config.Server.Media.Photo.Converting.resolution); Config.Server.Media.Photo.Converting.resolution = 1; expect(await PhotoProcessing.isValidConvertedPath(convertedPath)).to.be.false; } @@ -42,18 +45,18 @@ describe('PhotoProcessing', () => { for (let i = 0; i < Config.Client.Media.Thumbnail.thumbnailSizes.length; ++i) { const thSize = Config.Client.Media.Thumbnail.thumbnailSizes[i]; expect(await PhotoProcessing - .isValidConvertedPath(PhotoProcessing.generateThumbnailPath(photoPath, thSize))) + .isValidConvertedPath(PhotoProcessing.generateConvertedPath(photoPath, thSize))) .to.be.true; expect(await PhotoProcessing - .isValidConvertedPath(PhotoProcessing.generateThumbnailPath(photoPath + 'noPath', thSize))) + .isValidConvertedPath(PhotoProcessing.generateConvertedPath(photoPath + 'noPath', thSize))) .to.be.false; } expect(await PhotoProcessing - .isValidConvertedPath(PhotoProcessing.generateThumbnailPath(photoPath, 30))) + .isValidConvertedPath(PhotoProcessing.generateConvertedPath(photoPath, 30))) .to.be.false; });