self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2024-12-25 02:04:15 +02:00
Code Issues Releases Activity

implementing job history saving

Browse Source
This commit is contained in:
Patrik J. Braun 2019-12-29 00:35:41 +01:00
parent 47b1aa7b86
commit ddb734e64a
47 changed files with 479 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
benchmark/Benchmarks.ts
View File

@ -5,6 +5,7 @@ import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker'
import {IndexingManager} from '../src/backend/model/database/sql/IndexingManager';
import {SearchManager} from '../src/backend/model/database/sql/SearchManager';
import * as fs from 'fs';
import * as path from 'path';
import {SearchTypes} from '../src/common/entities/AutoCompleteItem';
import {Utils} from '../src/common/Utils';
import {GalleryManager} from '../src/backend/model/database/sql/GalleryManager';
@ -123,7 +124,7 @@ export class Benchmarks {
fs.unlinkSync(this.dbPath);
}
Config.Server.Database.type = ServerConfig.DatabaseType.sqlite;
Config.Server.Database.sqlite.storage = this.dbPath;
Config.Server.Database.dbFolder = path.dirname(this.dbPath);
await ObjectManagers.InitSQLManagers();
};

3
docker/alpine/Dockerfile.build
View File

@ -16,8 +16,7 @@ ENTRYPOINT ["npm", "start", "--", \
# after a extensive job (like video converting), pigallery calls gc, to clean up everthing as fast as possible
"--expose-gc", \
"--config-path=/app/data/config/config.json", \
"--Server-Database-sqlite-storage=/app/data/db/sqlite.db", \
"--Server-Database-memory-usersFile=/app/data/db/users.db", \
"--Server-Database-dbFolder=/app/data/db", \
"--Server-Media-folder=/app/data/images", \
"--Server-Media-tempFolder=/app/data/tmp"]
EXPOSE 80

3
docker/debian-stretch/Dockerfile.build
View File

@ -15,8 +15,7 @@ ENTRYPOINT ["npm", "start", "--", \
# after a extensive job (like video converting), pigallery calls gc, to clean up everthing as fast as possible
"--expose-gc", \
"--config-path=/app/data/config/config.json", \
"--Server-Database-sqlite-storage=/app/data/db/sqlite.db", \
"--Server-Database-memory-usersFile=/app/data/db/users.db", \
"--Server-Database-dbFolder=/app/data/db", \
"--Server-Media-folder=/app/data/images", \
"--Server-Media-tempFolder=/app/data/tmp"]
EXPOSE 80

3
docker/debian-stretch/selfcontained/Dockerfile
View File

@ -18,8 +18,7 @@ ENTRYPOINT ["npm", "start", "--", \
# after a extensive job (like video converting), pigallery calls gc, to clean up everthing as fast as possible
"--expose-gc", \
"--config-path=/app/data/config/config.json", \
"--Server-Database-sqlite-storage=/app/data/db/sqlite.db", \
"--Server-Database-memory-usersFile=/app/data/db/users.db", \
"--Server-Database-dbFolder=/app/data/db", \
"--Server-Media-folder=/app/data/images", \
"--Server-Media-tempFolder=/app/data/tmp"]
EXPOSE 80

5
src/backend/middlewares/thumbnail/PhotoConverterMWs.ts
View File

@ -15,7 +15,7 @@ export class PhotoConverterMWs {
}
const fullMediaPath = req.resultPipe;
const convertedVideo = PhotoProcessing.generateConvertedFilePath(fullMediaPath);
const convertedVideo = PhotoProcessing.generateConvertedPath(fullMediaPath, Config.Server.Media.Photo.Converting.resolution);
// check if transcoded video exist
if (fs.existsSync(convertedVideo) === true) {
@ -24,8 +24,7 @@ export class PhotoConverterMWs {
}
if (Config.Server.Media.Photo.Converting.onTheFly === true) {
req.resultPipe = await PhotoProcessing.convertPhoto(fullMediaPath,
Config.Server.Media.Photo.Converting.resolution);
req.resultPipe = await PhotoProcessing.convertPhoto(fullMediaPath);
return next();
}

4
src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts
View File

@ -149,7 +149,7 @@ export class ThumbnailGeneratorMWs {
const fullMediaPath = path.join(ProjectPath.ImageFolder, photos[i].directory.path, photos[i].directory.name, photos[i].name);
for (let j = 0; j < Config.Client.Media.Thumbnail.thumbnailSizes.length; j++) {
const size = Config.Client.Media.Thumbnail.thumbnailSizes[j];
const thPath = PhotoProcessing.generateThumbnailPath(fullMediaPath, size);
const thPath = PhotoProcessing.generateConvertedPath(fullMediaPath, size);
if (fs.existsSync(thPath) === true) {
if (typeof photos[i].readyThumbnails === 'undefined') {
photos[i].readyThumbnails = [];
@ -157,7 +157,7 @@ export class ThumbnailGeneratorMWs {
photos[i].readyThumbnails.push(size);
}
}
const iconPath = PhotoProcessing.generateThumbnailPath(fullMediaPath, Config.Client.Media.Thumbnail.iconSize);
const iconPath = PhotoProcessing.generateConvertedPath(fullMediaPath, Config.Client.Media.Thumbnail.iconSize);
if (fs.existsSync(iconPath) === true) {
photos[i].readyIcon = true;
}

3
src/backend/model/database/interfaces/IJobManager.ts
View File

@ -1,6 +1,5 @@
import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO';
import {JobDTO} from '../../../../common/entities/job/JobDTO';
import {JobLastRunDTO} from '../../../../common/entities/job/JobLastRunDTO';
export interface IJobManager {
@ -17,5 +16,5 @@ export interface IJobManager {
runSchedules(): void;
getJobLastRuns(): { [key: string]: { [key: string]: JobLastRunDTO } };
getJobLastRuns(): { [key: string]: JobProgressDTO };
}

3
src/backend/model/database/memory/UserManager.ts
View File

@ -3,6 +3,7 @@ import {IUserManager} from '../interfaces/IUserManager';
import {ProjectPath} from '../../../ProjectPath';
import {Utils} from '../../../../common/Utils';
import * as fs from 'fs';
import * as path from 'path';
import {PasswordHelper} from '../../PasswordHelper';
import {Config} from '../../../../common/config/private/Config';
@ -13,7 +14,7 @@ export class UserManager implements IUserManager {
constructor() {
this.dbPath = ProjectPath.getAbsolutePath(Config.Server.Database.memory.usersFile);
this.dbPath = path.join(ProjectPath.getAbsolutePath(Config.Server.Database.dbFolder), 'users.db');
if (fs.existsSync(this.dbPath)) {
this.loadDB();
}

6
src/backend/model/database/sql/SQLConnection.ts
View File

@ -17,6 +17,7 @@ import {FileEntity} from './enitites/FileEntity';
import {FaceRegionEntry} from './enitites/FaceRegionEntry';
import {PersonEntry} from './enitites/PersonEntry';
import {Utils} from '../../../../common/Utils';
import * as path from 'path';
import {ServerConfig} from '../../../../common/config/private/IPrivateConfig';
@ -178,11 +179,14 @@ export class SQLConnection {
} else if (config.type === ServerConfig.DatabaseType.sqlite) {
driver = {
type: 'sqlite',
database: ProjectPath.getAbsolutePath(config.sqlite.storage)
database: path.join(ProjectPath.getAbsolutePath(config.dbFolder), 'sqlite.db')
};
}
return driver;
}
public static getSQLiteDB(config: ServerConfig.DataBaseConfig){
return path.join(ProjectPath.getAbsolutePath(config.dbFolder), 'sqlite.db');
}
}

5
src/backend/model/diagnostics/ConfigDiagnostics.ts
View File

@ -31,10 +31,9 @@ export class ConfigDiagnostics {
}
if (databaseConfig.type !== ServerConfig.DatabaseType.sqlite) {
try {
await this.checkReadWritePermission(ProjectPath.getAbsolutePath(databaseConfig.sqlite.storage));
await this.checkReadWritePermission(SQLConnection.getSQLiteDB(databaseConfig));
} catch (e) {
throw new Error('Cannot read or write sqlite storage file: ' +
ProjectPath.getAbsolutePath(databaseConfig.sqlite.storage));
throw new Error('Cannot read or write sqlite storage file: ' + SQLConnection.getSQLiteDB(databaseConfig));
}
}
}

43
src/backend/model/fileprocessing/PhotoProcessing.ts
View File

@ -98,7 +98,7 @@ export class PhotoProcessing {
}
public static generateThumbnailPath(mediaPath: string, size: number): string {
public static generateConvertedPath(mediaPath: string, size: number): string {
const file = path.basename(mediaPath);
return path.join(ProjectPath.TranscodedFolder,
ProjectPath.getRelativePathToImages(path.dirname(mediaPath)),
@ -111,9 +111,6 @@ export class PhotoProcessing {
.digest('hex') + '_' + size + '.jpg');
}
public static generateConvertedFilePath(photoPath: string): string {
return this.generateThumbnailPath(photoPath, Config.Server.Media.Photo.Converting.resolution);
}
public static async isValidConvertedPath(convertedPath: string): Promise<boolean> {
const origFilePath = path.join(ProjectPath.ImageFolder,
@ -142,42 +139,35 @@ export class PhotoProcessing {
}
public static async convertPhoto(mediaPath: string, size: number) {
public static async convertPhoto(mediaPath: string) {
return this.generateThumbnail(mediaPath,
Config.Server.Media.Photo.Converting.resolution,
ThumbnailSourceType.Photo,
false);
}
static async convertedPhotoExist(mediaPath: string, size: number) {
// generate thumbnail path
const outPath = PhotoProcessing.generateConvertedFilePath(mediaPath);
const outPath = PhotoProcessing.generateConvertedPath(mediaPath, size);
// check if file already exist
try {
await fsp.access(outPath, fsConstants.R_OK);
return outPath;
return true;
} catch (e) {
}
// run on other thread
const input = <RendererInput>{
type: ThumbnailSourceType.Photo,
mediaPath: mediaPath,
size: size,
outPath: outPath,
makeSquare: false,
qualityPriority: Config.Server.Media.Thumbnail.qualityPriority
};
const outDir = path.dirname(input.outPath);
await fsp.mkdir(outDir, {recursive: true});
await this.taskQue.execute(input);
return outPath;
return false;
}
public static async generateThumbnail(mediaPath: string,
size: number,
sourceType: ThumbnailSourceType,
makeSquare: boolean) {
makeSquare: boolean): Promise<string> {
// generate thumbnail path
const outPath = PhotoProcessing.generateThumbnailPath(mediaPath, size);
const outPath = PhotoProcessing.generateConvertedPath(mediaPath, size);
// check if file already exist
@ -209,5 +199,6 @@ export class PhotoProcessing {
const extension = path.extname(fullPath).toLowerCase();
return SupportedFormats.WithDots.Photos.indexOf(extension) !== -1;
}
}

13
src/backend/model/fileprocessing/VideoProcessing.ts
View File

@ -40,6 +40,19 @@ export class VideoProcessing {
return true;
}
static async convertedVideoExist(videoPath: string): Promise<boolean> {
const outPath = this.generateConvertedFilePath(videoPath);
try {
await fsp.access(outPath, fsConstants.R_OK);
return true;
} catch (e) {
}
return false;
}
public static async convertVideo(videoPath: string): Promise<void> {

47
src/backend/model/jobs/JobManager.ts
View File

@ -1,50 +1,40 @@
import {IJobManager} from '../database/interfaces/IJobManager';
import {JobProgressDTO} from '../../../common/entities/job/JobProgressDTO';
import {JobProgressDTO, JobProgressStates} from '../../../common/entities/job/JobProgressDTO';
import {IJob} from './jobs/IJob';
import {JobRepository} from './JobRepository';
import {Config} from '../../../common/config/private/Config';
import {AfterJobTrigger, JobScheduleDTO, JobTriggerType} from '../../../common/entities/job/JobScheduleDTO';
import {Logger} from '../../Logger';
import {NotificationManager} from '../NotifocationManager';
import {JobLastRunDTO, JobLastRunState} from '../../../common/entities/job/JobLastRunDTO';
import {IJobListener} from './jobs/IJobListener';
import {JobProgress} from './jobs/JobProgress';
import {JobProgressManager} from './JobProgressManager';
declare var global: NodeJS.Global;
const LOG_TAG = '[JobManager]';
export class JobManager implements IJobManager {
export class JobManager implements IJobManager, IJobListener {
protected timers: { schedule: JobScheduleDTO, timer: NodeJS.Timeout }[] = [];
protected progressManager: JobProgressManager = null;
constructor() {
this.progressManager = new JobProgressManager();
this.runSchedules();
}
getProgresses(): { [id: string]: JobProgressDTO } {
const m: { [id: string]: JobProgressDTO } = {};
JobRepository.Instance.getAvailableJobs()
.filter(t => t.Progress)
.forEach(t => {
t.Progress.time.current = Date.now();
m[t.Name] = t.Progress;
});
return m;
return this.progressManager.Running;
}
getJobLastRuns(): { [key: string]: { [key: string]: JobLastRunDTO } } {
const m: { [id: string]: { [id: string]: JobLastRunDTO } } = {};
JobRepository.Instance.getAvailableJobs().forEach(t => {
m[t.Name] = t.LastRuns;
});
return m;
getJobLastRuns(): { [key: string]: JobProgressDTO } {
return this.progressManager.Finished;
}
async run<T>(jobName: string, config: T): Promise<void> {
const t = this.findJob(jobName);
if (t) {
await t.start(config, (status: JobLastRunState) => {
this.onJobFinished(t, status);
});
t.JobListener = this;
await t.start(config);
} else {
Logger.warn(LOG_TAG, 'cannot find job to start:' + jobName);
}
@ -53,14 +43,19 @@ export class JobManager implements IJobManager {
stop(jobName: string): void {
const t = this.findJob(jobName);
if (t) {
t.stop();
t.cancel();
} else {
Logger.warn(LOG_TAG, 'cannot find job to stop:' + jobName);
}
}
async onJobFinished(job: IJob<any>, status: JobLastRunState): Promise<void> {
if (status === JobLastRunState.canceled) { // if it was cancelled do not start the next one
onProgressUpdate = (progress: JobProgress): void => {
this.progressManager.onJobProgressUpdate(progress.toDTO());
};
onJobFinished = async (job: IJob<any>, state: JobProgressStates): Promise<void> => {
if (state !== JobProgressStates.finished) { // if it was not finished peacefully, do not start the next one
return;
}
const sch = Config.Server.Jobs.scheduled.find(s => s.jobName === job.Name);
@ -75,7 +70,7 @@ export class JobManager implements IJobManager {
}
}
}
}
};
getAvailableJobs(): IJob<any>[] {
return JobRepository.Instance.getAvailableJobs();

83
src/backend/model/jobs/JobProgressManager.ts Normal file
View File

@ -0,0 +1,83 @@
import {promises as fsp} from 'fs';
import * as path from 'path';
import {ProjectPath} from '../../ProjectPath';
import {Config} from '../../../common/config/private/Config';
import {JobProgressDTO, JobProgressStates} from '../../../common/entities/job/JobProgressDTO';
export class JobProgressManager {
db: { [key: string]: { progress: JobProgressDTO, timestamp: number } } = {};
private readonly dbPath: string;
private timer: NodeJS.Timeout = null;
constructor() {
this.dbPath = path.join(ProjectPath.getAbsolutePath(Config.Server.Database.dbFolder), 'jobs.db');
this.loadDB().catch(console.error);
}
get Running(): { [key: string]: JobProgressDTO } {
const m: { [key: string]: JobProgressDTO } = {};
for (const key of Object.keys(this.db)) {
if (this.db[key].progress.state === JobProgressStates.running) {
m[key] = this.db[key].progress;
m[key].time.end = Date.now();
}
}
return m;
}
get Finished(): { [key: string]: JobProgressDTO } {
const m: { [key: string]: JobProgressDTO } = {};
for (const key of Object.keys(this.db)) {
if (this.db[key].progress.state !== JobProgressStates.running) {
m[key] = this.db[key].progress;
}
}
return m;
}
onJobProgressUpdate(progress: JobProgressDTO) {
this.db[progress.HashName] = {progress: progress, timestamp: Date.now()};
this.delayedSave();
}
private async loadDB() {
try {
await fsp.access(this.dbPath);
} catch (e) {
return;
}
const data = await fsp.readFile(this.dbPath, 'utf8');
this.db = JSON.parse(data);
while (Object.keys(this.db).length > Config.Server.Jobs.maxSavedProgress) {
let min: string = null;
for (const key of Object.keys(this.db)) {
if (min === null || this.db[min].timestamp > this.db[key].timestamp) {
min = key;
}
}
delete this.db[min];
}
for (const key of Object.keys(this.db)) {
if (this.db[key].progress.state === JobProgressStates.running) {
this.db[key].progress.state = JobProgressStates.interrupted;
}
}
}
private async saveDB() {
await fsp.writeFile(this.dbPath, JSON.stringify(this.db));
}
private delayedSave() {
if (this.timer !== null) {
return;
}
this.timer = setTimeout(async () => {
this.saveDB().catch(console.error);
this.timer = null;
}, 1000);
}
}

7
src/backend/model/jobs/jobs/DBResetJob.ts
View File

@ -1,4 +1,3 @@
import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO';
import {ObjectManagers} from '../../ObjectManagers';
import {Config} from '../../../../common/config/private/Config';
import {ConfigTemplateEntry, DefaultsJobs} from '../../../../common/entities/job/JobDTO';
@ -19,9 +18,11 @@ export class DBRestJob extends Job {
protected async init() {
}
protected async step(): Promise<JobProgressDTO> {
protected async step(): Promise<boolean> {
this.Progress.Left = 1;
this.Progress.Processed++;
await ObjectManagers.getInstance().IndexingManager.resetDB();
return null;
return false;
}

25
src/backend/model/jobs/jobs/FileJob.ts
View File

@ -1,4 +1,3 @@
import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO';
import {ConfigTemplateEntry} from '../../../../common/entities/job/JobDTO';
import {Job} from './Job';
import * as path from 'path';
@ -53,14 +52,14 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
return files;
}
protected abstract async shouldProcess(file: FileDTO): Promise<boolean>;
protected abstract async processFile(file: FileDTO): Promise<void>;
protected async step(): Promise<JobProgressDTO> {
protected async step(): Promise<boolean> {
if (this.directoryQueue.length === 0 && this.fileQueue.length === 0) {
return null;
return false;
}
this.progress.time.current = Date.now();
if (this.directoryQueue.length > 0) {
if (this.config.indexedOnly === true &&
@ -71,24 +70,30 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
await this.loadADirectoryFromDisk();
}
} else if (this.fileQueue.length > 0) {
this.Progress.Left = this.fileQueue.length;
const file = this.fileQueue.shift();
this.progress.left = this.fileQueue.length;
this.progress.progress++;
const filePath = path.join(file.directory.path, file.directory.name, file.name);
this.progress.comment = 'processing: ' + filePath;
try {
if ((await this.shouldProcess(file)) === true) {
this.Progress.Processed++;
this.Progress.log('processing: ' + filePath);
await this.processFile(file);
} else {
this.Progress.log('skipping: ' + filePath);
this.Progress.Skipped++;
}
} catch (e) {
console.error(e);
Logger.error(LOG_TAG, 'Error during processing file:' + filePath + ', ' + e.toString());
this.Progress.log('Error during processing file:' + filePath + ', ' + e.toString());
}
}
return this.progress;
return true;
}
private async loadADirectoryFromDisk() {
const directory = this.directoryQueue.shift();
this.progress.comment = 'scanning directory: ' + directory;
this.Progress.log('scanning directory: ' + directory);
const scanned = await DiskManager.scanDirectory(directory, this.scanFilter);
for (let i = 0; i < scanned.directories.length; i++) {
this.directoryQueue.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
@ -106,7 +111,7 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
if (this.scanFilter.noVideo === true && this.scanFilter.noPhoto === true) {
return;
}
this.progress.comment = 'Loading files from db';
this.Progress.log('Loading files from db');
Logger.silly(LOG_TAG, 'Loading files from db');
const connection = await SQLConnection.getConnection();

12
src/backend/model/jobs/jobs/IJob.ts
View File

@ -1,16 +1,16 @@
import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO';
import {JobDTO} from '../../../../common/entities/job/JobDTO';
import {JobLastRunDTO, JobLastRunState} from '../../../../common/entities/job/JobLastRunDTO';
import {JobProgress} from './JobProgress';
import {IJobListener} from './IJobListener';
export interface IJob<T> extends JobDTO {
Name: string;
Supported: boolean;
Progress: JobProgressDTO;
LastRuns: { [key: string]: JobLastRunDTO };
Progress: JobProgress;
JobListener: IJobListener;
start(config: T, OnFinishCB: (status: JobLastRunState) => void): Promise<void>;
start(config: T): Promise<void>;
stop(): void;
cancel(): void;
toJSON(): JobDTO;
}

9
src/backend/model/jobs/jobs/IJobListener.ts Normal file
View File

@ -0,0 +1,9 @@
import {JobProgress} from './JobProgress';
import {IJob} from './IJob';
import {JobProgressStates} from '../../../../common/entities/job/JobProgressDTO';
export interface IJobListener {
onJobFinished(job: IJob<any>, state: JobProgressStates): void;
onProgressUpdate(progress: JobProgress): void;
}

19
src/backend/model/jobs/jobs/IndexingJob.ts
View File

@ -1,10 +1,10 @@
import {JobProgressDTO, JobState} from '../../../../common/entities/job/JobProgressDTO';
import {ObjectManagers} from '../../ObjectManagers';
import * as path from 'path';
import {Config} from '../../../../common/config/private/Config';
import {Job} from './Job';
import {ConfigTemplateEntry, DefaultsJobs} from '../../../../common/entities/job/JobDTO';
import {ServerConfig} from '../../../../common/config/private/IPrivateConfig';
import {JobProgressStates} from '../../../../common/entities/job/JobProgressDTO';
declare var global: NodeJS.Global;
const LOG_TAG = '[IndexingJob]';
@ -23,23 +23,22 @@ export class IndexingJob extends Job {
this.directoriesToIndex.push('/');
}
protected async step(): Promise<JobProgressDTO> {
protected async step(): Promise<boolean> {
if (this.directoriesToIndex.length === 0) {
return null;
return false;
}
const directory = this.directoriesToIndex.shift();
this.progress.comment = directory;
this.progress.left = this.directoriesToIndex.length;
this.Progress.log(directory);
this.Progress.Left = this.directoriesToIndex.length;
const scanned = await ObjectManagers.getInstance().IndexingManager.indexDirectory(directory);
if (this.state !== JobState.running) {
return null;
if (this.Progress.State !== JobProgressStates.running) {
return false;
}
this.progress.progress++;
this.progress.time.current = Date.now();
this.Progress.Processed++;
for (let i = 0; i < scanned.directories.length; i++) {
this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
}
return this.progress;
return true;
}

86
src/backend/model/jobs/jobs/Job.ts
View File

@ -1,8 +1,9 @@
import {JobProgressDTO, JobState} from '../../../../common/entities/job/JobProgressDTO';
import {Logger} from '../../../Logger';
import {IJob} from './IJob';
import {ConfigTemplateEntry, JobDTO} from '../../../../common/entities/job/JobDTO';
import {JobLastRunDTO, JobLastRunState} from '../../../../common/entities/job/JobLastRunDTO';
import {JobProgress} from './JobProgress';
import {IJobListener} from './IJobListener';
import {JobProgressStates} from '../../../../common/entities/job/JobProgressDTO';
declare const process: any;
declare const global: any;
@ -10,13 +11,15 @@ declare const global: any;
const LOG_TAG = '[JOB]';
export abstract class Job<T = void> implements IJob<T> {
protected progress: JobProgressDTO = null;
protected state = JobState.idle;
protected progress: JobProgress = null;
protected config: T;
protected prResolve: () => void;
protected IsInstant = false;
protected lastRuns: { [key: string]: JobLastRunDTO } = {};
private jobListener: IJobListener;
public set JobListener(value: IJobListener) {
this.jobListener = value;
}
public abstract get Supported(): boolean;
@ -24,45 +27,25 @@ export abstract class Job<T = void> implements IJob<T> {
public abstract get ConfigTemplate(): ConfigTemplateEntry[];
public get LastRuns(): { [key: string]: JobLastRunDTO } {
return this.lastRuns;
}
public get Progress(): JobProgressDTO {
public get Progress(): JobProgress {
return this.progress;
}
public start(config: T, onFinishCB: (status: JobLastRunState) => void): Promise<void> {
this.OnFinishCB = onFinishCB;
if (this.state === JobState.idle && this.Supported) {
public get CanRun() {
return this.Progress == null && this.Supported;
}
public start(config: T): Promise<void> {
if (this.CanRun) {
Logger.info(LOG_TAG, 'Running job: ' + this.Name);
this.config = config;
this.progress = {
progress: 0,
left: 0,
comment: '',
state: JobState.running,
time: {
start: Date.now(),
current: Date.now()
}
};
this.lastRuns[JSON.stringify(this.config)] = {
all: 0,
done: 0,
comment: '',
config: this.config,
state: JobLastRunState.finished,
time: {
start: Date.now(),
end: Date.now()
}
};
this.progress = new JobProgress(JobDTO.getHashName(this.Name, this.config));
this.progress.OnChange = this.jobListener.onProgressUpdate;
const pr = new Promise<void>((resolve) => {
this.prResolve = resolve;
});
this.init().catch(console.error);
this.state = JobState.running;
this.run();
if (!this.IsInstant) { // if instant, wait for execution, otherwise, return right away
return Promise.resolve();
@ -74,11 +57,9 @@ export abstract class Job<T = void> implements IJob<T> {
}
}
public stop(): void {
public cancel(): void {
Logger.info(LOG_TAG, 'Stopping job: ' + this.Name);
this.state = JobState.stopping;
this.progress.state = JobState.stopping;
this.lastRuns[JSON.stringify(this.config)].state = JobLastRunState.canceled;
this.Progress.State = JobProgressStates.cancelling;
}
public toJSON(): JobDTO {
@ -88,18 +69,19 @@ export abstract class Job<T = void> implements IJob<T> {
};
}
protected OnFinishCB = (status: JobLastRunState) => {
};
protected abstract async step(): Promise<JobProgressDTO>;
protected abstract async step(): Promise<boolean>;
protected abstract async init(): Promise<void>;
private onFinish(): void {
this.lastRuns[JSON.stringify(this.config)].all = this.progress.left + this.progress.progress;
this.lastRuns[JSON.stringify(this.config)].done = this.progress.progress;
this.lastRuns[JSON.stringify(this.config)].time.end = Date.now();
if (this.Progress.State === JobProgressStates.running) {
this.Progress.State = JobProgressStates.finished;
}
if (this.Progress.State === JobProgressStates.cancelling) {
this.Progress.State = JobProgressStates.canceled;
}
const finishState = this.Progress.State;
this.progress = null;
if (global.gc) {
global.gc();
@ -108,25 +90,19 @@ export abstract class Job<T = void> implements IJob<T> {
if (this.IsInstant) {
this.prResolve();
}
this.OnFinishCB(this.lastRuns[JSON.stringify(this.config)].state);
this.jobListener.onJobFinished(this, finishState);
}
private run() {
process.nextTick(async () => {
try {
if (this.state === JobState.idle) {
if (this.Progress == null || this.Progress.State !== JobProgressStates.running) {
return;
}
let prg = null;
if (this.state === JobState.running) {
prg = await this.step();
}
if (prg == null) { // finished
this.state = JobState.idle;
if (await this.step() === false) { // finished
this.onFinish();
return;
}
this.progress = prg;
this.run();
} catch (e) {
Logger.error(LOG_TAG, e);

102
src/backend/model/jobs/jobs/JobProgress.ts Normal file
View File

@ -0,0 +1,102 @@
import {JobProgressDTO, JobProgressStates} from '../../../../common/entities/job/JobProgressDTO';
export class JobProgress {
private steps = {
all: 0,
processed: 0,
skipped: 0,
};
private state = JobProgressStates.running;
private time = {
start: <number>Date.now(),
end: <number>null,
};
private logs: string[] = [];
constructor(public readonly HashName: string) {
}
set OnChange(val: (progress: JobProgress) => void) {
this.onChange = val;
}
get Skipped(): number {
return this.steps.skipped;
}
set Skipped(value: number) {
this.steps.skipped = value;
this.time.end = Date.now();
this.onChange(this);
}
get Processed(): number {
return this.steps.processed;
}
set Processed(value: number) {
this.steps.processed = value;
this.time.end = Date.now();
this.onChange(this);
}
get Left(): number {
return this.steps.all - this.steps.processed - this.steps.skipped;
}
set Left(value: number) {
this.steps.all = value + this.steps.skipped + this.steps.processed;
this.time.end = Date.now();
this.onChange(this);
}
get All(): number {
return this.steps.all;
}
set All(value: number) {
this.steps.all = value;
this.time.end = Date.now();
this.onChange(this);
}
get State(): JobProgressStates {
return this.state;
}
set State(value: JobProgressStates) {
this.state = value;
this.time.end = Date.now();
this.onChange(this);
}
get Logs(): string[] {
return this.logs;
}
onChange = (progress: JobProgress) => {
};
log(log: string) {
while (this.logs.length > 10) {
this.logs.shift();
}
this.logs.push(log);
this.onChange(this);
}
toDTO(): JobProgressDTO {
return {
HashName: this.HashName,
state: this.state,
time: {
start: this.time.start,
end: this.time.end
},
logs: this.logs,
steps: this.steps
};
}
}

12
src/backend/model/jobs/jobs/PhotoConvertingJob.ts
View File

@ -21,11 +21,15 @@ export class PhotoConvertingJob extends FileJob {
}
protected async shouldProcess(file: FileDTO): Promise<boolean> {
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
return !(await PhotoProcessing.convertedPhotoExist(mPath, Config.Server.Media.Photo.Converting.resolution));
}
protected async processFile(file: FileDTO): Promise<void> {
await PhotoProcessing.convertPhoto(path.join(ProjectPath.ImageFolder,
file.directory.path,
file.directory.name,
file.name), Config.Server.Media.Photo.Converting.resolution);
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
await PhotoProcessing.convertPhoto(mPath);
}

26
src/backend/model/jobs/jobs/TempFolderCleaningJob.ts
View File

@ -3,7 +3,6 @@ import * as path from 'path';
import * as util from 'util';
import {promises as fsp} from 'fs';
import {Job} from './Job';
import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO';
import {ProjectPath} from '../../../ProjectPath';
import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
import {VideoProcessing} from '../../fileprocessing/VideoProcessing';
@ -61,39 +60,38 @@ export class TempFolderCleaningJob extends Job {
const validFiles = [ProjectPath.TranscodedFolder, ProjectPath.FacesFolder];
for (let i = 0; i < files.length; ++i) {
if (validFiles.indexOf(files[i]) === -1) {
this.Progress.Processed++;
if ((await fsp.stat(files[i])).isDirectory()) {
await rimrafPR(files[i]);
} else {
await fsp.unlink(files[i]);
}
} else {
this.Progress.Skipped++;
}
}
this.progress.time.current = Date.now();
this.progress.comment = 'processing: ' + ProjectPath.TempFolder;
this.Progress.log('processing: ' + ProjectPath.TempFolder);
return this.progress;
return true;
}
protected async stepConvertedDirectory() {
this.progress.time.current = Date.now();
const filePath = this.directoryQueue.shift();
const stat = await fsp.stat(filePath);
this.progress.left = this.directoryQueue.length;
this.progress.progress++;
this.progress.comment = 'processing: ' + filePath;
this.Progress.Left = this.directoryQueue.length;
this.Progress.log('processing: ' + filePath);
if (stat.isDirectory()) {
if (await this.isValidDirectory(filePath) === false) {
this.Progress.Processed++;
await rimrafPR(filePath);
} else {
this.Progress.Skipped++;
this.directoryQueue = this.directoryQueue.concat(await this.readDir(filePath));
}
} else {
@ -101,12 +99,12 @@ export class TempFolderCleaningJob extends Job {
await fsp.unlink(filePath);
}
}
return this.progress;
return true;
}
protected async step(): Promise<JobProgressDTO> {
protected async step(): Promise<boolean> {
if (this.directoryQueue.length === 0) {
return null;
return false;
}
if (this.tempRootCleaned === false) {
this.tempRootCleaned = true;

21
src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts
View File

@ -7,12 +7,12 @@ import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
import {ThumbnailSourceType} from '../../threading/PhotoWorker';
import {MediaDTO} from '../../../../common/entities/MediaDTO';
import {FileDTO} from '../../../../common/entities/FileDTO';
import {JobLastRunState} from '../../../../common/entities/job/JobLastRunDTO';
const LOG_TAG = '[ThumbnailGenerationJob]';
export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOnly: boolean }> {
public readonly Name = DefaultsJobs[DefaultsJobs['Thumbnail Generation']];
constructor() {
@ -29,14 +29,14 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn
return true;
}
start(config: { sizes: number[], indexedOnly: boolean }, OnFinishCB: (status: JobLastRunState) => void): Promise<void> {
start(config: { sizes: number[], indexedOnly: boolean }): Promise<void> {
for (let i = 0; i < config.sizes.length; ++i) {
if (Config.Client.Media.Thumbnail.thumbnailSizes.indexOf(config.sizes[i]) === -1) {
throw new Error('unknown thumbnails size: ' + config.sizes[i] + '. Add it to the possible thumbnail sizes.');
}
}
return super.start(config, OnFinishCB);
return super.start(config);
}
protected async filterMediaFiles(files: FileDTO[]): Promise<FileDTO[]> {
@ -47,13 +47,22 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn
return undefined;
}
protected async processFile(media: FileDTO): Promise<void> {
protected async shouldProcess(file: FileDTO): Promise<boolean> {
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
for (let i = 0; i < this.config.sizes.length; ++i) {
if (!(await PhotoProcessing.convertedPhotoExist(mPath, this.config.sizes[i]))) {
return true;
}
}
}
const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name);
protected async processFile(file: FileDTO): Promise<void> {
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
for (let i = 0; i < this.config.sizes.length; ++i) {
await PhotoProcessing.generateThumbnail(mPath,
this.config.sizes[i],
MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo,
MediaDTO.isVideo(file) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo,
false);
}

10
src/backend/model/jobs/jobs/VideoConvertingJob.ts
View File

@ -20,12 +20,14 @@ export class VideoConvertingJob extends FileJob {
return Config.Client.Media.Video.enabled === true;
}
protected async shouldProcess(file: FileDTO): Promise<boolean> {
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
return !(await VideoProcessing.convertedVideoExist(mPath));
}
protected async processFile(file: FileDTO): Promise<void> {
await VideoProcessing.convertVideo(path.join(ProjectPath.ImageFolder,
file.directory.path,
file.directory.name,
file.name));
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
await VideoProcessing.convertVideo(mPath);
}

11
src/common/config/private/IPrivateConfig.ts
View File

@ -27,19 +27,19 @@ export module ServerConfig {
password: string;
}
/*
export interface SQLiteConfig {
storage: string;
}
export interface MemoryConfig {
usersFile: string;
}
}*/
export interface DataBaseConfig {
type: DatabaseType;
dbFolder: string;
mysql?: MySQLConfig;
sqlite?: SQLiteConfig;
memory?: MemoryConfig;
// sqlite?: SQLiteConfig;
// memory?: MemoryConfig;
}
export interface ThumbnailConfig {
@ -78,6 +78,7 @@ export module ServerConfig {
}
export interface JobConfig {
maxSavedProgress: number;
scheduled: JobScheduleDTO[];
}

8
src/common/config/private/PrivateConfigDefaultsClass.ts
View File

@ -43,18 +43,13 @@ export class PrivateConfigDefaultsClass extends PublicConfigClass implements IPr
photoMetadataSize: 512 * 1024,
Database: {
type: ServerConfig.DatabaseType.sqlite,
dbFolder: 'db',
mysql: {
host: '',
username: '',
password: '',
database: ''
},
sqlite: {
storage: 'sqlite.db'
},
memory: {
usersFile: 'user.db'
}
},
Sharing: {
@ -75,6 +70,7 @@ export class PrivateConfigDefaultsClass extends PublicConfigClass implements IPr
listingLimit: 1000
},
Jobs: {
maxSavedProgress: 10,
scheduled: [
{
name: DefaultsJobs[DefaultsJobs.Indexing],

6
src/common/entities/job/JobDTO.ts
View File

@ -21,3 +21,9 @@ export interface JobDTO {
Name: string;
ConfigTemplate: ConfigTemplateEntry[];
}
export module JobDTO {
export const getHashName = (jobName: string, config: any = {}) => {
return jobName + '-' + JSON.stringify(config);
};
}

15
src/common/entities/job/JobLastRunDTO.ts
View File

@ -1,15 +0,0 @@
export enum JobLastRunState {
finished = 1, canceled = 2
}
export interface JobLastRunDTO {
config: any;
done: number;
all: number;
state: JobLastRunState;
comment: string;
time: {
start: number,
end: number
};
}

18
src/common/entities/job/JobProgressDTO.ts
View File

@ -1,15 +1,19 @@
export enum JobState {
idle = 1, running = 2, stopping = 3
export enum JobProgressStates {
running = 1, cancelling = 2, interrupted = 3, canceled = 4, finished = 5
}
export interface JobProgressDTO {
progress: number;
left: number;
state: JobState;
comment: string;
HashName: string;
steps: {
all: number,
processed: number,
skipped: number,
};
state: JobProgressStates;
logs: string[];
time: {
start: number,
current: number
end: number
};
}

30
src/frontend/app/ui/settings/database/database.settings.component.html
View File

@ -21,6 +21,17 @@
</div>
</div>
<div class="form-group row">
<label class="col-md-2 control-label" for="dbFolder" i18n>Database folder</label>
<div class="col-md-10">
<input type="text" class="form-control" placeholder="db"
[(ngModel)]="settings.dbFolder" id="dbFolder" name="dbFolder" required>
</div>
<small class="form-text text-muted" i18n>
All file-based data will be stored here (sqlite database, user database in case of memory db, job history data)
</small>
</div>
<ng-container *ngIf="settings.type == DatabaseType.mysql">
<div class="form-group row">
@ -52,24 +63,7 @@
</div>
</div>
</ng-container>
<ng-container *ngIf="settings.type == DatabaseType.sqlite">
<div class="form-group row">
<label class="col-md-2 control-label" for="storage" i18n>Storage file</label>
<div class="col-md-10">
<input type="text" class="form-control" placeholder="sqlite.db"
[(ngModel)]="settings.sqlite.storage" id="storage" name="storage" required>
</div>
</div>
</ng-container>
<ng-container *ngIf="settings.type == DatabaseType.memory">
<div class="form-group row">
<label class="col-md-2 control-label" for="usersFile" i18n>User's file</label>
<div class="col-md-10">
<input type="text" class="form-control" placeholder="users.db"
[(ngModel)]="settings.memory.usersFile" id="usersFile" name="usersFile" required>
</div>
</div>
</ng-container>
<button class="btn btn-success float-right"
[disabled]="!settingsForm.form.valid || !changed || inProgress"

2
src/frontend/app/ui/settings/indexing/indexing.settings.component.html
View File

@ -123,7 +123,7 @@
</button>
<button class="btn btn-secondary ml-0"
*ngIf="Progress != null"
[disabled]="inProgress || Progress.state !== JobState.running"
[disabled]="inProgress || Progress.state !== JobProgressStates.running"
(click)="cancelIndexing()" i18n>Cancel converting
</button>
<button class="btn btn-danger ml-2"

8
src/frontend/app/ui/settings/indexing/indexing.settings.component.ts
View File

@ -8,9 +8,9 @@ import {SettingsComponent} from '../_abstract/abstract.settings.component';
import {Utils} from '../../../../../common/Utils';
import {I18n} from '@ngx-translate/i18n-polyfill';
import {ScheduledJobsService} from '../scheduled-jobs.service';
import {DefaultsJobs} from '../../../../../common/entities/job/JobDTO';
import {DefaultsJobs, JobDTO} from '../../../../../common/entities/job/JobDTO';
import {ServerConfig} from '../../../../../common/config/private/IPrivateConfig';
import {JobState} from '../../../../../common/entities/job/JobProgressDTO';
import {JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO';
@Component({
selector: 'app-settings-indexing',
@ -24,7 +24,7 @@ export class IndexingSettingsComponent extends SettingsComponent<ServerConfig.In
types: { key: number; value: string }[] = [];
JobState = JobState;
JobProgressStates = JobProgressStates;
constructor(_authService: AuthenticationService,
_navigation: NavigationService,
@ -44,7 +44,7 @@ export class IndexingSettingsComponent extends SettingsComponent<ServerConfig.In
}
get Progress() {
return this.jobsService.progress.value[DefaultsJobs[DefaultsJobs.Indexing]];
return this.jobsService.progress.value[JobDTO.getHashName(DefaultsJobs[DefaultsJobs.Indexing])];
}
get excludeFolderList(): string {

4
src/frontend/app/ui/settings/jobs/jobs.settings.component.html
View File

@ -36,7 +36,7 @@
</button>
<button class="btn btn-secondary job-control-button"
*ngIf="jobsService.progress.value[schedule.jobName]"
[disabled]="disableButtons || jobsService.progress.value[schedule.jobName].state !== JobState.running"
[disabled]="disableButtons || jobsService.progress.value[schedule.jobName].state !== JobProgressStates.running"
(click)="stop(schedule); $event.stopPropagation();"><span class="oi oi-media-stop"></span>
</button>
</div>
@ -129,7 +129,7 @@
</button>
<button class="btn btn-secondary float-right"
*ngIf="jobsService.progress.value[schedule.jobName]"
[disabled]="disableButtons || jobsService.progress.value[schedule.jobName].state !== JobState.running"
[disabled]="disableButtons || jobsService.progress.value[schedule.jobName].state !== JobProgressStates.running"
(click)="stop(schedule)" i18n>Stop
</button>
</div>

10
src/frontend/app/ui/settings/jobs/jobs.settings.component.ts
View File

@ -17,10 +17,10 @@ import {
} from '../../../../../common/entities/job/JobScheduleDTO';
import {Utils} from '../../../../../common/Utils';
import {ServerConfig} from '../../../../../common/config/private/IPrivateConfig';
import {ConfigTemplateEntry} from '../../../../../common/entities/job/JobDTO';
import {JobState} from '../../../../../common/entities/job/JobProgressDTO';
import {ConfigTemplateEntry, JobDTO} from '../../../../../common/entities/job/JobDTO';
import {Job} from '../../../../../backend/model/jobs/jobs/Job';
import {ModalDirective} from 'ngx-bootstrap/modal';
import {JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO';
@Component({
selector: 'app-settings-jobs',
@ -38,7 +38,7 @@ export class JobsSettingsComponent extends SettingsComponent<ServerConfig.JobCon
JobTriggerType = JobTriggerType;
periods: string[] = [];
showDetails: boolean[] = [];
JobState = JobState;
JobProgressStates = JobProgressStates;
newSchedule: JobScheduleDTO = {
name: '',
config: null,
@ -216,11 +216,11 @@ export class JobsSettingsComponent extends SettingsComponent<ServerConfig.JobCon
}
getProgress(schedule: JobScheduleDTO) {
return this.jobsService.progress.value[schedule.jobName];
return this.jobsService.progress.value[JobDTO.getHashName(schedule.jobName, schedule.config)];
}
getLastRun(schedule: JobScheduleDTO) {
return (this.jobsService.lastRuns.value[schedule.jobName] || {})[this.getConfigHash(schedule)];
return this.jobsService.lastRuns.value[JobDTO.getHashName(schedule.jobName, schedule.config)];
}
private getNextRunningDate(sch: JobScheduleDTO, list: JobScheduleDTO[], depth: number = 0): number {

38
src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html
View File

@ -2,26 +2,26 @@
<div class="col-md-2 col-12" i18n>
Last run:
</div>
<div class="col-md-4 col-12">
<span class="oi oi-clock" title="Run between" i18n-title aria-hidden="true"></span>
<div class="col-md-4 col-12" title="Run between" i18n-title >
<span class="oi oi-clock" aria-hidden="true"></span>
{{lastRun.time.start | date:'medium'}} - {{lastRun.time.end | date:'mediumTime'}}
</div>
<div class="col-md-3 col-6">
<span class="oi oi-check" title="done/all" i18n-title aria-hidden="true"></span>
{{lastRun.done}}/{{lastRun.all}}
<div class="col-md-3 col-6" title="processed+skipped/all" i18n-title>
<span class="oi oi-check" aria-hidden="true"></span>
{{lastRun.steps.processed}}+{{lastRun.steps.skipped}}/{{lastRun.steps.all}}
</div>
<div class="col-md-3 col-6">
<span class="oi oi-pulse" title="Status" i18n-title aria-hidden="true"></span>
{{JobLastRunState[lastRun.state]}}
<div class="col-md-3 col-6" title="Status" i18n-title >
<span class="oi oi-pulse" aria-hidden="true"></span>
{{JobProgressStates[lastRun.state]}}
</div>
</div>
<div *ngIf="progress">
<div class="form-group row">
<div class="col-md-12">
<input *ngIf="progress.state === JobState.running" type="text" class="form-control" disabled
[ngModel]="progress.comment" name="details">
<input *ngIf="progress.state === JobState.stopping" type="text" class="form-control" disabled value="Stopping"
<input *ngIf="progress.state === JobProgressStates.running" type="text" class="form-control" disabled
[ngModel]="progress.logs[progress.logs.length-1]" name="details">
<input *ngIf="progress.state === JobProgressStates.cancelling" type="text" class="form-control" disabled value="Cancelling: {{progress.logs[progress.logs.length-1]}}"
i18n-value name="details">
</div>
</div>
@ -30,22 +30,22 @@
<div class="form-group row progress-row ">
<div class="col-6 col-md-2 col-lg-1 text-md-right order-md-0" title="time elapsed" i18n-title>{{TimeElapsed| duration:':'}}</div>
<div class="col-6 col-md-2 col-lg-1 order-md-2 text-right text-md-left" title="time left" i18n-title>{{TimeAll| duration:':'}}</div>
<div class="progress col-md-8 col-lg-10 order-md-1">
<div class="progress col-md-8 col-lg-10 order-md-1"
title="processed:{{progress.steps.processed}}+ skipped:{{progress.steps.skipped}} / all:{{progress.steps.all}}">
<div
*ngIf="progress.progress + progress.left >0"
class="progress-bar d-inline-block progress-bar-success {{progress.state === JobState.stopping ? 'bg-secondary' : ''}}"
*ngIf="progress.steps.all >0"
class="progress-bar d-inline-block progress-bar-success {{progress.state === JobProgressStates.cancelling ? 'bg-secondary' : ''}}"
role="progressbar"
aria-valuenow="2"
aria-valuemin="0"
aria-valuemax="100"
style="min-width: 2em;"
title="{{progress.progress}}/{{progress.progress + progress.left}}"
[style.width.%]="(progress.progress/(progress.left+progress.progress))*100">
{{progress.progress}}/{{progress.progress + progress.left}}
[style.width.%]="(progress.steps.processed+progress.steps.skipped/(progress.steps.all))*100">
{{progress.steps.processed}}+{{progress.steps.skipped}}/{{progress.steps.all}}
</div>
<div
*ngIf="progress.progress + progress.left === 0"
class="progress-bar d-inline-block progress-bar-success progress-bar-striped progress-bar-animated {{progress.state === JobState.stopping ? 'bg-secondary' : ''}}"
*ngIf="progress.steps.all === 0"
class="progress-bar d-inline-block progress-bar-success progress-bar-striped progress-bar-animated {{progress.state === JobProgressStates.cancelling ? 'bg-secondary' : ''}}"
role="progressbar" aria-valuenow="100"
aria-valuemin="0" aria-valuemax="100" style="width: 100%">
</div>

16
src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.ts
View File

@ -1,7 +1,6 @@
import {Component, Input, OnChanges, OnDestroy} from '@angular/core';
import {JobProgressDTO, JobState} from '../../../../../../common/entities/job/JobProgressDTO';
import {JobProgressDTO, JobProgressStates} from '../../../../../../common/entities/job/JobProgressDTO';
import {Subscription, timer} from 'rxjs';
import {JobLastRunDTO, JobLastRunState} from '../../../../../../common/entities/job/JobLastRunDTO';
@Component({
selector: 'app-settings-job-progress',
@ -11,10 +10,9 @@ import {JobLastRunDTO, JobLastRunState} from '../../../../../../common/entities/
export class JobProgressComponent implements OnDestroy, OnChanges {
@Input() progress: JobProgressDTO;
@Input() lastRun: JobLastRunDTO;
JobState = JobState;
@Input() lastRun: JobProgressDTO;
JobProgressStates = JobProgressStates;
timeCurrentCopy: number;
JobLastRunState = JobLastRunState;
private timerSub: Subscription;
constructor() {
@ -24,15 +22,15 @@ export class JobProgressComponent implements OnDestroy, OnChanges {
if (!this.progress) {
return 0;
}
return (this.progress.time.current - this.progress.time.start) /
this.progress.progress * (this.progress.left + this.progress.progress);
return (this.progress.time.end - this.progress.time.start) /
(this.progress.steps.processed + this.progress.steps.skipped) * this.progress.steps.all;
}
get TimeLeft(): number {
if (!this.progress) {
return 0;
}
return (this.progress.time.current - this.progress.time.start) / this.progress.progress * this.progress.left;
return (this.progress.time.end - this.progress.time.start) / this.progress.steps.all;
}
get TimeElapsed() {
@ -46,7 +44,7 @@ export class JobProgressComponent implements OnDestroy, OnChanges {
if (!this.progress) {
return;
}
this.timeCurrentCopy = this.progress.time.current;
this.timeCurrentCopy = this.progress.time.end;
if (!this.timerSub) {
this.timerSub = timer(0, 1000).subscribe(() => {
if (this.progress) {

2
src/frontend/app/ui/settings/photo/photo.settings.component.html
View File

@ -111,7 +111,7 @@
</button>
<button class="btn btn-secondary float-left ml-0"
*ngIf="Progress != null"
[disabled]="inProgress || Progress.state !== JobState.running"
[disabled]="inProgress || Progress.state !== JobProgressStates.running"
(click)="cancelPhotoConverting()" i18n>Cancel converting
</button>

8
src/frontend/app/ui/settings/photo/photo.settings.component.ts
View File

@ -9,9 +9,9 @@ import {I18n} from '@ngx-translate/i18n-polyfill';
import {ScheduledJobsService} from '../scheduled-jobs.service';
import {ServerConfig} from '../../../../../common/config/private/IPrivateConfig';
import {Utils} from '../../../../../common/Utils';
import {DefaultsJobs} from '../../../../../common/entities/job/JobDTO';
import {DefaultsJobs, JobDTO} from '../../../../../common/entities/job/JobDTO';
import {ErrorDTO} from '../../../../../common/entities/Error';
import {JobState} from '../../../../../common/entities/job/JobProgressDTO';
import {JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO';
@Component({
@ -28,7 +28,7 @@ export class PhotoSettingsComponent extends SettingsComponent<{
}> {
resolutions = [720, 1080, 1440, 2160, 4320];
PhotoProcessingLib = ServerConfig.PhotoProcessingLib;
JobState = JobState;
JobProgressStates = JobProgressStates;
libTypes = Utils
.enumToArray(ServerConfig.PhotoProcessingLib).map((v) => {
@ -59,7 +59,7 @@ export class PhotoSettingsComponent extends SettingsComponent<{
get Progress() {
return this.jobsService.progress.value[DefaultsJobs[DefaultsJobs['Photo Converting']]];
return this.jobsService.progress.value[JobDTO.getHashName(DefaultsJobs[DefaultsJobs['Photo Converting']])];
}
async convertPhoto() {

19
src/frontend/app/ui/settings/scheduled-jobs.service.ts
View File

@ -1,15 +1,14 @@
import {EventEmitter, Injectable} from '@angular/core';
import {BehaviorSubject} from 'rxjs';
import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO';
import {JobProgressDTO, JobProgressStates} from '../../../../common/entities/job/JobProgressDTO';
import {NetworkService} from '../../model/network/network.service';
import {JobLastRunDTO} from '../../../../common/entities/job/JobLastRunDTO';
@Injectable()
export class ScheduledJobsService {
public progress: BehaviorSubject<{ [key: string]: JobProgressDTO }>;
public lastRuns: BehaviorSubject<{ [key: string]: { [key: string]: JobLastRunDTO } }>;
public lastRuns: BehaviorSubject<{ [key: string]: { [key: string]: JobProgressStates } }>;
public onJobFinish: EventEmitter<string> = new EventEmitter<string>();
timer: number = null;
private subscribers = 0;
@ -19,17 +18,6 @@ export class ScheduledJobsService {
this.lastRuns = new BehaviorSubject({});
}
public calcTimeElapsed(progress: JobProgressDTO) {
if (progress) {
return (progress.time.current - progress.time.start);
}
}
public calcTimeLeft(progress: JobProgressDTO) {
if (progress) {
return (progress.time.current - progress.time.start) / progress.progress * progress.left;
}
}
subscribeToProgress(): void {
this.incSubscribers();
@ -56,7 +44,7 @@ export class ScheduledJobsService {
protected async getProgress(): Promise<void> {
const prevPrg = this.progress.value;
this.progress.next(await this._networkService.getJson<{ [key: string]: JobProgressDTO }>('/admin/jobs/scheduled/progress'));
this.lastRuns.next(await this._networkService.getJson<{ [key: string]: { [key: string]: JobLastRunDTO } }>('/admin/jobs/scheduled/lastRun'));
this.lastRuns.next(await this._networkService.getJson<{ [key: string]: { [key: string]: JobProgressStates } }>('/admin/jobs/scheduled/lastRun'));
for (const prg in prevPrg) {
if (!this.progress.value.hasOwnProperty(prg)) {
this.onJobFinish.emit(prg);
@ -64,6 +52,7 @@ export class ScheduledJobsService {
}
}
protected getProgressPeriodically() {
if (this.timer != null || this.subscribers === 0) {
return;

2
src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html
View File

@ -85,7 +85,7 @@
</button>
<button class="btn btn-secondary float-left ml-0"
*ngIf="Progress != null"
[disabled]="inProgress || Progress.state !== JobState.running"
[disabled]="inProgress || Progress.state !== JobProgressStates.running"
(click)="cancelJob()" i18n>Cancel thumbnail generation
</button>

8
src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.ts
View File

@ -7,10 +7,10 @@ import {ClientConfig} from '../../../../../common/config/public/ConfigClass';
import {ThumbnailSettingsService} from './thumbnail.settings.service';
import {I18n} from '@ngx-translate/i18n-polyfill';
import {ServerConfig} from '../../../../../common/config/private/IPrivateConfig';
import {DefaultsJobs} from '../../../../../common/entities/job/JobDTO';
import {DefaultsJobs, JobDTO} from '../../../../../common/entities/job/JobDTO';
import {ErrorDTO} from '../../../../../common/entities/Error';
import {ScheduledJobsService} from '../scheduled-jobs.service';
import {JobState} from '../../../../../common/entities/job/JobProgressDTO';
import {JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO';
@Component({
selector: 'app-settings-thumbnail',
@ -22,7 +22,7 @@ import {JobState} from '../../../../../common/entities/job/JobProgressDTO';
export class ThumbnailSettingsComponent
extends SettingsComponent<{ server: ServerConfig.ThumbnailConfig, client: ClientConfig.ThumbnailConfig }>
implements OnInit {
JobState = JobState;
JobProgressStates = JobProgressStates;
constructor(_authService: AuthenticationService,
_navigation: NavigationService,
@ -49,7 +49,7 @@ export class ThumbnailSettingsComponent
}
get Progress() {
return this.jobsService.progress.value[DefaultsJobs[DefaultsJobs['Thumbnail Generation']]];
return this.jobsService.progress.value[JobDTO.getHashName(DefaultsJobs[DefaultsJobs['Thumbnail Generation']])];
}
ngOnInit() {

2
src/frontend/app/ui/settings/video/video.settings.component.html
View File

@ -136,7 +136,7 @@
</button>
<button class="btn btn-secondary float-left ml-0"
*ngIf="Progress != null"
[disabled]="inProgress || Progress.state !== JobState.running"
[disabled]="inProgress || Progress.state !== JobProgressStates.running"
(click)="cancelTranscoding()" i18n>Cancel transcoding
</button>

8
src/frontend/app/ui/settings/video/video.settings.component.ts
View File

@ -7,10 +7,10 @@ import {NotificationService} from '../../../model/notification.service';
import {ClientConfig} from '../../../../../common/config/public/ConfigClass';
import {I18n} from '@ngx-translate/i18n-polyfill';
import {ScheduledJobsService} from '../scheduled-jobs.service';
import {DefaultsJobs} from '../../../../../common/entities/job/JobDTO';
import {DefaultsJobs, JobDTO} from '../../../../../common/entities/job/JobDTO';
import {ErrorDTO} from '../../../../../common/entities/Error';
import {ServerConfig} from '../../../../../common/config/private/IPrivateConfig';
import { JobState } from '../../../../../common/entities/job/JobProgressDTO';
import {JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO';
@Component({
@ -27,7 +27,7 @@ export class VideoSettingsComponent extends SettingsComponent<{ server: ServerCo
formats: ServerConfig.formatType[] = ['mp4', 'webm'];
fps = [24, 25, 30, 48, 50, 60];
JobState = JobState;
JobProgressStates = JobProgressStates;
constructor(_authService: AuthenticationService,
_navigation: NavigationService,
@ -48,7 +48,7 @@ export class VideoSettingsComponent extends SettingsComponent<{ server: ServerCo
get Progress() {
return this.jobsService.progress.value[DefaultsJobs[DefaultsJobs['Video Converting']]];
return this.jobsService.progress.value[JobDTO.getHashName(DefaultsJobs[DefaultsJobs['Video Converting']])];
}
get bitRate(): number {

2
test/backend/SQLTestHelper.ts
View File

@ -63,7 +63,7 @@ export class SQLTestHelper {
await this.resetSQLite();
Config.Server.Database.type = ServerConfig.DatabaseType.sqlite;
Config.Server.Database.sqlite.storage = this.dbPath;
Config.Server.Database.dbFolder = path.dirname(this.dbPath);
}
private async initMySQL() {

2
test/backend/integration/model/sql/typeorm.ts
View File

@ -32,7 +32,7 @@ describe('Typeorm integration', () => {
}
Config.Server.Database.type = ServerConfig.DatabaseType.sqlite;
Config.Server.Database.sqlite.storage = dbPath;
Config.Server.Database.dbFolder = path.dirname(dbPath);
};

15
test/backend/unit/model/fileprocessing/PhotoProcessing.spec.ts
View File

@ -16,15 +16,18 @@ describe('PhotoProcessing', () => {
const photoPath = path.join(ProjectPath.ImageFolder, 'test_png.png');
expect(await PhotoProcessing
.isValidConvertedPath(PhotoProcessing.generateConvertedFilePath(photoPath)))
.isValidConvertedPath(PhotoProcessing.generateConvertedPath(photoPath,
Config.Server.Media.Photo.Converting.resolution)))
.to.be.true;
expect(await PhotoProcessing
.isValidConvertedPath(PhotoProcessing.generateConvertedFilePath(photoPath + 'noPath')))
.isValidConvertedPath(PhotoProcessing.generateConvertedPath(photoPath + 'noPath',
Config.Server.Media.Photo.Converting.resolution)))
.to.be.false;
{
const convertedPath = PhotoProcessing.generateConvertedFilePath(photoPath);
const convertedPath = PhotoProcessing.generateConvertedPath(photoPath,
Config.Server.Media.Photo.Converting.resolution);
Config.Server.Media.Photo.Converting.resolution = <any>1;
expect(await PhotoProcessing.isValidConvertedPath(convertedPath)).to.be.false;
}
@ -42,18 +45,18 @@ describe('PhotoProcessing', () => {
for (let i = 0; i < Config.Client.Media.Thumbnail.thumbnailSizes.length; ++i) {
const thSize = Config.Client.Media.Thumbnail.thumbnailSizes[i];
expect(await PhotoProcessing
.isValidConvertedPath(PhotoProcessing.generateThumbnailPath(photoPath, thSize)))
.isValidConvertedPath(PhotoProcessing.generateConvertedPath(photoPath, thSize)))
.to.be.true;
expect(await PhotoProcessing
.isValidConvertedPath(PhotoProcessing.generateThumbnailPath(photoPath + 'noPath', thSize)))
.isValidConvertedPath(PhotoProcessing.generateConvertedPath(photoPath + 'noPath', thSize)))
.to.be.false;
}
expect(await PhotoProcessing
.isValidConvertedPath(PhotoProcessing.generateThumbnailPath(photoPath, 30)))
.isValidConvertedPath(PhotoProcessing.generateConvertedPath(photoPath, 30)))
.to.be.false;
});