1
0
mirror of https://github.com/bpatrik/pigallery2.git synced 2025-10-30 23:57:43 +02:00

implementing basic task scheduling

This commit is contained in:
Patrik J. Braun
2019-07-27 22:56:12 +02:00
parent 83025f7b32
commit 3633168ee4
39 changed files with 984 additions and 153 deletions

View File

@@ -253,6 +253,7 @@ export class AdminMWs {
return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, 'Settings error: ' + JSON.stringify(err, null, ' '), err));
}
}
public static async updateFacesSettings(req: Request, res: Response, next: NextFunction) {
if ((typeof req.body === 'undefined') || (typeof req.body.settings === 'undefined')) {
return next(new ErrorDTO(ErrorCodes.INPUT_ERROR, 'settings is needed'));
@@ -439,6 +440,60 @@ export class AdminMWs {
}
public static startTask(req: Request, res: Response, next: NextFunction) {
try {
const id = req.params.id;
const taskConfig: any = req.body.config;
ObjectManagers.getInstance().TaskManager.start(id, taskConfig);
req.resultPipe = 'ok';
return next();
} catch (err) {
if (err instanceof Error) {
return next(new ErrorDTO(ErrorCodes.TASK_ERROR, 'Task error: ' + err.toString(), err));
}
return next(new ErrorDTO(ErrorCodes.TASK_ERROR, 'Task error: ' + JSON.stringify(err, null, ' '), err));
}
}
public static stopTask(req: Request, res: Response, next: NextFunction) {
try {
const id = req.params.id;
ObjectManagers.getInstance().TaskManager.stop(id);
req.resultPipe = 'ok';
return next();
} catch (err) {
if (err instanceof Error) {
return next(new ErrorDTO(ErrorCodes.TASK_ERROR, 'Task error: ' + err.toString(), err));
}
return next(new ErrorDTO(ErrorCodes.TASK_ERROR, 'Task error: ' + JSON.stringify(err, null, ' '), err));
}
}
public static getAvailableTasks(req: Request, res: Response, next: NextFunction) {
try {
req.resultPipe = ObjectManagers.getInstance().TaskManager.getAvailableTasks();
return next();
} catch (err) {
if (err instanceof Error) {
return next(new ErrorDTO(ErrorCodes.TASK_ERROR, 'Task error: ' + err.toString(), err));
}
return next(new ErrorDTO(ErrorCodes.TASK_ERROR, 'Task error: ' + JSON.stringify(err, null, ' '), err));
}
}
public static getTaskProgresses(req: Request, res: Response, next: NextFunction) {
try {
req.resultPipe = ObjectManagers.getInstance().TaskManager.getProgresses();
return next();
} catch (err) {
if (err instanceof Error) {
return next(new ErrorDTO(ErrorCodes.TASK_ERROR, 'Task error: ' + err.toString(), err));
}
return next(new ErrorDTO(ErrorCodes.TASK_ERROR, 'Task error: ' + JSON.stringify(err, null, ' '), err));
}
}
public static startIndexing(req: Request, res: Response, next: NextFunction) {
try {
const createThumbnails: boolean = (<IndexingDTO>req.body).createThumbnails || false;

View File

@@ -8,6 +8,7 @@ import {IIndexingTaskManager} from './interfaces/IIndexingTaskManager';
import {IIndexingManager} from './interfaces/IIndexingManager';
import {IPersonManager} from './interfaces/IPersonManager';
import {IVersionManager} from './interfaces/IVersionManager';
import {ITaskManager} from './interfaces/ITaskManager';
export class ObjectManagers {
@@ -21,7 +22,7 @@ export class ObjectManagers {
private _indexingTaskManager: IIndexingTaskManager;
private _personManager: IPersonManager;
private _versionManager: IVersionManager;
private _taskManager: ITaskManager;
get VersionManager(): IVersionManager {
@@ -88,6 +89,14 @@ export class ObjectManagers {
this._sharingManager = value;
}
get TaskManager(): ITaskManager {
return this._taskManager;
}
set TaskManager(value: ITaskManager) {
this._taskManager = value;
}
public static getInstance() {
if (this._instance === null) {
this._instance = new ObjectManagers();
@@ -110,6 +119,7 @@ export class ObjectManagers {
const IndexingManager = require('./memory/IndexingManager').IndexingManager;
const PersonManager = require('./memory/PersonManager').PersonManager;
const VersionManager = require('./memory/VersionManager').VersionManager;
const TaskManager = require('./tasks/TaskManager').TaskManager;
ObjectManagers.getInstance().GalleryManager = new GalleryManager();
ObjectManagers.getInstance().UserManager = new UserManager();
ObjectManagers.getInstance().SearchManager = new SearchManager();
@@ -118,6 +128,7 @@ export class ObjectManagers {
ObjectManagers.getInstance().IndexingManager = new IndexingManager();
ObjectManagers.getInstance().PersonManager = new PersonManager();
ObjectManagers.getInstance().VersionManager = new VersionManager();
ObjectManagers.getInstance().TaskManager = new TaskManager();
}
public static async InitSQLManagers() {
@@ -131,6 +142,7 @@ export class ObjectManagers {
const IndexingManager = require('./sql/IndexingManager').IndexingManager;
const PersonManager = require('./sql/PersonManager').PersonManager;
const VersionManager = require('./sql/VersionManager').VersionManager;
const TaskManager = require('./tasks/TaskManager').TaskManager;
ObjectManagers.getInstance().GalleryManager = new GalleryManager();
ObjectManagers.getInstance().UserManager = new UserManager();
ObjectManagers.getInstance().SearchManager = new SearchManager();
@@ -139,6 +151,7 @@ export class ObjectManagers {
ObjectManagers.getInstance().IndexingManager = new IndexingManager();
ObjectManagers.getInstance().PersonManager = new PersonManager();
ObjectManagers.getInstance().VersionManager = new VersionManager();
ObjectManagers.getInstance().TaskManager = new TaskManager();
Logger.debug('SQL DB inited');
}

View File

@@ -2,4 +2,6 @@ import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
export interface IIndexingManager {
indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO>;
resetDB(): Promise<void>;
}

View File

@@ -1,9 +1,9 @@
import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO';
export interface IIndexingTaskManager {
startIndexing(createThumbnails?: boolean): void;
getProgress(): IndexingProgressDTO;
getProgress(): TaskProgressDTO;
cancelIndexing(): void;

View File

@@ -0,0 +1,15 @@
import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO';
import {TaskScheduleDTO} from '../../../common/entities/task/TaskScheduleDTO';
import {TaskDTO} from '../../../common/entities/task/TaskDTO';
export interface ITaskManager {
start(taskId: string, config: any): void;
stop(taskId: string): void;
getProgresses(): { [key: string]: TaskProgressDTO };
getAvailableTasks(): TaskDTO[];
}

View File

@@ -7,5 +7,9 @@ export class IndexingManager implements IIndexingManager {
throw new Error('not supported by memory DB');
}
resetDB(): Promise<void> {
throw new Error('not supported by memory DB');
}
}

View File

@@ -1,5 +1,5 @@
import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager';
import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO';
export class IndexingTaskManager implements IIndexingTaskManager {
@@ -7,7 +7,7 @@ export class IndexingTaskManager implements IIndexingTaskManager {
throw new Error('not supported by memory DB');
}
getProgress(): IndexingProgressDTO {
getProgress(): TaskProgressDTO {
throw new Error('not supported by memory DB');
}

View File

@@ -16,6 +16,7 @@ import {FaceRegionEntry} from './enitites/FaceRegionEntry';
import {ObjectManagers} from '../ObjectManagers';
import {IIndexingManager} from '../interfaces/IIndexingManager';
import {DiskMangerWorker} from '../threading/DiskMangerWorker';
import {Logger} from '../../Logger';
const LOG_TAG = '[IndexingManager]';
@@ -44,6 +45,17 @@ export class IndexingManager implements IIndexingManager {
});
}
async resetDB(): Promise<void> {
Logger.info(LOG_TAG, 'Resetting DB');
const connection = await SQLConnection.getConnection();
return connection
.getRepository(DirectoryEntity)
.createQueryBuilder('directory')
.delete()
.execute().then(() => {
});
}
// Todo fix it, once typeorm support connection pools for sqlite
protected async queueForSave(scannedDirectory: DirectoryDTO) {
if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name &&

View File

@@ -1,5 +1,5 @@
import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager';
import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO';
import {ObjectManagers} from '../ObjectManagers';
import * as path from 'path';
import * as fs from 'fs';
@@ -16,7 +16,7 @@ const LOG_TAG = '[IndexingTaskManager]';
export class IndexingTaskManager implements IIndexingTaskManager {
directoriesToIndex: string[] = [];
indexingProgress: IndexingProgressDTO = null;
indexingProgress: TaskProgressDTO = null;
enabled = false;
private indexNewDirectory = async (createThumbnails: boolean = false) => {
if (this.directoriesToIndex.length === 0) {
@@ -27,13 +27,13 @@ export class IndexingTaskManager implements IIndexingTaskManager {
return;
}
const directory = this.directoriesToIndex.shift();
this.indexingProgress.current = directory;
this.indexingProgress.comment = directory;
this.indexingProgress.left = this.directoriesToIndex.length;
const scanned = await ObjectManagers.getInstance().IndexingManager.indexDirectory(directory);
if (this.enabled === false) {
return;
}
this.indexingProgress.indexed++;
this.indexingProgress.progress++;
this.indexingProgress.time.current = Date.now();
for (let i = 0; i < scanned.directories.length; i++) {
this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
@@ -72,9 +72,9 @@ export class IndexingTaskManager implements IIndexingTaskManager {
if (this.directoriesToIndex.length === 0 && this.enabled === false) {
Logger.info(LOG_TAG, 'Starting indexing');
this.indexingProgress = {
indexed: 0,
progress: 0,
left: 0,
current: '',
comment: '',
time: {
start: Date.now(),
current: Date.now()
@@ -88,7 +88,7 @@ export class IndexingTaskManager implements IIndexingTaskManager {
}
}
getProgress(): IndexingProgressDTO {
getProgress(): TaskProgressDTO {
return this.indexingProgress;
}

View File

@@ -0,0 +1,27 @@
import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO';
import {ObjectManagers} from '../ObjectManagers';
import {Config} from '../../../common/config/private/Config';
import {DatabaseType} from '../../../common/config/private/IPrivateConfig';
import {ConfigTemplateEntry, DefaultsTasks} from '../../../common/entities/task/TaskDTO';
import {Task} from './Task';
const LOG_TAG = '[DBRestTask]';
export class DBRestTask extends Task {
public readonly Name = DefaultsTasks[DefaultsTasks['Database Reset']];
public readonly ConfigTemplate: ConfigTemplateEntry[] = null;
public get Supported(): boolean {
return Config.Server.database.type !== DatabaseType.memory;
}
protected async init() {
}
protected async step(): Promise<TaskProgressDTO> {
await ObjectManagers.getInstance().IndexingManager.resetDB();
return null;
}
}

View File

@@ -0,0 +1,35 @@
import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO';
import {ConfigTemplateEntry, DefaultsTasks} from '../../../common/entities/task/TaskDTO';
import {Utils} from '../../../common/Utils';
import {Task} from './Task';
const LOG_TAG = '[DummyTask]';
export class DummyTask extends Task {
public readonly Name = DefaultsTasks[DefaultsTasks.Dummy];
counter = 0;
public readonly ConfigTemplate: ConfigTemplateEntry[] = null;
public get Supported(): boolean {
return true;
}
protected async init() {
this.counter = 0;
}
protected async step(): Promise<TaskProgressDTO> {
await Utils.wait(1000);
if (!this.running) {
return null;
}
this.counter++;
this.progress.progress = this.counter;
this.progress.left = Math.pow(10, Math.floor(Math.log10(this.counter)) + 1) - this.counter;
return this.progress;
}
}

View File

@@ -0,0 +1,12 @@
import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO';
import {TaskDTO} from '../../../common/entities/task/TaskDTO';
export interface ITask<T> extends TaskDTO {
Name: string;
Supported: boolean;
Progress: TaskProgressDTO;
start(config: T): void;
stop(): void;
}

View File

@@ -0,0 +1,84 @@
import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO';
import {ObjectManagers} from '../ObjectManagers';
import * as path from 'path';
import * as fs from 'fs';
import {Logger} from '../../Logger';
import {RendererInput, ThumbnailSourceType, ThumbnailWorker} from '../threading/ThumbnailWorker';
import {Config} from '../../../common/config/private/Config';
import {MediaDTO} from '../../../common/entities/MediaDTO';
import {ProjectPath} from '../../ProjectPath';
import {ThumbnailGeneratorMWs} from '../../middlewares/thumbnail/ThumbnailGeneratorMWs';
import {Task} from './Task';
import {DatabaseType} from '../../../common/config/private/IPrivateConfig';
import {ConfigTemplateEntry, DefaultsTasks} from '../../../common/entities/task/TaskDTO';
declare const global: any;
const LOG_TAG = '[IndexingTask]';
export class IndexingTask extends Task<{ createThumbnails: boolean }> {
public readonly Name = DefaultsTasks[DefaultsTasks.Indexing];
directoriesToIndex: string[] = [];
public readonly ConfigTemplate: ConfigTemplateEntry[] = [{
id: 'createThumbnails',
type: 'boolean',
name: 'With thumbnails',
defaultValue: false
}];
public get Supported(): boolean {
return Config.Server.database.type !== DatabaseType.memory;
}
protected async init() {
this.directoriesToIndex.push('/');
}
protected async step(): Promise<TaskProgressDTO> {
if (this.directoriesToIndex.length === 0) {
if (global.gc) {
global.gc();
}
return null;
}
const directory = this.directoriesToIndex.shift();
this.progress.comment = directory;
this.progress.left = this.directoriesToIndex.length;
const scanned = await ObjectManagers.getInstance().IndexingManager.indexDirectory(directory);
if (this.running === false) {
return null;
}
this.progress.progress++;
this.progress.time.current = Date.now();
for (let i = 0; i < scanned.directories.length; i++) {
this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
}
if (this.config.createThumbnails) {
for (let i = 0; i < scanned.media.length; i++) {
try {
const media = scanned.media[i];
const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name);
const thPath = path.join(ProjectPath.ThumbnailFolder,
ThumbnailGeneratorMWs.generateThumbnailName(mPath, Config.Client.Thumbnail.thumbnailSizes[0]));
if (fs.existsSync(thPath)) { // skip existing thumbnails
continue;
}
await ThumbnailWorker.render(<RendererInput>{
type: MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Image,
mediaPath: mPath,
size: Config.Client.Thumbnail.thumbnailSizes[0],
thPath: thPath,
makeSquare: false,
qualityPriority: Config.Server.thumbnail.qualityPriority
}, Config.Server.thumbnail.processingLibrary);
} catch (e) {
console.error(e);
Logger.error(LOG_TAG, 'Error during indexing job: ' + e.toString());
}
}
}
return this.progress;
}
}

View File

@@ -0,0 +1,75 @@
import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO';
import {Logger} from '../../Logger';
import {ITask} from './ITask';
import {ConfigTemplateEntry} from '../../../common/entities/task/TaskDTO';
declare const process: any;
export abstract class Task<T = void> implements ITask<T> {
protected progress: TaskProgressDTO;
protected running = false;
protected config: T;
public abstract get Supported(): boolean;
public abstract get Name(): string;
public abstract get ConfigTemplate(): ConfigTemplateEntry[];
public get Progress(): TaskProgressDTO {
return this.progress;
}
public start(config: T): void {
if (this.running === false && this.Supported) {
Logger.info('[Task]', 'running task:' + this.Name);
this.config = config;
this.progress = {
progress: 0,
left: 0,
comment: '',
time: {
start: Date.now(),
current: Date.now()
}
};
this.running = true;
this.init().catch(console.error);
this.run();
} else {
Logger.info('[Task]', 'Task already running: ' + this.Name);
}
}
public stop(): void {
Logger.info('[Task]', 'stopping task' + this.Name);
this.progress = null;
this.running = false;
}
protected abstract async step(): Promise<TaskProgressDTO>;
protected abstract async init(): Promise<void>;
private run() {
process.nextTick(async () => {
try {
if (!this.running) {
this.progress = null;
return;
}
this.progress = await this.step();
if (this.progress == null) { // finished
this.running = false;
return;
}
this.run();
} catch (e) {
Logger.error('[Task]', e);
}
});
}
}

View File

@@ -0,0 +1,40 @@
import {ITaskManager} from '../interfaces/ITaskManager';
import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO';
import {ITask} from './ITask';
import {TaskRepository} from './TaskRepository';
import {Config} from '../../../common/config/private/Config';
export class TaskManager implements ITaskManager {
getProgresses(): { [id: string]: TaskProgressDTO } {
const m: { [id: string]: TaskProgressDTO } = {};
TaskRepository.Instance.getAvailableTasks().forEach(t => m[t.Name] = t.Progress);
return m;
}
start(taskName: string, config: any): void {
const t = this.findTask(taskName);
if (t) {
t.start(config);
}
}
stop(taskName: string): void {
const t = this.findTask(taskName);
if (t) {
t.stop();
}
}
getAvailableTasks(): ITask<any>[] {
return TaskRepository.Instance.getAvailableTasks();
}
protected findTask(taskName: string): ITask<any> {
return this.getAvailableTasks().find(t => t.Name === taskName);
}
}

View File

@@ -0,0 +1,30 @@
import {ITask} from './ITask';
import {IndexingTask} from './IndexingTask';
import {DBRestTask} from './DBResetTask';
import {DummyTask} from './DummyTask';
export class TaskRepository {
private static instance: TaskRepository = null;
availableTasks: { [key: string]: ITask<any> } = {};
public static get Instance(): TaskRepository {
if (TaskRepository.instance == null) {
TaskRepository.instance = new TaskRepository();
}
return TaskRepository.instance;
}
getAvailableTasks(): ITask<any>[] {
return Object.values(this.availableTasks).filter(t => t.Supported);
}
register(task: ITask<any>) {
this.availableTasks[task.Name] = task;
}
}
TaskRepository.Instance.register(new IndexingTask());
TaskRepository.Instance.register(new DBRestTask());
TaskRepository.Instance.register(new DummyTask());

View File

@@ -9,6 +9,7 @@ export class AdminRouter {
this.addGetStatistic(app);
this.addGetDuplicates(app);
this.addTasks(app);
this.addIndexGallery(app);
this.addSettings(app);
}
@@ -21,6 +22,7 @@ export class AdminRouter {
RenderingMWs.renderResult
);
}
private static addGetDuplicates(app: Express) {
app.get('/api/admin/duplicates',
AuthenticationMWs.authenticate,
@@ -57,6 +59,33 @@ export class AdminRouter {
);
}
private static addTasks(app: Express) {
app.get('/api/admin/tasks/available',
AuthenticationMWs.authenticate,
AuthenticationMWs.authorise(UserRoles.Admin),
AdminMWs.getAvailableTasks,
RenderingMWs.renderResult
);
app.get('/api/admin/tasks/scheduled/progress',
AuthenticationMWs.authenticate,
AuthenticationMWs.authorise(UserRoles.Admin),
AdminMWs.getTaskProgresses,
RenderingMWs.renderResult
);
app.post('/api/admin/tasks/scheduled/:id/start',
AuthenticationMWs.authenticate,
AuthenticationMWs.authorise(UserRoles.Admin),
AdminMWs.startTask,
RenderingMWs.renderResult
);
app.post('/api/admin/tasks/scheduled/:id/stop',
AuthenticationMWs.authenticate,
AuthenticationMWs.authorise(UserRoles.Admin),
AdminMWs.stopTask,
RenderingMWs.renderResult
);
}
private static addSettings(app: Express) {
app.get('/api/settings',
AuthenticationMWs.authenticate,

View File

@@ -1,17 +0,0 @@
{
"compileOnSave": true,
"compilerOptions": {
"noImplicitAny": true,
"module": "commonjs",
"skipLibCheck": true,
"sourceMap": false,
"declaration": false,
"moduleResolution": "node",
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"target": "es6",
"typeRoots": [
"../node_modules/@types"
]
}
}

View File

@@ -1,4 +1,14 @@
export class Utils {
static GUID() {
const s4 = function () {
return Math.floor((1 + Math.random()) * 0x10000)
.toString(16)
.substring(1);
};
return s4() + s4() + '-' + s4() + s4();
}
static chunkArrays<T>(arr: T[], chunkSize: number): T[][] {
const R = [];
for (let i = 0; i < arr.length; i += chunkSize) {

View File

@@ -1,4 +1,5 @@
import {ClientConfig} from '../public/ConfigClass';
import {TaskScheduleDTO} from '../../entities/task/TaskScheduleDTO';
export enum DatabaseType {
memory = 1, mysql = 2, sqlite = 3
@@ -70,6 +71,10 @@ export interface LogConfig {
sqlLevel: SQLLogLevel;
}
export interface TaskConfig {
scheduled: TaskScheduleDTO[];
}
export interface ServerConfig {
port: number;
host: string;
@@ -83,6 +88,7 @@ export interface ServerConfig {
photoMetadataSize: number;
duplicates: DuplicatesConfig;
log: LogConfig;
tasks: TaskConfig;
}
export interface IPrivateConfig {

View File

@@ -12,6 +12,8 @@ import * as path from 'path';
import {ConfigLoader} from 'typeconfig';
import {Utils} from '../../Utils';
import {UserRoles} from '../../entities/UserDTO';
import {TaskScheduleDTO, TaskTriggerType} from '../../entities/task/TaskScheduleDTO';
import {Config} from './Config';
/**
* This configuration will be only at backend
@@ -61,6 +63,30 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
},
duplicates: {
listingLimit: 1000
},
tasks: {
scheduled: [
{
priority: 1,
taskName: 'indexing',
config: null,
trigger: {
type: TaskTriggerType.periodic,
time: {
offset: 0,
repeat: 10
}
}
},
{
priority: 2,
taskName: 'Database reset',
config: null,
trigger: {
type: TaskTriggerType.never
}
}
]
}
};
private ConfigLoader: any;
@@ -94,6 +120,20 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
throw new Error('Unknown Server.log.level, found: ' + this.Server.log.sqlLevel);
}
let updated = false;
Config.Server.tasks.scheduled.forEach((task: TaskScheduleDTO, i: number) => {
if (!task.id) {
task.id = Utils.GUID();
updated = true;
}
if (!task.name) {
task.name = task.taskName;
updated = true;
}
});
if (updated) {
this.save();
}
}
public save() {

View File

@@ -18,7 +18,8 @@ export enum ErrorCodes {
INPUT_ERROR = 12,
SETTINGS_ERROR = 13
SETTINGS_ERROR = 13,
TASK_ERROR = 14
}
export class ErrorDTO {

View File

@@ -1,28 +0,0 @@
export interface TaskType {
name: string;
parameter: any;
}
export enum TaskTriggerType {
scheduled, periodic
}
export interface TaskTrigger {
type: TaskTriggerType;
}
export interface ScheduledTaskTrigger extends TaskTrigger {
type: TaskTriggerType.scheduled;
time: number;
}
export interface PeriodicTaskTrigger extends TaskTrigger {
type: TaskTriggerType.periodic;
}
export interface TaskDTO {
priority: number;
type: TaskType;
trigger: TaskTrigger;
}

View File

@@ -1,9 +0,0 @@
export interface IndexingProgressDTO {
indexed: number;
left: number;
current: string;
time: {
start: number,
current: number
};
}

View File

@@ -0,0 +1,9 @@
export interface TaskProgressDTO {
progress: number;
left: number;
comment: string;
time: {
start: number,
current: number
};
}

View File

@@ -0,0 +1,23 @@
export type fieldType = 'string' | 'number' | 'boolean';
export enum DefaultsTasks {
Indexing, 'Database Reset', Dummy
}
export interface ConfigTemplateEntry {
id: string;
name: string;
type: fieldType;
defaultValue: any;
}
/*
export interface NestedFieldType {
[key: string]: fieldType | NestedFieldType;
}*/
export interface TaskDTO {
Name: string;
ConfigTemplate: ConfigTemplateEntry[];
}

View File

@@ -0,0 +1,33 @@
export enum TaskTriggerType {
never = 1, scheduled = 2, periodic = 3
}
export interface TaskTrigger {
type: TaskTriggerType;
}
export interface NeverTaskTrigger {
type: TaskTriggerType.never;
}
export interface ScheduledTaskTrigger extends TaskTrigger {
type: TaskTriggerType.scheduled;
time: number;
}
export interface PeriodicTaskTrigger extends TaskTrigger {
type: TaskTriggerType.periodic;
time: {
offset: number,
repeat: number
};
}
export interface TaskScheduleDTO {
priority: number;
name?: string;
id?: string;
taskName: string;
config: any;
trigger: NeverTaskTrigger | ScheduledTaskTrigger | PeriodicTaskTrigger;
}

View File

@@ -81,6 +81,8 @@ import {VersionService} from './model/version.service';
import {DirectoriesComponent} from './ui/gallery/directories/directories.component';
import {ControlsLightboxComponent} from './ui/gallery/lightbox/controls/controls.lightbox.gallery.component';
import {FacesSettingsComponent} from './ui/settings/faces/faces.settings.component';
import {TasksSettingsComponent} from './ui/settings/tasks/tasks.settings.component';
import {ScheduledTasksService} from './ui/settings/scheduled-tasks.service';
@Injectable()
@@ -166,8 +168,12 @@ export function translationsFactory(locale: string) {
InfoPanelLightboxComponent,
ControlsLightboxComponent,
RandomQueryBuilderGalleryComponent,
DirectoriesComponent,
// Face
FaceComponent,
// Duplicates
DuplicateComponent,
DuplicatesPhotoComponent,
// Settings
UserMangerSettingsComponent,
DatabaseSettingsComponent,
@@ -182,15 +188,14 @@ export function translationsFactory(locale: string) {
FacesSettingsComponent,
OtherSettingsComponent,
IndexingSettingsComponent,
DuplicateComponent,
DuplicatesPhotoComponent,
TasksSettingsComponent,
// Pipes
StringifyRole,
IconizeSortingMethod,
StringifySortingMethod,
FixOrientationPipe,
DurationPipe,
FileSizePipe,
DirectoriesComponent
FileSizePipe
],
providers: [
{provide: UrlSerializer, useClass: CustomUrlSerializer},
@@ -214,6 +219,7 @@ export function translationsFactory(locale: string) {
DuplicateService,
FacesService,
VersionService,
ScheduledTasksService,
{
provide: TRANSLATIONS,
useFactory: translationsFactory,

View File

@@ -66,5 +66,7 @@
[simplifiedMode]="simplifiedMode"></app-settings-faces>
<app-settings-indexing #indexing [hidden]="!indexing.hasAvailableSettings"
[simplifiedMode]="simplifiedMode"></app-settings-indexing>
<app-settings-tasks #tasks [hidden]="!tasks.hasAvailableSettings"
[simplifiedMode]="simplifiedMode"></app-settings-tasks>
</div>
</app-frame>

View File

@@ -69,8 +69,8 @@
)<br/>
<div *ngIf="_settingsService.progress.value != null">
<span class="progress-details" i18n>indexing</span>: {{_settingsService.progress.value.current}} <br/>
<div *ngIf="Progress != null">
<span class="progress-details" i18n>indexing</span>: {{Progress.comment}} <br/>
<span class="progress-details" i18n>elapsed</span>: {{TimeElapsed | duration}}<br/>
<span class="progress-details" i18n>left</span>: {{TimeLeft | duration}}
<div class="progress">
@@ -80,16 +80,16 @@
aria-valuemin="0"
aria-valuemax="100"
style="min-width: 2em;"
[style.width.%]="(_settingsService.progress.value.indexed/(_settingsService.progress.value.left+_settingsService.progress.value.indexed))*100">
{{_settingsService.progress.value.indexed}}
/{{_settingsService.progress.value.indexed + _settingsService.progress.value.left}}
[style.width.%]="(Progress.progress/(Progress.left+Progress.progress))*100">
{{Progress.progress}}
/{{Progress.progress + Progress.left}}
</div>
</div>
</div>
<div class="row justify-content-center buttons-row">
<button class="btn btn-success"
*ngIf="_settingsService.progress.value == null"
*ngIf="Progress == null"
[disabled]="inProgress"
title="Indexes the folders"
i18n-title
@@ -98,12 +98,12 @@
<button class="btn btn-primary"
title="Indexes the folders and also creates the thumbnails"
i18n-title
*ngIf="_settingsService.progress.value == null"
*ngIf="Progress == null"
[disabled]="inProgress"
(click)="index(true)" i18n>Index with Thumbnails
</button>
<button class="btn btn-secondary"
*ngIf="_settingsService.progress.value != null"
*ngIf="Progress != null"
[disabled]="inProgress"
(click)="cancelIndexing()" i18n>Cancel
</button>

View File

@@ -4,12 +4,13 @@ import {AuthenticationService} from '../../../model/network/authentication.servi
import {NavigationService} from '../../../model/navigation.service';
import {NotificationService} from '../../../model/notification.service';
import {ErrorDTO} from '../../../../../common/entities/Error';
import {interval, Observable} from 'rxjs';
import {IndexingConfig, ReIndexingSensitivity} from '../../../../../common/config/private/IPrivateConfig';
import {SettingsComponent} from '../_abstract/abstract.settings.component';
import {Utils} from '../../../../../common/Utils';
import {I18n} from '@ngx-translate/i18n-polyfill';
import {StatisticDTO} from '../../../../../common/entities/settings/StatisticDTO';
import {ScheduledTasksService} from '../scheduled-tasks.service';
import {DefaultsTasks} from '../../../../../common/entities/task/TaskDTO';
@Component({
selector: 'app-settings-indexing',
@@ -24,15 +25,11 @@ export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig,
types: { key: number; value: string }[] = [];
statistic: StatisticDTO;
private subscription: { timer: any, settings: any } = {
timer: null,
settings: null
};
private $counter: Observable<number> = null;
constructor(_authService: AuthenticationService,
_navigation: NavigationService,
_settingsService: IndexingSettingsService,
public tasksService: ScheduledTasksService,
notification: NotificationService,
i18n: I18n) {
@@ -46,43 +43,30 @@ export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig,
}
get TimeLeft() {
const prg = this._settingsService.progress.value;
return (prg.time.current - prg.time.start) / prg.indexed * prg.left;
get Progress() {
return this.tasksService.progress.value[DefaultsTasks[DefaultsTasks.Indexing]];
}
get TimeLeft(): number {
if (this.Progress) {
return (this.Progress.time.current - this.Progress.time.start) / this.Progress.progress * this.Progress.left;
}
}
get TimeElapsed() {
const prg = this._settingsService.progress.value;
return (prg.time.current - prg.time.start);
if (this.Progress) {
return (this.Progress.time.current - this.Progress.time.start);
}
}
updateProgress = async () => {
try {
const wasRunning = this._settingsService.progress.value !== null;
await (<IndexingSettingsService>this._settingsService).getProgress();
if (wasRunning && this._settingsService.progress.value === null) {
this.notification.success(this.i18n('Folder indexed'), this.i18n('Success'));
ngOnDestroy() {
super.ngOnDestroy();
this.tasksService.unsubscribeFromProgress();
}
} catch (err) {
if (this.subscription.timer != null) {
this.subscription.timer.unsubscribe();
this.subscription.timer = null;
}
}
if ((<IndexingSettingsService>this._settingsService).progress.value != null && this.subscription.timer == null) {
if (!this.$counter) {
this.$counter = interval(5000);
}
this.subscription.timer = this.$counter.subscribe((x) => this.updateProgress());
}
if ((<IndexingSettingsService>this._settingsService).progress.value == null && this.subscription.timer != null) {
this.subscription.timer.unsubscribe();
this.subscription.timer = null;
}
};
async ngOnInit() {
super.ngOnInit();
this.tasksService.subscribeToProgress();
this.types = Utils
.enumToArray(ReIndexingSensitivity);
this.types.forEach(v => {
@@ -98,30 +82,18 @@ export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig,
break;
}
});
this.updateProgress();
if (this._settingsService.isSupported()) {
this.statistic = await this._settingsService.getStatistic();
}
}
ngOnDestroy() {
super.ngOnDestroy();
if (this.subscription.timer != null) {
this.subscription.timer.unsubscribe();
this.subscription.timer = null;
}
if (this.subscription.settings != null) {
this.subscription.settings.unsubscribe();
this.subscription.settings = null;
}
}
async index(createThumbnails: boolean) {
this.inProgress = true;
this.error = '';
try {
await this._settingsService.index(createThumbnails);
this.updateProgress();
await this.tasksService.start(DefaultsTasks[DefaultsTasks.Indexing], {createThumbnails: !!createThumbnails});
await this.tasksService.forceUpdate();
this.notification.info(this.i18n('Folder indexing started'));
this.inProgress = false;
return true;
@@ -140,8 +112,8 @@ export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig,
this.inProgress = true;
this.error = '';
try {
await (<IndexingSettingsService>this._settingsService).cancel();
this._settingsService.progress.next(null);
await this.tasksService.stop(DefaultsTasks[DefaultsTasks.Indexing]);
await this.tasksService.forceUpdate();
this.notification.info(this.i18n('Folder indexing interrupted'));
this.inProgress = false;
return true;
@@ -160,7 +132,8 @@ export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig,
this.inProgress = true;
this.error = '';
try {
await (<IndexingSettingsService>this._settingsService).reset();
await this.tasksService.start(DefaultsTasks[DefaultsTasks['Database Reset']]);
await this.tasksService.forceUpdate();
this.notification.success(this.i18n('Database reset'), this.i18n('Success'));
this.inProgress = false;
return true;

View File

@@ -3,7 +3,7 @@ import {NetworkService} from '../../../model/network/network.service';
import {SettingsService} from '../settings.service';
import {AbstractSettingsService} from '../_abstract/abstract.settings.service';
import {DatabaseType, IndexingConfig} from '../../../../../common/config/private/IPrivateConfig';
import {IndexingProgressDTO} from '../../../../../common/entities/settings/IndexingProgressDTO';
import {TaskProgressDTO} from '../../../../../common/entities/settings/TaskProgressDTO';
import {BehaviorSubject} from 'rxjs';
import {IndexingDTO} from '../../../../../common/entities/settings/IndexingDTO';
import {StatisticDTO} from '../../../../../common/entities/settings/StatisticDTO';
@@ -12,12 +12,10 @@ import {StatisticDTO} from '../../../../../common/entities/settings/StatisticDTO
export class IndexingSettingsService extends AbstractSettingsService<IndexingConfig> {
public progress: BehaviorSubject<IndexingProgressDTO>;
constructor(private _networkService: NetworkService,
_settingsService: SettingsService) {
super(_settingsService);
this.progress = new BehaviorSubject(null);
}
public updateSettings(settings: IndexingConfig): Promise<void> {
@@ -29,22 +27,6 @@ export class IndexingSettingsService extends AbstractSettingsService<IndexingCon
return this._settingsService.settings.value.Server.database.type !== DatabaseType.memory;
}
public index(createThumbnails: boolean) {
return this._networkService.postJson('/admin/indexes/job', <IndexingDTO>{createThumbnails: createThumbnails});
}
public cancel() {
return this._networkService.deleteJson('/admin/indexes/job');
}
public async getProgress() {
this.progress.next(await this._networkService.getJson<IndexingProgressDTO>('/admin/indexes/job/progress'));
}
public reset() {
return this._networkService.deleteJson('/admin/indexes');
}
getStatistic() {
return this._networkService.getJson<StatisticDTO>('/admin/statistic');

View File

@@ -0,0 +1,63 @@
import {Injectable} from '@angular/core';
import {BehaviorSubject} from 'rxjs';
import {TaskProgressDTO} from '../../../../common/entities/settings/TaskProgressDTO';
import {NetworkService} from '../../model/network/network.service';
@Injectable()
export class ScheduledTasksService {
public progress: BehaviorSubject<{ [key: string]: TaskProgressDTO }>;
timer: number = null;
private subscribers = 0;
constructor(private _networkService: NetworkService) {
this.progress = new BehaviorSubject({});
}
subscribeToProgress(): void {
this.incSubscribers();
}
unsubscribeFromProgress(): void {
this.decSubscribers();
}
public forceUpdate() {
return this.getProgress();
}
public async start(id: string, config?: any) {
return await this._networkService.postJson('/admin/tasks/scheduled/' + id + '/start', {config: config});
}
public async stop(id: string) {
return await this._networkService.postJson('/admin/tasks/scheduled/' + id + '/stop');
}
protected async getProgress() {
return this.progress.next(await this._networkService.getJson<{ [key: string]: TaskProgressDTO }>('/admin/tasks/scheduled/progress'));
}
protected getProgressPeriodically() {
if (this.timer != null || this.subscribers === 0) {
return;
}
this.timer = window.setTimeout(async () => {
await this.getProgress();
this.timer = null;
this.getProgressPeriodically();
}, 5000);
}
private incSubscribers() {
this.subscribers++;
this.getProgressPeriodically();
}
private decSubscribers() {
this.subscribers--;
}
}

View File

@@ -114,6 +114,9 @@ export class SettingsService {
photoMetadataSize: 512 * 1024,
duplicates: {
listingLimit: 1000
},
tasks: {
scheduled: []
}
}
});

View File

@@ -0,0 +1,13 @@
.progress-details {
font-weight: bold;
}
.card {
margin-bottom: 1rem;
}
.button-delete {
margin-top: -5px;
margin-bottom: -5px;
}

View File

@@ -0,0 +1,117 @@
<form #settingsForm="ngForm" class="form-horizontal">
<div class="card mb-4">
<h5 class="card-header" i18n>
Tasks
</h5>
<div class="card-body">
<div [hidden]="!error" class="alert alert-danger" role="alert"><strong>Error: </strong>{{error}}</div>
<div *ngFor="let schedule of settings.scheduled;let i= index">
<div class="card bg-light">
<div class="card-header ">
<div class="d-flex justify-content-between">
{{schedule.name}}
<button class="btn btn-danger button-delete"
(click)="remove(schedule.taskName)" i18n><span class="oi oi-trash"></span>
</button>
</div>
</div>
<div class="card-body">
<div class="d-flex justify-content-between">
<div>
<select class="form-control" [(ngModel)]="schedule.taskName" [name]="'taskName'+i" required>
<option *ngFor="let availableTask of _settingsService.availableTasks | async"
[ngValue]="availableTask.Name">{{availableTask.Name}}
</option>
</select>
</div>
<div>
<button class="btn btn-success"
*ngIf="!tasksService.progress.value[schedule.taskName]"
[disabled]="disableButtons"
title="Trigger task run manually"
i18n-title
(click)="start(schedule)" i18n>Start
</button>
<button class="btn btn-secondary"
*ngIf="tasksService.progress.value[schedule.taskName]"
[disabled]="disableButtons"
(click)="stop(schedule)" i18n>Stop
</button>
</div>
</div>
<ng-container *ngIf="getConfigTemplate(schedule.taskName) ">
<hr/>
<div *ngFor="let configEntry of getConfigTemplate(schedule.taskName)">
<ng-container [ngSwitch]="configEntry.type">
<ng-container *ngSwitchCase="'boolean'">
<div class="form-group row">
<label class="col-md-2 control-label" [for]="configEntry.id"
i18n>{{configEntry.name}}</label>
<div class="col-md-10">
<bSwitch
id="enableThreading"
class="switch"
[name]="configEntry.id"
[id]="configEntry.id"
[switch-on-color]="'primary'"
[switch-inverse]="true"
[switch-off-text]="text.Disabled"
[switch-on-text]="text.Enabled"
[switch-handle-width]="100"
[switch-label-width]="20"
[(ngModel)]="schedule.config[configEntry.id]">
</bSwitch>
</div>
</div>
</ng-container>
<ng-container *ngSwitchCase="'string'">
<div class="form-group row" [hidden]="simplifiedMode">
<label class="col-md-2 control-label" [for]="configEntry.id"
i18n>{{configEntry.name}}</label>
<div class="col-md-10">
<input type="text" class="form-control" [name]="configEntry.id" [id]="configEntry.id"
[(ngModel)]="schedule.config[configEntry.id]" required>
</div>
</div>
</ng-container>
<ng-container *ngSwitchCase="'number'">
<div class="form-group row" [hidden]="simplifiedMode">
<label class="col-md-2 control-label" [for]="configEntry.id"
i18n>{{configEntry.name}}</label>
<div class="col-md-10">
<input type="number" class="form-control" [name]="configEntry.id" [id]="configEntry.id"
[(ngModel)]="schedule.config[configEntry.id]" required>
</div>
</div>
</ng-container>
</ng-container>
</div>
</ng-container>
</div>
<div class="card-footer bg-transparent" *ngIf="tasksService.progress.value[schedule.taskName]">
<span class="progress-details"
i18n>status</span>: {{tasksService.progress.value[schedule.taskName].comment}} <br/>
<span class="progress-details" i18n>elapsed</span>: {{getTimeElapsed(schedule.taskName) | duration}}<br/>
<span class="progress-details" i18n>left</span>: {{getTimeLeft(schedule.taskName) | duration}}
<div class="progress">
<div class="progress-bar progress-bar-success"
role="progressbar"
aria-valuenow="2"
aria-valuemin="0"
aria-valuemax="100"
style="min-width: 2em;"
[style.width.%]="(tasksService.progress.value[schedule.taskName].progress/(tasksService.progress.value[schedule.taskName].left+tasksService.progress.value[schedule.taskName].progress))*100">
{{tasksService.progress.value[schedule.taskName].progress}}
/{{tasksService.progress.value[schedule.taskName].progress + tasksService.progress.value[schedule.taskName].left}}
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</form>

View File

@@ -0,0 +1,136 @@
import {Component, OnChanges, OnDestroy, OnInit} from '@angular/core';
import {TasksSettingsService} from './tasks.settings.service';
import {AuthenticationService} from '../../../model/network/authentication.service';
import {NavigationService} from '../../../model/navigation.service';
import {NotificationService} from '../../../model/notification.service';
import {TaskConfig} from '../../../../../common/config/private/IPrivateConfig';
import {SettingsComponent} from '../_abstract/abstract.settings.component';
import {I18n} from '@ngx-translate/i18n-polyfill';
import {ErrorDTO} from '../../../../../common/entities/Error';
import {ScheduledTasksService} from '../scheduled-tasks.service';
import {TaskScheduleDTO} from '../../../../../common/entities/task/TaskScheduleDTO';
@Component({
selector: 'app-settings-tasks',
templateUrl: './tasks.settings.component.html',
styleUrls: ['./tasks.settings.component.css',
'./../_abstract/abstract.settings.component.css'],
providers: [TasksSettingsService],
})
export class TasksSettingsComponent extends SettingsComponent<TaskConfig, TasksSettingsService>
implements OnInit, OnDestroy, OnChanges {
disableButtons = false;
constructor(_authService: AuthenticationService,
_navigation: NavigationService,
_settingsService: TasksSettingsService,
public tasksService: ScheduledTasksService,
notification: NotificationService,
i18n: I18n) {
super(i18n('Tasks'),
_authService,
_navigation,
<any>_settingsService,
notification,
i18n,
s => s.Server.tasks);
this.hasAvailableSettings = !this.simplifiedMode;
}
ngOnChanges(): void {
this.hasAvailableSettings = !this.simplifiedMode;
}
getConfigTemplate(taskName: string) {
const task = this._settingsService.availableTasks.value.find(t => t.Name === taskName);
if (task && task.ConfigTemplate && task.ConfigTemplate.length > 0) {
const schedule = this.settings.scheduled.find(s => s.taskName === taskName);
if (schedule) {
schedule.config = schedule.config || {};
task.ConfigTemplate.forEach(ct => schedule.config[ct.id] = ct.defaultValue);
}
return task.ConfigTemplate;
}
return null;
}
ngOnInit() {
super.ngOnInit();
this.tasksService.subscribeToProgress();
this._settingsService.getAvailableTasks();
}
ngOnDestroy() {
super.ngOnDestroy();
this.tasksService.unsubscribeFromProgress();
}
getTimeLeft(id: string): number {
const prg = this.tasksService.progress.value[id];
if (!prg) {
return null;
}
return (prg.time.current - prg.time.start) / prg.progress * prg.left;
}
getTimeElapsed(id: string) {
const prg = this.tasksService.progress.value[id];
if (!prg) {
return null;
}
return (prg.time.current - prg.time.start);
}
public async start(schedule: TaskScheduleDTO) {
this.error = '';
try {
this.disableButtons = true;
await this.tasksService.start(schedule.taskName, schedule.config);
await this.tasksService.forceUpdate();
this.notification.info(this.i18n('Task') + ' ' + schedule.taskName + ' ' + this.i18n('started'));
return true;
} catch (err) {
console.log(err);
if (err.message) {
this.error = (<ErrorDTO>err).message;
}
} finally {
this.disableButtons = false;
}
return false;
}
public async stop(schedule: TaskScheduleDTO) {
this.error = '';
try {
this.disableButtons = true;
await this.tasksService.stop(schedule.taskName);
await this.tasksService.forceUpdate();
this.notification.info(this.i18n('Task') + ' ' + schedule.taskName + ' ' + this.i18n('stopped'));
return true;
} catch (err) {
console.log(err);
if (err.message) {
this.error = (<ErrorDTO>err).message;
}
} finally {
this.disableButtons = false;
}
return false;
}
remove(id: string) {
}
}

View File

@@ -0,0 +1,34 @@
import {Injectable} from '@angular/core';
import {NetworkService} from '../../../model/network/network.service';
import {SettingsService} from '../settings.service';
import {AbstractSettingsService} from '../_abstract/abstract.settings.service';
import {TaskConfig} from '../../../../../common/config/private/IPrivateConfig';
import {BehaviorSubject} from 'rxjs';
import {TaskDTO} from '../../../../../common/entities/task/TaskDTO';
@Injectable()
export class TasksSettingsService extends AbstractSettingsService<TaskConfig> {
public availableTasks: BehaviorSubject<TaskDTO[]>;
constructor(private _networkService: NetworkService,
_settingsService: SettingsService) {
super(_settingsService);
this.availableTasks = new BehaviorSubject([]);
}
public updateSettings(settings: TaskConfig): Promise<void> {
return this._networkService.putJson('/settings/tasks', {settings: settings});
}
public isSupported(): boolean {
return true;
}
public async getAvailableTasks() {
this.availableTasks.next(await this._networkService.getJson<TaskDTO[]>('/admin/tasks/available'));
}
}

View File

@@ -2,12 +2,13 @@
"compileOnSave": true,
"compilerOptions": {
"noImplicitAny": true,
"module": "commonjs",
"sourceMap": true,
"declaration": false,
"moduleResolution": "node",
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"target": "es5",
"target": "es6",
"typeRoots": [
"node_modules/@types"
],