self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2024-12-25 02:04:15 +02:00
Code Issues Releases Activity

improving jobs

Browse Source
This commit is contained in:
Patrik J. Braun 2019-12-28 11:58:40 +01:00
parent 08e3937292
commit 0c81cbce4b
15 changed files with 100 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gulpfile.ts
View File

@ -144,7 +144,7 @@ gulp.task('copy-package', function () {
json.buildTime = (new Date()).toISOString();
try {
json.buildCommitHash = require('child_process').execSync('git rev-parse HEAD');
json.buildCommitHash = require('child_process').execSync('git rev-parse HEAD').toString().trim();
} catch (e) {
}

14
src/backend/model/jobs/JobManager.ts
View File

@ -6,7 +6,7 @@ import {Config} from '../../../common/config/private/Config';
import {AfterJobTrigger, JobScheduleDTO, JobTriggerType} from '../../../common/entities/job/JobScheduleDTO';
import {Logger} from '../../Logger';
import {NotificationManager} from '../NotifocationManager';
import {JobLastRunDTO} from '../../../common/entities/job/JobLastRunDTO';
import {JobLastRunDTO, JobLastRunState} from '../../../common/entities/job/JobLastRunDTO';
declare var global: NodeJS.Global;
@ -42,8 +42,8 @@ export class JobManager implements IJobManager {
async run<T>(jobName: string, config: T): Promise<void> {
const t = this.findJob(jobName);
if (t) {
await t.start(config, () => {
this.onJobFinished(t);
await t.start(config, (status: JobLastRunState) => {
this.onJobFinished(t, status);
});
} else {
Logger.warn(LOG_TAG, 'cannot find job to start:' + jobName);
@ -54,15 +54,15 @@ export class JobManager implements IJobManager {
const t = this.findJob(jobName);
if (t) {
t.stop();
if (global.gc) {
global.gc();
}
} else {
Logger.warn(LOG_TAG, 'cannot find job to stop:' + jobName);
}
}
async onJobFinished(job: IJob<any>): Promise<void> {
async onJobFinished(job: IJob<any>, status: JobLastRunState): Promise<void> {
if (status === JobLastRunState.canceled) { // if it was cancelled do not start the next one
return;
}
const sch = Config.Server.Jobs.scheduled.find(s => s.jobName === job.Name);
if (sch) {
const children = Config.Server.Jobs.scheduled.filter(s => s.trigger.type === JobTriggerType.after &&

10
src/backend/model/jobs/jobs/FileJob.ts
View File

@ -74,12 +74,13 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
const file = this.fileQueue.shift();
this.progress.left = this.fileQueue.length;
this.progress.progress++;
this.progress.comment = 'processing: ' + path.join(file.directory.path, file.directory.name, file.name);
const filePath = path.join(file.directory.path, file.directory.name, file.name);
this.progress.comment = 'processing: ' + filePath;
try {
await this.processFile(file);
} catch (e) {
console.error(e);
Logger.error(LOG_TAG, 'Error during processing file.' + ', ' + e.toString());
Logger.error(LOG_TAG, 'Error during processing file:' + filePath + ', ' + e.toString());
}
}
return this.progress;
@ -105,6 +106,7 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
if (this.scanFilter.noVideo === true && this.scanFilter.noPhoto === true) {
return;
}
this.progress.comment = 'Loading files from db';
Logger.silly(LOG_TAG, 'Loading files from db');
const connection = await SQLConnection.getConnection();
@ -117,7 +119,9 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
usedEntity = VideoEntity;
}
const result = await connection.getRepository(usedEntity).createQueryBuilder('media')
const result = await connection
.getRepository(usedEntity)
.createQueryBuilder('media')
.select(['media.name', 'media.id'])
.leftJoinAndSelect('media.directory', 'directory')
.getMany();

4
src/backend/model/jobs/jobs/IJob.ts
View File

@ -1,6 +1,6 @@
import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO';
import {JobDTO} from '../../../../common/entities/job/JobDTO';
import {JobLastRunDTO} from '../../../../common/entities/job/JobLastRunDTO';
import {JobLastRunDTO, JobLastRunState} from '../../../../common/entities/job/JobLastRunDTO';
export interface IJob<T> extends JobDTO {
Name: string;
@ -8,7 +8,7 @@ export interface IJob<T> extends JobDTO {
Progress: JobProgressDTO;
LastRuns: { [key: string]: JobLastRunDTO };
start(config: T, OnFinishCB: () => void): Promise<void>;
start(config: T, OnFinishCB: (status: JobLastRunState) => void): Promise<void>;
stop(): void;

33
src/backend/model/jobs/jobs/Job.ts
View File

@ -32,7 +32,7 @@ export abstract class Job<T = void> implements IJob<T> {
return this.progress;
}
public start(config: T, onFinishCB: () => void): Promise<void> {
public start(config: T, onFinishCB: (status: JobLastRunState) => void): Promise<void> {
this.OnFinishCB = onFinishCB;
if (this.state === JobState.idle && this.Supported) {
Logger.info(LOG_TAG, 'Running job: ' + this.Name);
@ -47,6 +47,17 @@ export abstract class Job<T = void> implements IJob<T> {
current: Date.now()
}
};
this.lastRuns[JSON.stringify(this.config)] = {
all: 0,
done: 0,
comment: '',
config: this.config,
state: JobLastRunState.finished,
time: {
start: Date.now(),
end: Date.now()
}
};
const pr = new Promise<void>((resolve) => {
this.prResolve = resolve;
});
@ -67,6 +78,7 @@ export abstract class Job<T = void> implements IJob<T> {
Logger.info(LOG_TAG, 'Stopping job: ' + this.Name);
this.state = JobState.stopping;
this.progress.state = JobState.stopping;
this.lastRuns[JSON.stringify(this.config)].state = JobLastRunState.canceled;
}
public toJSON(): JobDTO {
@ -76,7 +88,7 @@ export abstract class Job<T = void> implements IJob<T> {
};
}
protected OnFinishCB = () => {
protected OnFinishCB = (status: JobLastRunState) => {
};
protected abstract async step(): Promise<JobProgressDTO>;
@ -84,17 +96,10 @@ export abstract class Job<T = void> implements IJob<T> {
protected abstract async init(): Promise<void>;
private onFinish(): void {
this.lastRuns[JSON.stringify(this.config)] = {
all: this.progress.left + this.progress.progress,
done: this.progress.progress,
comment: '',
config: this.config,
state: this.progress.state === JobState.stopping ? JobLastRunState.canceled : JobLastRunState.finished,
time: {
start: this.progress.time.start,
end: Date.now()
}
};
this.lastRuns[JSON.stringify(this.config)].all = this.progress.left + this.progress.progress;
this.lastRuns[JSON.stringify(this.config)].done = this.progress.progress;
this.lastRuns[JSON.stringify(this.config)].time.end = Date.now();
this.progress = null;
if (global.gc) {
global.gc();
@ -103,7 +108,7 @@ export abstract class Job<T = void> implements IJob<T> {
if (this.IsInstant) {
this.prResolve();
}
this.OnFinishCB();
this.OnFinishCB(this.lastRuns[JSON.stringify(this.config)].state);
}
private run() {

3
src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts
View File

@ -7,6 +7,7 @@ import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
import {ThumbnailSourceType} from '../../threading/PhotoWorker';
import {MediaDTO} from '../../../../common/entities/MediaDTO';
import {FileDTO} from '../../../../common/entities/FileDTO';
import {JobLastRunState} from '../../../../common/entities/job/JobLastRunDTO';
const LOG_TAG = '[ThumbnailGenerationJob]';
@ -28,7 +29,7 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn
return true;
}
start(config: { sizes: number[], indexedOnly: boolean }, OnFinishCB: () => void): Promise<void> {
start(config: { sizes: number[], indexedOnly: boolean }, OnFinishCB: (status: JobLastRunState) => void): Promise<void> {
for (let i = 0; i < config.sizes.length; ++i) {
if (Config.Client.Media.Thumbnail.thumbnailSizes.indexOf(config.sizes[i]) === -1) {
throw new Error('unknown thumbnails size: ' + config.sizes[i] + '. Add it to the possible thumbnail sizes.');

1
src/backend/server.ts
View File

@ -43,6 +43,7 @@ export class Server {
Config.Client.appVersion = require('../../package.json').version;
Config.Client.buildTime = require('../../package.json').buildTime;
Config.Client.buildCommitHash = require('../../package.json').buildCommitHash;
Config.Client.upTime = (new Date()).toISOString();
Logger.verbose(LOG_TAG, JSON.stringify(Config, null, '\t'));
this.app = _express();

4
src/common/config/public/ConfigClass.ts
View File

@ -91,6 +91,7 @@ export module ClientConfig {
}
export interface Config {
upTime: string;
appVersion: string;
buildTime: string;
buildCommitHash: string;
@ -115,13 +116,14 @@ export module ClientConfig {
/**
* These configuration will be available at frontend and backend too
*/
export class PublicConfigClass {
export class PublicConfigClass {
public Client: ClientConfig.Config = {
applicationTitle: 'PiGallery 2',
appVersion: '',
buildCommitHash: '',
buildTime: '',
upTime: '',
Media: {
Video: {
enabled: true

8
src/frontend/app/pipes/DurationPipe.ts
View File

@ -7,13 +7,18 @@ export class DurationPipe implements PipeTransform {
constructor(private i18n: I18n) {
}
transform(time: number): string {
transform(time: number, separator: ':' | 'string' = 'string'): string {
const h = Math.floor(time / 1000 / 60 / 60);
time %= 1000 * 60 * 60;
const m = Math.floor(time / 1000 / 60);
time %= 1000 * 60;
const s = Math.floor(time / 1000);
if (separator === ':') {
const leftPad = (x: any): string => String(x).length >= 2 ? x : leftPad(`0${x}`);
return [h || 0, m || 0, s || 0].map(leftPad).join(':');
}
let str = '';
if (h > 0) {
str += h + this.i18n({value: 'h', meaning: 'hour'});
@ -24,6 +29,7 @@ export class DurationPipe implements PipeTransform {
if (s > 0) {
str += s + this.i18n({value: 's', meaning: 'second'});
}
return str;
}
}

14
src/frontend/app/ui/admin/admin.component.html
View File

@ -106,9 +106,19 @@
[hidden]="!indexing.HasAvailableSettings"
[simplifiedMode]="simplifiedMode"></app-settings-indexing>
<app-settings-jobs #setting #jobs
[hidden]="!jobs.HasAvailableSettings"
[simplifiedMode]="simplifiedMode"></app-settings-jobs>
[hidden]="!jobs.HasAvailableSettings"
[simplifiedMode]="simplifiedMode"></app-settings-jobs>
</div>
</div>
<div class="row mb-2">
<div class="col-12">
<div class="text-right">
<ng-container i18n>Up time</ng-container><!--
-->: {{upTime | date:'medium'}}
</div>
</div>
</div>
</div>
</app-frame>

8
src/frontend/app/ui/admin/admin.component.ts
View File

@ -23,8 +23,7 @@ export class AdminComponent implements OnInit, AfterViewInit {
};
appVersion = Config.Client.appVersion;
versionExtra = '';
buildTime = Config.Client.buildTime;
buildCommitHash = Config.Client.buildCommitHash;
upTime = Config.Client.upTime;
@ViewChildren('setting') settingsComponents: QueryList<ISettingsComponent>;
@ViewChildren('setting', {read: ElementRef}) settingsComponents2: QueryList<ElementRef>;
contents: ISettingsComponent[] = [];
@ -36,6 +35,7 @@ export class AdminComponent implements OnInit, AfterViewInit {
public i18n: I18n) {
this.text.Advanced = i18n('Advanced');
this.text.Simplified = i18n('Simplified');
if (Config.Client.buildTime) {
this.versionExtra = i18n('Built at') + ': ' + formatDate(Config.Client.buildTime, 'medium', locale);
}
@ -51,12 +51,12 @@ export class AdminComponent implements OnInit, AfterViewInit {
scrollTo(i: number) {
PageHelper.ScrollY = this.settingsComponents2.toArray()[i].nativeElement.getBoundingClientRect().top +
PageHelper.ScrollY;
PageHelper.ScrollY;
}
ngOnInit() {
if (!this._authService.isAuthenticated()
|| this._authService.user.value.role < UserRoles.Admin) {
|| this._authService.user.value.role < UserRoles.Admin) {
this._navigation.toLogin();
return;
}

2
src/frontend/app/ui/settings/jobs/jobs.settings.component.css
View File

@ -3,7 +3,7 @@
margin-bottom: 1rem;
}
.button-delete {
.job-control-button {
margin-top: -5px;
margin-bottom: -5px;
}

25
src/frontend/app/ui/settings/jobs/jobs.settings.component.html
View File

@ -9,7 +9,8 @@
<div *ngFor="let schedule of sortedSchedules() as schedules; let i= index">
<div class="card bg-light {{shouldIdent(schedule,schedules[i-1])? 'ml-4' : ''}}">
<div class="card-header clickable" (click)="showDetails[schedule.name]=!showDetails[schedule.name]">
<div class="card-header clickable"
(click)="showDetails[schedule.name]=!showDetails[schedule.name]">
<div class="d-flex justify-content-between">
{{schedule.name}} @<!--
-->
@ -26,11 +27,24 @@
{{schedule.trigger.afterScheduleName}}
</ng-container>
</ng-container>
<button class="btn btn-danger button-delete" (click)="remove(i)"><span class="oi oi-trash"></span>
</button>
<div>
<button class="btn btn-danger job-control-button" (click)="remove(i)"><span class="oi oi-trash"></span>
</button>
<button class="btn btn-success job-control-button"
*ngIf="!jobsService.progress.value[schedule.jobName]"
[disabled]="disableButtons"
(click)="start(schedule); $event.stopPropagation();"><span class="oi oi-media-play"></span>
</button>
<button class="btn btn-secondary job-control-button"
*ngIf="jobsService.progress.value[schedule.jobName]"
[disabled]="disableButtons || jobsService.progress.value[schedule.jobName].state !== JobState.running"
(click)="stop(schedule); $event.stopPropagation();"><span class="oi oi-media-stop"></span>
</button>
</div>
</div>
</div>
<div class="card-body" [hidden]="!showDetails[schedule.name]">
<div class="row">
@ -192,8 +206,9 @@
<app-settings-job-progress
class="card-footer bg-transparent"
[progress]="jobsService.progress.value[schedule.jobName]"
[lastRun]="(jobsService.lastRuns.value[schedule.jobName] || {})[getConfigHash(schedule)]">
*ngIf="getProgress(schedule) || getLastRun(schedule)"
[progress]="getProgress(schedule)"
[lastRun]="getLastRun(schedule)">
</app-settings-job-progress>
</div>

10
src/frontend/app/ui/settings/jobs/jobs.settings.component.ts
View File

@ -186,7 +186,6 @@ export class JobsSettingsComponent extends SettingsComponent<ServerConfig.JobCon
configElement[id] = value.split(';')
.map((s: string) => parseInt(s, 10))
.filter((i: number) => !isNaN(i) && i > 0);
console.log(configElement[id]);
}
getNumberArray(configElement: any, id: string) {
@ -209,12 +208,21 @@ export class JobsSettingsComponent extends SettingsComponent<ServerConfig.JobCon
const count = this.settings.scheduled.filter(s => s.jobName === jobName).length;
this.newSchedule.name = count === 0 ? jobName : jobName + ' ' + (count + 1);
this.settings.scheduled.push(this.newSchedule);
this.jobModal.hide();
}
getConfigHash(schedule: JobScheduleDTO): string {
return JSON.stringify(schedule.config);
}
getProgress(schedule: JobScheduleDTO) {
return this.jobsService.progress.value[schedule.jobName];
}
getLastRun(schedule: JobScheduleDTO) {
return (this.jobsService.lastRuns.value[schedule.jobName] || {})[this.getConfigHash(schedule)];
}
private getNextRunningDate(sch: JobScheduleDTO, list: JobScheduleDTO[], depth: number = 0): number {
if (depth > list.length) {
return 0;

9
src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html
View File

@ -8,7 +8,7 @@
</div>
<div class="col-md-3 col-6">
<span class="oi oi-check" title="done/all" i18n-title aria-hidden="true"></span>
{{lastRun.all}}/{{lastRun.done}}
{{lastRun.done}}/{{lastRun.all}}
</div>
<div class="col-md-3 col-6">
<span class="oi oi-pulse" title="Status" i18n-title aria-hidden="true"></span>
@ -28,8 +28,9 @@
<div class="form-group row progress-row ">
<div class="col-1 text-right" title="time elapsed" i18n-title>{{TimeElapsed| duration}}</div>
<div class="progress col-10 ">
<div class="col-6 col-md-2 col-lg-1 text-md-right order-md-0" title="time elapsed" i18n-title>{{TimeElapsed| duration:':'}}</div>
<div class="col-6 col-md-2 col-lg-1 order-md-2 text-right text-md-left" title="time left" i18n-title>{{TimeAll| duration:':'}}</div>
<div class="progress col-md-8 col-lg-10 order-md-1">
<div
*ngIf="progress.progress + progress.left >0"
class="progress-bar d-inline-block progress-bar-success {{progress.state === JobState.stopping ? 'bg-secondary' : ''}}"
@ -38,6 +39,7 @@
aria-valuemin="0"
aria-valuemax="100"
style="min-width: 2em;"
title="{{progress.progress}}/{{progress.progress + progress.left}}"
[style.width.%]="(progress.progress/(progress.left+progress.progress))*100">
{{progress.progress}}/{{progress.progress + progress.left}}
</div>
@ -48,6 +50,5 @@
aria-valuemin="0" aria-valuemax="100" style="width: 100%">
</div>
</div>
<div class="col-1" title="time left" i18n-title>{{TimeAll| duration}}</div>
</div>
</div>