improving jobs

package.json

@@ -14,7 +14,7 @@
     "pretest": "tsc",
     "test": "ng test && mocha --recursive test/backend/unit && mocha --recursive test/backend/integration  && mocha --recursive test/common/unit ",
     "start": "node ./src/backend/index",
-    "run-dev": "ng build --aot --watch --output-path=./dist --i18n-locale hu --i18n-file src/frontend/translate/messages.hu.xlf --i18n-missing-translation warning",
+    "run-dev": "ng build --aot --watch --output-path=./dist --i18n-locale en --i18n-file src/frontend/translate/messages.en.xlf --i18n-missing-translation warning",
     "build-stats": "ng build --aot --prod --stats-json --output-path=./dist --i18n-locale en --i18n-file src/frontend/translate/messages.en.xlf --i18n-missing-translation warning",
     "merge-new-translation": "gulp merge-new-translation",
     "add-translation": "gulp add-translation"

src/backend/model/jobs/jobs/FileJob.ts

@@ -12,7 +12,7 @@ import {MediaEntity} from '../../database/sql/enitites/MediaEntity';
 import {PhotoEntity} from '../../database/sql/enitites/PhotoEntity';
 import {VideoEntity} from '../../database/sql/enitites/VideoEntity';
 import {backendTexts} from '../../../../common/BackendTexts';
-import DatabaseType = ServerConfig.DatabaseType;
+import {ProjectPath} from '../../../ProjectPath';
 
 declare var global: NodeJS.Global;
 
@@ -23,12 +23,12 @@ const LOG_TAG = '[FileJob]';
 export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly: boolean }> extends Job<S> {
   public readonly ConfigTemplate: ConfigTemplateEntry[] = [];
   directoryQueue: string[] = [];
-  fileQueue: FileDTO[] = [];
+  fileQueue: string[] = [];
 
 
   protected constructor(private scanFilter: DiskMangerWorker.DirectoryScanSettings) {
     super();
-    if (Config.Server.Database.type !== DatabaseType.memory) {
+    if (Config.Server.Database.type !== ServerConfig.DatabaseType.memory) {
       this.ConfigTemplate.push({
         id: 'indexedOnly',
         type: 'boolean',
@@ -54,9 +54,9 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
     return files;
   }
 
-  protected abstract async shouldProcess(file: FileDTO): Promise<boolean>;
+  protected abstract async shouldProcess(filePath: string): Promise<boolean>;
 
-  protected abstract async processFile(file: FileDTO): Promise<void>;
+  protected abstract async processFile(filePath: string): Promise<void>;
 
   protected async step(): Promise<boolean> {
     if (this.directoryQueue.length === 0 && this.fileQueue.length === 0) {
@@ -66,7 +66,7 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
     if (this.directoryQueue.length > 0) {
 
       if (this.config.indexedOnly === true &&
-        Config.Server.Database.type !== DatabaseType.memory) {
+        Config.Server.Database.type !== ServerConfig.DatabaseType.memory) {
         await this.loadAllMediaFilesFromDB();
         this.directoryQueue = [];
       } else {
@@ -74,13 +74,12 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
       }
     } else if (this.fileQueue.length > 0) {
       this.Progress.Left = this.fileQueue.length;
-      const file = this.fileQueue.shift();
+      const filePath = this.fileQueue.shift();
-      const filePath = path.join(file.directory.path, file.directory.name, file.name);
       try {
-        if ((await this.shouldProcess(file)) === true) {
+        if ((await this.shouldProcess(filePath)) === true) {
           this.Progress.Processed++;
           this.Progress.log('processing: ' + filePath);
-          await this.processFile(file);
+          await this.processFile(filePath);
         } else {
           this.Progress.log('skipping: ' + filePath);
           this.Progress.Skipped++;
@@ -102,10 +101,12 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
       this.directoryQueue.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
     }
     if (this.scanFilter.noVideo !== true || this.scanFilter.noVideo !== true) {
-      this.fileQueue.push(...await this.filterMediaFiles(scanned.media));
+      this.fileQueue.push(...(await this.filterMediaFiles(scanned.media))
+        .map(f => path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name)));
     }
     if (this.scanFilter.noMetaFile !== true) {
-      this.fileQueue.push(...await this.filterMetaFiles(scanned.metaFile));
+      this.fileQueue.push(...(await this.filterMetaFiles(scanned.metaFile))
+        .map(f => path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name)));
     }
   }
 
@@ -134,6 +135,7 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
       .leftJoinAndSelect('media.directory', 'directory')
       .getMany();
 
-    this.fileQueue.push(...await this.filterMediaFiles(result));
+    this.fileQueue.push(...(result
+      .map(f => path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name))));
   }
 }

src/backend/model/jobs/jobs/PhotoConvertingJob.ts

@@ -21,14 +21,12 @@ export class PhotoConvertingJob extends FileJob {
   }
 
 
-  protected async shouldProcess(file: FileDTO): Promise<boolean> {
+  protected async shouldProcess(mPath: string): Promise<boolean> {
-    const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
     return !(await PhotoProcessing.convertedPhotoExist(mPath, Config.Server.Media.Photo.Converting.resolution));
   }
 
 
-  protected async processFile(file: FileDTO): Promise<void> {
+  protected async processFile(mPath: string): Promise<void> {
-    const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
     await PhotoProcessing.convertPhoto(mPath);
   }
 

src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts

@@ -1,7 +1,5 @@
 import {Config} from '../../../../common/config/private/Config';
 import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
-import {ProjectPath} from '../../../ProjectPath';
-import * as path from 'path';
 import {FileJob} from './FileJob';
 import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
 import {ThumbnailSourceType} from '../../threading/PhotoWorker';
@@ -49,8 +47,7 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn
     return undefined;
   }
 
-  protected async shouldProcess(file: FileDTO): Promise<boolean> {
+  protected async shouldProcess(mPath: string): Promise<boolean> {
-    const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
     for (let i = 0; i < this.config.sizes.length; ++i) {
       if (!(await PhotoProcessing.convertedPhotoExist(mPath, this.config.sizes[i]))) {
         return true;
@@ -58,13 +55,11 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn
     }
   }
 
-  protected async processFile(file: FileDTO): Promise<void> {
+  protected async processFile(mPath: string): Promise<void> {
-
-    const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
     for (let i = 0; i < this.config.sizes.length; ++i) {
       await PhotoProcessing.generateThumbnail(mPath,
         this.config.sizes[i],
-        MediaDTO.isVideo(file) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo,
+        MediaDTO.isVideoPath(mPath) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo,
         false);
 
     }

src/backend/model/jobs/jobs/VideoConvertingJob.ts

@@ -1,12 +1,10 @@
 import {Config} from '../../../../common/config/private/Config';
 import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
-import {ProjectPath} from '../../../ProjectPath';
-import * as path from 'path';
 import {FileJob} from './FileJob';
 import {VideoProcessing} from '../../fileprocessing/VideoProcessing';
-import {FileDTO} from '../../../../common/entities/FileDTO';
 
 const LOG_TAG = '[VideoConvertingJob]';
+declare const global: any;
 
 
 export class VideoConvertingJob extends FileJob {
@@ -20,14 +18,15 @@ export class VideoConvertingJob extends FileJob {
     return Config.Client.Media.Video.enabled === true;
   }
 
-  protected async shouldProcess(file: FileDTO): Promise<boolean> {
+  protected async shouldProcess(mPath: string): Promise<boolean> {
-    const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
     return !(await VideoProcessing.convertedVideoExist(mPath));
   }
 
-  protected async processFile(file: FileDTO): Promise<void> {
+  protected async processFile(mPath: string): Promise<void> {
-    const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
     await VideoProcessing.convertVideo(mPath);
+    if (global.gc) {
+      global.gc();
+    }
   }
 
 

src/common/entities/MediaDTO.ts

@@ -65,6 +65,16 @@ export module MediaDTO {
     return false;
   };
 
+  export const isVideoPath = (path: string): boolean => {
+    const lower = path.toLowerCase();
+    for (const ext of SupportedFormats.WithDots.Videos) {
+      if (lower.endsWith(ext)) {
+        return true;
+      }
+    }
+    return false;
+  };
+
   export const isVideoTranscodingNeeded = (media: FileDTO): boolean => {
     const lower = media.name.toLowerCase();
     for (const ext of SupportedFormats.WithDots.TranscodeNeed.Videos) {

src/frontend/app/ui/settings/jobs/button/job-button.settings.component.ts

@@ -41,7 +41,7 @@ export class JobButtonComponent {
     this.error.emit('');
     try {
       await this.jobsService.start(this.jobName, this.config, this.soloRun);
-      this.notification.info(this.i18n('Job started') + ': ' + this.jobName);
+      this.notification.success(this.i18n('Job started') + ': ' + this.backendTextService.getJobName(this.jobName));
       return true;
     } catch (err) {
       console.log(err);
@@ -57,7 +57,7 @@ export class JobButtonComponent {
     this.error.emit('');
     try {
       await this.jobsService.stop(this.jobName);
-      this.notification.info(this.i18n('Job stopped') + ': ' + this.jobName);
+      this.notification.info(this.i18n('Job stopped') + ': ' + this.backendTextService.getJobName(this.jobName));
       return true;
     } catch (err) {
       console.log(err);

src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html

@@ -25,7 +25,7 @@
 
   <div class="input-group form-group">
     <input
-      *ngIf="progress.state === JobProgressStates.running" type="text" class="form-control" disabled
+      *ngIf="progress.state === JobProgressStates.running && progress.logs.length > 0" type="text" class="form-control" disabled
       [ngModel]="progress.logs[progress.logs.length-1].comment" name="details">
     <input
       *ngIf="progress.state === JobProgressStates.cancelling" type="text" class="form-control" disabled
@@ -112,7 +112,7 @@
       </div>
       <div class="card-body text-secondary">
         <ng-container *ngFor="let log of progress.logs; let i = index;">
-          <p class="card-text" *ngIf="(i==0 && log.id > 0) || ( i> 0 && progress.logs[i-1].id+1!=log.id)">
+          <p class="card-text" *ngIf="(i==0 && log.id > 0) || ( i> 0  && progress.logs[i-1].id+1!=log.id)">
             ...
           </p>
           <p class="card-text">

src/frontend/app/ui/settings/photo/photo.settings.component.html

@@ -100,7 +100,7 @@
       </button>
 
       <div [hidden]="!settings.client.Converting.enabled">
-        <app-settings-job-button class="mt-2 mt-md-2 float-left"
+        <app-settings-job-button class="mt-2 mt-md-0 float-left"
                                  [soloRun]="true"
                                  (error)="error=$event"
                                  [jobName]="jobName"></app-settings-job-button>

src/frontend/app/ui/settings/scheduled-jobs.service.ts

@@ -1,9 +1,12 @@
 import {EventEmitter, Injectable} from '@angular/core';
 import {BehaviorSubject} from 'rxjs';
-import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO';
+import {JobProgressDTO, JobProgressStates} from '../../../../common/entities/job/JobProgressDTO';
 import {NetworkService} from '../../model/network/network.service';
 import {JobScheduleDTO} from '../../../../common/entities/job/JobScheduleDTO';
 import {JobDTO} from '../../../../common/entities/job/JobDTO';
+import {BackendtextService} from '../../model/backendtext.service';
+import {NotificationService} from '../../model/notification.service';
+import {I18n} from '@ngx-translate/i18n-polyfill';
 
 @Injectable()
 export class ScheduledJobsService {
@@ -15,7 +18,10 @@ export class ScheduledJobsService {
   public jobStartingStopping: { [key: string]: boolean } = {};
   private subscribers = 0;
 
-  constructor(private _networkService: NetworkService) {
+  constructor(private _networkService: NetworkService,
+              private notification: NotificationService,
+              private backendTextService: BackendtextService,
+              private i18n: I18n) {
     this.progress = new BehaviorSubject({});
   }
 
@@ -39,6 +45,8 @@ export class ScheduledJobsService {
     this.jobStartingStopping[jobName] = true;
     await this._networkService.postJson('/admin/jobs/scheduled/' + jobName + '/' + (soloStart === true ? 'soloStart' : 'start'),
       {config: config});
+    // placeholder to force showing running job
+    this.addDummyProgress(jobName, config);
     delete this.jobStartingStopping[jobName];
     this.forceUpdate();
   }
@@ -53,14 +61,20 @@ export class ScheduledJobsService {
   protected async loadProgress(): Promise<void> {
     const prevPrg = this.progress.value;
     this.progress.next(await this._networkService.getJson<{ [key: string]: JobProgressDTO }>('/admin/jobs/scheduled/progress'));
-    for (const prg in prevPrg) {
+    for (const prg of Object.keys(prevPrg)) {
-      if (!this.progress.value.hasOwnProperty(prg)) {
+      if (!this.progress.value.hasOwnProperty(prg) ||
+        // state changed from running to finished
+        ((prevPrg[prg].state === JobProgressStates.running ||
+            prevPrg[prg].state === JobProgressStates.cancelling) &&
+          !(this.progress.value[prg].state === JobProgressStates.running ||
+            this.progress.value[prg].state === JobProgressStates.cancelling)
+        )) {
         this.onJobFinish.emit(prg);
+        this.notification.info(this.i18n('Job finished') + ': ' + this.backendTextService.getJobName(prevPrg[prg].jobName));
       }
     }
   }
 
-
   protected getProgressPeriodically() {
     if (this.timer != null || this.subscribers === 0) {
       return;
@@ -76,6 +90,25 @@ export class ScheduledJobsService {
     this.loadProgress().catch(console.error);
   }
 
+  private addDummyProgress(jobName: string, config: any) {
+    const prgs = this.progress.value;
+    prgs[JobDTO.getHashName(jobName, config)] = {
+      jobName: jobName,
+      state: JobProgressStates.running,
+      HashName: JobDTO.getHashName(jobName, config),
+      logs: [], steps: {
+        skipped: 0,
+        processed: 0,
+        all: 0
+      },
+      time: {
+        start: Date.now(),
+        end: Date.now()
+      }
+    };
+    this.progress.next(prgs);
+  }
+
   private incSubscribers() {
     this.subscribers++;
     this.getProgressPeriodically();

src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html

@@ -74,7 +74,7 @@
       </button>
 
 
-      <app-settings-job-button class="mt-2 mt-md-2 float-left"
+      <app-settings-job-button class="mt-2 mt-md-0 float-left"
                                [soloRun]="true"
                                (error)="error=$event"
                                [jobName]="jobName"

src/frontend/app/ui/settings/video/video.settings.component.html

@@ -127,7 +127,7 @@
               (click)="reset()" i18n>Reset
       </button>
 
-      <app-settings-job-button class="mt-2 mt-md-2 float-left"
+      <app-settings-job-button class="mt-2 mt-md-0 float-left"
                                [soloRun]="true"
                                (error)="error=$event"
                                [jobName]="jobName"></app-settings-job-button>

src/frontend/translate/messages.en.xlf

@@ -904,9 +904,7 @@
           <context context-type="sourcefile">app/ui/settings/thumbnail/thumbnail.settings.component.html</context>
           <context context-type="linenumber">59</context>
         </context-group>
-        <target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160
+        <target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels.</target>
-              pixels.
-            </target>
       </trans-unit>
       <trans-unit id="69149fe434cf1b0d9bafe5ead5d126101bb5162e" datatype="html">
         <source>Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or
@@ -2592,6 +2590,14 @@
         </context-group>
         <target>Random Photo</target>
       </trans-unit>
+      <trans-unit id="bf5ce15a43de0abbce2193b3061a208eb35f7195" datatype="html">
+        <source>Job finished</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/frontend/app/ui/settings/scheduled-jobs.service.ts</context>
+          <context context-type="linenumber">1</context>
+        </context-group>
+        <target>Job finished</target>
+      </trans-unit>
       <trans-unit id="45dda89cf029b7d7b457a8dff01dc4b9a6485816" datatype="html">
         <source>Thumbnail</source>
         <context-group purpose="location">

src/frontend/translate/messages.fr.xlf

@@ -904,9 +904,7 @@
           <context context-type="sourcefile">app/ui/settings/thumbnail/thumbnail.settings.component.html</context>
           <context context-type="linenumber">59</context>
         </context-group>
-        <target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160
+        <target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels.</target>
-              pixels.
-            </target>
       </trans-unit>
       <trans-unit id="69149fe434cf1b0d9bafe5ead5d126101bb5162e" datatype="html">
         <source>Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or
@@ -2592,6 +2590,14 @@
         </context-group>
         <target>Random Photo</target>
       </trans-unit>
+      <trans-unit id="bf5ce15a43de0abbce2193b3061a208eb35f7195" datatype="html">
+        <source>Job finished</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/frontend/app/ui/settings/scheduled-jobs.service.ts</context>
+          <context context-type="linenumber">1</context>
+        </context-group>
+        <target>Job finished</target>
+      </trans-unit>
       <trans-unit id="45dda89cf029b7d7b457a8dff01dc4b9a6485816" datatype="html">
         <source>Thumbnail</source>
         <context-group purpose="location">

src/frontend/translate/messages.hu.xlf

@@ -2590,6 +2590,14 @@
         </context-group>
         <target>Véletleg Fotó</target>
       </trans-unit>
+      <trans-unit id="bf5ce15a43de0abbce2193b3061a208eb35f7195" datatype="html">
+        <source>Job finished</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/frontend/app/ui/settings/scheduled-jobs.service.ts</context>
+          <context context-type="linenumber">1</context>
+        </context-group>
+        <target>Feladat végzett</target>
+      </trans-unit>
       <trans-unit id="45dda89cf029b7d7b457a8dff01dc4b9a6485816" datatype="html">
         <source>Thumbnail</source>
         <context-group purpose="location">

src/frontend/translate/messages.ro.xlf

@@ -904,9 +904,7 @@
           <context context-type="sourcefile">app/ui/settings/thumbnail/thumbnail.settings.component.html</context>
           <context context-type="linenumber">59</context>
         </context-group>
-        <target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160
+        <target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels.</target>
-              pixels.
-            </target>
       </trans-unit>
       <trans-unit id="69149fe434cf1b0d9bafe5ead5d126101bb5162e" datatype="html">
         <source>Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or
@@ -2592,6 +2590,14 @@
         </context-group>
         <target>Random Photo</target>
       </trans-unit>
+      <trans-unit id="bf5ce15a43de0abbce2193b3061a208eb35f7195" datatype="html">
+        <source>Job finished</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/frontend/app/ui/settings/scheduled-jobs.service.ts</context>
+          <context context-type="linenumber">1</context>
+        </context-group>
+        <target>Job finished</target>
+      </trans-unit>
       <trans-unit id="45dda89cf029b7d7b457a8dff01dc4b9a6485816" datatype="html">
         <source>Thumbnail</source>
         <context-group purpose="location">

src/frontend/translate/messages.ru.xlf

@@ -904,9 +904,7 @@
           <context context-type="sourcefile">app/ui/settings/thumbnail/thumbnail.settings.component.html</context>
           <context context-type="linenumber">59</context>
         </context-group>
-        <target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160
+        <target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels.</target>
-              pixels.
-            </target>
       </trans-unit>
       <trans-unit id="69149fe434cf1b0d9bafe5ead5d126101bb5162e" datatype="html">
         <source>Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or
@@ -2592,6 +2590,14 @@
         </context-group>
         <target>Random Photo</target>
       </trans-unit>
+      <trans-unit id="bf5ce15a43de0abbce2193b3061a208eb35f7195" datatype="html">
+        <source>Job finished</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/frontend/app/ui/settings/scheduled-jobs.service.ts</context>
+          <context context-type="linenumber">1</context>
+        </context-group>
+        <target>Job finished</target>
+      </trans-unit>
       <trans-unit id="45dda89cf029b7d7b457a8dff01dc4b9a6485816" datatype="html">
         <source>Thumbnail</source>
         <context-group purpose="location">