adding indexedOnly option to fileJobs

src/backend/model/jobs/JobManager.ts

@@ -63,10 +63,8 @@ export class JobManager implements IJobManager {
   }
 
   async onJobFinished(job: IJob<any>): Promise<void> {
-    console.log('onFinished' + job.Name);
     const sch = Config.Server.Jobs.scheduled.find(s => s.jobName === job.Name);
     if (sch) {
-      console.log('parent found' + sch.jobName);
       const children = Config.Server.Jobs.scheduled.filter(s => s.trigger.type === JobTriggerType.after &&
         (<AfterJobTrigger>s.trigger).afterScheduleName === sch.name);
       for (let i = 0; i < children.length; ++i) {

src/backend/model/jobs/jobs/FileJob.ts

@@ -4,8 +4,15 @@ import {Job} from './Job';
 import * as path from 'path';
 import {DiskManager} from '../../DiskManger';
 import {DiskMangerWorker} from '../../threading/DiskMangerWorker';
-import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO';
 import {Logger} from '../../../Logger';
+import {Config} from '../../../../common/config/private/Config';
+import {ServerConfig} from '../../../../common/config/private/IPrivateConfig';
+import {FileDTO} from '../../../../common/entities/FileDTO';
+import {SQLConnection} from '../../database/sql/SQLConnection';
+import {MediaEntity} from '../../database/sql/enitites/MediaEntity';
+import {PhotoEntity} from '../../database/sql/enitites/PhotoEntity';
+import {VideoEntity} from '../../database/sql/enitites/VideoEntity';
+import DatabaseType = ServerConfig.DatabaseType;
 
 declare var global: NodeJS.Global;
 
@@ -13,14 +20,22 @@ declare var global: NodeJS.Global;
 const LOG_TAG = '[FileJob]';
 
 
-export abstract class FileJob<T, S = void> extends Job<S> {
-  public readonly ConfigTemplate: ConfigTemplateEntry[] = null;
+export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly: boolean }> extends Job<S> {
+  public readonly ConfigTemplate: ConfigTemplateEntry[] = [];
   directoryQueue: string[] = [];
-  fileQueue: T[] = [];
+  fileQueue: FileDTO[] = [];
 
 
   protected constructor(private scanFilter: DiskMangerWorker.DirectoryScanSettings) {
     super();
+    if (Config.Server.Database.type !== DatabaseType.memory) {
+      this.ConfigTemplate.push({
+        id: 'indexedOnly',
+        type: 'boolean',
+        name: 'Only indexed files',
+        defaultValue: true
+      });
+    }
   }
 
   protected async init() {
@@ -29,9 +44,16 @@ export abstract class FileJob<T, S = void> extends Job<S> {
     this.directoryQueue.push('/');
   }
 
-  protected abstract async processDirectory(directory: DirectoryDTO): Promise<T[]>;
+  protected async filterMediaFiles(files: FileDTO[]): Promise<FileDTO[]> {
+    return files;
+  }
 
-  protected abstract async processFile(file: T): Promise<void>;
+
+  protected async filterMetaFiles(files: FileDTO[]): Promise<FileDTO[]> {
+    return files;
+  }
+
+  protected abstract async processFile(file: FileDTO): Promise<void>;
 
   protected async step(): Promise<JobProgressDTO> {
     if (this.directoryQueue.length === 0 && this.fileQueue.length === 0) {
@@ -40,18 +62,19 @@ export abstract class FileJob<T, S = void> extends Job<S> {
 
     this.progress.time.current = Date.now();
     if (this.directoryQueue.length > 0) {
-      const directory = this.directoryQueue.shift();
-      this.progress.comment = 'scanning directory: ' + directory;
-      const scanned = await DiskManager.scanDirectory(directory, this.scanFilter);
-      for (let i = 0; i < scanned.directories.length; i++) {
-        this.directoryQueue.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
+
+      if (this.config.indexedOnly === true &&
+        Config.Server.Database.type !== DatabaseType.memory) {
+        await this.loadAllMediaFilesFromDB();
+        this.directoryQueue = [];
+      } else {
+        await this.loadADirectoryFromDisk();
       }
-      this.fileQueue.push(...await this.processDirectory(scanned));
     } else if (this.fileQueue.length > 0) {
       const file = this.fileQueue.shift();
       this.progress.left = this.fileQueue.length;
       this.progress.progress++;
-      this.progress.comment = 'processing: ' + file;
+      this.progress.comment = 'processing: ' + path.join(file.directory.path, file.directory.name, file.name);
       try {
         await this.processFile(file);
       } catch (e) {
@@ -62,4 +85,43 @@ export abstract class FileJob<T, S = void> extends Job<S> {
     return this.progress;
   }
 
+  private async loadADirectoryFromDisk() {
+    const directory = this.directoryQueue.shift();
+    this.progress.comment = 'scanning directory: ' + directory;
+    const scanned = await DiskManager.scanDirectory(directory, this.scanFilter);
+    for (let i = 0; i < scanned.directories.length; i++) {
+      this.directoryQueue.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
+    }
+    if (this.scanFilter.noVideo !== true || this.scanFilter.noVideo !== true) {
+      this.fileQueue.push(...await this.filterMediaFiles(scanned.media));
+    }
+    if (this.scanFilter.noMetaFile !== true) {
+      this.fileQueue.push(...await this.filterMetaFiles(scanned.metaFile));
+    }
+  }
+
+  private async loadAllMediaFilesFromDB() {
+
+    if (this.scanFilter.noVideo === true && this.scanFilter.noPhoto === true) {
+      return;
+    }
+    Logger.silly(LOG_TAG, 'Loading files from db');
+
+    const connection = await SQLConnection.getConnection();
+
+    let usedEntity = MediaEntity;
+
+    if (this.scanFilter.noVideo === true) {
+      usedEntity = PhotoEntity;
+    } else if (this.scanFilter.noPhoto === true) {
+      usedEntity = VideoEntity;
+    }
+
+    const result = await connection.getRepository(usedEntity).createQueryBuilder('media')
+      .select(['media.name', 'media.id'])
+      .leftJoinAndSelect('media.directory', 'directory')
+      .getMany();
+
+    this.fileQueue.push(...await this.filterMediaFiles(result));
+  }
 }

src/backend/model/jobs/jobs/PhotoConvertingJob.ts

@@ -2,17 +2,14 @@ import {Config} from '../../../../common/config/private/Config';
 import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
 import {ProjectPath} from '../../../ProjectPath';
 import * as path from 'path';
-import * as fs from 'fs';
-import * as util from 'util';
 import {FileJob} from './FileJob';
-import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO';
 import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
+import {FileDTO} from '../../../../common/entities/FileDTO';
 
 const LOG_TAG = '[PhotoConvertingJob]';
-const existsPr = util.promisify(fs.exists);
 
 
-export class PhotoConvertingJob extends FileJob<string> {
+export class PhotoConvertingJob extends FileJob {
   public readonly Name = DefaultsJobs[DefaultsJobs['Photo Converting']];
 
   constructor() {
@@ -23,23 +20,12 @@ export class PhotoConvertingJob extends FileJob<string> {
     return Config.Client.Media.Photo.Converting.enabled === true;
   }
 
-  protected async processDirectory(directory: DirectoryDTO): Promise<string[]> {
-    const ret = [];
-    for (let i = 0; i < directory.media.length; ++i) {
-      const photoPath = path.join(ProjectPath.ImageFolder,
-        directory.media[i].directory.path,
-        directory.media[i].directory.name,
-        directory.media[i].name);
 
-      if (await existsPr(PhotoProcessing.generateConvertedFilePath(photoPath)) === false) {
-        ret.push(photoPath);
-      }
-    }
-    return ret;
-  }
-
-  protected async processFile(file: string): Promise<void> {
-    await PhotoProcessing.convertPhoto(file, Config.Server.Media.Photo.Converting.resolution);
+  protected async processFile(file: FileDTO): Promise<void> {
+    await PhotoProcessing.convertPhoto(path.join(ProjectPath.ImageFolder,
+      file.directory.path,
+      file.directory.name,
+      file.name), Config.Server.Media.Photo.Converting.resolution);
   }
 
 

src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts

@@ -1,34 +1,34 @@
 import {Config} from '../../../../common/config/private/Config';
-import {ConfigTemplateEntry, DefaultsJobs} from '../../../../common/entities/job/JobDTO';
+import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
 import {ProjectPath} from '../../../ProjectPath';
 import * as path from 'path';
 import {FileJob} from './FileJob';
-import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO';
 import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
 import {ThumbnailSourceType} from '../../threading/PhotoWorker';
 import {MediaDTO} from '../../../../common/entities/MediaDTO';
+import {FileDTO} from '../../../../common/entities/FileDTO';
 
 const LOG_TAG = '[ThumbnailGenerationJob]';
 
 
-export class ThumbnailGenerationJob extends FileJob<MediaDTO, { sizes: number[] }> {
+export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOnly: boolean }> {
   public readonly Name = DefaultsJobs[DefaultsJobs['Thumbnail Generation']];
-  public readonly ConfigTemplate: ConfigTemplateEntry[] = [{
-    id: 'sizes',
-    type: 'number-array',
-    name: 'Sizes to generate',
-    defaultValue: [Config.Client.Media.Thumbnail.thumbnailSizes[0]]
-  }];
 
   constructor() {
     super({noMetaFile: true});
+    this.ConfigTemplate.push({
+      id: 'sizes',
+      type: 'number-array',
+      name: 'Sizes to generate',
+      defaultValue: [Config.Client.Media.Thumbnail.thumbnailSizes[0]]
+    });
   }
 
   public get Supported(): boolean {
     return true;
   }
 
-  start(config: { sizes: number[] }, OnFinishCB: () => void): Promise<void> {
+  start(config: { sizes: number[], indexedOnly: boolean }, OnFinishCB: () => void): Promise<void> {
     for (let i = 0; i < config.sizes.length; ++i) {
       if (Config.Client.Media.Thumbnail.thumbnailSizes.indexOf(config.sizes[i]) === -1) {
         throw new Error('unknown thumbnails size: ' + config.sizes[i] + '. Add it to the possible thumbnail sizes.');
@@ -38,11 +38,15 @@ export class ThumbnailGenerationJob extends FileJob<MediaDTO, { sizes: number[]
     return super.start(config, OnFinishCB);
   }
 
-  protected async processDirectory(directory: DirectoryDTO): Promise<MediaDTO[]> {
-    return directory.media;
+  protected async filterMediaFiles(files: FileDTO[]): Promise<FileDTO[]> {
+    return files;
   }
 
-  protected async processFile(media: MediaDTO): Promise<void> {
+  protected async filterMetaFiles(files: FileDTO[]): Promise<FileDTO[]> {
+    return undefined;
+  }
+
+  protected async processFile(media: FileDTO): Promise<void> {
 
     const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name);
     for (let i = 0; i < this.config.sizes.length; ++i) {

src/backend/model/jobs/jobs/VideoConvertingJob.ts

@@ -2,17 +2,14 @@ import {Config} from '../../../../common/config/private/Config';
 import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
 import {ProjectPath} from '../../../ProjectPath';
 import * as path from 'path';
-import * as fs from 'fs';
-import * as util from 'util';
 import {FileJob} from './FileJob';
-import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO';
 import {VideoProcessing} from '../../fileprocessing/VideoProcessing';
+import {FileDTO} from '../../../../common/entities/FileDTO';
 
 const LOG_TAG = '[VideoConvertingJob]';
-const existsPr = util.promisify(fs.exists);
 
 
-export class VideoConvertingJob extends FileJob<string> {
+export class VideoConvertingJob extends FileJob {
   public readonly Name = DefaultsJobs[DefaultsJobs['Video Converting']];
 
   constructor() {
@@ -23,23 +20,12 @@ export class VideoConvertingJob extends FileJob<string> {
     return Config.Client.Media.Video.enabled === true;
   }
 
-  protected async processDirectory(directory: DirectoryDTO): Promise<string[]> {
-    const ret = [];
-    for (let i = 0; i < directory.media.length; ++i) {
-      const videoPath = path.join(ProjectPath.ImageFolder,
-        directory.media[i].directory.path,
-        directory.media[i].directory.name,
-        directory.media[i].name);
 
-      if (await existsPr(VideoProcessing.generateConvertedFilePath(videoPath)) === false) {
-        ret.push(videoPath);
-      }
-    }
-    return ret;
-  }
-
-  protected async processFile(file: string): Promise<void> {
-    await VideoProcessing.convertVideo(file);
+  protected async processFile(file: FileDTO): Promise<void> {
+    await VideoProcessing.convertVideo(path.join(ProjectPath.ImageFolder,
+      file.directory.path,
+      file.directory.name,
+      file.name));
   }
 
 

src/backend/model/threading/DiskMangerWorker.ts

@@ -126,7 +126,7 @@ export class DiskMangerWorker {
         d.isPartial = true;
         directory.directories.push(d);
       } else if (PhotoProcessing.isPhoto(fullFilePath)) {
-        if (settings.noPhoto) {
+        if (settings.noPhoto === true) {
           continue;
         }
         directory.media.push(<PhotoDTO>{
@@ -139,7 +139,7 @@ export class DiskMangerWorker {
           break;
         }
       } else if (VideoProcessing.isVideo(fullFilePath)) {
-        if (Config.Client.Media.Video.enabled === false || settings.noVideo) {
+        if (Config.Client.Media.Video.enabled === false || settings.noVideo === true) {
           continue;
         }
         try {
@@ -153,7 +153,7 @@ export class DiskMangerWorker {
         }
 
       } else if (DiskMangerWorker.isMetaFile(fullFilePath)) {
-        if (Config.Client.MetaFile.enabled === false || settings.noMetaFile) {
+        if (Config.Client.MetaFile.enabled === false || settings.noMetaFile === true) {
           continue;
         }
 

src/backend/model/threading/PhotoWorker.ts

@@ -15,7 +15,10 @@ export class PhotoWorker {
     if (input.type === ThumbnailSourceType.Photo) {
       return this.renderFromImage(input, renderer);
     }
-    return this.renderFromVideo(input);
+    if (input.type === ThumbnailSourceType.Video) {
+      return this.renderFromVideo(input);
+    }
+    throw new Error('Unsupported media type to render thumbnail:' + input.type);
   }
 
   public static renderFromImage(input: RendererInput, renderer: ServerConfig.PhotoProcessingLib): Promise<void> {

src/common/entities/MediaDTO.ts

@@ -51,11 +51,11 @@ export module MediaDTO {
 
   };
 
-  export const isPhoto = (media: MediaDTO): boolean => {
+  export const isPhoto = (media: FileDTO): boolean => {
     return !MediaDTO.isVideo(media);
   };
 
-  export const isVideo = (media: MediaDTO): boolean => {
+  export const isVideo = (media: FileDTO): boolean => {
     const lower = media.name.toLowerCase();
     for (const ext of SupportedFormats.WithDots.Videos) {
       if (lower.endsWith(ext)) {
@@ -65,7 +65,7 @@ export module MediaDTO {
     return false;
   };
 
-  export const isVideoTranscodingNeeded = (media: MediaDTO): boolean => {
+  export const isVideoTranscodingNeeded = (media: FileDTO): boolean => {
     const lower = media.name.toLowerCase();
     for (const ext of SupportedFormats.WithDots.TranscodeNeed.Videos) {
       if (lower.endsWith(ext)) {