From 08e3937292b8bcaca13eb1fef5a75e89b5fc7040 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Fri, 27 Dec 2019 19:45:11 +0100 Subject: [PATCH] adding indexedOnly option to fileJobs --- src/backend/model/jobs/JobManager.ts | 2 - src/backend/model/jobs/jobs/FileJob.ts | 88 ++++++++++++++++--- .../model/jobs/jobs/PhotoConvertingJob.ts | 28 ++---- .../model/jobs/jobs/ThumbnailGenerationJob.ts | 30 ++++--- .../model/jobs/jobs/VideoConvertingJob.ts | 28 ++---- .../model/threading/DiskMangerWorker.ts | 6 +- src/backend/model/threading/PhotoWorker.ts | 5 +- src/common/entities/MediaDTO.ts | 6 +- 8 files changed, 116 insertions(+), 77 deletions(-) diff --git a/src/backend/model/jobs/JobManager.ts b/src/backend/model/jobs/JobManager.ts index 09d1adcd..9a81a9ed 100644 --- a/src/backend/model/jobs/JobManager.ts +++ b/src/backend/model/jobs/JobManager.ts @@ -63,10 +63,8 @@ export class JobManager implements IJobManager { } async onJobFinished(job: IJob): Promise { - console.log('onFinished' + job.Name); const sch = Config.Server.Jobs.scheduled.find(s => s.jobName === job.Name); if (sch) { - console.log('parent found' + sch.jobName); const children = Config.Server.Jobs.scheduled.filter(s => s.trigger.type === JobTriggerType.after && (s.trigger).afterScheduleName === sch.name); for (let i = 0; i < children.length; ++i) { diff --git a/src/backend/model/jobs/jobs/FileJob.ts b/src/backend/model/jobs/jobs/FileJob.ts index 9f50f35d..8f18786b 100644 --- a/src/backend/model/jobs/jobs/FileJob.ts +++ b/src/backend/model/jobs/jobs/FileJob.ts @@ -4,8 +4,15 @@ import {Job} from './Job'; import * as path from 'path'; import {DiskManager} from '../../DiskManger'; import {DiskMangerWorker} from '../../threading/DiskMangerWorker'; -import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO'; import {Logger} from '../../../Logger'; +import {Config} from '../../../../common/config/private/Config'; +import {ServerConfig} from '../../../../common/config/private/IPrivateConfig'; +import {FileDTO} from '../../../../common/entities/FileDTO'; +import {SQLConnection} from '../../database/sql/SQLConnection'; +import {MediaEntity} from '../../database/sql/enitites/MediaEntity'; +import {PhotoEntity} from '../../database/sql/enitites/PhotoEntity'; +import {VideoEntity} from '../../database/sql/enitites/VideoEntity'; +import DatabaseType = ServerConfig.DatabaseType; declare var global: NodeJS.Global; @@ -13,14 +20,22 @@ declare var global: NodeJS.Global; const LOG_TAG = '[FileJob]'; -export abstract class FileJob extends Job { - public readonly ConfigTemplate: ConfigTemplateEntry[] = null; +export abstract class FileJob extends Job { + public readonly ConfigTemplate: ConfigTemplateEntry[] = []; directoryQueue: string[] = []; - fileQueue: T[] = []; + fileQueue: FileDTO[] = []; protected constructor(private scanFilter: DiskMangerWorker.DirectoryScanSettings) { super(); + if (Config.Server.Database.type !== DatabaseType.memory) { + this.ConfigTemplate.push({ + id: 'indexedOnly', + type: 'boolean', + name: 'Only indexed files', + defaultValue: true + }); + } } protected async init() { @@ -29,9 +44,16 @@ export abstract class FileJob extends Job { this.directoryQueue.push('/'); } - protected abstract async processDirectory(directory: DirectoryDTO): Promise; + protected async filterMediaFiles(files: FileDTO[]): Promise { + return files; + } - protected abstract async processFile(file: T): Promise; + + protected async filterMetaFiles(files: FileDTO[]): Promise { + return files; + } + + protected abstract async processFile(file: FileDTO): Promise; protected async step(): Promise { if (this.directoryQueue.length === 0 && this.fileQueue.length === 0) { @@ -40,18 +62,19 @@ export abstract class FileJob extends Job { this.progress.time.current = Date.now(); if (this.directoryQueue.length > 0) { - const directory = this.directoryQueue.shift(); - this.progress.comment = 'scanning directory: ' + directory; - const scanned = await DiskManager.scanDirectory(directory, this.scanFilter); - for (let i = 0; i < scanned.directories.length; i++) { - this.directoryQueue.push(path.join(scanned.directories[i].path, scanned.directories[i].name)); + + if (this.config.indexedOnly === true && + Config.Server.Database.type !== DatabaseType.memory) { + await this.loadAllMediaFilesFromDB(); + this.directoryQueue = []; + } else { + await this.loadADirectoryFromDisk(); } - this.fileQueue.push(...await this.processDirectory(scanned)); } else if (this.fileQueue.length > 0) { const file = this.fileQueue.shift(); this.progress.left = this.fileQueue.length; this.progress.progress++; - this.progress.comment = 'processing: ' + file; + this.progress.comment = 'processing: ' + path.join(file.directory.path, file.directory.name, file.name); try { await this.processFile(file); } catch (e) { @@ -62,4 +85,43 @@ export abstract class FileJob extends Job { return this.progress; } + private async loadADirectoryFromDisk() { + const directory = this.directoryQueue.shift(); + this.progress.comment = 'scanning directory: ' + directory; + const scanned = await DiskManager.scanDirectory(directory, this.scanFilter); + for (let i = 0; i < scanned.directories.length; i++) { + this.directoryQueue.push(path.join(scanned.directories[i].path, scanned.directories[i].name)); + } + if (this.scanFilter.noVideo !== true || this.scanFilter.noVideo !== true) { + this.fileQueue.push(...await this.filterMediaFiles(scanned.media)); + } + if (this.scanFilter.noMetaFile !== true) { + this.fileQueue.push(...await this.filterMetaFiles(scanned.metaFile)); + } + } + + private async loadAllMediaFilesFromDB() { + + if (this.scanFilter.noVideo === true && this.scanFilter.noPhoto === true) { + return; + } + Logger.silly(LOG_TAG, 'Loading files from db'); + + const connection = await SQLConnection.getConnection(); + + let usedEntity = MediaEntity; + + if (this.scanFilter.noVideo === true) { + usedEntity = PhotoEntity; + } else if (this.scanFilter.noPhoto === true) { + usedEntity = VideoEntity; + } + + const result = await connection.getRepository(usedEntity).createQueryBuilder('media') + .select(['media.name', 'media.id']) + .leftJoinAndSelect('media.directory', 'directory') + .getMany(); + + this.fileQueue.push(...await this.filterMediaFiles(result)); + } } diff --git a/src/backend/model/jobs/jobs/PhotoConvertingJob.ts b/src/backend/model/jobs/jobs/PhotoConvertingJob.ts index 772461dd..ebd9895e 100644 --- a/src/backend/model/jobs/jobs/PhotoConvertingJob.ts +++ b/src/backend/model/jobs/jobs/PhotoConvertingJob.ts @@ -2,17 +2,14 @@ import {Config} from '../../../../common/config/private/Config'; import {DefaultsJobs} from '../../../../common/entities/job/JobDTO'; import {ProjectPath} from '../../../ProjectPath'; import * as path from 'path'; -import * as fs from 'fs'; -import * as util from 'util'; import {FileJob} from './FileJob'; -import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO'; import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing'; +import {FileDTO} from '../../../../common/entities/FileDTO'; const LOG_TAG = '[PhotoConvertingJob]'; -const existsPr = util.promisify(fs.exists); -export class PhotoConvertingJob extends FileJob { +export class PhotoConvertingJob extends FileJob { public readonly Name = DefaultsJobs[DefaultsJobs['Photo Converting']]; constructor() { @@ -23,23 +20,12 @@ export class PhotoConvertingJob extends FileJob { return Config.Client.Media.Photo.Converting.enabled === true; } - protected async processDirectory(directory: DirectoryDTO): Promise { - const ret = []; - for (let i = 0; i < directory.media.length; ++i) { - const photoPath = path.join(ProjectPath.ImageFolder, - directory.media[i].directory.path, - directory.media[i].directory.name, - directory.media[i].name); - if (await existsPr(PhotoProcessing.generateConvertedFilePath(photoPath)) === false) { - ret.push(photoPath); - } - } - return ret; - } - - protected async processFile(file: string): Promise { - await PhotoProcessing.convertPhoto(file, Config.Server.Media.Photo.Converting.resolution); + protected async processFile(file: FileDTO): Promise { + await PhotoProcessing.convertPhoto(path.join(ProjectPath.ImageFolder, + file.directory.path, + file.directory.name, + file.name), Config.Server.Media.Photo.Converting.resolution); } diff --git a/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts b/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts index d0a96d8e..80d75a06 100644 --- a/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts +++ b/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts @@ -1,34 +1,34 @@ import {Config} from '../../../../common/config/private/Config'; -import {ConfigTemplateEntry, DefaultsJobs} from '../../../../common/entities/job/JobDTO'; +import {DefaultsJobs} from '../../../../common/entities/job/JobDTO'; import {ProjectPath} from '../../../ProjectPath'; import * as path from 'path'; import {FileJob} from './FileJob'; -import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO'; import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing'; import {ThumbnailSourceType} from '../../threading/PhotoWorker'; import {MediaDTO} from '../../../../common/entities/MediaDTO'; +import {FileDTO} from '../../../../common/entities/FileDTO'; const LOG_TAG = '[ThumbnailGenerationJob]'; -export class ThumbnailGenerationJob extends FileJob { +export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOnly: boolean }> { public readonly Name = DefaultsJobs[DefaultsJobs['Thumbnail Generation']]; - public readonly ConfigTemplate: ConfigTemplateEntry[] = [{ - id: 'sizes', - type: 'number-array', - name: 'Sizes to generate', - defaultValue: [Config.Client.Media.Thumbnail.thumbnailSizes[0]] - }]; constructor() { super({noMetaFile: true}); + this.ConfigTemplate.push({ + id: 'sizes', + type: 'number-array', + name: 'Sizes to generate', + defaultValue: [Config.Client.Media.Thumbnail.thumbnailSizes[0]] + }); } public get Supported(): boolean { return true; } - start(config: { sizes: number[] }, OnFinishCB: () => void): Promise { + start(config: { sizes: number[], indexedOnly: boolean }, OnFinishCB: () => void): Promise { for (let i = 0; i < config.sizes.length; ++i) { if (Config.Client.Media.Thumbnail.thumbnailSizes.indexOf(config.sizes[i]) === -1) { throw new Error('unknown thumbnails size: ' + config.sizes[i] + '. Add it to the possible thumbnail sizes.'); @@ -38,11 +38,15 @@ export class ThumbnailGenerationJob extends FileJob { - return directory.media; + protected async filterMediaFiles(files: FileDTO[]): Promise { + return files; } - protected async processFile(media: MediaDTO): Promise { + protected async filterMetaFiles(files: FileDTO[]): Promise { + return undefined; + } + + protected async processFile(media: FileDTO): Promise { const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name); for (let i = 0; i < this.config.sizes.length; ++i) { diff --git a/src/backend/model/jobs/jobs/VideoConvertingJob.ts b/src/backend/model/jobs/jobs/VideoConvertingJob.ts index 4848be48..5caf03a1 100644 --- a/src/backend/model/jobs/jobs/VideoConvertingJob.ts +++ b/src/backend/model/jobs/jobs/VideoConvertingJob.ts @@ -2,17 +2,14 @@ import {Config} from '../../../../common/config/private/Config'; import {DefaultsJobs} from '../../../../common/entities/job/JobDTO'; import {ProjectPath} from '../../../ProjectPath'; import * as path from 'path'; -import * as fs from 'fs'; -import * as util from 'util'; import {FileJob} from './FileJob'; -import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO'; import {VideoProcessing} from '../../fileprocessing/VideoProcessing'; +import {FileDTO} from '../../../../common/entities/FileDTO'; const LOG_TAG = '[VideoConvertingJob]'; -const existsPr = util.promisify(fs.exists); -export class VideoConvertingJob extends FileJob { +export class VideoConvertingJob extends FileJob { public readonly Name = DefaultsJobs[DefaultsJobs['Video Converting']]; constructor() { @@ -23,23 +20,12 @@ export class VideoConvertingJob extends FileJob { return Config.Client.Media.Video.enabled === true; } - protected async processDirectory(directory: DirectoryDTO): Promise { - const ret = []; - for (let i = 0; i < directory.media.length; ++i) { - const videoPath = path.join(ProjectPath.ImageFolder, - directory.media[i].directory.path, - directory.media[i].directory.name, - directory.media[i].name); - if (await existsPr(VideoProcessing.generateConvertedFilePath(videoPath)) === false) { - ret.push(videoPath); - } - } - return ret; - } - - protected async processFile(file: string): Promise { - await VideoProcessing.convertVideo(file); + protected async processFile(file: FileDTO): Promise { + await VideoProcessing.convertVideo(path.join(ProjectPath.ImageFolder, + file.directory.path, + file.directory.name, + file.name)); } diff --git a/src/backend/model/threading/DiskMangerWorker.ts b/src/backend/model/threading/DiskMangerWorker.ts index dfa5cc8a..a6dfa1e9 100644 --- a/src/backend/model/threading/DiskMangerWorker.ts +++ b/src/backend/model/threading/DiskMangerWorker.ts @@ -126,7 +126,7 @@ export class DiskMangerWorker { d.isPartial = true; directory.directories.push(d); } else if (PhotoProcessing.isPhoto(fullFilePath)) { - if (settings.noPhoto) { + if (settings.noPhoto === true) { continue; } directory.media.push({ @@ -139,7 +139,7 @@ export class DiskMangerWorker { break; } } else if (VideoProcessing.isVideo(fullFilePath)) { - if (Config.Client.Media.Video.enabled === false || settings.noVideo) { + if (Config.Client.Media.Video.enabled === false || settings.noVideo === true) { continue; } try { @@ -153,7 +153,7 @@ export class DiskMangerWorker { } } else if (DiskMangerWorker.isMetaFile(fullFilePath)) { - if (Config.Client.MetaFile.enabled === false || settings.noMetaFile) { + if (Config.Client.MetaFile.enabled === false || settings.noMetaFile === true) { continue; } diff --git a/src/backend/model/threading/PhotoWorker.ts b/src/backend/model/threading/PhotoWorker.ts index 34730827..a597ce0b 100644 --- a/src/backend/model/threading/PhotoWorker.ts +++ b/src/backend/model/threading/PhotoWorker.ts @@ -15,7 +15,10 @@ export class PhotoWorker { if (input.type === ThumbnailSourceType.Photo) { return this.renderFromImage(input, renderer); } - return this.renderFromVideo(input); + if (input.type === ThumbnailSourceType.Video) { + return this.renderFromVideo(input); + } + throw new Error('Unsupported media type to render thumbnail:' + input.type); } public static renderFromImage(input: RendererInput, renderer: ServerConfig.PhotoProcessingLib): Promise { diff --git a/src/common/entities/MediaDTO.ts b/src/common/entities/MediaDTO.ts index 7a78e6e7..71833484 100644 --- a/src/common/entities/MediaDTO.ts +++ b/src/common/entities/MediaDTO.ts @@ -51,11 +51,11 @@ export module MediaDTO { }; - export const isPhoto = (media: MediaDTO): boolean => { + export const isPhoto = (media: FileDTO): boolean => { return !MediaDTO.isVideo(media); }; - export const isVideo = (media: MediaDTO): boolean => { + export const isVideo = (media: FileDTO): boolean => { const lower = media.name.toLowerCase(); for (const ext of SupportedFormats.WithDots.Videos) { if (lower.endsWith(ext)) { @@ -65,7 +65,7 @@ export module MediaDTO { return false; }; - export const isVideoTranscodingNeeded = (media: MediaDTO): boolean => { + export const isVideoTranscodingNeeded = (media: FileDTO): boolean => { const lower = media.name.toLowerCase(); for (const ext of SupportedFormats.WithDots.TranscodeNeed.Videos) { if (lower.endsWith(ext)) {