From e2864117b276d667ae756f6b7d9b715807363632 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Thu, 26 Dec 2019 21:03:10 +0100 Subject: [PATCH] implementing TempFolderCleaningJob --- package-lock.json | 137 +------------ package.json | 6 +- src/backend/ProjectPath.ts | 12 +- src/backend/middlewares/GalleryMWs.ts | 2 +- .../thumbnail/ThumbnailGeneratorMWs.ts | 1 - .../model/fileprocessing/PhotoProcessing.ts | 74 +++++-- .../model/fileprocessing/VideoProcessing.ts | 69 +++++-- src/backend/model/jobs/JobRepository.ts | 7 +- src/backend/model/jobs/jobs/FileJob.ts | 9 +- src/backend/model/jobs/jobs/IndexingJob.ts | 3 - src/backend/model/jobs/jobs/Job.ts | 11 +- .../model/jobs/jobs/TempFolderCleaningJob.ts | 118 +++++++++++ .../model/jobs/jobs/ThumbnailGenerationJob.ts | 3 - .../model/jobs/jobs/VideoConvertingJob.ts | 2 +- .../model/threading/DiskMangerWorker.ts | 192 ++++++++---------- src/common/entities/job/JobDTO.ts | 7 +- .../settings/jobs/jobs.settings.component.ts | 15 +- .../fileprocessing/PhotoProcessing.spec.ts | 61 ++++++ .../fileprocessing/VideoProcessing.spec.ts | 45 ++++ 19 files changed, 468 insertions(+), 306 deletions(-) create mode 100644 src/backend/model/jobs/jobs/TempFolderCleaningJob.ts create mode 100644 test/backend/unit/model/fileprocessing/PhotoProcessing.spec.ts create mode 100644 test/backend/unit/model/fileprocessing/VideoProcessing.spec.ts diff --git a/package-lock.json b/package-lock.json index 1d6b3e7c..f8337368 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "pigallery2", - "version": "1.7.8", + "version": "1.7.9", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -4002,6 +4002,16 @@ "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==", "dev": true }, + "@types/rimraf": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/rimraf/-/rimraf-2.0.3.tgz", + "integrity": "sha512-dZfyfL/u9l/oi984hEXdmAjX3JHry7TLWw43u1HQ8HhPv6KtfxnrZ3T/bleJ0GEvnk9t5sM7eePkgMqz3yBcGg==", + "dev": true, + "requires": { + "@types/glob": "*", + "@types/node": "*" + } + }, "@types/selenium-webdriver": { "version": "3.0.14", "resolved": "https://registry.npmjs.org/@types/selenium-webdriver/-/selenium-webdriver-3.0.14.tgz", @@ -4472,12 +4482,6 @@ "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" }, - "any-shell-escape": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/any-shell-escape/-/any-shell-escape-0.1.1.tgz", - "integrity": "sha1-1Vq5ciRMcaml4asIefML8RCAaVk=", - "dev": true - }, "anymatch": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", @@ -8725,15 +8729,6 @@ "parse-filepath": "^1.0.1" } }, - "first-chunk-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/first-chunk-stream/-/first-chunk-stream-2.0.0.tgz", - "integrity": "sha1-G97NuOCDwGZLkZRVgVd6Q6nzHXA=", - "dev": true, - "requires": { - "readable-stream": "^2.0.2" - } - }, "flagged-respawn": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/flagged-respawn/-/flagged-respawn-1.0.1.tgz", @@ -9990,90 +9985,6 @@ } } }, - "gulp-git": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/gulp-git/-/gulp-git-2.10.0.tgz", - "integrity": "sha512-AYh0xXpKdDYS+ftCuyF9+LFXoltjtFlpfKITTCKDI0LunztpwVuHFtp31SvRSFVZikvRHTHUGMZ9Z0TnXjDIxQ==", - "dev": true, - "requires": { - "any-shell-escape": "^0.1.1", - "fancy-log": "^1.3.2", - "lodash.template": "^4.4.0", - "plugin-error": "^1.0.1", - "require-dir": "^1.0.0", - "strip-bom-stream": "^3.0.0", - "through2": "^2.0.3", - "vinyl": "^2.0.1" - }, - "dependencies": { - "ansi-colors": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-1.1.0.tgz", - "integrity": "sha512-SFKX67auSNoVR38N3L+nvsPjOE0bybKTYbkf5tRvushrAPQ9V75huw0ZxBkKVeRU9kqH3d6HA4xTckbwZ4ixmA==", - "dev": true, - "requires": { - "ansi-wrap": "^0.1.0" - } - }, - "clone-stats": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-1.0.0.tgz", - "integrity": "sha1-s3gt/4u1R04Yuba/D9/ngvh3doA=", - "dev": true - }, - "lodash.template": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.template/-/lodash.template-4.5.0.tgz", - "integrity": "sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A==", - "dev": true, - "requires": { - "lodash._reinterpolate": "^3.0.0", - "lodash.templatesettings": "^4.0.0" - } - }, - "lodash.templatesettings": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz", - "integrity": "sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ==", - "dev": true, - "requires": { - "lodash._reinterpolate": "^3.0.0" - } - }, - "plugin-error": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/plugin-error/-/plugin-error-1.0.1.tgz", - "integrity": "sha512-L1zP0dk7vGweZME2i+EeakvUNqSrdiI3F91TwEoYiGrAfUXmVv6fJIq4g82PAXxNsWOp0J7ZqQy/3Szz0ajTxA==", - "dev": true, - "requires": { - "ansi-colors": "^1.0.1", - "arr-diff": "^4.0.0", - "arr-union": "^3.1.0", - "extend-shallow": "^3.0.2" - } - }, - "replace-ext": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/replace-ext/-/replace-ext-1.0.0.tgz", - "integrity": "sha1-3mMSg3P8v3w8z6TeWkgMRaZ5WOs=", - "dev": true - }, - "vinyl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-2.2.0.tgz", - "integrity": "sha512-MBH+yP0kC/GQ5GwBqrTPTzEfiiLjta7hTtvQtbxBgTeSXsmKQRQecjibMbxIXzVT3Y9KJK+drOz1/k+vsu8Nkg==", - "dev": true, - "requires": { - "clone": "^2.1.1", - "clone-buffer": "^1.0.0", - "clone-stats": "^1.0.0", - "cloneable-readable": "^1.0.0", - "remove-trailing-separator": "^1.0.1", - "replace-ext": "^1.0.0" - } - } - } - }, "gulp-json-editor": { "version": "2.5.4", "resolved": "https://registry.npmjs.org/gulp-json-editor/-/gulp-json-editor-2.5.4.tgz", @@ -15678,12 +15589,6 @@ } } }, - "require-dir": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/require-dir/-/require-dir-1.2.0.tgz", - "integrity": "sha512-LY85DTSu+heYgDqq/mK+7zFHWkttVNRXC9NKcKGyuGLdlsfbjEPrIEYdCVrx6hqnJb+xSu3Lzaoo8VnmOhhjNA==", - "dev": true - }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -15788,7 +15693,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.0.tgz", "integrity": "sha512-NDGVxTsjqfunkds7CqsOiEnxln4Bo7Nddl3XhS4pXg5OzwkLqJ971ZVAAnB+DDLnF76N+VnDEiBHaVV8I06SUg==", - "dev": true, "requires": { "glob": "^7.1.3" } @@ -17050,25 +16954,6 @@ "is-utf8": "^0.2.0" } }, - "strip-bom-buf": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/strip-bom-buf/-/strip-bom-buf-1.0.0.tgz", - "integrity": "sha1-HLRar1dTD0yvhsf3UXnSyaUd1XI=", - "dev": true, - "requires": { - "is-utf8": "^0.2.1" - } - }, - "strip-bom-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-bom-stream/-/strip-bom-stream-3.0.0.tgz", - "integrity": "sha1-lWvMXYRDD2klapDtgjdlzYWOFZw=", - "dev": true, - "requires": { - "first-chunk-stream": "^2.0.0", - "strip-bom-buf": "^1.0.0" - } - }, "strip-eof": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", diff --git a/package.json b/package.json index 38c3bff0..f985baac 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pigallery2", - "version": "1.7.8", + "version": "1.7.9", "description": "This is a photo gallery optimised for running low resource servers (especially on raspberry pi)", "author": "Patrik J. Braun", "homepage": "https://github.com/bpatrik/PiGallery2", @@ -39,6 +39,7 @@ "jimp": "0.9.3", "locale": "0.1.0", "reflect-metadata": "0.1.13", + "rimraf": "^3.0.0", "sqlite3": "4.1.1", "ts-exif-parser": "0.1.4", "ts-node-iptc": "1.0.11", @@ -75,6 +76,7 @@ "@types/image-size": "0.8.0", "@types/jasmine": "3.5.0", "@types/node": "12.12.14", + "@types/rimraf": "^2.0.3", "@types/sharp": "0.23.1", "@types/winston": "2.4.4", "@yaga/leaflet-ng2": "1.0.0", @@ -135,6 +137,6 @@ "sharp": "0.23.4" }, "engines": { - "node": ">=10 <13.0" + "node": ">=10.17 <13.0" } } diff --git a/src/backend/ProjectPath.ts b/src/backend/ProjectPath.ts index 84df68ed..8193c1ba 100644 --- a/src/backend/ProjectPath.ts +++ b/src/backend/ProjectPath.ts @@ -5,7 +5,7 @@ import {Config} from '../common/config/private/Config'; class ProjectPathClass { public Root: string; public ImageFolder: string; - public ThumbnailFolder: string; + public TempFolder: string; public TranscodedFolder: string; public FacesFolder: string; public FrontendFolder: string; @@ -30,13 +30,13 @@ class ProjectPathClass { this.Root = path.join(__dirname, '/../../'); this.FrontendFolder = path.join(this.Root, 'dist'); this.ImageFolder = this.getAbsolutePath(Config.Server.Media.folder); - this.ThumbnailFolder = this.getAbsolutePath(Config.Server.Media.tempFolder); - this.TranscodedFolder = path.join(this.ThumbnailFolder, 'tc'); - this.FacesFolder = path.join(this.ThumbnailFolder, 'f'); + this.TempFolder = this.getAbsolutePath(Config.Server.Media.tempFolder); + this.TranscodedFolder = path.join(this.TempFolder, 'tc'); + this.FacesFolder = path.join(this.TempFolder, 'f'); // create thumbnail folder if not exist - if (!fs.existsSync(this.ThumbnailFolder)) { - fs.mkdirSync(this.ThumbnailFolder); + if (!fs.existsSync(this.TempFolder)) { + fs.mkdirSync(this.TempFolder); } } diff --git a/src/backend/middlewares/GalleryMWs.ts b/src/backend/middlewares/GalleryMWs.ts index 198876d4..49b7180c 100644 --- a/src/backend/middlewares/GalleryMWs.ts +++ b/src/backend/middlewares/GalleryMWs.ts @@ -195,7 +195,7 @@ export class GalleryMWs { } req.resultPipe = fullMediaPath; - const convertedVideo = VideoProcessing.generateConvertedFileName(fullMediaPath); + const convertedVideo = VideoProcessing.generateConvertedFilePath(fullMediaPath); // check if transcoded video exist if (fs.existsSync(convertedVideo) === true) { diff --git a/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts b/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts index 472773cf..27505755 100644 --- a/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts +++ b/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts @@ -145,7 +145,6 @@ export class ThumbnailGeneratorMWs { } private static addThInfoToPhotos(photos: MediaDTO[]) { - const thumbnailFolder = ProjectPath.ThumbnailFolder; for (let i = 0; i < photos.length; i++) { const fullMediaPath = path.join(ProjectPath.ImageFolder, photos[i].directory.path, photos[i].directory.name, photos[i].name); for (let j = 0; j < Config.Client.Media.Thumbnail.thumbnailSizes.length; j++) { diff --git a/src/backend/model/fileprocessing/PhotoProcessing.ts b/src/backend/model/fileprocessing/PhotoProcessing.ts index e3e393bd..2d58f843 100644 --- a/src/backend/model/fileprocessing/PhotoProcessing.ts +++ b/src/backend/model/fileprocessing/PhotoProcessing.ts @@ -1,5 +1,5 @@ import * as path from 'path'; -import * as fs from 'fs'; +import {constants as fsConstants, promises as fsp} from 'fs'; import * as os from 'os'; import * as crypto from 'crypto'; import {ProjectPath} from '../../ProjectPath'; @@ -9,6 +9,7 @@ import {PhotoWorker, RendererInput, ThumbnailSourceType} from '../threading/Phot import {ITaskExecuter, TaskExecuter} from '../threading/TaskExecuter'; import {ServerConfig} from '../../../common/config/private/IPrivateConfig'; import {FaceRegion, PhotoDTO} from '../../../common/entities/PhotoDTO'; +import {SupportedFormats} from '../../../common/SupportedFormats'; export class PhotoProcessing { @@ -60,8 +61,11 @@ export class PhotoProcessing { // check if thumbnail already exist - if (fs.existsSync(thPath) === true) { + try { + await fsp.access(thPath, fsConstants.R_OK); return null; + } catch (e) { + } @@ -95,11 +99,10 @@ export class PhotoProcessing { public static generateThumbnailPath(mediaPath: string, size: number): string { - const extension = path.extname(mediaPath); - const file = path.basename(mediaPath, extension); + const file = path.basename(mediaPath); return path.join(ProjectPath.TranscodedFolder, - ProjectPath.getRelativePathToImages(path.dirname(mediaPath)), file + - '_' + size + '.jpg'); + ProjectPath.getRelativePathToImages(path.dirname(mediaPath)), + file + '_' + size + '.jpg'); } public static generatePersonThumbnailPath(mediaPath: string, faceRegion: FaceRegion, size: number): string { @@ -108,14 +111,34 @@ export class PhotoProcessing { .digest('hex') + '_' + size + '.jpg'); } - public static generateConvertedFilePath(photoPath: string): string { - const extension = path.extname(photoPath); - const file = path.basename(photoPath, extension); - const postfix = Config.Server.Media.Photo.Converting.resolution; - return path.join(ProjectPath.TranscodedFolder, - ProjectPath.getRelativePathToImages(path.dirname(photoPath)), file + - '_' + postfix + '.jpg'); + return this.generateThumbnailPath(photoPath, Config.Server.Media.Photo.Converting.resolution); + } + + public static async isValidConvertedPath(convertedPath: string): Promise { + const origFilePath = path.join(ProjectPath.ImageFolder, + path.relative(ProjectPath.TranscodedFolder, + convertedPath.substring(0, convertedPath.lastIndexOf('_')))); + + const sizeStr = convertedPath.substring(convertedPath.lastIndexOf('_') + 1, + convertedPath.length - path.extname(convertedPath).length); + + const size = parseInt(sizeStr, 10); + + if ((size + '').length !== sizeStr.length || + (Config.Client.Media.Thumbnail.thumbnailSizes.indexOf(size) === -1 && + Config.Server.Media.Photo.Converting.resolution !== size)) { + return false; + } + + try { + await fsp.access(origFilePath, fsConstants.R_OK); + } catch (e) { + return false; + } + + + return true; } @@ -125,8 +148,10 @@ export class PhotoProcessing { // check if file already exist - if (fs.existsSync(outPath) === true) { + try { + await fsp.access(outPath, fsConstants.R_OK); return outPath; + } catch (e) { } @@ -141,9 +166,8 @@ export class PhotoProcessing { }; const outDir = path.dirname(input.outPath); - if (!fs.existsSync(outDir)) { - fs.mkdirSync(outDir, {recursive: true}); - } + + await fsp.mkdir(outDir, {recursive: true}); await this.taskQue.execute(input); return outPath; } @@ -156,9 +180,11 @@ export class PhotoProcessing { const outPath = PhotoProcessing.generateThumbnailPath(mediaPath, size); - // check if thumbnail already exist - if (fs.existsSync(outPath) === true) { + // check if file already exist + try { + await fsp.access(outPath, fsConstants.R_OK); return outPath; + } catch (e) { } @@ -173,11 +199,15 @@ export class PhotoProcessing { }; const outDir = path.dirname(input.outPath); - if (!fs.existsSync(outDir)) { - fs.mkdirSync(outDir, {recursive: true}); - } + + await fsp.mkdir(outDir, {recursive: true}); await this.taskQue.execute(input); return outPath; } + + public static isPhoto(fullPath: string) { + const extension = path.extname(fullPath).toLowerCase(); + return SupportedFormats.WithDots.Photos.indexOf(extension) !== -1; + } } diff --git a/src/backend/model/fileprocessing/VideoProcessing.ts b/src/backend/model/fileprocessing/VideoProcessing.ts index f7009471..a06161d8 100644 --- a/src/backend/model/fileprocessing/VideoProcessing.ts +++ b/src/backend/model/fileprocessing/VideoProcessing.ts @@ -1,38 +1,56 @@ import * as path from 'path'; -import * as fs from 'fs'; -import * as util from 'util'; -import {ITaskExecuter, TaskExecuter} from '../../model/threading/TaskExecuter'; -import {VideoConverterInput, VideoConverterWorker} from '../../model/threading/VideoConverterWorker'; -import {MetadataLoader} from '../../model/threading/MetadataLoader'; +import {constants as fsConstants, promises as fsp} from 'fs'; +import {ITaskExecuter, TaskExecuter} from '../threading/TaskExecuter'; +import {VideoConverterInput, VideoConverterWorker} from '../threading/VideoConverterWorker'; +import {MetadataLoader} from '../threading/MetadataLoader'; import {Config} from '../../../common/config/private/Config'; import {ProjectPath} from '../../ProjectPath'; +import {SupportedFormats} from '../../../common/SupportedFormats'; -const existPr = util.promisify(fs.exists); export class VideoProcessing { private static taskQue: ITaskExecuter = new TaskExecuter(1, (input => VideoConverterWorker.convert(input))); - - public static generateConvertedFileName(videoPath: string): string { - const extension = path.extname(videoPath); - const file = path.basename(videoPath, extension); - const postfix = Math.round(Config.Server.Media.Video.transcoding.bitRate / 1024) + 'k' + - Config.Server.Media.Video.transcoding.codec.toString().toLowerCase() + - Config.Server.Media.Video.transcoding.resolution; + public static generateConvertedFilePath(videoPath: string): string { return path.join(ProjectPath.TranscodedFolder, - ProjectPath.getRelativePathToImages(path.dirname(videoPath)), file + - '_' + postfix + '.' + Config.Server.Media.Video.transcoding.format); + ProjectPath.getRelativePathToImages(path.dirname(videoPath)), + path.basename(videoPath) + '_' + this.getConvertedFilePostFix()); + } + + public static async isValidConvertedPath(convertedPath: string): Promise { + + const origFilePath = path.join(ProjectPath.ImageFolder, + path.relative(ProjectPath.TranscodedFolder, + convertedPath.substring(0, convertedPath.lastIndexOf('_')))); + + const postfix = convertedPath.substring(convertedPath.lastIndexOf('_') + 1, convertedPath.length); + + if (postfix !== this.getConvertedFilePostFix()) { + return false; + } + + try { + await fsp.access(origFilePath, fsConstants.R_OK); + } catch (e) { + return false; + } + + + return true; } public static async convertVideo(videoPath: string): Promise { - const outPath = this.generateConvertedFileName(videoPath); + const outPath = this.generateConvertedFilePath(videoPath); - if (await existPr(outPath)) { + try { + await fsp.access(outPath, fsConstants.R_OK); return; + } catch (e) { } + const metaData = await MetadataLoader.loadVideoMetadata(videoPath); const renderInput: VideoConverterInput = { @@ -56,12 +74,23 @@ export class VideoProcessing { } const outDir = path.dirname(renderInput.output.path); - if (!fs.existsSync(outDir)) { - fs.mkdirSync(outDir, {recursive: true}); - } + await fsp.mkdir(outDir, {recursive: true}); await VideoProcessing.taskQue.execute(renderInput); } + + public static isVideo(fullPath: string) { + const extension = path.extname(fullPath).toLowerCase(); + return SupportedFormats.WithDots.Videos.indexOf(extension) !== -1; + } + + protected static getConvertedFilePostFix(): string { + return Math.round(Config.Server.Media.Video.transcoding.bitRate / 1024) + 'k' + + Config.Server.Media.Video.transcoding.codec.toString().toLowerCase() + + Config.Server.Media.Video.transcoding.resolution + + '.' + Config.Server.Media.Video.transcoding.format.toLowerCase(); + } + } diff --git a/src/backend/model/jobs/JobRepository.ts b/src/backend/model/jobs/JobRepository.ts index b9d9c438..0a9c9648 100644 --- a/src/backend/model/jobs/JobRepository.ts +++ b/src/backend/model/jobs/JobRepository.ts @@ -4,6 +4,7 @@ import {DBRestJob} from './jobs/DBResetJob'; import {VideoConvertingJob} from './jobs/VideoConvertingJob'; import {PhotoConvertingJob} from './jobs/PhotoConvertingJob'; import {ThumbnailGenerationJob} from './jobs/ThumbnailGenerationJob'; +import {TempFolderCleaningJob} from './jobs/TempFolderCleaningJob'; export class JobRepository { @@ -21,7 +22,10 @@ export class JobRepository { return Object.values(this.availableJobs).filter(t => t.Supported); } - register(job: IJob) { + register(job: IJob): void { + if (typeof this.availableJobs[job.Name] !== 'undefined') { + throw new Error('Job already exist:' + job.Name); + } this.availableJobs[job.Name] = job; } } @@ -32,3 +36,4 @@ JobRepository.Instance.register(new DBRestJob()); JobRepository.Instance.register(new VideoConvertingJob()); JobRepository.Instance.register(new PhotoConvertingJob()); JobRepository.Instance.register(new ThumbnailGenerationJob()); +JobRepository.Instance.register(new TempFolderCleaningJob()); diff --git a/src/backend/model/jobs/jobs/FileJob.ts b/src/backend/model/jobs/jobs/FileJob.ts index 4954168b..4cdcf54c 100644 --- a/src/backend/model/jobs/jobs/FileJob.ts +++ b/src/backend/model/jobs/jobs/FileJob.ts @@ -1,4 +1,4 @@ -import {JobProgressDTO, JobState} from '../../../../common/entities/settings/JobProgressDTO'; +import {JobProgressDTO} from '../../../../common/entities/settings/JobProgressDTO'; import {ConfigTemplateEntry} from '../../../../common/entities/job/JobDTO'; import {Job} from './Job'; import * as path from 'path'; @@ -6,7 +6,6 @@ import {DiskManager} from '../../DiskManger'; import {DiskMangerWorker} from '../../threading/DiskMangerWorker'; import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO'; import {Logger} from '../../../Logger'; -import {MediaDTO} from '../../../../common/entities/MediaDTO'; declare var global: NodeJS.Global; @@ -35,11 +34,7 @@ export abstract class FileJob extends Job { protected abstract async processFile(file: T): Promise; protected async step(): Promise { - if ((this.directoryQueue.length === 0 && this.fileQueue.length === 0) - || this.state !== JobState.running) { - if (global.gc) { - global.gc(); - } + if (this.directoryQueue.length === 0 && this.fileQueue.length === 0) { return null; } diff --git a/src/backend/model/jobs/jobs/IndexingJob.ts b/src/backend/model/jobs/jobs/IndexingJob.ts index 7e34d996..ee94ddd6 100644 --- a/src/backend/model/jobs/jobs/IndexingJob.ts +++ b/src/backend/model/jobs/jobs/IndexingJob.ts @@ -25,9 +25,6 @@ export class IndexingJob extends Job { protected async step(): Promise { if (this.directoriesToIndex.length === 0) { - if (global.gc) { - global.gc(); - } return null; } const directory = this.directoriesToIndex.shift(); diff --git a/src/backend/model/jobs/jobs/Job.ts b/src/backend/model/jobs/jobs/Job.ts index 415c67d0..2ec7262c 100644 --- a/src/backend/model/jobs/jobs/Job.ts +++ b/src/backend/model/jobs/jobs/Job.ts @@ -2,6 +2,7 @@ import {JobProgressDTO, JobState} from '../../../../common/entities/settings/Job import {Logger} from '../../../Logger'; import {IJob} from './IJob'; import {ConfigTemplateEntry, JobDTO} from '../../../../common/entities/job/JobDTO'; +import * as rimraf from 'rimraf'; declare const process: any; @@ -77,6 +78,9 @@ export abstract class Job implements IJob { private onFinish(): void { this.progress = null; + if (global.gc) { + global.gc(); + } Logger.info(LOG_TAG, 'Job finished: ' + this.Name); if (this.IsInstant) { this.prResolve(); @@ -87,10 +91,13 @@ export abstract class Job implements IJob { process.nextTick(async () => { try { if (this.state === JobState.idle) { - this.progress = null; return; } - this.progress = await this.step(); + if (this.state === JobState.running) { + this.progress = await this.step(); + } else { + this.progress = null; + } if (this.progress == null) { // finished this.state = JobState.idle; this.onFinish(); diff --git a/src/backend/model/jobs/jobs/TempFolderCleaningJob.ts b/src/backend/model/jobs/jobs/TempFolderCleaningJob.ts new file mode 100644 index 00000000..070ff37a --- /dev/null +++ b/src/backend/model/jobs/jobs/TempFolderCleaningJob.ts @@ -0,0 +1,118 @@ +import {ConfigTemplateEntry, DefaultsJobs} from '../../../../common/entities/job/JobDTO'; +import * as path from 'path'; +import * as util from 'util'; +import {promises as fsp} from 'fs'; +import {Job} from './Job'; +import {JobProgressDTO} from '../../../../common/entities/settings/JobProgressDTO'; +import {ProjectPath} from '../../../ProjectPath'; +import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing'; +import {VideoProcessing} from '../../fileprocessing/VideoProcessing'; +import * as rimraf from 'rimraf'; + +const LOG_TAG = '[TempFolderCleaningJob]'; + +const rimrafPR = util.promisify(rimraf); + + +export class TempFolderCleaningJob extends Job { + public readonly Name = DefaultsJobs[DefaultsJobs['Temp Folder Cleaning']]; + public readonly ConfigTemplate: ConfigTemplateEntry[] = null; + public readonly Supported = true; + directoryQueue: string[] = []; + private tempRootCleaned = false; + + + protected async init() { + this.tempRootCleaned = false; + this.directoryQueue = []; + this.directoryQueue.push(ProjectPath.TranscodedFolder); + } + + + protected async isValidFile(filePath: string): Promise { + if (PhotoProcessing.isPhoto(filePath)) { + return PhotoProcessing.isValidConvertedPath(filePath); + } + + if (VideoProcessing.isVideo(filePath)) { + return VideoProcessing.isValidConvertedPath(filePath); + } + + return false; + } + + protected async isValidDirectory(filePath: string): Promise { + const originalPath = path.join(ProjectPath.ImageFolder, + path.relative(ProjectPath.TranscodedFolder, filePath)); + try { + await fsp.access(originalPath); + return true; + } catch (e) { + } + return false; + } + + protected async readDir(dirPath: string): Promise { + return (await fsp.readdir(dirPath)).map(f => path.normalize(path.join(dirPath, f))); + } + + protected async stepTempDirectory() { + const files = await this.readDir(ProjectPath.TempFolder); + const validFiles = [ProjectPath.TranscodedFolder, ProjectPath.FacesFolder]; + for (let i = 0; i < files.length; ++i) { + if (validFiles.indexOf(files[i]) === -1) { + if ((await fsp.stat(files[i])).isDirectory()) { + await rimrafPR(files[i]); + } else { + await fsp.unlink(files[i]); + } + } + } + + this.progress.time.current = Date.now(); + + this.progress.comment = 'processing: ' + ProjectPath.TempFolder; + + return this.progress; + + + } + + protected async stepConvertedDirectory() { + + + this.progress.time.current = Date.now(); + + + const filePath = this.directoryQueue.shift(); + const stat = await fsp.stat(filePath); + + this.progress.left = this.directoryQueue.length; + this.progress.progress++; + this.progress.comment = 'processing: ' + filePath; + if (stat.isDirectory()) { + if (await this.isValidDirectory(filePath) === false) { + await rimrafPR(filePath); + } else { + this.directoryQueue = this.directoryQueue.concat(await this.readDir(filePath)); + } + } else { + if (await this.isValidFile(filePath) === false) { + await fsp.unlink(filePath); + } + } + return this.progress; + } + + protected async step(): Promise { + if (this.directoryQueue.length === 0) { + return null; + } + if (this.tempRootCleaned === false) { + this.tempRootCleaned = true; + return this.stepTempDirectory(); + } + return this.stepConvertedDirectory(); + } + +} diff --git a/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts b/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts index 9a1fd2e6..e41c4ea2 100644 --- a/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts +++ b/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts @@ -2,8 +2,6 @@ import {Config} from '../../../../common/config/private/Config'; import {ConfigTemplateEntry, DefaultsJobs} from '../../../../common/entities/job/JobDTO'; import {ProjectPath} from '../../../ProjectPath'; import * as path from 'path'; -import * as fs from 'fs'; -import * as util from 'util'; import {FileJob} from './FileJob'; import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO'; import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing'; @@ -11,7 +9,6 @@ import {ThumbnailSourceType} from '../../threading/PhotoWorker'; import {MediaDTO} from '../../../../common/entities/MediaDTO'; const LOG_TAG = '[ThumbnailGenerationJob]'; -const existsPr = util.promisify(fs.exists); export class ThumbnailGenerationJob extends FileJob { diff --git a/src/backend/model/jobs/jobs/VideoConvertingJob.ts b/src/backend/model/jobs/jobs/VideoConvertingJob.ts index 15da4fd5..4848be48 100644 --- a/src/backend/model/jobs/jobs/VideoConvertingJob.ts +++ b/src/backend/model/jobs/jobs/VideoConvertingJob.ts @@ -31,7 +31,7 @@ export class VideoConvertingJob extends FileJob { directory.media[i].directory.name, directory.media[i].name); - if (await existsPr(VideoProcessing.generateConvertedFileName(videoPath)) === false) { + if (await existsPr(VideoProcessing.generateConvertedFilePath(videoPath)) === false) { ret.push(videoPath); } } diff --git a/src/backend/model/threading/DiskMangerWorker.ts b/src/backend/model/threading/DiskMangerWorker.ts index 7c3bef7a..dfa5cc8a 100644 --- a/src/backend/model/threading/DiskMangerWorker.ts +++ b/src/backend/model/threading/DiskMangerWorker.ts @@ -1,5 +1,4 @@ -import * as fs from 'fs'; -import {Stats} from 'fs'; +import {promises as fsp, Stats} from 'fs'; import * as path from 'path'; import {DirectoryDTO} from '../../../common/entities/DirectoryDTO'; import {PhotoDTO} from '../../../common/entities/PhotoDTO'; @@ -10,6 +9,8 @@ import {FileDTO} from '../../../common/entities/FileDTO'; import {MetadataLoader} from './MetadataLoader'; import {Logger} from '../../Logger'; import {SupportedFormats} from '../../../common/SupportedFormats'; +import {VideoProcessing} from '../fileprocessing/VideoProcessing'; +import {PhotoProcessing} from '../fileprocessing/PhotoProcessing'; const LOG_TAG = '[DiskMangerWorker]'; @@ -41,7 +42,7 @@ export class DiskMangerWorker { return path.basename(name); } - public static excludeDir(name: string, relativeDirectoryName: string, absoluteDirectoryName: string) { + public static async excludeDir(name: string, relativeDirectoryName: string, absoluteDirectoryName: string) { if (Config.Server.Indexing.excludeFolderList.length === 0 || Config.Server.Indexing.excludeFileList.length === 0) { return false; @@ -70,122 +71,105 @@ export class DiskMangerWorker { for (let j = 0; j < Config.Server.Indexing.excludeFileList.length; j++) { const exclude = Config.Server.Indexing.excludeFileList[j]; - if (fs.existsSync(path.join(absoluteName, exclude))) { + try { + await fsp.access(path.join(absoluteName, exclude)); return true; + } catch (e) { } } return false; } - public static scanDirectory(relativeDirectoryName: string, settings: DiskMangerWorker.DirectoryScanSettings = {}): Promise { - return new Promise((resolve, reject) => { - relativeDirectoryName = this.normalizeDirPath(relativeDirectoryName); - const directoryName = DiskMangerWorker.dirName(relativeDirectoryName); - const directoryParent = this.pathFromRelativeDirName(relativeDirectoryName); - const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName); + public static async scanDirectory(relativeDirectoryName: string, + settings: DiskMangerWorker.DirectoryScanSettings = {}): Promise { - const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); - const directory: DirectoryDTO = { - id: null, - parent: null, - name: directoryName, - path: directoryParent, - lastModified: this.calcLastModified(stat), - lastScanned: Date.now(), - directories: [], - isPartial: false, - mediaCount: 0, - media: [], - metaFile: [] - }; - fs.readdir(absoluteDirectoryName, async (err, list: string[]) => { - if (err) { - return reject(err); + relativeDirectoryName = this.normalizeDirPath(relativeDirectoryName); + const directoryName = DiskMangerWorker.dirName(relativeDirectoryName); + const directoryParent = this.pathFromRelativeDirName(relativeDirectoryName); + const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName); + + const stat = await fsp.stat(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); + const directory: DirectoryDTO = { + id: null, + parent: null, + name: directoryName, + path: directoryParent, + lastModified: this.calcLastModified(stat), + lastScanned: Date.now(), + directories: [], + isPartial: false, + mediaCount: 0, + media: [], + metaFile: [] + }; + const list = await fsp.readdir(absoluteDirectoryName); + for (let i = 0; i < list.length; i++) { + const file = list[i]; + const fullFilePath = path.normalize(path.join(absoluteDirectoryName, file)); + if ((await fsp.stat(fullFilePath)).isDirectory()) { + if (settings.noDirectory === true || + await DiskMangerWorker.excludeDir(file, relativeDirectoryName, absoluteDirectoryName)) { + continue; + } + + // create preview directory + const d = await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file), + { + maxPhotos: Config.Server.Indexing.folderPreviewSize, + noMetaFile: true, + noVideo: true, + noDirectory: true + } + ); + d.lastScanned = 0; // it was not a fully scan + d.isPartial = true; + directory.directories.push(d); + } else if (PhotoProcessing.isPhoto(fullFilePath)) { + if (settings.noPhoto) { + continue; + } + directory.media.push({ + name: file, + directory: null, + metadata: await MetadataLoader.loadPhotoMetadata(fullFilePath) + }); + + if (settings.maxPhotos && directory.media.length > settings.maxPhotos) { + break; + } + } else if (VideoProcessing.isVideo(fullFilePath)) { + if (Config.Client.Media.Video.enabled === false || settings.noVideo) { + continue; } try { - for (let i = 0; i < list.length; i++) { - const file = list[i]; - const fullFilePath = path.normalize(path.join(absoluteDirectoryName, file)); - if (fs.statSync(fullFilePath).isDirectory()) { - if (settings.noDirectory === true || - DiskMangerWorker.excludeDir(file, relativeDirectoryName, absoluteDirectoryName)) { - continue; - } - - // create preview directory - const d = await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file), - { - maxPhotos: Config.Server.Indexing.folderPreviewSize, - noMetaFile: true, - noVideo: true, - noDirectory: true - } - ); - d.lastScanned = 0; // it was not a fully scan - d.isPartial = true; - directory.directories.push(d); - } else if (DiskMangerWorker.isImage(fullFilePath)) { - if (settings.noPhoto) { - continue; - } - directory.media.push({ - name: file, - directory: null, - metadata: await MetadataLoader.loadPhotoMetadata(fullFilePath) - }); - - if (settings.maxPhotos && directory.media.length > settings.maxPhotos) { - break; - } - } else if (DiskMangerWorker.isVideo(fullFilePath)) { - if (Config.Client.Media.Video.enabled === false || settings.noVideo) { - continue; - } - try { - directory.media.push({ - name: file, - directory: null, - metadata: await MetadataLoader.loadVideoMetadata(fullFilePath) - }); - } catch (e) { - Logger.warn('Media loading error, skipping: ' + file + ', reason: ' + e.toString()); - } - - } else if (DiskMangerWorker.isMetaFile(fullFilePath)) { - if (Config.Client.MetaFile.enabled === false || settings.noMetaFile) { - continue; - } - - directory.metaFile.push({ - name: file, - directory: null, - }); - - } - } - - directory.mediaCount = directory.media.length; - - return resolve(directory); - } catch (err) { - return reject({error: err}); + directory.media.push({ + name: file, + directory: null, + metadata: await MetadataLoader.loadVideoMetadata(fullFilePath) + }); + } catch (e) { + Logger.warn('Media loading error, skipping: ' + file + ', reason: ' + e.toString()); } - }); - }); + } else if (DiskMangerWorker.isMetaFile(fullFilePath)) { + if (Config.Client.MetaFile.enabled === false || settings.noMetaFile) { + continue; + } + directory.metaFile.push({ + name: file, + directory: null, + }); + + } + } + + directory.mediaCount = directory.media.length; + + return directory; } - private static isImage(fullPath: string) { - const extension = path.extname(fullPath).toLowerCase(); - return SupportedFormats.WithDots.Photos.indexOf(extension) !== -1; - } - - private static isVideo(fullPath: string) { - const extension = path.extname(fullPath).toLowerCase(); - return SupportedFormats.WithDots.Videos.indexOf(extension) !== -1; - } private static isMetaFile(fullPath: string) { const extension = path.extname(fullPath).toLowerCase(); diff --git a/src/common/entities/job/JobDTO.ts b/src/common/entities/job/JobDTO.ts index e2f98144..2ecd9757 100644 --- a/src/common/entities/job/JobDTO.ts +++ b/src/common/entities/job/JobDTO.ts @@ -2,7 +2,12 @@ export type fieldType = 'string' | 'number' | 'boolean' | 'number-array'; export enum DefaultsJobs { - Indexing = 1, 'Database Reset' = 2, 'Video Converting' = 3, 'Photo Converting' = 4, 'Thumbnail Generation' = 5 + Indexing = 1, + 'Database Reset' = 2, + 'Video Converting' = 3, + 'Photo Converting' = 4, + 'Thumbnail Generation' = 5, + 'Temp Folder Cleaning' = 6 } export interface ConfigTemplateEntry { diff --git a/src/frontend/app/ui/settings/jobs/jobs.settings.component.ts b/src/frontend/app/ui/settings/jobs/jobs.settings.component.ts index 86671604..ccf48124 100644 --- a/src/frontend/app/ui/settings/jobs/jobs.settings.component.ts +++ b/src/frontend/app/ui/settings/jobs/jobs.settings.component.ts @@ -8,11 +8,11 @@ import {I18n} from '@ngx-translate/i18n-polyfill'; import {ErrorDTO} from '../../../../../common/entities/Error'; import {ScheduledJobsService} from '../scheduled-jobs.service'; import { + JobScheduleDTO, + JobTriggerType, NeverJobTrigger, PeriodicJobTrigger, - ScheduledJobTrigger, - JobScheduleDTO, - JobTriggerType + ScheduledJobTrigger } from '../../../../../common/entities/job/JobScheduleDTO'; import {Utils} from '../../../../../common/Utils'; import {ServerConfig} from '../../../../../common/config/private/IPrivateConfig'; @@ -64,7 +64,6 @@ export class JobsSettingsComponent extends SettingsComponent t.Name === JobName); if (job && job.ConfigTemplate && job.ConfigTemplate.length > 0) { @@ -131,7 +130,9 @@ export class JobsSettingsComponent extends SettingsComponent t.Name === schedule.jobName); schedule.config = schedule.config || {}; - job.ConfigTemplate.forEach(ct => schedule.config[ct.id] = ct.defaultValue); + if (job.ConfigTemplate) { + job.ConfigTemplate.forEach(ct => schedule.config[ct.id] = ct.defaultValue); + } } addNewJob() { @@ -146,7 +147,9 @@ export class JobsSettingsComponent extends SettingsComponent t.Name === jobName); newSchedule.config = newSchedule.config || {}; - job.ConfigTemplate.forEach(ct => newSchedule.config[ct.id] = ct.defaultValue); + if (job.ConfigTemplate) { + job.ConfigTemplate.forEach(ct => newSchedule.config[ct.id] = ct.defaultValue); + } this.settings.scheduled.push(newSchedule); } diff --git a/test/backend/unit/model/fileprocessing/PhotoProcessing.spec.ts b/test/backend/unit/model/fileprocessing/PhotoProcessing.spec.ts new file mode 100644 index 00000000..6528143c --- /dev/null +++ b/test/backend/unit/model/fileprocessing/PhotoProcessing.spec.ts @@ -0,0 +1,61 @@ +import {expect} from 'chai'; +import {Config} from '../../../../../src/common/config/private/Config'; +import {ProjectPath} from '../../../../../src/backend/ProjectPath'; +import * as path from 'path'; +import {PhotoProcessing} from '../../../../../src/backend/model/fileprocessing/PhotoProcessing'; + + +describe('PhotoProcessing', () => { + + // tslint:disable:no-unused-expression + it('should generate converted file path', async () => { + + Config.load(); + Config.Client.Media.Thumbnail.thumbnailSizes = []; + ProjectPath.ImageFolder = path.join(__dirname, './../../assets'); + const photoPath = path.join(ProjectPath.ImageFolder, 'test_png.png'); + + expect(await PhotoProcessing + .isValidConvertedPath(PhotoProcessing.generateConvertedFilePath(photoPath))) + .to.be.true; + + expect(await PhotoProcessing + .isValidConvertedPath(PhotoProcessing.generateConvertedFilePath(photoPath + 'noPath'))) + .to.be.false; + + { + const convertedPath = PhotoProcessing.generateConvertedFilePath(photoPath); + Config.Server.Media.Photo.Converting.resolution = 1; + expect(await PhotoProcessing.isValidConvertedPath(convertedPath)).to.be.false; + } + }); + + // tslint:disable:no-unused-expression + it('should generate converted thumbnail path', async () => { + + Config.load(); + Config.Server.Media.Photo.Converting.resolution = null; + Config.Client.Media.Thumbnail.thumbnailSizes = [10, 20]; + ProjectPath.ImageFolder = path.join(__dirname, './../../assets'); + const photoPath = path.join(ProjectPath.ImageFolder, 'test_png.png'); + + for (let i = 0; i < Config.Client.Media.Thumbnail.thumbnailSizes.length; ++i) { + const thSize = Config.Client.Media.Thumbnail.thumbnailSizes[i]; + expect(await PhotoProcessing + .isValidConvertedPath(PhotoProcessing.generateThumbnailPath(photoPath, thSize))) + .to.be.true; + + + expect(await PhotoProcessing + .isValidConvertedPath(PhotoProcessing.generateThumbnailPath(photoPath + 'noPath', thSize))) + .to.be.false; + } + + + expect(await PhotoProcessing + .isValidConvertedPath(PhotoProcessing.generateThumbnailPath(photoPath, 30))) + .to.be.false; + + }); + +}); diff --git a/test/backend/unit/model/fileprocessing/VideoProcessing.spec.ts b/test/backend/unit/model/fileprocessing/VideoProcessing.spec.ts new file mode 100644 index 00000000..89ceae76 --- /dev/null +++ b/test/backend/unit/model/fileprocessing/VideoProcessing.spec.ts @@ -0,0 +1,45 @@ +import {expect} from 'chai'; +import {VideoProcessing} from '../../../../../src/backend/model/fileprocessing/VideoProcessing'; +import {Config} from '../../../../../src/common/config/private/Config'; +import {ProjectPath} from '../../../../../src/backend/ProjectPath'; +import * as path from 'path'; + + +describe('VideoProcessing', () => { + + // tslint:disable:no-unused-expression + it('should generate converted file path', async () => { + + ProjectPath.ImageFolder = path.join(__dirname, './../../assets'); + const videoPath = path.join(ProjectPath.ImageFolder, 'video.mp4'); + expect(await VideoProcessing + .isValidConvertedPath(VideoProcessing.generateConvertedFilePath(videoPath))) + .to.be.true; + + expect(await VideoProcessing + .isValidConvertedPath(VideoProcessing.generateConvertedFilePath(videoPath + 'noPath'))) + .to.be.false; + + { + const convertedPath = VideoProcessing.generateConvertedFilePath(videoPath); + Config.Server.Media.Video.transcoding.bitRate = 10; + expect(await VideoProcessing.isValidConvertedPath(convertedPath)).to.be.false; + } + { + const convertedPath = VideoProcessing.generateConvertedFilePath(videoPath); + Config.Server.Media.Video.transcoding.codec = 'codec_text'; + expect(await VideoProcessing.isValidConvertedPath(convertedPath)).to.be.false; + } + { + const convertedPath = VideoProcessing.generateConvertedFilePath(videoPath); + Config.Server.Media.Video.transcoding.format = 'format_test'; + expect(await VideoProcessing.isValidConvertedPath(convertedPath)).to.be.false; + } + { + const convertedPath = VideoProcessing.generateConvertedFilePath(videoPath); + Config.Server.Media.Video.transcoding.resolution = 1; + expect(await VideoProcessing.isValidConvertedPath(convertedPath)).to.be.false; + } + }); + +});