self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2024-12-21 01:22:08 +02:00
Code Issues Releases Activity

remove threading #641

Browse Source
This commit is contained in:
Patrik J. Braun 2023-10-14 17:54:21 +02:00
parent 4dc431fe22
commit 2b0d1a96a6
48 changed files with 283 additions and 582 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
benchmark/BenchmarkRunner.ts
View File

@ -1,6 +1,6 @@
import {Config} from '../src/common/config/private/Config';
import {ObjectManagers} from '../src/backend/model/ObjectManagers';
import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker';
import {DiskManager} from '../src/backend/model/fileaccess/DiskManager';
import {IndexingManager} from '../src/backend/model/database/IndexingManager';
import * as path from 'path';
import * as fs from 'fs';
@ -84,7 +84,7 @@ export class BenchmarkRunner {
}
async bmSaveDirectory(): Promise<BenchmarkResult[]> {
const dir = await DiskMangerWorker.scanDirectory(this.biggestDirPath);
const dir = await DiskManager.scanDirectory(this.biggestDirPath);
const bm = new Benchmark('Saving directory to DB', null,
(): Promise<void> => this.resetDB(), null,
async (): Promise<void> => {
@ -110,7 +110,7 @@ export class BenchmarkRunner {
});
bm.addAStep({
name: 'Scanning directory',
fn: async (): Promise<ContentWrapper> => new ContentWrapper(await DiskMangerWorker.scanDirectory(this.biggestDirPath))
fn: async (): Promise<ContentWrapper> => new ContentWrapper(await DiskManager.scanDirectory(this.biggestDirPath))
});
return await bm.run(this.RUNS);
}
@ -285,7 +285,6 @@ export class BenchmarkRunner {
private resetDB = async (): Promise<void> => {
Config.Server.Threading.enabled = false;
await ObjectManagers.reset();
await fs.promises.rm(ProjectPath.DBFolder, {recursive: true, force: true});
Config.Database.type = DatabaseType.sqlite;
@ -294,7 +293,6 @@ export class BenchmarkRunner {
};
private async setupDB(): Promise<void> {
Config.Server.Threading.enabled = false;
await this.resetDB();
await new Promise<void>((resolve, reject): void => {
try {

8
src/backend/index.ts
View File

@ -1,6 +1,4 @@
import * as cluster from 'cluster';
import {Server} from './server';
import {Worker} from './model/threading/Worker';
import {ConfigDiagnostics} from './model/diagnostics/ConfigDiagnostics';
@ -13,9 +11,5 @@ if ((process.argv || []).includes('--run-diagnostics')) {
process.exit(0);
});
} else {
if ((cluster as any).isMaster) {
new Server();
} else {
Worker.process();
}
new Server();
}

2
src/backend/middlewares/GalleryMWs.ts
View File

@ -11,7 +11,7 @@ import {Config} from '../../common/config/private/Config';
import {UserDTOUtils} from '../../common/entities/UserDTO';
import {MediaDTO, MediaDTOUtils} from '../../common/entities/MediaDTO';
import {QueryParams} from '../../common/QueryParams';
import {VideoProcessing} from '../model/fileprocessing/VideoProcessing';
import {VideoProcessing} from '../model/fileaccess/fileprocessing/VideoProcessing';
import {SearchQueryDTO, SearchQueryTypes,} from '../../common/entities/SearchQueryDTO';
import {LocationLookupException} from '../exceptions/LocationLookupException';
import {SupportedFormats} from '../../common/SupportedFormats';

2
src/backend/middlewares/MetaFileMWs.ts
View File

@ -1,7 +1,7 @@
import {NextFunction, Request, Response} from 'express';
import * as fs from 'fs';
import {Config} from '../../common/config/private/Config';
import {GPXProcessing} from '../model/fileprocessing/GPXProcessing';
import {GPXProcessing} from '../model/fileaccess/fileprocessing/GPXProcessing';
import {Logger} from '../Logger';
const LOG_TAG = 'MetaFileMWs';

2
src/backend/middlewares/thumbnail/PhotoConverterMWs.ts
View File

@ -1,6 +1,6 @@
import {NextFunction, Request, Response} from 'express';
import * as fs from 'fs';
import {PhotoProcessing} from '../../model/fileprocessing/PhotoProcessing';
import {PhotoProcessing} from '../../model/fileaccess/fileprocessing/PhotoProcessing';
import {Config} from '../../../common/config/private/Config';
import {ErrorCodes, ErrorDTO} from '../../../common/entities/Error';

4
src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts
View File

@ -6,9 +6,9 @@ import {ContentWrapper} from '../../../common/entities/ConentWrapper';
import {DirectoryPathDTO, ParentDirectoryDTO, SubDirectoryDTO,} from '../../../common/entities/DirectoryDTO';
import {ProjectPath} from '../../ProjectPath';
import {Config} from '../../../common/config/private/Config';
import {ThumbnailSourceType} from '../../model/threading/PhotoWorker';
import {ThumbnailSourceType} from '../../model/fileaccess/PhotoWorker';
import {MediaDTO} from '../../../common/entities/MediaDTO';
import {PhotoProcessing} from '../../model/fileprocessing/PhotoProcessing';
import {PhotoProcessing} from '../../model/fileaccess/fileprocessing/PhotoProcessing';
import {ServerTime} from '../ServerTimingMWs';
import {PersonEntry} from '../../model/database/enitites/PersonEntry';

51
src/backend/model/DiskManger.ts
View File

@ -1,51 +0,0 @@
import {ParentDirectoryDTO,} from '../../common/entities/DirectoryDTO';
import {Logger} from '../Logger';
import {Config} from '../../common/config/private/Config';
import {DiskManagerTH} from './threading/ThreadPool';
import {DirectoryScanSettings, DiskMangerWorker,} from './threading/DiskMangerWorker';
import {FileDTO} from '../../common/entities/FileDTO';
const LOG_TAG = '[DiskManager]';
export class DiskManager {
static threadPool: DiskManagerTH = null;
public static init(): void {
if (Config.Server.Threading.enabled === true) {
DiskManager.threadPool = new DiskManagerTH(1);
}
}
/**
* List all files in a folder as fast as possible
*/
public static async scanDirectoryNoMetadata(
relativeDirectoryName: string,
settings: DirectoryScanSettings = {}
): Promise<ParentDirectoryDTO<FileDTO>> {
settings.noMetadata = true;
return this.scanDirectory(relativeDirectoryName, settings);
}
public static async scanDirectory(
relativeDirectoryName: string,
settings: DirectoryScanSettings = {}
): Promise<ParentDirectoryDTO> {
Logger.silly(LOG_TAG, 'scanning directory:', relativeDirectoryName);
let directory: ParentDirectoryDTO;
if (Config.Server.Threading.enabled === true) {
directory = await DiskManager.threadPool.execute(
relativeDirectoryName,
settings
);
} else {
directory = (await DiskMangerWorker.scanDirectory(
relativeDirectoryName,
settings
)) as ParentDirectoryDTO;
}
return directory;
}
}

18
src/backend/model/database/CoverManager.ts
View File

@ -1,7 +1,7 @@
import {Config} from '../../../common/config/private/Config';
import {Brackets, SelectQueryBuilder, WhereExpression} from 'typeorm';
import {MediaEntity} from './enitites/MediaEntity';
import {DiskMangerWorker} from '../threading/DiskMangerWorker';
import {DiskManager} from '../fileaccess/DiskManager';
import {ObjectManagers} from '../ObjectManagers';
import {DatabaseType} from '../../../common/config/private/PrivateConfig';
import {SQLConnection} from './SQLConnection';
@ -37,7 +37,7 @@ export class CoverManager implements IObjectManager {
public async onNewDataVersion(changedDir: ParentDirectoryDTO): Promise<void> {
// Invalidating Album cover
let fullPath = DiskMangerWorker.normalizeDirPath(
let fullPath = DiskManager.normalizeDirPath(
path.join(changedDir.path, changedDir.name)
);
const query = (await SQLConnection.getConnection())
@ -46,10 +46,10 @@ export class CoverManager implements IObjectManager {
.set({validCover: false});
let i = 0;
const root = DiskMangerWorker.pathFromRelativeDirName('.');
const root = DiskManager.pathFromRelativeDirName('.');
while (fullPath !== root) {
const name = DiskMangerWorker.dirName(fullPath);
const parentPath = DiskMangerWorker.pathFromRelativeDirName(fullPath);
const name = DiskManager.dirName(fullPath);
const parentPath = DiskManager.pathFromRelativeDirName(fullPath);
fullPath = parentPath;
++i;
query.orWhere(
@ -67,8 +67,8 @@ export class CoverManager implements IObjectManager {
query.orWhere(
new Brackets((q: WhereExpression) => {
const param: { [key: string]: string } = {};
param['name' + i] = DiskMangerWorker.dirName('.');
param['path' + i] = DiskMangerWorker.pathFromRelativeDirName('.');
param['name' + i] = DiskManager.dirName('.');
param['path' + i] = DiskManager.pathFromRelativeDirName('.');
q.where(`path = :path${i}`, param);
q.andWhere(`name = :name${i}`, param);
})
@ -157,11 +157,11 @@ export class CoverManager implements IObjectManager {
});
if (Config.Database.type === DatabaseType.mysql) {
q.orWhere('directory.path like :path || \'%\'', {
path: DiskMangerWorker.pathFromParent(dir),
path: DiskManager.pathFromParent(dir),
});
} else {
q.orWhere('directory.path GLOB :path', {
path: DiskMangerWorker.pathFromParent(dir)
path: DiskManager.pathFromParent(dir)
// glob escaping. see https://github.com/bpatrik/pigallery2/issues/621
.replaceAll('[', '[[]') + '*',
});

6
src/backend/model/database/GalleryManager.ts
View File

@ -9,11 +9,11 @@ import {Config} from '../../../common/config/private/Config';
import {Connection} from 'typeorm';
import {MediaEntity} from './enitites/MediaEntity';
import {VideoEntity} from './enitites/VideoEntity';
import {DiskMangerWorker} from '../threading/DiskMangerWorker';
import {Logger} from '../../Logger';
import {ObjectManagers} from '../ObjectManagers';
import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO';
import {ReIndexingSensitivity} from '../../../common/config/private/PrivateConfig';
import { DiskManager } from '../fileaccess/DiskManager';
const LOG_TAG = '[GalleryManager]';
@ -22,7 +22,7 @@ export class GalleryManager {
name: string;
parent: string;
} {
relativeDirectoryName = DiskMangerWorker.normalizeDirPath(
relativeDirectoryName = DiskManager.normalizeDirPath(
relativeDirectoryName
);
return {
@ -57,7 +57,7 @@ export class GalleryManager {
const stat = fs.statSync(
path.join(ProjectPath.ImageFolder, relativeDirectoryName)
);
const lastModified = DiskMangerWorker.calcLastModified(stat);
const lastModified = DiskManager.calcLastModified(stat);
// If it seems that the content did not change, do not work on it
if (

7
src/backend/model/database/IndexingManager.ts
View File

@ -1,7 +1,6 @@
import {DirectoryBaseDTO, DirectoryDTOUtils, DirectoryPathDTO, ParentDirectoryDTO} from '../../../common/entities/DirectoryDTO';
import {DirectoryEntity} from './enitites/DirectoryEntity';
import {SQLConnection} from './SQLConnection';
import {DiskManager} from '../DiskManger';
import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity';
import {Utils} from '../../../common/Utils';
import {PhotoMetadata,} from '../../../common/entities/PhotoDTO';
@ -13,7 +12,6 @@ import {FileEntity} from './enitites/FileEntity';
import {FileDTO} from '../../../common/entities/FileDTO';
import {NotificationManager} from '../NotifocationManager';
import {ObjectManagers} from '../ObjectManagers';
import {DiskMangerWorker} from '../threading/DiskMangerWorker';
import {Logger} from '../../Logger';
import {ServerPG2ConfMap, ServerSidePG2ConfAction,} from '../../../common/PG2ConfMap';
import {ProjectPath} from '../../ProjectPath';
@ -24,6 +22,7 @@ import {PersonEntry} from './enitites/PersonEntry';
import {PersonJunctionTable} from './enitites/PersonJunctionTable';
import {MDFileEntity} from './enitites/MDFileEntity';
import {MDFileDTO} from '../../../common/entities/MDFileDTO';
import {DiskManager} from '../fileaccess/DiskManager';
const LOG_TAG = '[IndexingManager]';
@ -242,9 +241,9 @@ export class IndexingManager {
.update(DirectoryEntity)
.set({parent: currentDirId as unknown})
.where('path = :path', {
path: DiskMangerWorker.pathFromParent(scannedDirectory),
path: DiskManager.pathFromParent(scannedDirectory),
})
.andWhere('name NOT LIKE :root', {root: DiskMangerWorker.dirName('.')})
.andWhere('name NOT LIKE :root', {root: DiskManager.dirName('.')})
.andWhere('parent IS NULL')
.execute();

124
src/backend/model/threading/DiskMangerWorker.ts → src/backend/model/fileaccess/DiskManager.ts
View File

@ -6,15 +6,18 @@ import {ProjectPath} from '../../ProjectPath';
import {Config} from '../../../common/config/private/Config';
import {VideoDTO} from '../../../common/entities/VideoDTO';
import {FileDTO} from '../../../common/entities/FileDTO';
import {MetadataLoader} from './MetadataLoader';
import {Logger} from '../../Logger';
import {VideoProcessing} from '../fileprocessing/VideoProcessing';
import {PhotoProcessing} from '../fileprocessing/PhotoProcessing';
import {VideoProcessing} from './fileprocessing/VideoProcessing';
import {PhotoProcessing} from './fileprocessing/PhotoProcessing';
import {Utils} from '../../../common/Utils';
import {GPXProcessing} from '../fileprocessing/GPXProcessing';
import {GPXProcessing} from './fileprocessing/GPXProcessing';
import {MDFileDTO} from '../../../common/entities/MDFileDTO';
import {MetadataLoader} from './MetadataLoader';
export class DiskMangerWorker {
const LOG_TAG = '[DiskManager]';
export class DiskManager {
public static calcLastModified(stat: Stats): number {
return Math.max(stat.ctime.getTime(), stat.mtime.getTime());
}
@ -25,15 +28,15 @@ export class DiskMangerWorker {
public static pathFromRelativeDirName(relativeDirectoryName: string): string {
return path.join(
path.dirname(this.normalizeDirPath(relativeDirectoryName)),
path.sep
path.dirname(this.normalizeDirPath(relativeDirectoryName)),
path.sep
);
}
public static pathFromParent(parent: { path: string; name: string }): string {
return path.join(
this.normalizeDirPath(path.join(parent.path, parent.name)),
path.sep
this.normalizeDirPath(path.join(parent.path, parent.name)),
path.sep
);
}
@ -45,13 +48,13 @@ export class DiskMangerWorker {
}
public static async excludeDir(
name: string,
relativeDirectoryName: string,
absoluteDirectoryName: string
name: string,
relativeDirectoryName: string,
absoluteDirectoryName: string
): Promise<boolean> {
if (
Config.Indexing.excludeFolderList.length === 0 &&
Config.Indexing.excludeFileList.length === 0
Config.Indexing.excludeFolderList.length === 0 &&
Config.Indexing.excludeFileList.length === 0
) {
return false;
}
@ -87,30 +90,31 @@ export class DiskMangerWorker {
}
public static async scanDirectoryNoMetadata(
relativeDirectoryName: string,
settings: DirectoryScanSettings = {}
relativeDirectoryName: string,
settings: DirectoryScanSettings = {}
): Promise<ParentDirectoryDTO<FileDTO>> {
settings.noMetadata = true;
return (await this.scanDirectory(
relativeDirectoryName,
settings
relativeDirectoryName,
settings
)) as ParentDirectoryDTO<FileDTO>;
}
public static async scanDirectory(
relativeDirectoryName: string,
settings: DirectoryScanSettings = {}
relativeDirectoryName: string,
settings: DirectoryScanSettings = {}
): Promise<ParentDirectoryDTO> {
Logger.silly(LOG_TAG, 'scanning directory:', relativeDirectoryName);
relativeDirectoryName = this.normalizeDirPath(relativeDirectoryName);
const directoryName = DiskMangerWorker.dirName(relativeDirectoryName);
const directoryName = DiskManager.dirName(relativeDirectoryName);
const directoryParent = this.pathFromRelativeDirName(relativeDirectoryName);
const absoluteDirectoryName = path.join(
ProjectPath.ImageFolder,
relativeDirectoryName
ProjectPath.ImageFolder,
relativeDirectoryName
);
const stat = await fsp.stat(
path.join(ProjectPath.ImageFolder, relativeDirectoryName)
path.join(ProjectPath.ImageFolder, relativeDirectoryName)
);
const directory: ParentDirectoryDTO = {
id: null,
@ -132,36 +136,36 @@ export class DiskMangerWorker {
// nothing to scan, we are here for the empty dir
if (
settings.noPhoto === true &&
settings.noMetaFile === true &&
settings.noVideo === true
settings.noPhoto === true &&
settings.noMetaFile === true &&
settings.noVideo === true
) {
return directory;
}
const list = await fsp.readdir(absoluteDirectoryName);
for (const file of list) {
const fullFilePath = path.normalize(
path.join(absoluteDirectoryName, file)
path.join(absoluteDirectoryName, file)
);
if ((await fsp.stat(fullFilePath)).isDirectory()) {
if (
settings.noDirectory === true ||
settings.coverOnly === true ||
(await DiskMangerWorker.excludeDir(
file,
relativeDirectoryName,
absoluteDirectoryName
))
settings.noDirectory === true ||
settings.coverOnly === true ||
(await DiskManager.excludeDir(
file,
relativeDirectoryName,
absoluteDirectoryName
))
) {
continue;
}
// create cover directory
const d = (await DiskMangerWorker.scanDirectory(
path.join(relativeDirectoryName, file),
{
coverOnly: true,
}
const d = (await DiskManager.scanDirectory(
path.join(relativeDirectoryName, file),
{
coverOnly: true,
}
)) as SubDirectoryDTO;
directory.directories.push(d);
@ -174,9 +178,9 @@ export class DiskMangerWorker {
name: file,
directory: null,
metadata:
settings.noMetadata === true
? null
: await MetadataLoader.loadPhotoMetadata(fullFilePath),
settings.noMetadata === true
? null
: await MetadataLoader.loadPhotoMetadata(fullFilePath),
} as PhotoDTO;
if (!directory.cover) {
@ -197,9 +201,9 @@ export class DiskMangerWorker {
}
} else if (VideoProcessing.isVideo(fullFilePath)) {
if (
Config.Media.Video.enabled === false ||
settings.noVideo === true ||
settings.coverOnly === true
Config.Media.Video.enabled === false ||
settings.noVideo === true ||
settings.coverOnly === true
) {
continue;
}
@ -208,23 +212,23 @@ export class DiskMangerWorker {
name: file,
directory: null,
metadata:
settings.noMetadata === true
? null
: await MetadataLoader.loadVideoMetadata(fullFilePath),
settings.noMetadata === true
? null
: await MetadataLoader.loadVideoMetadata(fullFilePath),
} as VideoDTO);
} catch (e) {
Logger.warn(
'Media loading error, skipping: ' +
file +
', reason: ' +
e.toString()
'Media loading error, skipping: ' +
file +
', reason: ' +
e.toString()
);
}
} else if (GPXProcessing.isMetaFile(fullFilePath)) {
if (
!DiskMangerWorker.isEnabledMetaFile(fullFilePath) ||
settings.noMetaFile === true ||
settings.coverOnly === true
!DiskManager.isEnabledMetaFile(fullFilePath) ||
settings.noMetaFile === true ||
settings.coverOnly === true
) {
continue;
}
@ -242,13 +246,13 @@ export class DiskMangerWorker {
directory.oldestMedia = Number.MIN_SAFE_INTEGER;
directory.media.forEach((m) => {
directory.youngestMedia = Math.min(m.metadata.creationDate, directory.youngestMedia);
directory.oldestMedia = Math.max(m.metadata.creationDate, directory.oldestMedia);
}
directory.youngestMedia = Math.min(m.metadata.creationDate, directory.youngestMedia);
directory.oldestMedia = Math.max(m.metadata.creationDate, directory.oldestMedia);
}
);
directory.metaFile.forEach(mf => {
if (DiskMangerWorker.isMarkdown(mf.name)) {
if (DiskManager.isMarkdown(mf.name)) {
(mf as MDFileDTO).date = directory.youngestMedia;
}
});

0
src/backend/model/threading/MetadataLoader.ts → src/backend/model/fileaccess/MetadataLoader.ts
View File

0
src/backend/model/threading/PhotoWorker.ts → src/backend/model/fileaccess/PhotoWorker.ts
View File

0
src/backend/model/threading/TaskExecuter.ts → src/backend/model/fileaccess/TaskExecuter.ts
View File

0
src/backend/model/threading/TaskQue.ts → src/backend/model/fileaccess/TaskQue.ts
View File

0
src/backend/model/threading/VideoConverterWorker.ts → src/backend/model/fileaccess/VideoConverterWorker.ts
View File

6
src/backend/model/fileprocessing/GPXProcessing.ts → src/backend/model/fileaccess/fileprocessing/GPXProcessing.ts
View File

@ -1,9 +1,9 @@
import * as path from 'path';
import {constants as fsConstants, promises as fsp} from 'fs';
import * as xml2js from 'xml2js';
import {ProjectPath} from '../../ProjectPath';
import {Config} from '../../../common/config/private/Config';
import {SupportedFormats} from '../../../common/SupportedFormats';
import {ProjectPath} from '../../../ProjectPath';
import {Config} from '../../../../common/config/private/Config';
import {SupportedFormats} from '../../../../common/SupportedFormats';
type gpxEntry = { '$': { lat: string, lon: string }, ele?: string[], time?: string[], extensions?: unknown };

33
src/backend/model/fileprocessing/PhotoProcessing.ts → src/backend/model/fileaccess/fileprocessing/PhotoProcessing.ts
View File

@ -2,14 +2,14 @@ import * as path from 'path';
import {constants as fsConstants, promises as fsp} from 'fs';
import * as os from 'os';
import * as crypto from 'crypto';
import {ProjectPath} from '../../ProjectPath';
import {Config} from '../../../common/config/private/Config';
import {MediaRendererInput, PhotoWorker, SvgRendererInput, ThumbnailSourceType,} from '../threading/PhotoWorker';
import {ITaskExecuter, TaskExecuter} from '../threading/TaskExecuter';
import {FaceRegion, PhotoDTO} from '../../../common/entities/PhotoDTO';
import {SupportedFormats} from '../../../common/SupportedFormats';
import {PersonEntry} from '../database/enitites/PersonEntry';
import {SVGIconConfig} from '../../../common/config/public/ClientConfig';
import {ProjectPath} from '../../../ProjectPath';
import {Config} from '../../../../common/config/private/Config';
import {MediaRendererInput, PhotoWorker, SvgRendererInput, ThumbnailSourceType,} from '../PhotoWorker';
import {ITaskExecuter, TaskExecuter} from '../TaskExecuter';
import {FaceRegion, PhotoDTO} from '../../../../common/entities/PhotoDTO';
import {SupportedFormats} from '../../../../common/SupportedFormats';
import {PersonEntry} from '../../database/enitites/PersonEntry';
import {SVGIconConfig} from '../../../../common/config/public/ClientConfig';
export class PhotoProcessing {
private static initDone = false;
@ -21,19 +21,10 @@ export class PhotoProcessing {
return;
}
if (Config.Server.Threading.enabled === true) {
if (Config.Server.Threading.thumbnailThreads > 0) {
Config.Media.Thumbnail.concurrentThumbnailGenerations =
Config.Server.Threading.thumbnailThreads;
} else {
Config.Media.Thumbnail.concurrentThumbnailGenerations = Math.max(
1,
os.cpus().length - 1
);
}
} else {
Config.Media.Thumbnail.concurrentThumbnailGenerations = 1;
}
Config.Media.Thumbnail.concurrentThumbnailGenerations = Math.max(
1,
os.cpus().length - 1
);
this.taskQue = new TaskExecuter(
Config.Media.Thumbnail.concurrentThumbnailGenerations,

68
src/backend/model/fileprocessing/VideoProcessing.ts → src/backend/model/fileaccess/fileprocessing/VideoProcessing.ts
View File

@ -1,41 +1,41 @@
import * as path from 'path';
import {constants as fsConstants, promises as fsp} from 'fs';
import {ITaskExecuter, TaskExecuter} from '../threading/TaskExecuter';
import {VideoConverterInput, VideoConverterWorker,} from '../threading/VideoConverterWorker';
import {MetadataLoader} from '../threading/MetadataLoader';
import {Config} from '../../../common/config/private/Config';
import {ProjectPath} from '../../ProjectPath';
import {SupportedFormats} from '../../../common/SupportedFormats';
import {ITaskExecuter, TaskExecuter} from '../TaskExecuter';
import {VideoConverterInput, VideoConverterWorker,} from '../VideoConverterWorker';
import {MetadataLoader} from '../MetadataLoader';
import {Config} from '../../../../common/config/private/Config';
import {ProjectPath} from '../../../ProjectPath';
import {SupportedFormats} from '../../../../common/SupportedFormats';
export class VideoProcessing {
private static taskQue: ITaskExecuter<VideoConverterInput, void> =
new TaskExecuter(
1,
(input): Promise<void> => VideoConverterWorker.convert(input)
);
new TaskExecuter(
1,
(input): Promise<void> => VideoConverterWorker.convert(input)
);
public static generateConvertedFilePath(videoPath: string): string {
return path.join(
ProjectPath.TranscodedFolder,
ProjectPath.getRelativePathToImages(path.dirname(videoPath)),
path.basename(videoPath) + '_' + this.getConvertedFilePostFix()
ProjectPath.TranscodedFolder,
ProjectPath.getRelativePathToImages(path.dirname(videoPath)),
path.basename(videoPath) + '_' + this.getConvertedFilePostFix()
);
}
public static async isValidConvertedPath(
convertedPath: string
convertedPath: string
): Promise<boolean> {
const origFilePath = path.join(
ProjectPath.ImageFolder,
path.relative(
ProjectPath.TranscodedFolder,
convertedPath.substring(0, convertedPath.lastIndexOf('_'))
)
ProjectPath.ImageFolder,
path.relative(
ProjectPath.TranscodedFolder,
convertedPath.substring(0, convertedPath.lastIndexOf('_'))
)
);
const postfix = convertedPath.substring(
convertedPath.lastIndexOf('_') + 1,
convertedPath.length
convertedPath.lastIndexOf('_') + 1,
convertedPath.length
);
if (postfix !== this.getConvertedFilePostFix()) {
@ -82,8 +82,8 @@ export class VideoProcessing {
output: {
path: outPath,
codec: Config.Media.Video.transcoding.format === 'mp4' ?
Config.Media.Video.transcoding.mp4Codec :
Config.Media.Video.transcoding.webmCodec,
Config.Media.Video.transcoding.mp4Codec :
Config.Media.Video.transcoding.webmCodec,
format: Config.Media.Video.transcoding.format,
crf: Config.Media.Video.transcoding.crf,
preset: Config.Media.Video.transcoding.preset,
@ -93,17 +93,17 @@ export class VideoProcessing {
if (metaData.bitRate > Config.Media.Video.transcoding.bitRate) {
renderInput.output.bitRate =
Config.Media.Video.transcoding.bitRate;
Config.Media.Video.transcoding.bitRate;
}
if (metaData.fps > Config.Media.Video.transcoding.fps) {
renderInput.output.fps = Config.Media.Video.transcoding.fps;
}
if (
Config.Media.Video.transcoding.resolution < metaData.size.height
Config.Media.Video.transcoding.resolution < metaData.size.height
) {
renderInput.output.resolution =
Config.Media.Video.transcoding.resolution;
Config.Media.Video.transcoding.resolution;
}
const outDir = path.dirname(renderInput.output.path);
@ -119,14 +119,14 @@ export class VideoProcessing {
protected static getConvertedFilePostFix(): string {
return (
Math.round(Config.Media.Video.transcoding.bitRate / 1024) +
'k' +
(Config.Media.Video.transcoding.format === 'mp4' ?
Config.Media.Video.transcoding.mp4Codec :
Config.Media.Video.transcoding.webmCodec).toString().toLowerCase() +
Config.Media.Video.transcoding.resolution +
'.' +
Config.Media.Video.transcoding.format.toLowerCase()
Math.round(Config.Media.Video.transcoding.bitRate / 1024) +
'k' +
(Config.Media.Video.transcoding.format === 'mp4' ?
Config.Media.Video.transcoding.mp4Codec :
Config.Media.Video.transcoding.webmCodec).toString().toLowerCase() +
Config.Media.Video.transcoding.resolution +
'.' +
Config.Media.Video.transcoding.format.toLowerCase()
);
}
}

121
src/backend/model/jobs/jobs/FileJob.ts
View File

@ -1,8 +1,6 @@
import {ConfigTemplateEntry} from '../../../../common/entities/job/JobDTO';
import {Job} from './Job';
import * as path from 'path';
import {DiskManager} from '../../DiskManger';
import {DirectoryScanSettings} from '../../threading/DiskMangerWorker';
import {Logger} from '../../../Logger';
import {Config} from '../../../../common/config/private/Config';
import {FileDTO} from '../../../../common/entities/FileDTO';
@ -14,6 +12,7 @@ import {backendTexts} from '../../../../common/BackendTexts';
import {ProjectPath} from '../../../ProjectPath';
import {FileEntity} from '../../database/enitites/FileEntity';
import {DirectoryBaseDTO, DirectoryDTOUtils} from '../../../../common/entities/DirectoryDTO';
import {DirectoryScanSettings, DiskManager} from '../../fileaccess/DiskManager';
const LOG_TAG = '[FileJob]';
@ -69,10 +68,10 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
protected async step(): Promise<boolean> {
if (
this.fileQueue.length === 0 &&
((this.directoryQueue.length === 0 && !this.config.indexedOnly) ||
(this.config.indexedOnly &&
this.DBProcessing.hasMoreMedia === false))) {
this.fileQueue.length === 0 &&
((this.directoryQueue.length === 0 && !this.config.indexedOnly) ||
(this.config.indexedOnly &&
this.DBProcessing.hasMoreMedia === false))) {
return false;
}
@ -113,11 +112,11 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
} catch (e) {
console.error(e);
Logger.error(
LOG_TAG,
'Error during processing file:' + filePath + ', ' + e.toString()
LOG_TAG,
'Error during processing file:' + filePath + ', ' + e.toString()
);
this.Progress.log(
'Error during processing file:' + filePath + ', ' + e.toString()
'Error during processing file:' + filePath + ', ' + e.toString()
);
}
@ -128,8 +127,8 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
const directory = this.directoryQueue.shift();
this.Progress.log('scanning directory: ' + directory);
const scanned = await DiskManager.scanDirectoryNoMetadata(
directory,
this.scanFilter
directory,
this.scanFilter
);
for (const item of scanned.directories) {
this.directoryQueue.push(path.join(item.path, item.name));
@ -144,12 +143,12 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
}
for (const item of scannedAndFiltered) {
this.fileQueue.push(
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
);
}
}
@ -162,12 +161,12 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
}
for (const item of scannedAndFiltered) {
this.fileQueue.push(
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
);
}
}
@ -175,8 +174,8 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
private async loadMediaFilesFromDB(): Promise<void> {
if (this.scanFilter.noVideo === true &&
this.scanFilter.noPhoto === true &&
this.scanFilter.noMetaFile === true) {
this.scanFilter.noPhoto === true &&
this.scanFilter.noMetaFile === true) {
return;
}
@ -190,7 +189,7 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
};
const connection = await SQLConnection.getConnection();
if (!this.scanFilter.noVideo ||
!this.scanFilter.noPhoto) {
!this.scanFilter.noPhoto) {
let usedEntity = MediaEntity;
@ -201,13 +200,13 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
}
const result = await connection
.getRepository(usedEntity)
.createQueryBuilder('media')
.select(['media.name', 'directory.name', 'directory.path'])
.leftJoin('media.directory', 'directory')
.offset(this.DBProcessing.mediaLoaded)
.limit(Config.Jobs.mediaProcessingBatchSize)
.getMany();
.getRepository(usedEntity)
.createQueryBuilder('media')
.select(['media.name', 'directory.name', 'directory.path'])
.leftJoin('media.directory', 'directory')
.offset(this.DBProcessing.mediaLoaded)
.limit(Config.Jobs.mediaProcessingBatchSize)
.getMany();
hasMoreFile.media = result.length > 0;
this.DBProcessing.mediaLoaded += result.length;
@ -219,25 +218,25 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
}
for (const item of scannedAndFiltered) {
this.fileQueue.push(
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
);
}
}
if (!this.scanFilter.noMetaFile) {
const result = await connection
.getRepository(FileEntity)
.createQueryBuilder('file')
.select(['file.name', 'directory.name', 'directory.path'])
.leftJoin('file.directory', 'directory')
.offset(this.DBProcessing.mediaLoaded)
.limit(Config.Jobs.mediaProcessingBatchSize)
.getMany();
.getRepository(FileEntity)
.createQueryBuilder('file')
.select(['file.name', 'directory.name', 'directory.path'])
.leftJoin('file.directory', 'directory')
.offset(this.DBProcessing.mediaLoaded)
.limit(Config.Jobs.mediaProcessingBatchSize)
.getMany();
hasMoreFile.metafile = result.length > 0;
@ -250,12 +249,12 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
}
for (const item of scannedAndFiltered) {
this.fileQueue.push(
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
);
}
}
@ -264,14 +263,14 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
private async countMediaFromDB(): Promise<number> {
if (this.scanFilter.noVideo === true &&
this.scanFilter.noPhoto === true &&
this.scanFilter.noMetaFile === true) {
this.scanFilter.noPhoto === true &&
this.scanFilter.noMetaFile === true) {
return;
}
let count = 0;
const connection = await SQLConnection.getConnection();
if (!this.scanFilter.noVideo ||
!this.scanFilter.noPhoto) {
!this.scanFilter.noPhoto) {
let usedEntity = MediaEntity;
@ -282,16 +281,16 @@ export abstract class FileJob<S extends { indexedOnly?: boolean } = { indexedOnl
}
count += await connection
.getRepository(usedEntity)
.createQueryBuilder('media')
.getCount();
.getRepository(usedEntity)
.createQueryBuilder('media')
.getCount();
}
if (!this.scanFilter.noMetaFile) {
count += await connection
.getRepository(FileEntity)
.createQueryBuilder('file')
.getCount();
.getRepository(FileEntity)
.createQueryBuilder('file')
.getCount();
}
return count;

2
src/backend/model/jobs/jobs/GPXCompressionJob.ts
View File

@ -1,9 +1,9 @@
import {Config} from '../../../../common/config/private/Config';
import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
import {FileJob} from './FileJob';
import {GPXProcessing} from '../../fileprocessing/GPXProcessing';
import {FileDTO} from '../../../../common/entities/FileDTO';
import {Logger} from '../../../Logger';
import {GPXProcessing} from '../../fileaccess/fileprocessing/GPXProcessing';
export class GPXCompressionJob extends FileJob {
public readonly Name = DefaultsJobs[DefaultsJobs['GPX Compression']];

2
src/backend/model/jobs/jobs/GPXCompressionResetJob.ts
View File

@ -1,6 +1,6 @@
import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
import {TempFolderCleaningJob} from './TempFolderCleaningJob';
import {GPXProcessing} from '../../fileprocessing/GPXProcessing';
import {GPXProcessing} from '../../fileaccess/fileprocessing/GPXProcessing';
/**
* Deletes all gpx file from the tmp folder

4
src/backend/model/jobs/jobs/IndexingJob.ts
View File

@ -4,12 +4,12 @@ import * as fs from 'fs';
import {Job} from './Job';
import {ConfigTemplateEntry, DefaultsJobs,} from '../../../../common/entities/job/JobDTO';
import {JobProgressStates} from '../../../../common/entities/job/JobProgressDTO';
import {DiskMangerWorker} from '../../threading/DiskMangerWorker';
import {ProjectPath} from '../../../ProjectPath';
import {backendTexts} from '../../../../common/BackendTexts';
import {ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO';
import {Logger} from '../../../Logger';
import {FileDTO} from '../../../../common/entities/FileDTO';
import {DiskManager} from '../../fileaccess/DiskManager';
const LOG_TAG = '[IndexingJob]';
@ -62,7 +62,7 @@ export class IndexingJob<
if (this.config.indexChangesOnly) {
const stat = fs.statSync(absDirPath);
const lastModified = DiskMangerWorker.calcLastModified(stat);
const lastModified = DiskManager.calcLastModified(stat);
scanned = await ObjectManagers.getInstance().GalleryManager.selectDirStructure(directory);
// If not modified and it was scanned before, dir is up-to-date
if (

6
src/backend/model/jobs/jobs/PhotoConvertingJob.ts
View File

@ -1,7 +1,7 @@
import {Config} from '../../../../common/config/private/Config';
import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
import {FileJob} from './FileJob';
import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
import {PhotoProcessing} from '../../fileaccess/fileprocessing/PhotoProcessing';
export class PhotoConvertingJob extends FileJob {
public readonly Name = DefaultsJobs[DefaultsJobs['Photo Converting']];
@ -16,8 +16,8 @@ export class PhotoConvertingJob extends FileJob {
protected async shouldProcess(mPath: string): Promise<boolean> {
return !(await PhotoProcessing.convertedPhotoExist(
mPath,
Config.Media.Photo.Converting.resolution
mPath,
Config.Media.Photo.Converting.resolution
));
}

14
src/backend/model/jobs/jobs/TempFolderCleaningJob.ts
View File

@ -3,9 +3,9 @@ import * as path from 'path';
import * as fs from 'fs';
import {Job} from './Job';
import {ProjectPath} from '../../../ProjectPath';
import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
import {VideoProcessing} from '../../fileprocessing/VideoProcessing';
import {GPXProcessing} from '../../fileprocessing/GPXProcessing';
import {GPXProcessing} from '../../fileaccess/fileprocessing/GPXProcessing';
import {PhotoProcessing} from '../../fileaccess/fileprocessing/PhotoProcessing';
import {VideoProcessing} from '../../fileaccess/fileprocessing/VideoProcessing';
export class TempFolderCleaningJob extends Job {
public readonly Name = DefaultsJobs[DefaultsJobs['Temp Folder Cleaning']];
@ -38,8 +38,8 @@ export class TempFolderCleaningJob extends Job {
protected async isValidDirectory(filePath: string): Promise<boolean> {
const originalPath = path.join(
ProjectPath.ImageFolder,
path.relative(ProjectPath.TranscodedFolder, filePath)
ProjectPath.ImageFolder,
path.relative(ProjectPath.TranscodedFolder, filePath)
);
try {
await fs.promises.access(originalPath);
@ -52,7 +52,7 @@ export class TempFolderCleaningJob extends Job {
protected async readDir(dirPath: string): Promise<string[]> {
return (await fs.promises.readdir(dirPath)).map((f) =>
path.normalize(path.join(dirPath, f))
path.normalize(path.join(dirPath, f))
);
}
@ -91,7 +91,7 @@ export class TempFolderCleaningJob extends Job {
this.Progress.log('skipping: ' + filePath);
this.Progress.Skipped++;
this.directoryQueue = this.directoryQueue.concat(
await this.readDir(filePath)
await this.readDir(filePath)
);
}
} else {

4
src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts
View File

@ -1,8 +1,8 @@
import {Config} from '../../../../common/config/private/Config';
import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
import {FileJob} from './FileJob';
import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
import {ThumbnailSourceType} from '../../threading/PhotoWorker';
import {PhotoProcessing} from '../../fileaccess/fileprocessing/PhotoProcessing';
import {ThumbnailSourceType} from '../../fileaccess/PhotoWorker';
import {MediaDTOUtils} from '../../../../common/entities/MediaDTO';
import {FileDTO} from '../../../../common/entities/FileDTO';
import {backendTexts} from '../../../../common/BackendTexts';

2
src/backend/model/jobs/jobs/VideoConvertingJob.ts
View File

@ -1,7 +1,7 @@
import {Config} from '../../../../common/config/private/Config';
import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
import {FileJob} from './FileJob';
import {VideoProcessing} from '../../fileprocessing/VideoProcessing';
import {VideoProcessing} from '../../fileaccess/fileprocessing/VideoProcessing';
declare const global: any;

36
src/backend/model/mediamessengers/EmailMediaMessenger.ts
View File

@ -1,8 +1,8 @@
import {createTransport, Transporter} from 'nodemailer';
import {MediaDTO, MediaDTOUtils} from '../../../common/entities/MediaDTO';
import {Config} from '../../../common/config/private/Config';
import {PhotoProcessing} from '../fileprocessing/PhotoProcessing';
import {ThumbnailSourceType} from '../threading/PhotoWorker';
import {PhotoProcessing} from '../fileaccess/fileprocessing/PhotoProcessing';
import {ThumbnailSourceType} from '../fileaccess/PhotoWorker';
import {ProjectPath} from '../../ProjectPath';
import * as path from 'path';
import {PhotoMetadata} from '../../../common/entities/PhotoDTO';
@ -27,10 +27,10 @@ export class EmailMediaMessenger {
private async getThumbnail(m: MediaDTO) {
return await PhotoProcessing.generateThumbnail(
path.join(ProjectPath.ImageFolder, m.directory.path, m.directory.name, m.name),
Config.Media.Thumbnail.thumbnailSizes[0],
MediaDTOUtils.isPhoto(m) ? ThumbnailSourceType.Photo : ThumbnailSourceType.Video,
false
path.join(ProjectPath.ImageFolder, m.directory.path, m.directory.name, m.name),
Config.Media.Thumbnail.thumbnailSizes[0],
MediaDTOUtils.isPhoto(m) ? ThumbnailSourceType.Photo : ThumbnailSourceType.Video,
false
);
}
@ -42,22 +42,22 @@ export class EmailMediaMessenger {
const attachments = [];
const htmlStart = '<h1 style="text-align: center; margin-bottom: 2em">' + Config.Server.applicationTitle + '</h1>\n' +
'<h3>' + mailSettings.text + '</h3>\n' +
'<table style="margin-left: auto; margin-right: auto;">\n' +
' <tbody>\n';
'<h3>' + mailSettings.text + '</h3>\n' +
'<table style="margin-left: auto; margin-right: auto;">\n' +
' <tbody>\n';
const htmlEnd = ' </tr>\n' +
' </tbody>\n' +
'</table>';
' </tbody>\n' +
'</table>';
let htmlMiddle = '';
const numberOfColumns = media.length >= 6 ? 3 : 2;
for (let i = 0; i < media.length; ++i) {
const thPath = await this.getThumbnail(media[i]);
const linkUrl = Utils.concatUrls(Config.Server.publicUrl, '/gallery/', encodeURIComponent(path.join(media[i].directory.path, media[i].directory.name))) +
'?' + QueryParams.gallery.photo + '=' + encodeURIComponent(media[i].name);
'?' + QueryParams.gallery.photo + '=' + encodeURIComponent(media[i].name);
const location = (media[i].metadata as PhotoMetadata).positionData?.country ?
(media[i].metadata as PhotoMetadata).positionData?.country :
((media[i].metadata as PhotoMetadata).positionData?.city ?
(media[i].metadata as PhotoMetadata).positionData?.city : '');
(media[i].metadata as PhotoMetadata).positionData?.country :
((media[i].metadata as PhotoMetadata).positionData?.city ?
(media[i].metadata as PhotoMetadata).positionData?.city : '');
const caption = (new Date(media[i].metadata.creationDate)).getFullYear() + (location ? ', ' + location : '');
attachments.push({
filename: media[i].name,
@ -68,9 +68,9 @@ export class EmailMediaMessenger {
htmlMiddle += '<tr>';
}
htmlMiddle += '<td>\n' +
' <a style="display: block;text-align: center;" href="' + linkUrl + '"><img alt="' + media[i].name + '" style="max-width: 200px; max-height: 150px; height:auto; width:auto;" src="cid:img' + i + '"/></a>\n' +
caption +
' </td>\n';
' <a style="display: block;text-align: center;" href="' + linkUrl + '"><img alt="' + media[i].name + '" style="max-width: 200px; max-height: 150px; height:auto; width:auto;" src="cid:img' + i + '"/></a>\n' +
caption +
' </td>\n';
if (i % numberOfColumns == (numberOfColumns - 1) || i === media.length - 1) {
htmlMiddle += '</tr>';

119
src/backend/model/threading/ThreadPool.ts
View File

@ -1,119 +0,0 @@
import * as cluster from 'cluster';
import {Worker} from 'cluster';
import {Logger} from '../../Logger';
import {DiskManagerTask, WorkerMessage, WorkerTask, WorkerTaskTypes,} from './Worker';
import {ParentDirectoryDTO} from '../../../common/entities/DirectoryDTO';
import {TaskQue, TaskQueEntry} from './TaskQue';
import {ITaskExecuter} from './TaskExecuter';
import {DirectoryScanSettings} from './DiskMangerWorker';
interface WorkerWrapper<O> {
worker: Worker;
poolTask: TaskQueEntry<WorkerTask, O>;
}
const LOG_TAG = '[ThreadPool]';
export class ThreadPool<O> {
public static WorkerCount = 0;
private workers: WorkerWrapper<O>[] = [];
private taskQue = new TaskQue<WorkerTask, O>();
constructor(private size: number) {
Logger.silly(LOG_TAG, 'Creating thread pool with', size, 'workers');
for (let i = 0; i < size; i++) {
this.startWorker();
}
}
protected executeTask(task: WorkerTask): Promise<O> {
const promise = this.taskQue.add(task).promise.obj;
this.run();
return promise;
}
private run = (): void => {
if (this.taskQue.isEmpty()) {
return;
}
const worker = this.getFreeWorker();
if (worker == null) {
return;
}
const poolTask = this.taskQue.get();
worker.poolTask = poolTask;
worker.worker.send(poolTask.data);
};
private getFreeWorker(): null | WorkerWrapper<O> {
for (const worker of this.workers) {
if (worker.poolTask == null) {
return worker;
}
}
return null;
}
private startWorker(): void {
const worker = {
poolTask: null,
worker: (cluster as any).fork(),
} as WorkerWrapper<O>;
this.workers.push(worker);
worker.worker.on('online', (): void => {
ThreadPool.WorkerCount++;
Logger.debug(
LOG_TAG,
'Worker ' + worker.worker.process.pid + ' is online, worker count:',
ThreadPool.WorkerCount
);
});
worker.worker.on('exit', (code, signal): void => {
ThreadPool.WorkerCount--;
Logger.warn(
LOG_TAG,
'Worker ' +
worker.worker.process.pid +
' died with code: ' +
code +
', and signal: ' +
signal +
', worker count:',
ThreadPool.WorkerCount
);
Logger.debug(LOG_TAG, 'Starting a new worker');
this.startWorker();
});
worker.worker.on('message', (msg: WorkerMessage<O>): void => {
if (worker.poolTask == null) {
throw new Error('No worker task after worker task is completed');
}
if (msg.error) {
worker.poolTask.promise.reject(msg.error);
} else {
worker.poolTask.promise.resolve(msg.result);
}
this.taskQue.ready(worker.poolTask);
worker.poolTask = null;
this.run();
});
}
}
export class DiskManagerTH
extends ThreadPool<ParentDirectoryDTO>
implements ITaskExecuter<string, ParentDirectoryDTO> {
execute(
relativeDirectoryName: string,
settings: DirectoryScanSettings = {}
): Promise<ParentDirectoryDTO> {
return super.executeTask({
type: WorkerTaskTypes.diskManager,
relativeDirectoryName,
settings,
} as DiskManagerTask);
}
}

71
src/backend/model/threading/Worker.ts
View File

@ -1,71 +0,0 @@
import {DirectoryScanSettings, DiskMangerWorker} from './DiskMangerWorker';
import {Logger} from '../../Logger';
import {MediaRendererInput, PhotoWorker} from './PhotoWorker';
import {Utils} from '../../../common/Utils';
import {MediaDTO} from '../../../common/entities/MediaDTO';
import {ParentDirectoryDTO} from '../../../common/entities/DirectoryDTO';
declare const process: NodeJS.Process;
const LOG_TAG = '[Worker]';
export class Worker {
public static process<O extends void | ParentDirectoryDTO<MediaDTO>>(): void {
Logger.debug(LOG_TAG, 'Worker is waiting for tasks');
process.on('message', async (task: WorkerTask) => {
try {
let result = null;
switch (task.type) {
case WorkerTaskTypes.diskManager:
result = await DiskMangerWorker.scanDirectory(
(task as DiskManagerTask).relativeDirectoryName,
(task as DiskManagerTask).settings
);
if (global.gc) {
global.gc();
}
break;
case WorkerTaskTypes.thumbnail:
result = await PhotoWorker.render((task as ThumbnailTask).input);
break;
default:
throw new Error('Unknown worker task type');
}
process.send({
error: null,
result,
} as WorkerMessage<O>);
} catch (err) {
process.send({error: err, result: null});
}
});
}
}
export enum WorkerTaskTypes {
thumbnail = 1,
diskManager = 2,
}
export interface WorkerTask {
type: WorkerTaskTypes;
}
export interface DiskManagerTask extends WorkerTask {
relativeDirectoryName: string;
settings: DirectoryScanSettings;
}
export interface ThumbnailTask extends WorkerTask {
input: MediaRendererInput;
}
export const WorkerTask = {
equals: (t1: WorkerTask, t2: WorkerTask): boolean => {
return Utils.equalsFilter(t1, t2);
},
};
export interface WorkerMessage<O> {
error: Error;
result: O;
}

2
src/backend/routes/GalleryRouter.ts
View File

@ -4,7 +4,7 @@ import {GalleryMWs} from '../middlewares/GalleryMWs';
import {RenderingMWs} from '../middlewares/RenderingMWs';
import {ThumbnailGeneratorMWs} from '../middlewares/thumbnail/ThumbnailGeneratorMWs';
import {UserRoles} from '../../common/entities/UserDTO';
import {ThumbnailSourceType} from '../model/threading/PhotoWorker';
import {ThumbnailSourceType} from '../model/fileaccess/PhotoWorker';
import {VersionMWs} from '../middlewares/VersionMWs';
import {SupportedFormats} from '../../common/SupportedFormats';
import {PhotoConverterMWs} from '../middlewares/thumbnail/PhotoConverterMWs';

2
src/backend/routes/PublicRouter.ts
View File

@ -11,7 +11,7 @@ import {UserDTO} from '../../common/entities/UserDTO';
import {ServerTimeEntry} from '../middlewares/ServerTimingMWs';
import {ClientConfig, TAGS} from '../../common/config/public/ClientConfig';
import {QueryParams} from '../../common/QueryParams';
import {PhotoProcessing} from '../model/fileprocessing/PhotoProcessing';
import {PhotoProcessing} from '../model/fileaccess/fileprocessing/PhotoProcessing';
declare global {
// eslint-disable-next-line @typescript-eslint/no-namespace

60
src/backend/server.ts
View File

@ -10,12 +10,11 @@ import * as locale from 'locale';
import {ObjectManagers} from './model/ObjectManagers';
import {Logger} from './Logger';
import {LoggerRouter} from './routes/LoggerRouter';
import {DiskManager} from './model/DiskManger';
import {ConfigDiagnostics} from './model/diagnostics/ConfigDiagnostics';
import {Localizations} from './model/Localizations';
import {CookieNames} from '../common/CookieNames';
import {Router} from './routes/Router';
import {PhotoProcessing} from './model/fileprocessing/PhotoProcessing';
import {PhotoProcessing} from './model/fileaccess/fileprocessing/PhotoProcessing';
import * as _csrf from 'csurf';
import {Event} from '../common/event/Event';
import {QueryParams} from '../common/QueryParams';
@ -39,8 +38,8 @@ export class Server {
constructor() {
if (!(process.env.NODE_ENV === 'production')) {
Logger.info(
LOG_TAG,
'Running in DEBUG mode, set env variable NODE_ENV=production to disable '
LOG_TAG,
'Running in DEBUG mode, set env variable NODE_ENV=production to disable '
);
}
this.init().catch(console.error);
@ -54,13 +53,13 @@ export class Server {
Logger.info(LOG_TAG, 'running diagnostics...');
await ConfigDiagnostics.runDiagnostics();
Logger.verbose(
LOG_TAG,
'using config from ' +
(
ConfigClassBuilder.attachPrivateInterface(Config)
.__options as ConfigClassOptions<ServerConfig>
).configPath +
':'
LOG_TAG,
'using config from ' +
(
ConfigClassBuilder.attachPrivateInterface(Config)
.__options as ConfigClassOptions<ServerConfig>
).configPath +
':'
);
Logger.verbose(LOG_TAG, JSON.stringify(Config.toJSON({attachDescription: false}), null, '\t'));
@ -75,10 +74,10 @@ export class Server {
*/
this.app.use(
session({
name: CookieNames.session,
keys: Config.Server.sessionSecret,
})
session({
name: CookieNames.session,
keys: Config.Server.sessionSecret,
})
);
/**
@ -90,29 +89,28 @@ export class Server {
const csuf: any = _csrf();
csuf.unless = unless;
this.app.use(
csuf.unless((req: Request) => {
return (
Config.Users.authenticationRequired === false ||
[Config.Server.apiPath + '/user/login', Config.Server.apiPath + '/user/logout', Config.Server.apiPath + '/share/login'].indexOf(
req.originalUrl
) !== -1 ||
(Config.Sharing.enabled === true &&
!!req.query[QueryParams.gallery.sharingKey_query])
);
})
csuf.unless((req: Request) => {
return (
Config.Users.authenticationRequired === false ||
[Config.Server.apiPath + '/user/login', Config.Server.apiPath + '/user/logout', Config.Server.apiPath + '/share/login'].indexOf(
req.originalUrl
) !== -1 ||
(Config.Sharing.enabled === true &&
!!req.query[QueryParams.gallery.sharingKey_query])
);
})
);
// enable token generation but do not check it
this.app.post(
[Config.Server.apiPath + '/user/login', Config.Server.apiPath + '/share/login'],
_csrf({ignoreMethods: ['POST']})
[Config.Server.apiPath + '/user/login', Config.Server.apiPath + '/share/login'],
_csrf({ignoreMethods: ['POST']})
);
this.app.get(
[Config.Server.apiPath + '/user/me', Config.Server.apiPath + '/share/:' + QueryParams.gallery.sharingKey_params],
_csrf({ignoreMethods: ['GET']})
[Config.Server.apiPath + '/user/me', Config.Server.apiPath + '/share/:' + QueryParams.gallery.sharingKey_params],
_csrf({ignoreMethods: ['GET']})
);
DiskManager.init();
PhotoProcessing.init();
Localizations.init();
@ -176,7 +174,7 @@ export class Server {
private onListening = () => {
const addr = this.server.address();
const bind =
typeof addr === 'string' ? 'pipe ' + addr : 'port ' + addr.port;
typeof addr === 'string' ? 'pipe ' + addr : 'port ' + addr.port;
Logger.info(LOG_TAG, 'Listening on ' + bind);
};

32
src/common/config/private/PrivateConfig.ts
View File

@ -477,30 +477,6 @@ export class ServerIndexingConfig {
excludeFileList: string[] = [];
}
@SubConfigClass({softReadonly: true})
export class ServerThreadingConfig {
@ConfigProperty({
tags:
{
name: $localize`Threading`,
uiResetNeeded: {server: true},
priority: ConfigPriority.underTheHood,
} as TAGS,
description: $localize`[Deprecated, will be removed in the next release] Runs directory scanning and thumbnail generation in a different thread.`
})
enabled: boolean = false;
@ConfigProperty({
tags:
{
name: $localize`Thumbnail threads`,
uiResetNeeded: {server: true},
priority: ConfigPriority.underTheHood
},
description: $localize`Number of threads that are used to generate thumbnails. If 0, number of 'CPU cores -1' threads will be used.`,
})
thumbnailThreads: number = 0; // if zero-> CPU count -1
}
@SubConfigClass({softReadonly: true})
export class ServerDuplicatesConfig {
@ConfigProperty({
@ -1028,14 +1004,6 @@ export class ServerServiceConfig extends ClientServiceConfig {
})
host: string = '0.0.0.0';
@ConfigProperty({
tags: {
name: $localize`Threading`,
priority: ConfigPriority.underTheHood,
}
})
Threading: ServerThreadingConfig = new ServerThreadingConfig();
@ConfigProperty({
tags: {
name: $localize`Logs`,

12
test/TestHelper.ts
View File

@ -19,7 +19,7 @@ import {
} from '../src/common/entities/PhotoDTO';
import {DirectoryBaseDTO, DirectoryPathDTO} from '../src/common/entities/DirectoryDTO';
import {FileDTO} from '../src/common/entities/FileDTO';
import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker';
import {DiskManager} from '../src/backend/model/fileaccess/DiskManager';
export class TestHelper {
@ -29,7 +29,7 @@ export class TestHelper {
const dir = new DirectoryEntity();
dir.name = name;
dir.path = DiskMangerWorker.pathFromParent({path: '', name: '.'});
dir.path = DiskManager.pathFromParent({path: '', name: '.'});
dir.mediaCount = 0;
dir.youngestMedia = 10;
dir.oldestMedia = 1000;
@ -40,7 +40,7 @@ export class TestHelper {
dir.lastScanned = 1656069687773;
// dir.parent = null;
if (parent !== null) {
dir.path = DiskMangerWorker.pathFromParent(parent);
dir.path = DiskManager.pathFromParent(parent);
parent.directories.push(dir);
}
return dir;
@ -299,8 +299,8 @@ export class TestHelper {
const dir: DirectoryBaseDTO = {
id: null,
name: DiskMangerWorker.dirName(forceStr || Math.random().toString(36).substring(7)),
path: DiskMangerWorker.pathFromParent({path: '', name: '.'}),
name: DiskManager.dirName(forceStr || Math.random().toString(36).substring(7)),
path: DiskManager.pathFromParent({path: '', name: '.'}),
mediaCount: 0,
youngestMedia: 10,
oldestMedia: 1000,
@ -314,7 +314,7 @@ export class TestHelper {
parent
};
if (parent !== null) {
dir.path = DiskMangerWorker.pathFromParent(parent);
dir.path = DiskManager.pathFromParent(parent);
parent.directories.push(dir);
}
return dir;

4
test/backend/DBTestHelper.ts
View File

@ -6,7 +6,7 @@ import {DatabaseType, LogLevel} from '../../src/common/config/private/PrivateCon
import {ProjectPath} from '../../src/backend/ProjectPath';
import {DirectoryBaseDTO, ParentDirectoryDTO, SubDirectoryDTO} from '../../src/common/entities/DirectoryDTO';
import {ObjectManagers} from '../../src/backend/model/ObjectManagers';
import {DiskMangerWorker} from '../../src/backend/model/threading/DiskMangerWorker';
import {DiskManager} from '../../src/backend/model/fileaccess/DiskManager';
import {IndexingManager} from '../../src/backend/model/database/IndexingManager';
import {GalleryManager} from '../../src/backend/model/database/GalleryManager';
import {Connection} from 'typeorm';
@ -135,7 +135,7 @@ export class DBTestHelper {
for (let i = 0; i < d.directories.length; i++) {
d.directories[i] = await gm.getParentDirFromId(connection,
(await gm.getDirIdAndTime(connection, d.directories[i].name,
path.join(DiskMangerWorker.pathFromParent(d), path.sep))).id);
path.join(DiskManager.pathFromParent(d), path.sep))).id);
await populateDir(d.directories[i]);
}
};

1
test/backend/integration/routers/GalleryRouter.ts
View File

@ -30,7 +30,6 @@ describe('GalleryRouter', (sqlHelper: DBTestHelper) => {
const setUp = async () => {
await sqlHelper.initDB();
Config.Users.authenticationRequired = false;
Config.Server.Threading.enabled = false;
Config.Media.Video.enabled = true;
Config.Media.folder = path.join(__dirname, '../../assets');
Config.Media.tempFolder = tempDir;

1
test/backend/integration/routers/PublicRouter.ts
View File

@ -34,7 +34,6 @@ describe('PublicRouter', () => {
const setUp = async () => {
await fs.promises.rm(tempDir, {recursive: true, force: true});
Config.Users.authenticationRequired = true;
Config.Server.Threading.enabled = false;
Config.Sharing.enabled = true;
Config.Database.type = DatabaseType.sqlite;
Config.Database.dbFolder = tempDir;

1
test/backend/integration/routers/SharingRouter.ts
View File

@ -35,7 +35,6 @@ describe('SharingRouter', () => {
const setUp = async () => {
await fs.promises.rm(tempDir, {recursive: true, force: true});
Config.Users.authenticationRequired = true;
Config.Server.Threading.enabled = false;
Config.Sharing.enabled = true;
Config.Database.type = DatabaseType.sqlite;
Config.Database.dbFolder = tempDir;

1
test/backend/integration/routers/UserRouter.ts
View File

@ -35,7 +35,6 @@ describe('UserRouter', () => {
let server: Server;
const setUp = async () => {
await fs.promises.rm(tempDir, {recursive: true, force: true});
Config.Server.Threading.enabled = false;
Config.Database.type = DatabaseType.sqlite;
Config.Database.dbFolder = tempDir;
ProjectPath.reset();

1
test/backend/integration/routers/admin/SettingsRouter.ts
View File

@ -19,7 +19,6 @@ describe('SettingsRouter', () => {
const tempDir = path.join(__dirname, '../../tmp');
beforeEach(async () => {
await fs.promises.rm(tempDir, {recursive: true, force: true});
Config.Server.Threading.enabled = false;
Config.Database.type = DatabaseType.sqlite;
Config.Database.dbFolder = tempDir;
ProjectPath.reset();

2
test/backend/unit/model/fileprocessing/PhotoProcessing.spec.ts
View File

@ -2,7 +2,7 @@ import {expect} from 'chai';
import {Config} from '../../../../../src/common/config/private/Config';
import {ProjectPath} from '../../../../../src/backend/ProjectPath';
import * as path from 'path';
import {PhotoProcessing} from '../../../../../src/backend/model/fileprocessing/PhotoProcessing';
import {PhotoProcessing} from '../../../../../src/backend/model/fileaccess/fileprocessing/PhotoProcessing';
describe('PhotoProcessing', () => {

2
test/backend/unit/model/fileprocessing/VideoProcessing.spec.ts
View File

@ -1,5 +1,5 @@
import {expect} from 'chai';
import {VideoProcessing} from '../../../../../src/backend/model/fileprocessing/VideoProcessing';
import {VideoProcessing} from '../../../../../src/backend/model/fileaccess/fileprocessing/VideoProcessing';
import {Config} from '../../../../../src/common/config/private/Config';
import {ProjectPath} from '../../../../../src/backend/ProjectPath';
import * as path from 'path';

10
test/backend/unit/model/sql/IndexingManager.spec.ts
View File

@ -11,15 +11,14 @@ import {FileDTO} from '../../../../../src/common/entities/FileDTO';
import {IndexingManager} from '../../../../../src/backend/model/database/IndexingManager';
import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers';
import {DBTestHelper} from '../../../DBTestHelper';
import {DiskMangerWorker} from '../../../../../src/backend/model/threading/DiskMangerWorker';
import {ReIndexingSensitivity} from '../../../../../src/common/config/private/PrivateConfig';
import {SearchQueryTypes, TextSearch, TextSearchQueryMatchTypes} from '../../../../../src/common/entities/SearchQueryDTO';
import {ProjectPath} from '../../../../../src/backend/ProjectPath';
import * as path from 'path';
import {DiskManager} from '../../../../../src/backend/model/DiskManger';
import {AlbumManager} from '../../../../../src/backend/model/database/AlbumManager';
import {SortByTypes} from '../../../../../src/common/entities/SortingMethods';
import {ClientSortingConfig} from '../../../../../src/common/config/public/ClientConfig';
import { DiskManager } from '../../../../../src/backend/model/fileaccess/DiskManager';
// eslint-disable-next-line @typescript-eslint/no-var-requires
const deepEqualInAnyOrder = require('deep-equal-in-any-order');
@ -162,7 +161,6 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
ProjectPath.reset();
ProjectPath.ImageFolder = path.join(__dirname, '/../../../assets');
Config.Server.Threading.enabled = false;
await ObjectManagers.getInstance().IndexingManager.indexDirectory('.');
if (ObjectManagers.getInstance().IndexingManager.IsSavingInProgress) {
@ -353,7 +351,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
const subDir = TestHelper.getRandomizedDirectoryEntry(null, 'subDir');
subDir.path = DiskMangerWorker.pathFromParent(parent);
subDir.path = DiskManager.pathFromParent(parent);
const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0);
sp1.metadata.rating = 5;
const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0);
@ -391,7 +389,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
const subDir = TestHelper.getRandomizedDirectoryEntry(null, 'subDir');
subDir.path = DiskMangerWorker.pathFromParent(parent);
subDir.path = DiskManager.pathFromParent(parent);
const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0);
sp1.metadata.rating = 5;
const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0);
@ -636,7 +634,6 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
}) as any).timeout(40000);
it('should save .md with date', async () => {
Config.Server.Threading.enabled = false;
Config.Album.enabled = true;
Config.Faces.enabled = true;
@ -721,7 +718,6 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
it('.saved_searches.pg2conf', async () => {
Config.Server.Threading.enabled = false;
Config.Album.enabled = true;
Config.Faces.enabled = true;

4
test/backend/unit/model/threading/DiskManagerWorker.spec.ts
View File

@ -1,10 +1,10 @@
import {expect} from 'chai';
import {DiskMangerWorker} from '../../../../../src/backend/model/threading/DiskMangerWorker';
import * as path from 'path';
import {Config} from '../../../../../src/common/config/private/Config';
import {ProjectPath} from '../../../../../src/backend/ProjectPath';
import {Utils} from '../../../../../src/common/Utils';
import {DatabaseType} from '../../../../../src/common/config/private/PrivateConfig';
import {DiskManager} from '../../../../../src/backend/model/fileaccess/DiskManager';
declare const before: any;
@ -21,7 +21,7 @@ describe('DiskMangerWorker', () => {
it('should parse metadata', async () => {
Config.Media.folder = path.join(__dirname, '/../../../assets');
ProjectPath.ImageFolder = path.join(__dirname, '/../../../assets');
const dir = await DiskMangerWorker.scanDirectory('/');
const dir = await DiskManager.scanDirectory('/');
// should match the number of media (photo/video) files in the assets folder
expect(dir.media.length).to.be.equals(10);
const expected = require(path.join(__dirname, '/../../../assets/test image öüóőúéáű-.,.json'));

6
test/backend/unit/model/threading/MetaDataLoader.spec.ts
View File

@ -1,10 +1,10 @@
/* eslint-disable @typescript-eslint/no-var-requires */
import {expect} from 'chai';
import {MetadataLoader} from '../../../../../src/backend/model/threading/MetadataLoader';
import {MetadataLoader} from '../../../../../src/backend/model/fileaccess/MetadataLoader';
import {Utils} from '../../../../../src/common/Utils';
import * as path from 'path';
import * as fs from 'fs';
import {PhotoProcessing} from '../../../../../src/backend/model/fileprocessing/PhotoProcessing';
import {PhotoProcessing} from '../../../../../src/backend/model/fileaccess/fileprocessing/PhotoProcessing';
import {Config} from '../../../../../src/common/config/private/Config';
import {DatabaseType} from '../../../../../src/common/config/private/PrivateConfig';
@ -143,7 +143,7 @@ describe('MetadataLoader', () => {
delete expected.duration;
expect(Utils.clone(data)).to.be.deep.equal(expected);
});
it('should load mkv', async () => {
const data = await MetadataLoader.loadVideoMetadata(path.join(__dirname, '/../../../assets/video_mkv.mkv'));
const expected = require(path.join(__dirname, '/../../../assets/video_mkv.json'));

2
test/backend/unit/model/threading/TaskExecuter.spec.ts
View File

@ -1,5 +1,5 @@
import {expect} from 'chai';
import {TaskExecuter} from '../../../../../src/backend/model/threading/TaskExecuter';
import {TaskExecuter} from '../../../../../src/backend/model/fileaccess/TaskExecuter';
describe('TaskExecuter', () => {

2
test/backend/unit/model/threading/TaskQue.spec.ts
View File

@ -1,5 +1,5 @@
import {expect} from 'chai';
import {TaskQue} from '../../../../../src/backend/model/threading/TaskQue';
import {TaskQue} from '../../../../../src/backend/model/fileaccess/TaskQue';
describe('TaskQue', () => {