self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2025-10-30 23:57:43 +02:00
Code Issues Releases Activity

Implementing cluster based threading

Browse Source 
removing threads dependency
This commit is contained in:
Braun Patrik
2017-07-04 10:24:20 +02:00
parent a7d9bc81c5
commit 6fb57a7b0a
13 changed files with 651 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/index.ts Normal file
View File

@@ -0,0 +1,10 @@
import * as cluster from "cluster";
import {Worker} from "./model/threading/Worker";
import {Server} from "./server";
if (cluster.isMaster) {
new Server();
} else {
Worker.process();
}

2
backend/middlewares/GalleryMWs.ts
View File

@@ -84,7 +84,7 @@ export class GalleryMWs {
//check if thumbnail already exist
if (fs.existsSync(fullImagePath) === false) {
return next(new Error(ErrorCodes.GENERAL_ERROR, "no such file :" + fullImagePath));
return next(new Error(ErrorCodes.GENERAL_ERROR, "no such file:" + fullImagePath));
}
req.resultPipe = fullImagePath;

170
backend/middlewares/thumbnail/THRenderers.ts
View File

@@ -1,170 +0,0 @@
import {Metadata, SharpInstance} from "sharp";
import {Dimensions, State} from "gm";
import {Logger} from "../../Logger";
import {Error, ErrorCodes} from "../../../common/entities/Error";
export module ThumbnailRenderers {
export interface RendererInput {
imagePath: string;
size: number;
makeSquare: boolean;
thPath: string;
qualityPriority: boolean,
__dirname: string;
}
export const jimp = (input: RendererInput, done) => {
//generate thumbnail
const Jimp = require("jimp");
Jimp.read(input.imagePath).then((image) => {
const Logger = require(input.__dirname + "/../../Logger").Logger;
Logger.silly("[JimpThRenderer] rendering thumbnail:", input.imagePath);
/**
* newWidth * newHeight = size*size
* newHeight/newWidth = height/width
*
* newHeight = (height/width)*newWidth
* newWidth * newWidth = (size*size) / (height/width)
*
* @type {number}
*/
const ratio = image.bitmap.height / image.bitmap.width;
const algo = input.qualityPriority == true ? Jimp.RESIZE_BEZIER : Jimp.RESIZE_NEAREST_NEIGHBOR;
if (input.makeSquare == false) {
let newWidth = Math.sqrt((input.size * input.size) / ratio);
image.resize(newWidth, Jimp.AUTO, algo);
} else {
image.resize(input.size / Math.min(ratio, 1), Jimp.AUTO, algo);
image.crop(0, 0, input.size, input.size);
}
image.quality(60); // set JPEG quality
image.write(input.thPath, () => { // save
return done();
});
}).catch(function (err) {
const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
return done(new Error(ErrorCodes.GENERAL_ERROR, err));
});
};
export const sharp = (input: RendererInput, done) => {
//generate thumbnail
const sharp = require("sharp");
const image: SharpInstance = sharp(input.imagePath);
image
.metadata()
.then((metadata: Metadata) => {
// const Logger = require(input.__dirname + "/../../Logger").Logger;
Logger.silly("[SharpThRenderer] rendering thumbnail:", input.imagePath);
/**
* newWidth * newHeight = size*size
* newHeight/newWidth = height/width
*
* newHeight = (height/width)*newWidth
* newWidth * newWidth = (size*size) / (height/width)
*
* @type {number}
*/
try {
const ratio = metadata.height / metadata.width;
const kernel = input.qualityPriority == true ? sharp.kernel.lanczos3 : sharp.kernel.nearest;
const interpolator = input.qualityPriority == true ? sharp.interpolator.bicubic : sharp.interpolator.nearest;
if (input.makeSquare == false) {
const newWidth = Math.round(Math.sqrt((input.size * input.size) / ratio));
image.resize(newWidth, null, {
kernel: kernel,
interpolator: interpolator
});
} else {
image
.resize(input.size, input.size, {
kernel: kernel,
interpolator: interpolator
})
.crop(sharp.strategy.center);
}
image
.jpeg()
.toFile(input.thPath).then(() => {
return done();
}).catch(function (err) {
// const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
// const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
console.error(err);
return done(new Error(ErrorCodes.GENERAL_ERROR, err));
});
} catch (err) {
// const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
// const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
console.error(err);
return done(new Error(ErrorCodes.GENERAL_ERROR, err));
}
});
};
export const gm = (input: RendererInput, done) => {
//generate thumbnail
const gm = require("gm");
let image: State = gm(input.imagePath);
image
.size((err, value: Dimensions) => {
if (err) {
const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
return done(new Error(ErrorCodes.GENERAL_ERROR, err));
}
const Logger = require(input.__dirname + "/../../Logger").Logger;
Logger.silly("[GMThRenderer] rendering thumbnail:", input.imagePath);
/**
* newWidth * newHeight = size*size
* newHeight/newWidth = height/width
*
* newHeight = (height/width)*newWidth
* newWidth * newWidth = (size*size) / (height/width)
*
* @type {number}
*/
try {
const ratio = value.height / value.width;
const filter = input.qualityPriority == true ? 'Lanczos' : 'Point';
image.filter(filter);
if (input.makeSquare == false) {
const newWidth = Math.round(Math.sqrt((input.size * input.size) / ratio));
image = image.resize(newWidth);
} else {
image = image.resize(input.size, input.size)
.crop(input.size, input.size);
}
image.write(input.thPath, (err) => {
if (err) {
const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
return done(new Error(ErrorCodes.GENERAL_ERROR, err));
}
return done();
});
} catch (err) {
const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
return done(new Error(ErrorCodes.GENERAL_ERROR, err));
}
});
};
}

65
backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts
View File

@@ -9,16 +9,16 @@ import {ContentWrapper} from "../../../common/entities/ConentWrapper";
import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
import {ProjectPath} from "../../ProjectPath";
import {PhotoDTO} from "../../../common/entities/PhotoDTO";
import {ThumbnailRenderers} from "./THRenderers";
import {Config} from "../../../common/config/private/Config";
import {ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig";
import RendererInput = ThumbnailRenderers.RendererInput;
import {ThumbnailTH} from "../../model/threading/ThreadPool";
import {RendererFactory, RendererInput} from "../../model/threading/ThumbnailWoker";
export class ThumbnailGeneratorMWs {
private static initDone = false;
private static ThumbnailFunction = null;
private static thPool = null;
private static ThumbnailFunction: (input: RendererInput) => Promise<void> = null;
private static threadPool: ThumbnailTH = null;
public static init() {
if (this.initDone == true) {
@@ -26,28 +26,18 @@ export class ThumbnailGeneratorMWs {
}
Config.Client.concurrentThumbnailGenerations = 1;
switch (Config.Server.thumbnail.processingLibrary) {
case ThumbnailProcessingLib.Jimp:
this.ThumbnailFunction = ThumbnailRenderers.jimp;
break;
case ThumbnailProcessingLib.gm:
this.ThumbnailFunction = ThumbnailRenderers.gm;
break;
case ThumbnailProcessingLib.sharp:
this.ThumbnailFunction = ThumbnailRenderers.sharp;
break;
default:
throw "Unknown thumbnail processing lib";
if (Config.Server.enableThreading == true ||
Config.Server.thumbnail.processingLibrary != ThumbnailProcessingLib.Jimp) {
Config.Client.concurrentThumbnailGenerations = Math.max(1, os.cpus().length - 1);
} else {
Config.Client.concurrentThumbnailGenerations = 1;
}
this.ThumbnailFunction = RendererFactory.build(Config.Server.thumbnail.processingLibrary);
if (Config.Server.enableThreading == true &&
Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.Jimp) {
Config.Client.concurrentThumbnailGenerations = Math.max(1, os.cpus().length - 1);
const Pool = require('threads').Pool;
this.thPool = new Pool(Config.Client.concurrentThumbnailGenerations);
this.thPool.run(this.ThumbnailFunction);
this.threadPool = new ThumbnailTH(Config.Client.concurrentThumbnailGenerations);
}
this.initDone = true;
@@ -138,8 +128,7 @@ export class ThumbnailGeneratorMWs {
}
private static generateImage(imagePath: string, size: number, makeSquare: boolean, req: Request, res: Response, next: NextFunction) {
ThumbnailGeneratorMWs.init();
private static async generateImage(imagePath: string, size: number, makeSquare: boolean, req: Request, res: Response, next: NextFunction) {
//generate thumbnail path
let thPath = path.join(ProjectPath.ThumbnailFolder, ThumbnailGeneratorMWs.generateThumbnailName(imagePath, size));
@@ -157,30 +146,24 @@ export class ThumbnailGeneratorMWs {
}
//run on other thread
let input = <RendererInput>{
imagePath: imagePath,
size: size,
thPath: thPath,
makeSquare: makeSquare,
qualityPriority: Config.Server.thumbnail.qualityPriority,
__dirname: __dirname,
qualityPriority: Config.Server.thumbnail.qualityPriority
};
if (this.thPool !== null) {
this.thPool.send(input)
.on('done', (out) => {
return next(out);
}).on('error', (error) => {
console.log(error);
return next(new Error(ErrorCodes.THUMBNAIL_GENERATION_ERROR, error));
});
} else {
try {
ThumbnailGeneratorMWs.ThumbnailFunction(input, out => next(out));
} catch (error) {
console.log(error);
return next(new Error(ErrorCodes.THUMBNAIL_GENERATION_ERROR, error));
try {
if (this.threadPool !== null) {
await this.threadPool.execute(input);
return next();
} else {
await ThumbnailGeneratorMWs.ThumbnailFunction(input);
return next();
}
} catch (error) {
console.log(error);
return next(new Error(ErrorCodes.THUMBNAIL_GENERATION_ERROR, error));
}
}

76
backend/model/DiskManger.ts
View File

@@ -1,66 +1,44 @@
///<reference path="exif.d.ts"/>
import * as path from "path";
import {DirectoryDTO} from "../../common/entities/DirectoryDTO";
import {ProjectPath} from "../ProjectPath";
import {Logger} from "../Logger";
import {diskManagerTask, DiskManagerTask} from "./DiskMangerTask";
import {Config} from "../../common/config/private/Config";
import {DiskManagerTH} from "./threading/ThreadPool";
import {DiskMangerWorker} from "./threading/DiskMangerWorker";
const Pool = require('threads').Pool;
const pool = new Pool(1);
const LOG_TAG = "[DiskManager]";
pool.run(diskManagerTask);
export class DiskManager {
public static scanDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
return new Promise((resolve, reject) => {
Logger.silly(LOG_TAG, "scanning directory:", relativeDirectoryName);
let directoryName = path.basename(relativeDirectoryName);
let directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
let absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
static threadPool: DiskManagerTH = null;
let input = <DiskManagerTask.PoolInput>{
relativeDirectoryName,
directoryName,
directoryParent,
absoluteDirectoryName
};
public static init() {
if (Config.Server.enableThreading == true) {
DiskManager.threadPool = new DiskManagerTH(1);
}
}
let done = (error: any, result: DirectoryDTO) => {
if (error || !result) {
return reject(error);
}
let addDirs = (dir: DirectoryDTO) => {
dir.photos.forEach((ph) => {
ph.directory = dir;
});
dir.directories.forEach((d) => {
addDirs(d);
});
};
addDirs(result);
return resolve(result);
};
let error = (error) => {
return reject(error);
};
public static async scanDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
Logger.silly(LOG_TAG, "scanning directory:", relativeDirectoryName);
if (Config.Server.enableThreading == true) {
pool.send(input).on('done', done).on('error', error);
} else {
try {
diskManagerTask(input, done);
} catch (err) {
error(err);
}
}
});
let directory: DirectoryDTO = null;
if (Config.Server.enableThreading == true) {
directory = await DiskManager.threadPool.execute(relativeDirectoryName);
} else {
directory = await DiskMangerWorker.scanDirectory(relativeDirectoryName);
}
let addDirs = (dir: DirectoryDTO) => {
dir.photos.forEach((ph) => {
ph.directory = dir;
});
dir.directories.forEach((d) => {
addDirs(d);
});
};
addDirs(directory);
return directory;
}
}

202
backend/model/DiskMangerTask.ts
View File

@@ -1,202 +0,0 @@
///<reference path="exif.d.ts"/>
import {DirectoryDTO} from "../../common/entities/DirectoryDTO";
import {CameraMetadata, GPSMetadata, ImageSize, PhotoDTO, PhotoMetadata} from "../../common/entities/PhotoDTO";
import {Logger} from "../Logger";
const LOG_TAG = "[DiskManagerTask]";
export const diskManagerTask = (input: DiskManagerTask.PoolInput, done) => {
const fs = require("fs");
const path = require("path");
const mime = require("mime");
const iptc = require("node-iptc");
const exif_parser = require("exif-parser");
let isImage = (fullPath: string) => {
let imageMimeTypes = [
'image/bmp',
'image/gif',
'image/jpeg',
'image/png',
'image/pjpeg',
'image/tiff',
'image/webp',
'image/x-tiff',
'image/x-windows-bmp'
];
let extension = mime.lookup(fullPath);
return imageMimeTypes.indexOf(extension) !== -1;
};
let loadPhotoMetadata = (fullPath: string): Promise<PhotoMetadata> => {
return new Promise<PhotoMetadata>((resolve: (metadata: PhotoMetadata) => void, reject) => {
fs.readFile(fullPath, function (err, data) {
if (err) {
return reject({file: fullPath, error: err});
}
const metadata: PhotoMetadata = <PhotoMetadata>{
keywords: {},
cameraData: {},
positionData: null,
size: {},
creationDate: {}
};
try {
try {
const exif = exif_parser.create(data).parse();
metadata.cameraData = <CameraMetadata> {
ISO: exif.tags.ISO,
model: exif.tags.Modeol,
maker: exif.tags.Make,
fStop: exif.tags.FNumber,
exposure: exif.tags.ExposureTime,
focalLength: exif.tags.FocalLength,
lens: exif.tags.LensModel,
};
if (!isNaN(exif.tags.GPSLatitude) || exif.tags.GPSLongitude || exif.tags.GPSAltitude) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.GPSData = <GPSMetadata> {
latitude: exif.tags.GPSLatitude,
longitude: exif.tags.GPSLongitude,
altitude: exif.tags.GPSAltitude
};
}
metadata.size = <ImageSize> {width: exif.imageSize.width, height: exif.imageSize.height};
} catch (err) {
Logger.info(LOG_TAG, "Error parsing exif", fullPath);
metadata.size = <ImageSize> {width: 1, height: 1};
}
try {
const iptcData = iptc(data);
//Decode characters to UTF8
const decode = (s: any) => {
for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
((a = s[i][c](0)) & 0x80) &&
(s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
);
return s.join("");
};
if (iptcData.country_or_primary_location_name || iptcData.province_or_state || iptcData.city) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.country = iptcData.country_or_primary_location_name;
metadata.positionData.state = iptcData.province_or_state;
metadata.positionData.city = iptcData.city;
}
metadata.keywords = <string[]> (iptcData.keywords || []).map((s: string) => decode(s));
metadata.creationDate = <number> iptcData.date_time ? iptcData.date_time.getTime() : 0;
} catch (err) {
Logger.info(LOG_TAG, "Error parsing iptc data", fullPath);
}
return resolve(metadata);
} catch (err) {
return reject({file: fullPath, error: err});
}
});
}
);
}
;
let parseDir = (directoryInfo: {
relativeDirectoryName: string,
directoryName: string,
directoryParent: string,
absoluteDirectoryName: string
}, maxPhotos: number = null, photosOnly: boolean = false): Promise<DirectoryDTO> => {
return new Promise<DirectoryDTO>((resolve, reject) => {
let promises: Array<Promise<any>> = [];
let directory = <DirectoryDTO>{
name: directoryInfo.directoryName,
path: directoryInfo.directoryParent,
lastUpdate: Date.now(),
directories: [],
photos: []
};
fs.readdir(directoryInfo.absoluteDirectoryName, (err, list) => {
if (err) {
return reject(err);
}
try {
for (let i = 0; i < list.length; i++) {
let file = list[i];
let fullFilePath = path.normalize(path.resolve(directoryInfo.absoluteDirectoryName, file));
if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
let promise = parseDir({
relativeDirectoryName: path.join(directoryInfo.relativeDirectoryName, path.sep),
directoryName: file,
directoryParent: path.join(directoryInfo.relativeDirectoryName, path.sep),
absoluteDirectoryName: fullFilePath
},
5, true
).then((dir) => {
directory.directories.push(dir);
});
promises.push(promise);
} else if (isImage(fullFilePath)) {
let promise = loadPhotoMetadata(fullFilePath).then((photoMetadata) => {
directory.photos.push(<PhotoDTO>{
name: file,
directory: null,
metadata: photoMetadata
});
});
promises.push(promise);
if (maxPhotos != null && promises.length > maxPhotos) {
break;
}
}
}
Promise.all(promises).then(() => {
return resolve(directory);
}).catch((err) => {
return reject({directoryInfo: directoryInfo, error: err});
});
} catch (err) {
return reject({directoryInfo: directoryInfo, error: err});
}
});
});
};
parseDir(input).then((dir) => {
done(null, dir);
}).catch((err) => {
done(err, null);
});
}
;
export module DiskManagerTask {
export interface PoolInput {
relativeDirectoryName: string;
directoryName: string;
directoryParent: string;
absoluteDirectoryName: string;
}
}

162
backend/model/threading/DiskMangerWorker.ts Normal file
View File

@@ -0,0 +1,162 @@
///<reference path="../exif.d.ts"/>
import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
import {CameraMetadata, GPSMetadata, ImageSize, PhotoDTO, PhotoMetadata} from "../../../common/entities/PhotoDTO";
import {Logger} from "../../Logger";
import * as fs from "fs";
import * as path from "path";
import * as mime from "mime";
import * as iptc from "node-iptc";
import * as exif_parser from "exif-parser";
import {ProjectPath} from "../../ProjectPath";
const LOG_TAG = "[DiskManagerTask]";
export class DiskMangerWorker {
private static isImage(fullPath: string) {
let imageMimeTypes = [
'image/bmp',
'image/gif',
'image/jpeg',
'image/png',
'image/pjpeg',
'image/tiff',
'image/webp',
'image/x-tiff',
'image/x-windows-bmp'
];
let extension = mime.lookup(fullPath);
return imageMimeTypes.indexOf(extension) !== -1;
}
private static loadPhotoMetadata(fullPath: string): Promise<PhotoMetadata> {
return new Promise<PhotoMetadata>((resolve, reject) => {
fs.readFile(fullPath, function (err, data) {
if (err) {
return reject({file: fullPath, error: err});
}
const metadata: PhotoMetadata = <PhotoMetadata>{
keywords: {},
cameraData: {},
positionData: null,
size: {},
creationDate: 0
};
try {
try {
const exif = exif_parser.create(data).parse();
metadata.cameraData = <CameraMetadata> {
ISO: exif.tags.ISO,
model: exif.tags.Modeol,
maker: exif.tags.Make,
fStop: exif.tags.FNumber,
exposure: exif.tags.ExposureTime,
focalLength: exif.tags.FocalLength,
lens: exif.tags.LensModel,
};
if (!isNaN(exif.tags.GPSLatitude) || exif.tags.GPSLongitude || exif.tags.GPSAltitude) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.GPSData = <GPSMetadata> {
latitude: exif.tags.GPSLatitude,
longitude: exif.tags.GPSLongitude,
altitude: exif.tags.GPSAltitude
};
}
metadata.size = <ImageSize> {width: exif.imageSize.width, height: exif.imageSize.height};
} catch (err) {
Logger.info(LOG_TAG, "Error parsing exif", fullPath);
metadata.size = <ImageSize> {width: 1, height: 1};
}
try {
const iptcData = iptc(data);
//Decode characters to UTF8
const decode = (s: any) => {
for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
((a = s[i][c](0)) & 0x80) &&
(s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
);
return s.join("");
};
if (iptcData.country_or_primary_location_name || iptcData.province_or_state || iptcData.city) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.country = iptcData.country_or_primary_location_name;
metadata.positionData.state = iptcData.province_or_state;
metadata.positionData.city = iptcData.city;
}
metadata.keywords = <string[]> (iptcData.keywords || []).map((s: string) => decode(s));
metadata.creationDate = <number> iptcData.date_time ? iptcData.date_time.getTime() : 0;
} catch (err) {
Logger.info(LOG_TAG, "Error parsing iptc data", fullPath);
}
return resolve(metadata);
} catch (err) {
return reject({file: fullPath, error: err});
}
});
}
);
}
public static scanDirectory(relativeDirectoryName: string, maxPhotos: number = null, photosOnly: boolean = false): Promise<DirectoryDTO> {
return new Promise<DirectoryDTO>((resolve, reject) => {
const directoryName = path.basename(relativeDirectoryName);
const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
// let promises: Array<Promise<any>> = [];
let directory = <DirectoryDTO>{
name: directoryName,
path: directoryParent,
lastUpdate: Date.now(),
directories: [],
photos: []
};
fs.readdir(absoluteDirectoryName, async (err, list) => {
if (err) {
return reject(err);
}
try {
for (let i = 0; i < list.length; i++) {
let file = list[i];
let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file));
if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
directory.directories.push(await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file),
5, true
));
} else if (DiskMangerWorker.isImage(fullFilePath)) {
directory.photos.push(<PhotoDTO>{
name: file,
directory: null,
metadata: await DiskMangerWorker.loadPhotoMetadata(fullFilePath)
});
if (maxPhotos != null && directory.photos.length > maxPhotos) {
break;
}
}
}
return resolve(directory);
} catch (err) {
return reject({error: err});
}
});
});
}
}

110
backend/model/threading/ThreadPool.ts Normal file
View File

@@ -0,0 +1,110 @@
import * as cluster from "cluster";
import {Logger} from "../../Logger";
import {DiskManagerTask, ThumbnailTask, WorkerMessage, WorkerTask, WorkerTaskTypes} from "./Worker";
import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
import {RendererInput} from "./ThumbnailWoker";
import {Config} from "../../../common/config/private/Config";
interface PoolTask {
task: WorkerTask;
promise: { resolve: Function, reject: Function };
}
interface WorkerWrapper {
worker: cluster.Worker;
poolTask: PoolTask;
}
export class ThreadPool {
public static WorkerCount = 0;
private workers: WorkerWrapper[] = [];
private tasks: PoolTask[] = [];
constructor(private size: number) {
Logger.silly("Creating thread pool with", size, "workers");
for (let i = 0; i < size; i++) {
this.startWorker();
}
}
private startWorker() {
const worker = <WorkerWrapper>{poolTask: null, worker: cluster.fork()};
this.workers.push(worker);
worker.worker.on('online', () => {
ThreadPool.WorkerCount++;
Logger.debug('Worker ' + worker.worker.process.pid + ' is online, worker count:', ThreadPool.WorkerCount);
});
worker.worker.on('exit', (code, signal) => {
ThreadPool.WorkerCount--;
Logger.warn('Worker ' + worker.worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal + ", worker count:", ThreadPool.WorkerCount);
Logger.debug('Starting a new worker');
this.startWorker();
});
worker.worker.on("message", (msg: WorkerMessage) => {
if (worker.poolTask == null) {
throw "No worker task after worker task is completed"
}
if (msg.error) {
worker.poolTask.promise.reject(msg.error);
} else {
worker.poolTask.promise.resolve(msg.result);
}
worker.poolTask = null;
this.run();
});
}
protected executeTask<T>(task: WorkerTask): Promise<T> {
return new Promise((resolve: Function, reject: Function) => {
this.tasks.push({task: task, promise: {resolve: resolve, reject: reject}});
this.run();
});
}
private getFreeWorker() {
for (let i = 0; i < this.workers.length; i++) {
if (this.workers[i].poolTask == null) {
return this.workers[i];
}
}
return null;
}
private run = () => {
if (this.tasks.length == 0) {
return;
}
const worker = this.getFreeWorker();
if (worker == null) {
return;
}
const poolTask = this.tasks.pop();
worker.poolTask = poolTask;
worker.worker.send(poolTask.task);
};
}
export class DiskManagerTH extends ThreadPool {
execute(relativeDirectoryName: string): Promise<DirectoryDTO> {
return super.executeTask(<DiskManagerTask>{
type: WorkerTaskTypes.diskManager,
relativeDirectoryName: relativeDirectoryName
});
}
}
export class ThumbnailTH extends ThreadPool {
execute(input: RendererInput): Promise<void> {
return super.executeTask(<ThumbnailTask>{
type: WorkerTaskTypes.thumbnail,
input: input,
renderer: Config.Server.thumbnail.processingLibrary
});
}
}

170
backend/model/threading/ThumbnailWoker.ts Normal file
View File

@@ -0,0 +1,170 @@
import {Metadata, SharpInstance} from "sharp";
import {Dimensions, State} from "gm";
import {Logger} from "../../Logger";
import {ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig";
export class ThumbnailWoker {
private static renderer: (input: RendererInput) => Promise<void> = null;
private static rendererType = null;
public static render(input: RendererInput, renderer: ThumbnailProcessingLib): Promise<void> {
if (ThumbnailWoker.rendererType != renderer) {
ThumbnailWoker.renderer = RendererFactory.build(renderer);
ThumbnailWoker.rendererType = renderer;
}
return ThumbnailWoker.renderer(input);
}
}
export interface RendererInput {
imagePath: string;
size: number;
makeSquare: boolean;
thPath: string;
qualityPriority: boolean
}
export class RendererFactory {
public static build(renderer: ThumbnailProcessingLib): (input: RendererInput) => Promise<void> {
switch (renderer) {
case ThumbnailProcessingLib.Jimp:
return RendererFactory.Jimp();
case ThumbnailProcessingLib.gm:
return RendererFactory.Gm();
case ThumbnailProcessingLib.sharp:
return RendererFactory.Sharp();
}
throw "unknown renderer"
}
public static Jimp() {
const Jimp = require("jimp");
return async (input: RendererInput): Promise<void> => {
//generate thumbnail
Logger.silly("[JimpThRenderer] rendering thumbnail:", input.imagePath);
const image = await Jimp.read(input.imagePath);
/**
* newWidth * newHeight = size*size
* newHeight/newWidth = height/width
*
* newHeight = (height/width)*newWidth
* newWidth * newWidth = (size*size) / (height/width)
*
* @type {number}
*/
const ratio = image.bitmap.height / image.bitmap.width;
const algo = input.qualityPriority == true ? Jimp.RESIZE_BEZIER : Jimp.RESIZE_NEAREST_NEIGHBOR;
if (input.makeSquare == false) {
let newWidth = Math.sqrt((input.size * input.size) / ratio);
image.resize(newWidth, Jimp.AUTO, algo);
} else {
image.resize(input.size / Math.min(ratio, 1), Jimp.AUTO, algo);
image.crop(0, 0, input.size, input.size);
}
image.quality(60); // set JPEG quality
await new Promise((resolve, reject) => {
image.write(input.thPath, (err) => { // save
if (err) {
return reject(err);
}
resolve();
});
});
};
}
public static Sharp() {
const sharp = require("sharp");
return async (input: RendererInput): Promise<void> => {
Logger.silly("[SharpThRenderer] rendering thumbnail:", input.imagePath);
const image: SharpInstance = sharp(input.imagePath);
const metadata: Metadata = await image.metadata();
/**
* newWidth * newHeight = size*size
* newHeight/newWidth = height/width
*
* newHeight = (height/width)*newWidth
* newWidth * newWidth = (size*size) / (height/width)
*
* @type {number}
*/
const ratio = metadata.height / metadata.width;
const kernel = input.qualityPriority == true ? sharp.kernel.lanczos3 : sharp.kernel.nearest;
const interpolator = input.qualityPriority == true ? sharp.interpolator.bicubic : sharp.interpolator.nearest;
if (input.makeSquare == false) {
const newWidth = Math.round(Math.sqrt((input.size * input.size) / ratio));
image.resize(newWidth, null, {
kernel: kernel,
interpolator: interpolator
});
} else {
image
.resize(input.size, input.size, {
kernel: kernel,
interpolator: interpolator
})
.crop(sharp.strategy.center);
}
await image.jpeg().toFile(input.thPath);
};
}
public static Gm() {
const gm = require("gm");
return (input: RendererInput): Promise<void> => {
return new Promise((resolve, reject) => {
Logger.silly("[GMThRenderer] rendering thumbnail:", input.imagePath);
let image: State = gm(input.imagePath);
image.size((err, value: Dimensions) => {
if (err) {
return reject(err);
}
/**
* newWidth * newHeight = size*size
* newHeight/newWidth = height/width
*
* newHeight = (height/width)*newWidth
* newWidth * newWidth = (size*size) / (height/width)
*
* @type {number}
*/
try {
const ratio = value.height / value.width;
const filter = input.qualityPriority == true ? 'Lanczos' : 'Point';
image.filter(filter);
if (input.makeSquare == false) {
const newWidth = Math.round(Math.sqrt((input.size * input.size) / ratio));
image = image.resize(newWidth);
} else {
image = image.resize(input.size, input.size)
.crop(input.size, input.size);
}
image.write(input.thPath, (err) => {
if (err) {
return reject(err);
}
return resolve();
});
} catch (err) {
return reject(err);
}
});
});
};
}
}

57
backend/model/threading/Worker.ts Normal file
View File

@@ -0,0 +1,57 @@
import {DiskMangerWorker} from "./DiskMangerWorker";
import {Logger} from "../../Logger";
import {RendererInput, ThumbnailWoker} from "./ThumbnailWoker";
import {ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig";
export class Worker {
public static process() {
Logger.debug("Worker is waiting for tasks");
process.on('message', async (task: WorkerTask) => {
try {
let result = null;
switch (task.type) {
case WorkerTaskTypes.diskManager:
result = await DiskMangerWorker.scanDirectory((<DiskManagerTask>task).relativeDirectoryName);
break;
case WorkerTaskTypes.thumbnail:
result = await ThumbnailWoker.render((<ThumbnailTask>task).input, (<ThumbnailTask>task).renderer);
break;
default:
Logger.error("Unknown worker task type");
throw "Unknown worker task type";
}
process.send(<WorkerMessage>{
error: null,
result: result
});
} catch (err) {
process.send({error: err, result: null});
}
});
}
}
export enum WorkerTaskTypes{
thumbnail, diskManager
}
export interface WorkerTask {
type: WorkerTaskTypes;
}
export interface DiskManagerTask extends WorkerTask {
relativeDirectoryName: string;
}
export interface ThumbnailTask extends WorkerTask {
input: RendererInput;
renderer: ThumbnailProcessingLib;
}
export interface WorkerMessage {
error: any;
result: any;
}

127
backend/server.ts
View File

@@ -14,20 +14,26 @@ import {Config} from "../common/config/private/Config";
import {DatabaseType, ThumbnailProcessingLib} from "../common/config/private/IPrivateConfig";
import {LoggerRouter} from "./routes/LoggerRouter";
import {ProjectPath} from "./ProjectPath";
import {ThumbnailGeneratorMWs} from "./middlewares/thumbnail/ThumbnailGeneratorMWs";
import {DiskManager} from "./model/DiskManger";
const LOG_TAG = "[server]";
export class Server {
private debug: any;
private app: any;
private server: any;
constructor() {
if (process.env.DEBUG) {
Logger.debug(LOG_TAG, "Running in DEBUG mode");
}
this.init();
}
async init() {
Logger.info(LOG_TAG, "config:");
Logger.info(LOG_TAG, "running diagnostics...");
await this.runDiagnostics();
Logger.info(LOG_TAG, "using config:");
Logger.info(LOG_TAG, JSON.stringify(Config, null, '\t'));
this.app = _express();
@@ -57,59 +63,8 @@ export class Server {
// for parsing application/json
this.app.use(_bodyParser.json());
if (Config.Server.database.type == DatabaseType.mysql) {
try {
await ObjectManagerRepository.InitMySQLManagers();
} catch (err) {
Logger.warn(LOG_TAG, "[MYSQL error]", err);
Logger.warn(LOG_TAG, "Error during initializing mysql falling back to memory DB");
Config.setDatabaseType(DatabaseType.memory);
await ObjectManagerRepository.InitMemoryManagers();
}
} else {
await ObjectManagerRepository.InitMemoryManagers();
}
if (Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.sharp) {
try {
const sharp = require("sharp");
sharp();
} catch (err) {
Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] sharp module error: ", err);
Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
" 'Sharp' node module is not found." +
" Falling back to JS based thumbnail generation");
Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
}
}
if (Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.gm) {
try {
const gm = require("gm");
gm(ProjectPath.FrontendFolder + "/assets/icon.png").size((err, value) => {
console.log(err, value);
if (!err) {
return;
}
Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] gm module error: ", err);
Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
" 'gm' node module is not found." +
" Falling back to JS based thumbnail generation");
Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
});
} catch (err) {
Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] gm module error: ", err);
Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
" 'gm' node module is not found." +
" Falling back to JS based thumbnail generation");
Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
}
}
DiskManager.init();
ThumbnailGeneratorMWs.init();
PublicRouter.route(this.app);
@@ -135,6 +90,61 @@ export class Server {
}
async runDiagnostics() {
if (Config.Server.database.type == DatabaseType.mysql) {
try {
await ObjectManagerRepository.InitMySQLManagers();
} catch (err) {
Logger.warn(LOG_TAG, "[MYSQL error]", err);
Logger.warn(LOG_TAG, "Error during initializing mysql falling back to memory DB");
Config.setDatabaseType(DatabaseType.memory);
await ObjectManagerRepository.InitMemoryManagers();
}
} else {
await ObjectManagerRepository.InitMemoryManagers();
}
if (Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.sharp) {
try {
const sharp = require("sharp");
sharp();
} catch (err) {
Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] sharp module error: ", err);
Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
" 'Sharp' node module is not found." +
" Falling back to JS based thumbnail generation");
Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
}
}
if (Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.gm) {
try {
const gm = require("gm");
gm(ProjectPath.FrontendFolder + "/assets/icon.png").size((err, value) => {
if (!err) {
return;
}
Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] gm module error: ", err);
Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
" 'gm' node module is not found." +
" Falling back to JS based thumbnail generation");
Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
});
} catch (err) {
Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] gm module error: ", err);
Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
" 'gm' node module is not found." +
" Falling back to JS based thumbnail generation");
Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
}
}
}
/**
* Event listener for HTTP server "error" event.
@@ -178,8 +188,7 @@ export class Server {
}
if (process.env.DEBUG) {
Logger.debug(LOG_TAG, "Running in DEBUG mode");
}
new Server();

4
frontend/app/gallery/thumnailLoader.service.ts
View File

@@ -96,7 +96,7 @@ export class ThumbnailLoaderService {
let thumbnailTaskEntity = {priority: priority, listener: listener, parentTask: thTask};
//add to task
//add to poolTask
thTask.taskEntities.push(thumbnailTaskEntity);
if (thTask.inProgress == true) {
listener.onStartedLoading();
@@ -144,7 +144,7 @@ export class ThumbnailLoaderService {
let i = this.que.indexOf(task);
if (i == -1) {
if (task.taskEntities.length !== 0) {
console.error("ThumbnailLoader: can't find task to remove");
console.error("ThumbnailLoader: can't find poolTask to remove");
}
return;
}

40
package.json
View File

@@ -5,13 +5,13 @@
"author": "Patrik J. Braun",
"homepage": "https://github.com/bpatrik/PiGallery2",
"license": "MIT",
"main": "./backend/server.js",
"bin": "./backend/server.js",
"main": "./backend/index.js",
"bin": "./backend/index.js",
"scripts": {
"build": "ng build",
"pretest": "tsc",
"test": "ng test --single-run && mocha --recursive test/backend/unit",
"start": "node ./backend/server",
"start": "node ./backend/index",
"ng": "ng",
"lint": "ng lint",
"e2e": "ng e2e"
@@ -35,43 +35,39 @@
"mysql": "^2.13.0",
"node-iptc": "^1.0.4",
"reflect-metadata": "^0.1.10",
"threads": "^0.8.1",
"typeconfig": "^1.0.1",
"typeorm": "0.0.11",
"winston": "^2.3.1"
},
"devDependencies": {
"@agm/core": "^1.0.0-beta.0",
"@angular/cli": "1.2.0",
"@angular/common": "~4.2.5",
"@angular/compiler": "~4.2.5",
"@angular/compiler-cli": "^4.2.5",
"@angular/core": "~4.2.5",
"@angular/forms": "~4.2.5",
"@angular/http": "~4.2.5",
"@angular/language-service": "^4.2.5",
"@angular/platform-browser": "~4.2.5",
"@angular/platform-browser-dynamic": "~4.2.5",
"@angular/router": "~4.2.5",
"@types/express": "^4.0.36",
"@types/express-session": "1.15.0",
"@types/gm": "^1.17.31",
"@types/jasmine": "^2.5.53",
"@types/jimp": "^0.2.1",
"@types/node": "^8.0.7",
"@types/sharp": "^0.17.2",
"ng2-cookies": "^1.0.12",
"ng2-slim-loading-bar": "^4.0.0",
"intl": "^1.2.5",
"core-js": "^2.4.1",
"zone.js": "^0.8.12",
"rxjs": "^5.4.1",
"@types/winston": "^2.3.3",
"@agm/core": "^1.0.0-beta.0",
"@angular/common": "~4.2.5",
"@angular/compiler": "~4.2.5",
"@angular/core": "~4.2.5",
"@angular/forms": "~4.2.5",
"@angular/http": "~4.2.5",
"@angular/platform-browser": "~4.2.5",
"@angular/platform-browser-dynamic": "~4.2.5",
"@angular/router": "~4.2.5",
"chai": "^4.0.2",
"codelyzer": "~3.1.1",
"core-js": "^2.4.1",
"ejs-loader": "^0.3.0",
"gulp": "^3.9.1",
"gulp-typescript": "^3.1.7",
"gulp-zip": "^4.0.0",
"intl": "^1.2.5",
"jasmine-core": "^2.6.4",
"jasmine-spec-reporter": "~4.1.1",
"karma": "^1.7.0",
@@ -84,15 +80,19 @@
"karma-systemjs": "^0.16.0",
"merge2": "^1.1.0",
"mocha": "^3.4.2",
"ng2-cookies": "^1.0.12",
"ng2-slim-loading-bar": "^4.0.0",
"phantomjs-prebuilt": "^2.1.14",
"protractor": "^5.1.2",
"remap-istanbul": "^0.9.5",
"rimraf": "^2.6.1",
"run-sequence": "^2.0.0",
"rxjs": "^5.4.1",
"ts-helpers": "^1.1.2",
"ts-node": "~3.1.0",
"tslint": "^5.4.3",
"typescript": "^2.4.1"
"typescript": "^2.4.1",
"zone.js": "^0.8.12"
},
"optionalDependencies": {
"gm": "^1.23.0",