From 29c3e580d461c0b2fed7325c3d3c60ce0a319765 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Wed, 30 Dec 2020 21:54:07 +0100 Subject: [PATCH] improving benchmark --- benchmark/BenchmarkRunner.ts | 64 ++++++++++++-- benchmark/README.md | 87 ------------------- benchmark/index.ts | 22 ++--- src/backend/middlewares/admin/AdminMWs.ts | 3 + .../model/database/sql/GalleryManager.ts | 1 + .../model/database/sql/IPersonManager.ts | 5 ++ .../model/database/sql/PersonManager.ts | 11 ++- 7 files changed, 83 insertions(+), 110 deletions(-) create mode 100644 src/backend/model/database/sql/IPersonManager.ts diff --git a/benchmark/BenchmarkRunner.ts b/benchmark/BenchmarkRunner.ts index 75d9aaa5..2c47e49a 100644 --- a/benchmark/BenchmarkRunner.ts +++ b/benchmark/BenchmarkRunner.ts @@ -4,6 +4,7 @@ import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker' import {IndexingManager} from '../src/backend/model/database/sql/IndexingManager'; import {SearchManager} from '../src/backend/model/database/sql/SearchManager'; import * as util from 'util'; +import * as path from 'path'; import * as rimraf from 'rimraf'; import {SearchTypes} from '../src/common/entities/AutoCompleteItem'; import {Utils} from '../src/common/Utils'; @@ -20,6 +21,8 @@ import {JobProgress} from '../src/backend/model/jobs/jobs/JobProgress'; import {GalleryMWs} from '../src/backend/middlewares/GalleryMWs'; import {UserDTO, UserRoles} from '../src/common/entities/UserDTO'; import {ContentWrapper} from '../src/common/entities/ConentWrapper'; +import {GalleryManager} from '../src/backend/model/database/sql/GalleryManager'; +import {PersonManager} from '../src/backend/model/database/sql/PersonManager'; const rimrafPR = util.promisify(rimraf); @@ -39,15 +42,17 @@ export class BMIndexingManager extends IndexingManager { } export class BenchmarkRunner { - + inited = false; + private biggestPath: string = null; constructor(public RUNS: number) { } async bmSaveDirectory(): Promise { + await this.init(); await this.resetDB(); - const dir = await DiskMangerWorker.scanDirectory('./'); + const dir = await DiskMangerWorker.scanDirectory(this.biggestPath); const bm = new Benchmark('Saving directory to DB', null, () => this.resetDB()); bm.addAStep({ name: 'Saving directory to DB', @@ -60,15 +65,17 @@ export class BenchmarkRunner { } async bmScanDirectory(): Promise { + await this.init(); const bm = new Benchmark('Scanning directory'); bm.addAStep({ name: 'Scanning directory', - fn: async () => new ContentWrapper(await DiskMangerWorker.scanDirectory('./')) + fn: async () => new ContentWrapper(await DiskMangerWorker.scanDirectory(this.biggestPath)) }); return await bm.run(this.RUNS); } async bmListDirectory(): Promise { + await this.init(); await this.setupDB(); Config.Server.Indexing.reIndexingSensitivity = ServerConfig.ReIndexingSensitivity.low; const bm = new Benchmark('List directory', @@ -79,7 +86,7 @@ export class BenchmarkRunner { }); bm.addAStep({ name: 'List directory', - fn: (input) => this.nextToPromise(GalleryMWs.listDirectory, input, {directory: '/'}) + fn: (input) => this.nextToPromise(GalleryMWs.listDirectory, input, {directory: this.biggestPath}) }); bm.addAStep({ name: 'Add Thumbnail information', @@ -163,6 +170,54 @@ export class BenchmarkRunner { return await bm.run(this.RUNS); } + async getStatistic() { + await this.setupDB(); + const gm = new GalleryManager(); + const pm = new PersonManager(); + + const renderDataSize = (size: number) => { + const postFixes = ['B', 'KB', 'MB', 'GB', 'TB']; + let index = 0; + while (size > 1000 && index < postFixes.length - 1) { + size /= 1000; + index++; + } + return size.toFixed(2) + postFixes[index]; + }; + return 'directories: ' + await gm.countDirectories() + + ', photos: ' + await gm.countPhotos() + + ', videos: ' + await gm.countVideos() + + ', diskUsage : ' + renderDataSize(await gm.countMediaSize()) + + ', persons : ' + await pm.countFaces() + + ', unique persons (faces): ' + (await pm.getAll()).length; + + } + + private async init() { + if (this.inited === false) { + await this.setupDB(); + + const gm = new GalleryManager(); + let biggest = 0; + let biggestPath = '/'; + const queue = ['/']; + while (queue.length > 0) { + const dirPath = queue.shift(); + const dir = await gm.listDirectory(dirPath); + dir.directories.forEach(d => queue.push(path.join(d.path + d.name))); + if (biggest < dir.media.length) { + biggestPath = path.join(dir.path + dir.name); + biggest = dir.media.length; + } + } + this.biggestPath = biggestPath; + console.log('updating path of biggest dir to: ' + this.biggestPath); + this.inited = true; + } + return this.biggestPath; + + } + private nextToPromise(fn: (req: any, res: any, next: Function) => void, input?: any, params = {}) { return new Promise((resolve, reject) => { const request = { @@ -211,5 +266,4 @@ export class BenchmarkRunner { } }); } - } diff --git a/benchmark/README.md b/benchmark/README.md index 81e5266d..46edac06 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -2,93 +2,6 @@ These results are created mostly for development, but the results are public for curious users. -## PiGallery2 v1.8.2, 30.12.2020 -**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm - -**Gallery**: directories: 0 media: 341, faces: 65 - -| Action | Sub action | Action details | Average Duration | Details | -|:------:|:----------:|:--------------:|:----------------:|:-------:| -| **Scanning directory** | | | 1526.7 ms | media: 341, directories:0 | -| **Saving directory to DB** | | | 679.9 ms | - | -| **List directory** | | | 61.1 ms | media: 341, directories:0 | -| | List directory | | 39.7 ms | media: 341, directories:0 | -| | Add Thumbnail information | | 19.3 ms | media: 341, directories:0 | -| | Clean Up Gallery Result | | 2.0 ms | media: 341, directories:0 | -| **Listing Faces** | | | 8.4 ms | items: 1 | -| | List Persons | | 1.0 ms | items: 1 | -| | Add sample photo | | 7.1 ms | items: 1 | -| | Add thumbnail info | | 0.1 ms | items: 1 | -| | Remove sample photo | | 0.0 ms | items: 1 | -| **Searching** | | `a` as `directory` | 3.3 ms | media: 0, directories:0 | -| **Searching** | | `a` as `person` | 11.6 ms | media: 65, directories:0 | -| **Searching** | | `a` as `keyword` | 38.0 ms | media: 339, directories:0 | -| **Searching** | | `a` as `position` | 31.7 ms | media: 282, directories:0 | -| **Searching** | | `a` as `photo` | 3.2 ms | media: 0, directories:0 | -| **Searching** | | `a` as `video` | 3.2 ms | media: 0, directories:0 | -| **Searching** | | `a` as `any` | 39.3 ms | media: 339, directories:0 | -| **Instant search** | | `a` | 6.7 ms | media: 10, directories:0 | -| **Auto complete** | | `a` | 6.7 ms | items: 10 | -*Measurements run 2 times and an average was calculated. - - -## PiGallery2 v1.8.2, 30.12.2020 -**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm - -**Gallery**: directories: 0 media: 341, faces: 65 - -| Action | Sub action | Action details | Average Duration | Details | -|:------:|:----------:|:--------------:|:----------------:|:-------:| -| **Scanning directory** | | | 1459.3 ms | media: 341, directories:0 | -| **Saving directory to DB** | | | 654.9 ms | - | -| **List directory** | | | 70.4 ms | - | -| | List directory | | 42.1 ms | - | -| | Add Thumbnail information | | 25.8 ms | - | -| | Clean Up Gallery Result | | 2.3 ms | - | -| **Listing Faces** | | | 10.5 ms | items: 1 | -| | List Persons | | 1.0 ms | items: 1 | -| | Add sample photo | | 9.1 ms | items: 1 | -| | Add thumbnail info | | 0.2 ms | items: 1 | -| | Remove sample photo | | 0.0 ms | items: 1 | -| **Searching** | | `a` as `directory` | 3.3 ms | - | -| **Searching** | | `a` as `person` | 11.7 ms | media: 65, directories:0 | -| **Searching** | | `a` as `keyword` | 40.4 ms | media: 339, directories:0 | -| **Searching** | | `a` as `position` | 30.4 ms | media: 282, directories:0 | -| **Searching** | | `a` as `photo` | 2.7 ms | - | -| **Searching** | | `a` as `video` | 3.5 ms | - | -| **Searching** | | `a` as `any` | 36.9 ms | media: 339, directories:0 | -| **Instant search** | | `a` | 5.4 ms | media: 10, directories:0 | -| **Auto complete** | | `a` | 6.7 ms | items: 10 | -*Measurements run 2 times and an average was calculated. - - -## PiGallery2 v1.8.2, 30.12.2020 -**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm - -**Gallery**: directories: 0 media: 341, faces: 65 - -| Action | Sub action | Action details | Average Duration | Details | -|:------:|:----------:|:--------------:|:----------------:|:-------:| -| Scanning directory | | | 1746.5 ms | media: 341, directories:0 | -| Saving directory to DB | | | 994.1 ms | - | -| Scanning directory | | | 65.3 ms | media: 341, directories:0 | -| Listing Faces | | | 19.0 ms | items: 1 | -| | List Persons | | 1.9 ms | items: 1 | -| | Add sample photo | | 16.6 ms | items: 1 | -| | Add thumbnail info | | 0.3 ms | items: 1 | -| | Remove sample photo | | 0.0 ms | items: 1 | -| Searching | | `a` as `directory` | 4.1 ms | - | -| Searching | | `a` as `person` | 16.1 ms | media: 65, directories:0 | -| Searching | | `a` as `keyword` | 41.6 ms | media: 339, directories:0 | -| Searching | | `a` as `position` | 67.1 ms | media: 282, directories:0 | -| Searching | | `a` as `photo` | 5.4 ms | - | -| Searching | | `a` as `video` | 4.3 ms | - | -| Searching | | `a` as `any` | 53.5 ms | media: 339, directories:0 | -| Instant search | | `a` | 5.3 ms | media: 10, directories:0 | -| Auto complete | | `a` | 7.2 ms | items: 10 | -*Measurements run 2 times and an average was calculated. - - ## PiGallery2 v1.5.8, 26.01.2019 **System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm diff --git a/benchmark/index.ts b/benchmark/index.ts index c73cb165..62e017a1 100644 --- a/benchmark/index.ts +++ b/benchmark/index.ts @@ -3,10 +3,7 @@ import {ProjectPath} from '../src/backend/ProjectPath'; import {BenchmarkResult, BenchmarkRunner} from './BenchmarkRunner'; import {SearchTypes} from '../src/common/entities/AutoCompleteItem'; import {Utils} from '../src/common/Utils'; -import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker'; import {BMConfig} from './BMConfig'; -import {GalleryManager} from '../src/backend/model/database/sql/GalleryManager'; -import {PersonManager} from '../src/backend/model/database/sql/PersonManager'; Config.Server.Media.folder = BMConfig.path; @@ -18,22 +15,15 @@ const printLine = (text: string) => { resultsText += text + '\n'; }; -const printHeader = async () => { + +const printHeader = async (statistic: string) => { const dt = new Date(); printLine('## PiGallery2 v' + require('./../package.json').version + ', ' + Utils.zeroPrefix(dt.getDate(), 2) + '.' + Utils.zeroPrefix(dt.getMonth() + 1, 2) + '.' + dt.getFullYear()); printLine('**System**: ' + BMConfig.system); - const dir = await DiskMangerWorker.scanDirectory('./'); - const gm = new GalleryManager(); - const pm = new PersonManager(); - printLine('\n**Gallery**: directories: ' + - dir.directories.length + - ' media: ' + dir.media.length + - // @ts-ignore - ', all persons: ' + dir.media.reduce((p, c) => p + (c.metadata.faces || []).length, 0) + - ',unique persons (faces): ' + (await pm.getAll()).length + '\n'); + printLine('\n**Gallery**: ' + statistic + '\n'); }; @@ -60,8 +50,8 @@ const printResult = (result: BenchmarkResult, actionDetails: string = '', isSubR printLine('| | ' + result.name + ' | ' + actionDetails + ' | ' + (result.duration).toFixed(1) + ' ms | ' + details + ' |'); } else { - printLine('| **' + result.name + '** | | ' + actionDetails + - ' | ' + (result.duration).toFixed(1) + ' ms | ' + details + ' |'); + printLine('| **' + result.name + '** | | **' + actionDetails + + '** | **' + (result.duration).toFixed(1) + ' ms** | **' + details + '** |'); } if (result.subBenchmarks && result.subBenchmarks.length > 1) { for (let i = 0; i < result.subBenchmarks.length; i++) { @@ -76,7 +66,7 @@ const run = async () => { const bm = new BenchmarkRunner(RUNS); // header - await printHeader(); + await printHeader(await bm.getStatistic()); printTableHeader(); printResult(await bm.bmScanDirectory()); diff --git a/src/backend/middlewares/admin/AdminMWs.ts b/src/backend/middlewares/admin/AdminMWs.ts index a2fc04f1..3596b261 100644 --- a/src/backend/middlewares/admin/AdminMWs.ts +++ b/src/backend/middlewares/admin/AdminMWs.ts @@ -4,6 +4,7 @@ import {ObjectManagers} from '../../model/ObjectManagers'; import {Config} from '../../../common/config/private/Config'; import {ISQLGalleryManager} from '../../model/database/sql/IGalleryManager'; import {ServerConfig} from '../../../common/config/private/PrivateConfig'; +import {ISQLPersonManager} from '../../model/database/sql/IPersonManager'; export class AdminMWs { @@ -15,12 +16,14 @@ export class AdminMWs { const galleryManager = ObjectManagers.getInstance().GalleryManager; + const personManager = ObjectManagers.getInstance().PersonManager; try { req.resultPipe = { directories: await galleryManager.countDirectories(), photos: await galleryManager.countPhotos(), videos: await galleryManager.countVideos(), diskUsage: await galleryManager.countMediaSize(), + faces: await personManager.countFaces(), }; return next(); } catch (err) { diff --git a/src/backend/model/database/sql/GalleryManager.ts b/src/backend/model/database/sql/GalleryManager.ts index 59faf04b..6d66fcda 100644 --- a/src/backend/model/database/sql/GalleryManager.ts +++ b/src/backend/model/database/sql/GalleryManager.ts @@ -156,6 +156,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { return sum || 0; } + async countPhotos(): Promise { const connection = await SQLConnection.getConnection(); return await connection.getRepository(PhotoEntity) diff --git a/src/backend/model/database/sql/IPersonManager.ts b/src/backend/model/database/sql/IPersonManager.ts new file mode 100644 index 00000000..82fb0449 --- /dev/null +++ b/src/backend/model/database/sql/IPersonManager.ts @@ -0,0 +1,5 @@ +import {IPersonManager} from '../interfaces/IPersonManager'; + +export interface ISQLPersonManager extends IPersonManager { + countFaces(): Promise; +} diff --git a/src/backend/model/database/sql/PersonManager.ts b/src/backend/model/database/sql/PersonManager.ts index 851b8741..cf04e238 100644 --- a/src/backend/model/database/sql/PersonManager.ts +++ b/src/backend/model/database/sql/PersonManager.ts @@ -1,4 +1,3 @@ -import {IPersonManager} from '../interfaces/IPersonManager'; import {SQLConnection} from './SQLConnection'; import {PersonEntry} from './enitites/PersonEntry'; import {PhotoDTO} from '../../../../common/entities/PhotoDTO'; @@ -7,9 +6,10 @@ import {FaceRegionEntry} from './enitites/FaceRegionEntry'; import {PersonDTO} from '../../../../common/entities/PersonDTO'; import {Utils} from '../../../../common/Utils'; import {SelectQueryBuilder} from 'typeorm'; +import {ISQLPersonManager} from './IPersonManager'; -export class PersonManager implements IPersonManager { +export class PersonManager implements ISQLPersonManager { samplePhotos: { [key: string]: PhotoDTO } = {}; persons: PersonEntry[] = []; @@ -96,6 +96,13 @@ export class PersonManager implements IPersonManager { } + async countFaces(): Promise { + const connection = await SQLConnection.getConnection(); + return await connection.getRepository(FaceRegionEntry) + .createQueryBuilder('faceRegion') + .getCount(); + } + async get(name: string): Promise { let person = this.persons.find(p => p.name === name);