improving benchmark

benchmark/BenchmarkRunner.ts

@@ -4,6 +4,7 @@ import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker'
 import {IndexingManager} from '../src/backend/model/database/sql/IndexingManager';
 import {SearchManager} from '../src/backend/model/database/sql/SearchManager';
 import * as util from 'util';
+import * as path from 'path';
 import * as rimraf from 'rimraf';
 import {SearchTypes} from '../src/common/entities/AutoCompleteItem';
 import {Utils} from '../src/common/Utils';
@@ -20,6 +21,8 @@ import {JobProgress} from '../src/backend/model/jobs/jobs/JobProgress';
 import {GalleryMWs} from '../src/backend/middlewares/GalleryMWs';
 import {UserDTO, UserRoles} from '../src/common/entities/UserDTO';
 import {ContentWrapper} from '../src/common/entities/ConentWrapper';
+import {GalleryManager} from '../src/backend/model/database/sql/GalleryManager';
+import {PersonManager} from '../src/backend/model/database/sql/PersonManager';
 
 const rimrafPR = util.promisify(rimraf);
 
@@ -39,15 +42,17 @@ export class BMIndexingManager extends IndexingManager {
 }
 
 export class BenchmarkRunner {
-
+  inited = false;
+  private biggestPath: string = null;
 
   constructor(public RUNS: number) {
 
   }
 
   async bmSaveDirectory(): Promise<BenchmarkResult> {
+    await this.init();
     await this.resetDB();
-    const dir = await DiskMangerWorker.scanDirectory('./');
+    const dir = await DiskMangerWorker.scanDirectory(this.biggestPath);
     const bm = new Benchmark('Saving directory to DB', null, () => this.resetDB());
     bm.addAStep({
       name: 'Saving directory to DB',
@@ -60,15 +65,17 @@ export class BenchmarkRunner {
   }
 
   async bmScanDirectory(): Promise<BenchmarkResult> {
+    await this.init();
     const bm = new Benchmark('Scanning directory');
     bm.addAStep({
       name: 'Scanning directory',
-      fn: async () => new ContentWrapper(await DiskMangerWorker.scanDirectory('./'))
+      fn: async () => new ContentWrapper(await DiskMangerWorker.scanDirectory(this.biggestPath))
     });
     return await bm.run(this.RUNS);
   }
 
   async bmListDirectory(): Promise<BenchmarkResult> {
+    await this.init();
     await this.setupDB();
     Config.Server.Indexing.reIndexingSensitivity = ServerConfig.ReIndexingSensitivity.low;
     const bm = new Benchmark('List directory',
@@ -79,7 +86,7 @@ export class BenchmarkRunner {
       });
     bm.addAStep({
       name: 'List directory',
-      fn: (input) => this.nextToPromise(GalleryMWs.listDirectory, input, {directory: '/'})
+      fn: (input) => this.nextToPromise(GalleryMWs.listDirectory, input, {directory: this.biggestPath})
     });
     bm.addAStep({
       name: 'Add Thumbnail information',
@@ -163,6 +170,54 @@ export class BenchmarkRunner {
     return await bm.run(this.RUNS);
   }
 
+  async getStatistic() {
+    await this.setupDB();
+    const gm = new GalleryManager();
+    const pm = new PersonManager();
+
+    const renderDataSize = (size: number) => {
+      const postFixes = ['B', 'KB', 'MB', 'GB', 'TB'];
+      let index = 0;
+      while (size > 1000 && index < postFixes.length - 1) {
+        size /= 1000;
+        index++;
+      }
+      return size.toFixed(2) + postFixes[index];
+    };
+    return 'directories: ' + await gm.countDirectories() +
+      ', photos: ' + await gm.countPhotos() +
+      ', videos: ' + await gm.countVideos() +
+      ', diskUsage : ' + renderDataSize(await gm.countMediaSize()) +
+      ', persons : ' + await pm.countFaces() +
+      ', unique persons (faces): ' + (await pm.getAll()).length;
+
+  }
+
+  private async init() {
+    if (this.inited === false) {
+      await this.setupDB();
+
+      const gm = new GalleryManager();
+      let biggest = 0;
+      let biggestPath = '/';
+      const queue = ['/'];
+      while (queue.length > 0) {
+        const dirPath = queue.shift();
+        const dir = await gm.listDirectory(dirPath);
+        dir.directories.forEach(d => queue.push(path.join(d.path + d.name)));
+        if (biggest < dir.media.length) {
+          biggestPath = path.join(dir.path + dir.name);
+          biggest = dir.media.length;
+        }
+      }
+      this.biggestPath = biggestPath;
+      console.log('updating path of biggest dir to: ' + this.biggestPath);
+      this.inited = true;
+    }
+    return this.biggestPath;
+
+  }
+
   private nextToPromise(fn: (req: any, res: any, next: Function) => void, input?: any, params = {}) {
     return new Promise<void>((resolve, reject) => {
       const request = {
@@ -211,5 +266,4 @@ export class BenchmarkRunner {
       }
     });
   }
-
 }

benchmark/README.md

@@ -2,93 +2,6 @@
 
 These results are created mostly for development, but the results are public for curious users.
 
-## PiGallery2 v1.8.2, 30.12.2020
-**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm
-
-**Gallery**: directories: 0 media: 341, faces: 65
-
-| Action | Sub action | Action details | Average Duration | Details |
-|:------:|:----------:|:--------------:|:----------------:|:-------:|
-| **Scanning directory** | |  | 1526.7 ms | media: 341, directories:0 |
-| **Saving directory to DB** | |  | 679.9 ms | - |
-| **List directory** | |  | 61.1 ms | media: 341, directories:0 |
-| | List directory |  | 39.7 ms | media: 341, directories:0 |
-| | Add Thumbnail information |  | 19.3 ms | media: 341, directories:0 |
-| | Clean Up Gallery Result |  | 2.0 ms | media: 341, directories:0 |
-| **Listing Faces** | |  | 8.4 ms | items: 1 |
-| | List Persons |  | 1.0 ms | items: 1 |
-| | Add sample photo |  | 7.1 ms | items: 1 |
-| | Add thumbnail info |  | 0.1 ms | items: 1 |
-| | Remove sample photo |  | 0.0 ms | items: 1 |
-| **Searching** | | `a` as `directory` | 3.3 ms | media: 0, directories:0 |
-| **Searching** | | `a` as `person` | 11.6 ms | media: 65, directories:0 |
-| **Searching** | | `a` as `keyword` | 38.0 ms | media: 339, directories:0 |
-| **Searching** | | `a` as `position` | 31.7 ms | media: 282, directories:0 |
-| **Searching** | | `a` as `photo` | 3.2 ms | media: 0, directories:0 |
-| **Searching** | | `a` as `video` | 3.2 ms | media: 0, directories:0 |
-| **Searching** | | `a` as `any` | 39.3 ms | media: 339, directories:0 |
-| **Instant search** | | `a` | 6.7 ms | media: 10, directories:0 |
-| **Auto complete** | | `a` | 6.7 ms | items: 10 |
-*Measurements run 2 times and an average was calculated.
-
-
-## PiGallery2 v1.8.2, 30.12.2020
-**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm
-
-**Gallery**: directories: 0 media: 341, faces: 65
-
-| Action | Sub action | Action details | Average Duration | Details |
-|:------:|:----------:|:--------------:|:----------------:|:-------:|
-| **Scanning directory** | |  | 1459.3 ms | media: 341, directories:0 |
-| **Saving directory to DB** | |  | 654.9 ms | - |
-| **List directory** | |  | 70.4 ms | - |
-| | List directory |  | 42.1 ms | - |
-| | Add Thumbnail information |  | 25.8 ms | - |
-| | Clean Up Gallery Result |  | 2.3 ms | - |
-| **Listing Faces** | |  | 10.5 ms | items: 1 |
-| | List Persons |  | 1.0 ms | items: 1 |
-| | Add sample photo |  | 9.1 ms | items: 1 |
-| | Add thumbnail info |  | 0.2 ms | items: 1 |
-| | Remove sample photo |  | 0.0 ms | items: 1 |
-| **Searching** | | `a` as `directory` | 3.3 ms | - |
-| **Searching** | | `a` as `person` | 11.7 ms | media: 65, directories:0 |
-| **Searching** | | `a` as `keyword` | 40.4 ms | media: 339, directories:0 |
-| **Searching** | | `a` as `position` | 30.4 ms | media: 282, directories:0 |
-| **Searching** | | `a` as `photo` | 2.7 ms | - |
-| **Searching** | | `a` as `video` | 3.5 ms | - |
-| **Searching** | | `a` as `any` | 36.9 ms | media: 339, directories:0 |
-| **Instant search** | | `a` | 5.4 ms | media: 10, directories:0 |
-| **Auto complete** | | `a` | 6.7 ms | items: 10 |
-*Measurements run 2 times and an average was calculated.
-
-
-## PiGallery2 v1.8.2, 30.12.2020
-**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm
-
-**Gallery**: directories: 0 media: 341, faces: 65
-
-| Action | Sub action | Action details | Average Duration | Details |
-|:------:|:----------:|:--------------:|:----------------:|:-------:|
-| Scanning directory | |  | 1746.5 ms | media: 341, directories:0 |
-| Saving directory to DB | |  | 994.1 ms | - |
-| Scanning directory | |  | 65.3 ms | media: 341, directories:0 |
-| Listing Faces | |  | 19.0 ms | items: 1 |
-| | List Persons |  | 1.9 ms | items: 1 |
-| | Add sample photo |  | 16.6 ms | items: 1 |
-| | Add thumbnail info |  | 0.3 ms | items: 1 |
-| | Remove sample photo |  | 0.0 ms | items: 1 |
-| Searching | | `a` as `directory` | 4.1 ms | - |
-| Searching | | `a` as `person` | 16.1 ms | media: 65, directories:0 |
-| Searching | | `a` as `keyword` | 41.6 ms | media: 339, directories:0 |
-| Searching | | `a` as `position` | 67.1 ms | media: 282, directories:0 |
-| Searching | | `a` as `photo` | 5.4 ms | - |
-| Searching | | `a` as `video` | 4.3 ms | - |
-| Searching | | `a` as `any` | 53.5 ms | media: 339, directories:0 |
-| Instant search | | `a` | 5.3 ms | media: 10, directories:0 |
-| Auto complete | | `a` | 7.2 ms | items: 10 |
-*Measurements run 2 times and an average was calculated.
-
-
 ## PiGallery2 v1.5.8, 26.01.2019
 
 **System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm

benchmark/index.ts

@@ -3,10 +3,7 @@ import {ProjectPath} from '../src/backend/ProjectPath';
 import {BenchmarkResult, BenchmarkRunner} from './BenchmarkRunner';
 import {SearchTypes} from '../src/common/entities/AutoCompleteItem';
 import {Utils} from '../src/common/Utils';
-import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker';
 import {BMConfig} from './BMConfig';
-import {GalleryManager} from '../src/backend/model/database/sql/GalleryManager';
-import {PersonManager} from '../src/backend/model/database/sql/PersonManager';
 
 
 Config.Server.Media.folder = BMConfig.path;
@@ -18,22 +15,15 @@ const printLine = (text: string) => {
   resultsText += text + '\n';
 };
 
-const printHeader = async () => {
+
+const printHeader = async (statistic: string) => {
   const dt = new Date();
   printLine('## PiGallery2 v' + require('./../package.json').version +
     ', ' + Utils.zeroPrefix(dt.getDate(), 2) +
     '.' + Utils.zeroPrefix(dt.getMonth() + 1, 2) +
     '.' + dt.getFullYear());
   printLine('**System**: ' + BMConfig.system);
-  const dir = await DiskMangerWorker.scanDirectory('./');
-  const gm = new GalleryManager();
-  const pm = new PersonManager();
-  printLine('\n**Gallery**: directories: ' +
-    dir.directories.length +
-    ' media: ' + dir.media.length +
-    // @ts-ignore
-    ', all persons: ' + dir.media.reduce((p, c) => p + (c.metadata.faces || []).length, 0) +
-    ',unique persons (faces): ' + (await pm.getAll()).length + '\n');
+  printLine('\n**Gallery**: ' + statistic + '\n');
 };
 
 
@@ -60,8 +50,8 @@ const printResult = (result: BenchmarkResult, actionDetails: string = '', isSubR
     printLine('| | ' + result.name + ' | ' + actionDetails +
       ' | ' + (result.duration).toFixed(1) + ' ms | ' + details + ' |');
   } else {
-    printLine('| **' + result.name + '** | | ' + actionDetails +
-      ' | ' + (result.duration).toFixed(1) + ' ms | ' + details + ' |');
+    printLine('| **' + result.name + '** | | **' + actionDetails +
+      '** | **' + (result.duration).toFixed(1) + ' ms** | **' + details + '** |');
   }
   if (result.subBenchmarks && result.subBenchmarks.length > 1) {
     for (let i = 0; i < result.subBenchmarks.length; i++) {
@@ -76,7 +66,7 @@ const run = async () => {
   const bm = new BenchmarkRunner(RUNS);
 
   // header
-  await printHeader();
+  await printHeader(await bm.getStatistic());
   printTableHeader();
 
   printResult(await bm.bmScanDirectory());

src/backend/middlewares/admin/AdminMWs.ts

@@ -4,6 +4,7 @@ import {ObjectManagers} from '../../model/ObjectManagers';
 import {Config} from '../../../common/config/private/Config';
 import {ISQLGalleryManager} from '../../model/database/sql/IGalleryManager';
 import {ServerConfig} from '../../../common/config/private/PrivateConfig';
+import {ISQLPersonManager} from '../../model/database/sql/IPersonManager';
 
 
 export class AdminMWs {
@@ -15,12 +16,14 @@ export class AdminMWs {
 
 
     const galleryManager = <ISQLGalleryManager>ObjectManagers.getInstance().GalleryManager;
+    const personManager = <ISQLPersonManager>ObjectManagers.getInstance().PersonManager;
     try {
       req.resultPipe = {
         directories: await galleryManager.countDirectories(),
         photos: await galleryManager.countPhotos(),
         videos: await galleryManager.countVideos(),
         diskUsage: await galleryManager.countMediaSize(),
+        faces: await personManager.countFaces(),
       };
       return next();
     } catch (err) {

src/backend/model/database/sql/GalleryManager.ts

@@ -156,6 +156,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
     return sum || 0;
   }
 
+
   async countPhotos(): Promise<number> {
     const connection = await SQLConnection.getConnection();
     return await connection.getRepository(PhotoEntity)

src/backend/model/database/sql/IPersonManager.ts

@@ -0,0 +1,5 @@
+import {IPersonManager} from '../interfaces/IPersonManager';
+
+export interface ISQLPersonManager extends IPersonManager {
+  countFaces(): Promise<number>;
+}

src/backend/model/database/sql/PersonManager.ts

@@ -1,4 +1,3 @@
-import {IPersonManager} from '../interfaces/IPersonManager';
 import {SQLConnection} from './SQLConnection';
 import {PersonEntry} from './enitites/PersonEntry';
 import {PhotoDTO} from '../../../../common/entities/PhotoDTO';
@@ -7,9 +6,10 @@ import {FaceRegionEntry} from './enitites/FaceRegionEntry';
 import {PersonDTO} from '../../../../common/entities/PersonDTO';
 import {Utils} from '../../../../common/Utils';
 import {SelectQueryBuilder} from 'typeorm';
+import {ISQLPersonManager} from './IPersonManager';
 
 
-export class PersonManager implements IPersonManager {
+export class PersonManager implements ISQLPersonManager {
   samplePhotos: { [key: string]: PhotoDTO } = {};
   persons: PersonEntry[] = [];
 
@@ -96,6 +96,13 @@ export class PersonManager implements IPersonManager {
   }
 
 
+  async countFaces(): Promise<number> {
+    const connection = await SQLConnection.getConnection();
+    return await connection.getRepository(FaceRegionEntry)
+      .createQueryBuilder('faceRegion')
+      .getCount();
+  }
+
   async get(name: string): Promise<PersonEntry> {
 
     let person = this.persons.find(p => p.name === name);