improving sql, fixing tests

backend/middlewares/AdminMWs.ts

@@ -399,7 +399,7 @@ export class AdminMWs {
   public static startIndexing(req: Request, res: Response, next: NextFunction) {
     try {
       const createThumbnails: boolean = (<IndexingDTO>req.body).createThumbnails || false;
-      ObjectManagerRepository.getInstance().IndexingManager.startIndexing(createThumbnails);
+      ObjectManagerRepository.getInstance().IndexingTaskManager.startIndexing(createThumbnails);
       req.resultPipe = 'ok';
       return next();
     } catch (err) {
@@ -413,7 +413,7 @@ export class AdminMWs {
 
   public static getIndexingProgress(req: Request, res: Response, next: NextFunction) {
     try {
-      req.resultPipe = ObjectManagerRepository.getInstance().IndexingManager.getProgress();
+      req.resultPipe = ObjectManagerRepository.getInstance().IndexingTaskManager.getProgress();
       return next();
     } catch (err) {
       if (err instanceof Error) {
@@ -425,7 +425,7 @@ export class AdminMWs {
 
   public static cancelIndexing(req: Request, res: Response, next: NextFunction) {
     try {
-      ObjectManagerRepository.getInstance().IndexingManager.cancelIndexing();
+      ObjectManagerRepository.getInstance().IndexingTaskManager.cancelIndexing();
       req.resultPipe = 'ok';
       return next();
     } catch (err) {
@@ -438,7 +438,7 @@ export class AdminMWs {
 
   public static async resetIndexes(req: Express.Request, res: Response, next: NextFunction) {
     try {
-      await ObjectManagerRepository.getInstance().IndexingManager.reset();
+      await ObjectManagerRepository.getInstance().IndexingTaskManager.reset();
       req.resultPipe = 'ok';
       return next();
     } catch (err) {

backend/model/ObjectManagerRepository.ts

@@ -4,7 +4,9 @@ import {ISearchManager} from './interfaces/ISearchManager';
 import {SQLConnection} from './sql/SQLConnection';
 import {ISharingManager} from './interfaces/ISharingManager';
 import {Logger} from '../Logger';
+import {IIndexingTaskManager} from './interfaces/IIndexingTaskManager';
 import {IIndexingManager} from './interfaces/IIndexingManager';
+import {IPersonManager} from './interfaces/IPersonManager';
 
 export class ObjectManagerRepository {
 
@@ -15,6 +17,16 @@ export class ObjectManagerRepository {
   private _searchManager: ISearchManager;
   private _sharingManager: ISharingManager;
   private _indexingManager: IIndexingManager;
+  private _indexingTaskManager: IIndexingTaskManager;
+  private _personManager: IPersonManager;
+
+  get PersonManager(): IPersonManager {
+    return this._personManager;
+  }
+
+  set PersonManager(value: IPersonManager) {
+    this._personManager = value;
+  }
 
   get IndexingManager(): IIndexingManager {
     return this._indexingManager;
@@ -24,19 +36,14 @@ export class ObjectManagerRepository {
     this._indexingManager = value;
   }
 
-  public static getInstance() {
+  get IndexingTaskManager(): IIndexingTaskManager {
-    if (this._instance === null) {
+    return this._indexingTaskManager;
-      this._instance = new ObjectManagerRepository();
-    }
-    return this._instance;
   }
 
-  public static async reset() {
+  set IndexingTaskManager(value: IIndexingTaskManager) {
-    await SQLConnection.close();
+    this._indexingTaskManager = value;
-    this._instance = null;
   }
 
-
   get GalleryManager(): IGalleryManager {
     return this._galleryManager;
   }
@@ -69,18 +76,34 @@ export class ObjectManagerRepository {
     this._sharingManager = value;
   }
 
+  public static getInstance() {
+    if (this._instance === null) {
+      this._instance = new ObjectManagerRepository();
+    }
+    return this._instance;
+  }
+
+  public static async reset() {
+    await SQLConnection.close();
+    this._instance = null;
+  }
+
   public static async InitMemoryManagers() {
     await ObjectManagerRepository.reset();
     const GalleryManager = require('./memory/GalleryManager').GalleryManager;
     const UserManager = require('./memory/UserManager').UserManager;
     const SearchManager = require('./memory/SearchManager').SearchManager;
     const SharingManager = require('./memory/SharingManager').SharingManager;
+    const IndexingTaskManager = require('./memory/IndexingTaskManager').IndexingTaskManager;
     const IndexingManager = require('./memory/IndexingManager').IndexingManager;
+    const PersonManager = require('./memory/PersonManager').PersonManager;
     ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
     ObjectManagerRepository.getInstance().UserManager = new UserManager();
     ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
     ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
+    ObjectManagerRepository.getInstance().IndexingTaskManager = new IndexingTaskManager();
     ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager();
+    ObjectManagerRepository.getInstance().PersonManager = new PersonManager();
   }
 
   public static async InitSQLManagers() {
@@ -90,12 +113,16 @@ export class ObjectManagerRepository {
     const UserManager = require('./sql/UserManager').UserManager;
     const SearchManager = require('./sql/SearchManager').SearchManager;
     const SharingManager = require('./sql/SharingManager').SharingManager;
+    const IndexingTaskManager = require('./sql/IndexingManager').IndexingTaskManager;
     const IndexingManager = require('./sql/IndexingManager').IndexingManager;
+    const PersonManager = require('./sql/PersonManager').PersonManager;
     ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
     ObjectManagerRepository.getInstance().UserManager = new UserManager();
     ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
     ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
+    ObjectManagerRepository.getInstance().IndexingTaskManager = new IndexingTaskManager();
     ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager();
+    ObjectManagerRepository.getInstance().PersonManager = new PersonManager();
     Logger.debug('SQL DB inited');
   }
 

backend/model/interfaces/IIndexingManager.ts

@@ -1,11 +1,5 @@
-import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
+import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
 
 export interface IIndexingManager {
-  startIndexing(createThumbnails?: boolean): void;
+  indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO>;
-
-  getProgress(): IndexingProgressDTO;
-
-  cancelIndexing(): void;
-
-  reset(): Promise<void>;
 }

backend/model/interfaces/IIndexingTaskManager.ts

@@ -0,0 +1,11 @@
+import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
+
+export interface IIndexingTaskManager {
+  startIndexing(createThumbnails?: boolean): void;
+
+  getProgress(): IndexingProgressDTO;
+
+  cancelIndexing(): void;
+
+  reset(): Promise<void>;
+}

backend/model/interfaces/IPersonManager.ts

@@ -0,0 +1,7 @@
+import {PersonEntry} from '../sql/enitites/PersonEntry';
+
+export interface IPersonManager {
+  get(name: string): Promise<PersonEntry>;
+
+  saveAll(names: string[]): Promise<void>;
+}

backend/model/memory/IndexingManager.ts

@@ -1,21 +1,11 @@
 import {IIndexingManager} from '../interfaces/IIndexingManager';
-import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
+import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
 
 export class IndexingManager implements IIndexingManager {
 
-  startIndexing(): void {
+  indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
     throw new Error('not supported by memory DB');
   }
 
-  getProgress(): IndexingProgressDTO {
-    throw new Error('not supported by memory DB');
-  }
 
-  cancelIndexing(): void {
-    throw new Error('not supported by memory DB');
-  }
-
-  reset(): Promise<void> {
-    throw new Error('Method not implemented.');
-  }
 }

backend/model/memory/IndexingTaskManager.ts

@@ -0,0 +1,21 @@
+import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager';
+import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
+
+export class IndexingTaskManager implements IIndexingTaskManager {
+
+  startIndexing(): void {
+    throw new Error('not supported by memory DB');
+  }
+
+  getProgress(): IndexingProgressDTO {
+    throw new Error('not supported by memory DB');
+  }
+
+  cancelIndexing(): void {
+    throw new Error('not supported by memory DB');
+  }
+
+  reset(): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+}

backend/model/memory/PersonManager.ts

@@ -0,0 +1,11 @@
+import {IPersonManager} from '../interfaces/IPersonManager';
+
+export class IndexingTaskManager implements IPersonManager {
+  get(name: string): Promise<any> {
+    throw new Error('not supported by memory DB');
+  }
+
+  saveAll(names: string[]): Promise<void> {
+    throw new Error('not supported by memory DB');
+  }
+}

backend/model/sql/GalleryManager.ts

@@ -4,94 +4,25 @@ import * as path from 'path';
 import * as fs from 'fs';
 import {DirectoryEntity} from './enitites/DirectoryEntity';
 import {SQLConnection} from './SQLConnection';
-import {DiskManager} from '../DiskManger';
+import {PhotoEntity} from './enitites/PhotoEntity';
-import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity';
-import {Utils} from '../../../common/Utils';
 import {ProjectPath} from '../../ProjectPath';
 import {Config} from '../../../common/config/private/Config';
 import {ISQLGalleryManager} from './IGalleryManager';
 import {DatabaseType, ReIndexingSensitivity} from '../../../common/config/private/IPrivateConfig';
-import {FaceRegion, PhotoDTO, PhotoMetadata} from '../../../common/entities/PhotoDTO';
+import {PhotoDTO} from '../../../common/entities/PhotoDTO';
 import {OrientationType} from '../../../common/entities/RandomQueryDTO';
-import {Brackets, Connection, Transaction, TransactionRepository, Repository} from 'typeorm';
+import {Brackets, Connection} from 'typeorm';
 import {MediaEntity} from './enitites/MediaEntity';
-import {MediaDTO} from '../../../common/entities/MediaDTO';
 import {VideoEntity} from './enitites/VideoEntity';
-import {FileEntity} from './enitites/FileEntity';
-import {FileDTO} from '../../../common/entities/FileDTO';
-import {NotificationManager} from '../NotifocationManager';
 import {DiskMangerWorker} from '../threading/DiskMangerWorker';
 import {Logger} from '../../Logger';
 import {FaceRegionEntry} from './enitites/FaceRegionEntry';
-import {PersonEntry} from './enitites/PersonEntry';
+import {ObjectManagerRepository} from '../ObjectManagerRepository';
 
 const LOG_TAG = '[GalleryManager]';
 
 export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
-  private savingQueue: DirectoryDTO[] = [];
-  private isSaving = false;
-
-  protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<DirectoryEntity> {
-    const query = connection
-      .getRepository(DirectoryEntity)
-      .createQueryBuilder('directory')
-      .where('directory.name = :name AND directory.path = :path', {
-        name: directoryName,
-        path: directoryParent
-      })
-      .leftJoinAndSelect('directory.directories', 'directories')
-      .leftJoinAndSelect('directory.media', 'media');
-
-    if (Config.Client.MetaFile.enabled === true) {
-      query.leftJoinAndSelect('directory.metaFile', 'metaFile');
-    }
-
-    return await query.getOne();
-  }
-
-  protected async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise<void> {
-    if (dir.media) {
-      const indexedFaces = await connection.getRepository(FaceRegionEntry)
-        .createQueryBuilder('face')
-        .leftJoinAndSelect('face.media', 'media')
-        .where('media.directory = :directory', {
-          directory: dir.id
-        })
-        .leftJoinAndSelect('face.person', 'person')
-        .getMany();
-      for (let i = 0; i < dir.media.length; i++) {
-        dir.media[i].directory = dir;
-        dir.media[i].readyThumbnails = [];
-        dir.media[i].readyIcon = false;
-        (<PhotoDTO>dir.media[i]).metadata.faces = indexedFaces
-          .filter(fe => fe.media.id === dir.media[i].id)
-          .map(f => ({box: f.box, name: f.person.name}));
-      }
-
-    }
-    if (dir.directories) {
-      for (let i = 0; i < dir.directories.length; i++) {
-        dir.directories[i].media = await connection
-          .getRepository(MediaEntity)
-          .createQueryBuilder('media')
-          .where('media.directory = :dir', {
-            dir: dir.directories[i].id
-          })
-          .orderBy('media.metadata.creationDate', 'ASC')
-          .limit(Config.Server.indexing.folderPreviewSize)
-          .getMany();
-        dir.directories[i].isPartial = true;
-
-        for (let j = 0; j < dir.directories[i].media.length; j++) {
-          dir.directories[i].media[j].directory = dir.directories[i];
-          dir.directories[i].media[j].readyThumbnails = [];
-          dir.directories[i].media[j].readyIcon = false;
-        }
-      }
-    }
-  }
-
 
   public async listDirectory(relativeDirectoryName: string,
                              knownLastModified?: number,
@@ -124,7 +55,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
       if (dir.lastModified !== lastModified) {
         Logger.silly(LOG_TAG, 'Reindexing reason: lastModified mismatch: known: '
           + dir.lastModified + ', current:' + lastModified);
-        return this.indexDirectory(relativeDirectoryName);
+        return ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName);
       }
 
 
@@ -136,7 +67,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
         Logger.silly(LOG_TAG, 'lazy reindexing reason: cache timeout: lastScanned: '
           + (Date.now() - dir.lastScanned) + ', cachedFolderTimeout:' + Config.Server.indexing.cachedFolderTimeout);
-        this.indexDirectory(relativeDirectoryName).catch((err) => {
+        ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName).catch((err) => {
           console.error(err);
         });
       }
@@ -146,33 +77,11 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
     // never scanned (deep indexed), do it and return with it
     Logger.silly(LOG_TAG, 'Reindexing reason: never scanned');
-    return this.indexDirectory(relativeDirectoryName);
+    return ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName);
 
 
   }
 
-
-  public indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
-    return new Promise(async (resolve, reject) => {
-      try {
-        const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName);
-
-        // returning with the result
-        scannedDirectory.media.forEach(p => p.readyThumbnails = []);
-        resolve(scannedDirectory);
-
-        this.queueForSave(scannedDirectory).catch(console.error);
-
-      } catch (error) {
-        NotificationManager.warning('Unknown indexing error for: ' + relativeDirectoryName, error.toString());
-        console.error(error);
-        return reject(error);
-      }
-
-    });
-  }
-
-
   public async getRandomPhoto(queryFilter: RandomQuery): Promise<PhotoDTO> {
     const connection = await SQLConnection.getConnection();
     const photosRepository = connection.getRepository(PhotoEntity);
@@ -230,236 +139,6 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
   }
 
-  // Todo fix it, once typeorm support connection pools ofr sqlite
-  protected async queueForSave(scannedDirectory: DirectoryDTO) {
-    if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name &&
-      dir.path === scannedDirectory.path &&
-      dir.lastModified === scannedDirectory.lastModified &&
-      dir.lastScanned === scannedDirectory.lastScanned &&
-      (dir.media || dir.media.length) === (scannedDirectory.media || scannedDirectory.media.length) &&
-      (dir.metaFile || dir.metaFile.length) === (scannedDirectory.metaFile || scannedDirectory.metaFile.length)) !== -1) {
-      return;
-    }
-    this.savingQueue.push(scannedDirectory);
-    while (this.isSaving === false && this.savingQueue.length > 0) {
-      await this.saveToDB(this.savingQueue[0]);
-      this.savingQueue.shift();
-    }
-
-  }
-
-  protected async saveToDB(scannedDirectory: DirectoryDTO) {
-    console.log('saving');
-    this.isSaving = true;
-    try {
-      const connection = await SQLConnection.getConnection();
-
-      // saving to db
-      const directoryRepository = connection.getRepository(DirectoryEntity);
-      const mediaRepository = connection.getRepository(MediaEntity);
-      const fileRepository = connection.getRepository(FileEntity);
-
-
-      let currentDir: DirectoryEntity = await directoryRepository.createQueryBuilder('directory')
-        .where('directory.name = :name AND directory.path = :path', {
-          name: scannedDirectory.name,
-          path: scannedDirectory.path
-        }).getOne();
-      if (!!currentDir) {// Updated parent dir (if it was in the DB previously)
-        currentDir.lastModified = scannedDirectory.lastModified;
-        currentDir.lastScanned = scannedDirectory.lastScanned;
-        currentDir.mediaCount = scannedDirectory.mediaCount;
-        currentDir = await directoryRepository.save(currentDir);
-      } else {
-        currentDir = await directoryRepository.save(<DirectoryEntity>scannedDirectory);
-      }
-
-      // TODO: fix when first opened directory is not root
-      // save subdirectories
-      const childDirectories = await directoryRepository.createQueryBuilder('directory')
-        .where('directory.parent = :dir', {
-          dir: currentDir.id
-        }).getMany();
-
-      for (let i = 0; i < scannedDirectory.directories.length; i++) {
-        // Was this child Dir already indexed before?
-        let directory: DirectoryEntity = null;
-        for (let j = 0; j < childDirectories.length; j++) {
-          if (childDirectories[j].name === scannedDirectory.directories[i].name) {
-            directory = childDirectories[j];
-            childDirectories.splice(j, 1);
-            break;
-          }
-        }
-
-        if (directory != null) { // update existing directory
-          if (!directory.parent || !directory.parent.id) { // set parent if not set yet
-            directory.parent = currentDir;
-            delete directory.media;
-            await directoryRepository.save(directory);
-          }
-        } else { // dir does not exists yet
-          scannedDirectory.directories[i].parent = currentDir;
-          (<DirectoryEntity>scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet
-          const d = await directoryRepository.save(<DirectoryEntity>scannedDirectory.directories[i]);
-          for (let j = 0; j < scannedDirectory.directories[i].media.length; j++) {
-            scannedDirectory.directories[i].media[j].directory = d;
-          }
-
-          await this.saveMedia(connection, scannedDirectory.directories[i].media);
-        }
-      }
-
-      // Remove child Dirs that are not anymore in the parent dir
-      await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)});
-
-      // save media
-      const indexedMedia = (await mediaRepository.createQueryBuilder('media')
-        .where('media.directory = :dir', {
-          dir: currentDir.id
-        })
-        .getMany());
-
-
-      const mediaToSave = [];
-      for (let i = 0; i < scannedDirectory.media.length; i++) {
-        let media: MediaDTO = null;
-        for (let j = 0; j < indexedMedia.length; j++) {
-          if (indexedMedia[j].name === scannedDirectory.media[i].name) {
-            media = indexedMedia[j];
-            indexedMedia.splice(j, 1);
-            break;
-          }
-        }
-
-
-        if (media == null) { // not in DB yet
-          scannedDirectory.media[i].directory = null;
-          media = Utils.clone(scannedDirectory.media[i]);
-          scannedDirectory.media[i].directory = scannedDirectory;
-          media.directory = currentDir;
-          mediaToSave.push(media);
-        } else {
-          delete (<PhotoMetadata>media.metadata).faces;
-
-          if (!Utils.equalsFilter(media.metadata, scannedDirectory.media[i].metadata)) {
-            media.metadata = scannedDirectory.media[i].metadata;
-            mediaToSave.push(media);
-          }
-        }
-        const scannedFaces = (<PhotoMetadata>scannedDirectory.media[i].metadata).faces;
-        delete (<PhotoMetadata>scannedDirectory.media[i].metadata).faces;
-
-        const mediaEntry = await this.saveAMedia(connection, media);
-
-        await this.saveFaces(connection, mediaEntry, scannedFaces);
-      }
-      // await this.saveMedia(connection, mediaToSave);
-      await mediaRepository.remove(indexedMedia);
-
-
-      // save files
-      const indexedMetaFiles = await fileRepository.createQueryBuilder('file')
-        .where('file.directory = :dir', {
-          dir: currentDir.id
-        }).getMany();
-
-
-      const metaFilesToSave = [];
-      for (let i = 0; i < scannedDirectory.metaFile.length; i++) {
-        let metaFile: FileDTO = null;
-        for (let j = 0; j < indexedMetaFiles.length; j++) {
-          if (indexedMetaFiles[j].name === scannedDirectory.metaFile[i].name) {
-            metaFile = indexedMetaFiles[j];
-            indexedMetaFiles.splice(j, 1);
-            break;
-          }
-        }
-        if (metaFile == null) { // not in DB yet
-          scannedDirectory.metaFile[i].directory = null;
-          metaFile = Utils.clone(scannedDirectory.metaFile[i]);
-          scannedDirectory.metaFile[i].directory = scannedDirectory;
-          metaFile.directory = currentDir;
-          metaFilesToSave.push(metaFile);
-        }
-      }
-      await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)});
-      await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)});
-    } catch (e) {
-      throw e;
-    } finally {
-      this.isSaving = false;
-    }
-  }
-
-  protected async saveFaces(connection: Connection, media: MediaEntity, scannedFaces: FaceRegion[]) {
-
-    const faceRepository = connection.getRepository(FaceRegionEntry);
-    const personRepository = connection.getRepository(PersonEntry);
-    const indexedPersons = await personRepository.createQueryBuilder('person').getMany();
-
-    const indexedFaces = await faceRepository.createQueryBuilder('face')
-      .where('face.media = :media', {
-        media: media.id
-      })
-      .leftJoinAndSelect('face.person', 'person')
-      .getMany();
-
-
-    const getPerson = async (name: string) => {
-      let person = indexedPersons.find(p => p.name === name);
-      if (!person) {
-        person = <any>await personRepository.save({name: name});
-        indexedPersons.push(person);
-      }
-      return person;
-    };
-
-    const faceToSave = [];
-    for (let i = 0; i < scannedFaces.length; i++) {
-      let face: FaceRegionEntry = null;
-      for (let j = 0; j < indexedFaces.length; j++) {
-        if (indexedFaces[j].box.height === scannedFaces[i].box.height &&
-          indexedFaces[j].box.width === scannedFaces[i].box.width &&
-          indexedFaces[j].box.x === scannedFaces[i].box.x &&
-          indexedFaces[j].box.y === scannedFaces[i].box.y &&
-          indexedFaces[j].person.name === scannedFaces[i].name) {
-          face = indexedFaces[j];
-          indexedFaces.splice(j, 1);
-          break;
-        }
-      }
-
-      if (face == null) {
-        (<FaceRegionEntry>scannedFaces[i]).person = await getPerson(scannedFaces[i].name);
-        (<FaceRegionEntry>scannedFaces[i]).media = media;
-        //   console.log('inserting', (<FaceRegionEntry>scannedFaces[i]).person, (<FaceRegionEntry>scannedFaces[i]).media);
-        //  console.log('inserting', (<FaceRegionEntry>scannedFaces[i]).person.id, (<FaceRegionEntry>scannedFaces[i]).media.id);
-        faceToSave.push(scannedFaces[i]);
-      }
-    }
-    await faceRepository.save(faceToSave, {chunk: Math.max(Math.ceil(faceToSave.length / 500), 1)});
-    await faceRepository.remove(indexedFaces, {chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1)});
-
-  }
-
-  protected async saveAMedia(connection: Connection, media: MediaDTO): Promise<MediaEntity> {
-    if (MediaDTO.isPhoto(media)) {
-      return await <any>connection.getRepository(PhotoEntity).save(media);
-    }
-    return await <any>connection.getRepository(VideoEntity).save(media);
-  }
-
-  protected async saveMedia(connection: Connection, mediaList: MediaDTO[]): Promise<MediaEntity[]> {
-    const chunked = Utils.chunkArrays(mediaList, 100);
-    let list: MediaEntity[] = [];
-    for (let i = 0; i < chunked.length; i++) {
-      list = list.concat(await connection.getRepository(PhotoEntity).save(<PhotoEntity[]>chunked[i].filter(m => MediaDTO.isPhoto(m))));
-      list = list.concat(await connection.getRepository(VideoEntity).save(<VideoEntity[]>chunked[i].filter(m => MediaDTO.isVideo(m))));
-    }
-    return list;
-  }
-
   async countDirectories(): Promise<number> {
     const connection = await SQLConnection.getConnection();
     return await connection.getRepository(DirectoryEntity)
@@ -490,5 +169,68 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
       .getCount();
   }
 
+  protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<DirectoryEntity> {
+    const query = connection
+      .getRepository(DirectoryEntity)
+      .createQueryBuilder('directory')
+      .where('directory.name = :name AND directory.path = :path', {
+        name: directoryName,
+        path: directoryParent
+      })
+      .leftJoinAndSelect('directory.directories', 'directories')
+      .leftJoinAndSelect('directory.media', 'media');
+
+    if (Config.Client.MetaFile.enabled === true) {
+      query.leftJoinAndSelect('directory.metaFile', 'metaFile');
+    }
+
+    return await query.getOne();
+  }
+
+  protected async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise<void> {
+    if (dir.media) {
+      const indexedFaces = await connection.getRepository(FaceRegionEntry)
+        .createQueryBuilder('face')
+        .leftJoinAndSelect('face.media', 'media')
+        .where('media.directory = :directory', {
+          directory: dir.id
+        })
+        .leftJoinAndSelect('face.person', 'person')
+        .select(['face.id',  'face.box.x',
+          'face.box.y', 'face.box.width', 'face.box.height',
+          'media.id',  'person.name', 'person.id'])
+        .getMany();
+      for (let i = 0; i < dir.media.length; i++) {
+        dir.media[i].directory = dir;
+        dir.media[i].readyThumbnails = [];
+        dir.media[i].readyIcon = false;
+        (<PhotoDTO>dir.media[i]).metadata.faces = indexedFaces
+          .filter(fe => fe.media.id === dir.media[i].id)
+          .map(f => ({box: f.box, name: f.person.name}));
+      }
+
+    }
+    if (dir.directories) {
+      for (let i = 0; i < dir.directories.length; i++) {
+        dir.directories[i].media = await connection
+          .getRepository(MediaEntity)
+          .createQueryBuilder('media')
+          .where('media.directory = :dir', {
+            dir: dir.directories[i].id
+          })
+          .orderBy('media.metadata.creationDate', 'ASC')
+          .limit(Config.Server.indexing.folderPreviewSize)
+          .getMany();
+        dir.directories[i].isPartial = true;
+
+        for (let j = 0; j < dir.directories[i].media.length; j++) {
+          dir.directories[i].media[j].directory = dir.directories[i];
+          dir.directories[i].media[j].readyThumbnails = [];
+          dir.directories[i].media[j].readyIcon = false;
+        }
+      }
+    }
+  }
+
 
 }

backend/model/sql/IGalleryManager.ts

@@ -6,8 +6,6 @@ export interface ISQLGalleryManager extends IGalleryManager {
                 knownLastModified?: number,
                 knownLastScanned?: number): Promise<DirectoryDTO>;
 
-  indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO>;
-
   countDirectories(): Promise<number>;
 
   countPhotos(): Promise<number>;

backend/model/sql/IndexingManager.ts

@@ -1,119 +1,325 @@
-import {IIndexingManager} from '../interfaces/IIndexingManager';
+import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
-import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
-import {ObjectManagerRepository} from '../ObjectManagerRepository';
-import {ISQLGalleryManager} from './IGalleryManager';
-import * as path from 'path';
-import * as fs from 'fs';
-import {SQLConnection} from './SQLConnection';
 import {DirectoryEntity} from './enitites/DirectoryEntity';
-import {Logger} from '../../Logger';
+import {SQLConnection} from './SQLConnection';
-import {RendererInput, ThumbnailSourceType, ThumbnailWorker} from '../threading/ThumbnailWorker';
+import {DiskManager} from '../DiskManger';
-import {Config} from '../../../common/config/private/Config';
+import {PhotoEntity} from './enitites/PhotoEntity';
+import {Utils} from '../../../common/Utils';
+import {FaceRegion, PhotoMetadata} from '../../../common/entities/PhotoDTO';
+import {Connection,  Repository} from 'typeorm';
+import {MediaEntity} from './enitites/MediaEntity';
 import {MediaDTO} from '../../../common/entities/MediaDTO';
-import {ProjectPath} from '../../ProjectPath';
+import {VideoEntity} from './enitites/VideoEntity';
-import {ThumbnailGeneratorMWs} from '../../middlewares/thumbnail/ThumbnailGeneratorMWs';
+import {FileEntity} from './enitites/FileEntity';
+import {FileDTO} from '../../../common/entities/FileDTO';
+import {NotificationManager} from '../NotifocationManager';
+import {FaceRegionEntry} from './enitites/FaceRegionEntry';
+import {ObjectManagerRepository} from '../ObjectManagerRepository';
 
 const LOG_TAG = '[IndexingManager]';
 
-export class IndexingManager implements IIndexingManager {
+export class IndexingManager {
-  directoriesToIndex: string[] = [];
+
-  indexingProgress: IndexingProgressDTO = null;
+  private savingQueue: DirectoryDTO[] = [];
-  enabled = false;
+  private isSaving = false;
-  private indexNewDirectory = async (createThumbnails: boolean = false) => {
+
-    if (this.directoriesToIndex.length === 0) {
+  public indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
-      this.indexingProgress = null;
+    return new Promise(async (resolve, reject) => {
-      if (global.gc) {
+      try {
-        global.gc();
+        const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName);
-      }
+
-      return;
+        // returning with the result
-    }
+        scannedDirectory.media.forEach(p => p.readyThumbnails = []);
-    const directory = this.directoriesToIndex.shift();
+        resolve(scannedDirectory);
-    this.indexingProgress.current = directory;
+
-    this.indexingProgress.left = this.directoriesToIndex.length;
+        this.queueForSave(scannedDirectory).catch(console.error);
-    const scanned = await (<ISQLGalleryManager>ObjectManagerRepository.getInstance().GalleryManager).indexDirectory(directory);
+
-    if (this.enabled === false) {
+      } catch (error) {
-      return;
+        NotificationManager.warning('Unknown indexing error for: ' + relativeDirectoryName, error.toString());
-    }
+        console.error(error);
-    this.indexingProgress.indexed++;
+        return reject(error);
-    this.indexingProgress.time.current = Date.now();
-    for (let i = 0; i < scanned.directories.length; i++) {
-      this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
-    }
-    if (createThumbnails) {
-      for (let i = 0; i < scanned.media.length; i++) {
-        try {
-          const media = scanned.media[i];
-          const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name);
-          const thPath = path.join(ProjectPath.ThumbnailFolder,
-            ThumbnailGeneratorMWs.generateThumbnailName(mPath, Config.Client.Thumbnail.thumbnailSizes[0]));
-          if (fs.existsSync(thPath)) { // skip existing thumbnails
-            continue;
-          }
-          await ThumbnailWorker.render(<RendererInput>{
-            type: MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Image,
-            mediaPath: mPath,
-            size: Config.Client.Thumbnail.thumbnailSizes[0],
-            thPath: thPath,
-            makeSquare: false,
-            qualityPriority: Config.Server.thumbnail.qualityPriority
-          }, Config.Server.thumbnail.processingLibrary);
-        } catch (e) {
-          console.error(e);
-          Logger.error(LOG_TAG, 'Error during indexing job: ' + e.toString());
-        }
       }
 
-    }
-    process.nextTick(() => {
-      this.indexNewDirectory(createThumbnails);
     });
-  };
+  }
+
+  // Todo fix it, once typeorm support connection pools ofr sqlite
+  protected async queueForSave(scannedDirectory: DirectoryDTO) {
+    if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name &&
+      dir.path === scannedDirectory.path &&
+      dir.lastModified === scannedDirectory.lastModified &&
+      dir.lastScanned === scannedDirectory.lastScanned &&
+      (dir.media || dir.media.length) === (scannedDirectory.media || scannedDirectory.media.length) &&
+      (dir.metaFile || dir.metaFile.length) === (scannedDirectory.metaFile || scannedDirectory.metaFile.length)) !== -1) {
+      return;
+    }
+    this.savingQueue.push(scannedDirectory);
+    while (this.isSaving === false && this.savingQueue.length > 0) {
+      await this.saveToDB(this.savingQueue[0]);
+      this.savingQueue.shift();
+    }
+
+  }
+
+  protected async saveParentDir(connection: Connection, scannedDirectory: DirectoryDTO): Promise<number> {
+    const directoryRepository = connection.getRepository(DirectoryEntity);
+
+    const currentDir: DirectoryEntity = await directoryRepository.createQueryBuilder('directory')
+      .where('directory.name = :name AND directory.path = :path', {
+        name: scannedDirectory.name,
+        path: scannedDirectory.path
+      }).getOne();
+    if (!!currentDir) {// Updated parent dir (if it was in the DB previously)
+      currentDir.lastModified = scannedDirectory.lastModified;
+      currentDir.lastScanned = scannedDirectory.lastScanned;
+      currentDir.mediaCount = scannedDirectory.mediaCount;
+      await directoryRepository.save(currentDir);
+      return currentDir.id;
 
-  startIndexing(createThumbnails: boolean = false): void {
-    if (this.directoriesToIndex.length === 0 && this.enabled === false) {
-      Logger.info(LOG_TAG, 'Starting indexing');
-      this.indexingProgress = {
-        indexed: 0,
-        left: 0,
-        current: '',
-        time: {
-          start: Date.now(),
-          current: Date.now()
-        }
-      };
-      this.directoriesToIndex.push('/');
-      this.enabled = true;
-      this.indexNewDirectory(createThumbnails);
     } else {
-      Logger.info(LOG_TAG, 'Already indexing..');
+      return (await directoryRepository.insert(<DirectoryEntity>{
+        mediaCount: scannedDirectory.mediaCount,
+        lastModified: scannedDirectory.lastModified,
+        lastScanned: scannedDirectory.lastScanned,
+        name: scannedDirectory.name,
+        path: scannedDirectory.path
+      })).identifiers[0].id;
     }
   }
 
-  getProgress(): IndexingProgressDTO {
+  protected async saveChildDirs(connection: Connection, currentDirId: number, scannedDirectory: DirectoryDTO) {
-    return this.indexingProgress;
+    const directoryRepository = connection.getRepository(DirectoryEntity);
+    // TODO: fix when first opened directory is not root
+    // save subdirectories
+    const childDirectories = await directoryRepository.createQueryBuilder('directory')
+      .where('directory.parent = :dir', {
+        dir: currentDirId
+      }).getMany();
+
+    for (let i = 0; i < scannedDirectory.directories.length; i++) {
+      // Was this child Dir already indexed before?
+      let directory: DirectoryEntity = null;
+      for (let j = 0; j < childDirectories.length; j++) {
+        if (childDirectories[j].name === scannedDirectory.directories[i].name) {
+          directory = childDirectories[j];
+          childDirectories.splice(j, 1);
+          break;
+        }
+      }
+
+      if (directory != null) { // update existing directory
+        if (!directory.parent || !directory.parent.id) { // set parent if not set yet
+          directory.parent = <any>{id: currentDirId};
+          delete directory.media;
+          await directoryRepository.save(directory);
+        }
+      } else { // dir does not exists yet
+        scannedDirectory.directories[i].parent = <any>{id: currentDirId};
+        (<DirectoryEntity>scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet
+        const d = await directoryRepository.insert(<DirectoryEntity>scannedDirectory.directories[i]);
+
+        await this.saveMedia(connection, d.identifiers[0].id, scannedDirectory.directories[i].media);
+      }
+    }
+
+    // Remove child Dirs that are not anymore in the parent dir
+    await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)});
+
   }
 
-  cancelIndexing(): void {
+  protected async saveMetaFiles(connection: Connection, currentDirID: number, scannedDirectory: DirectoryDTO) {
-    Logger.info(LOG_TAG, 'Canceling indexing');
+    const fileRepository = connection.getRepository(FileEntity);
-    this.directoriesToIndex = [];
+    // save files
-    this.indexingProgress = null;
+    const indexedMetaFiles = await fileRepository.createQueryBuilder('file')
-    this.enabled = false;
+      .where('file.directory = :dir', {
-    if (global.gc) {
+        dir: currentDirID
-      global.gc();
+      }).getMany();
+
+
+    const metaFilesToSave = [];
+    for (let i = 0; i < scannedDirectory.metaFile.length; i++) {
+      let metaFile: FileDTO = null;
+      for (let j = 0; j < indexedMetaFiles.length; j++) {
+        if (indexedMetaFiles[j].name === scannedDirectory.metaFile[i].name) {
+          metaFile = indexedMetaFiles[j];
+          indexedMetaFiles.splice(j, 1);
+          break;
+        }
+      }
+      if (metaFile == null) { // not in DB yet
+        scannedDirectory.metaFile[i].directory = null;
+        metaFile = Utils.clone(scannedDirectory.metaFile[i]);
+        scannedDirectory.metaFile[i].directory = scannedDirectory;
+        metaFile.directory = <any>{id: currentDirID};
+        metaFilesToSave.push(metaFile);
+      }
+    }
+    await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)});
+    await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)});
+  }
+
+  protected async saveMedia(connection: Connection, parentDirId: number, media: MediaDTO[]) {
+    const mediaRepository = connection.getRepository(MediaEntity);
+    const photoRepository = connection.getRepository(PhotoEntity);
+    const videoRepository = connection.getRepository(VideoEntity);
+    // save media
+    let indexedMedia = (await mediaRepository.createQueryBuilder('media')
+      .where('media.directory = :dir', {
+        dir: parentDirId
+      })
+      .getMany());
+
+    const mediaChange: any = {
+      saveP: [],
+      saveV: [],
+      insertP: [],
+      insertV: []
+    };
+    const facesPerPhoto: { faces: FaceRegionEntry[], mediaName: string }[] = [];
+    for (let i = 0; i < media.length; i++) {
+      let mediaItem: MediaEntity = null;
+      for (let j = 0; j < indexedMedia.length; j++) {
+        if (indexedMedia[j].name === media[i].name) {
+          mediaItem = indexedMedia[j];
+          indexedMedia.splice(j, 1);
+          break;
+        }
+      }
+
+      const scannedFaces = (<PhotoMetadata>media[i].metadata).faces || [];
+      delete (<PhotoMetadata>media[i].metadata).faces;
+
+      // let mediaItemId: number = null;
+      if (mediaItem == null) { // not in DB yet
+        media[i].directory = null;
+        mediaItem = <any>Utils.clone(media[i]);
+        mediaItem.directory = <any>{id: parentDirId};
+        (MediaDTO.isPhoto(mediaItem) ? mediaChange.insertP : mediaChange.insertV).push(mediaItem);
+      } else {
+        delete (<PhotoMetadata>mediaItem.metadata).faces;
+        if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) {
+          mediaItem.metadata = <any>media[i].metadata;
+          (MediaDTO.isPhoto(mediaItem) ? mediaChange.saveP : mediaChange.saveV).push(mediaItem);
+
+        }
+      }
+
+      facesPerPhoto.push({faces: scannedFaces as FaceRegionEntry[], mediaName: mediaItem.name});
+    }
+
+    await this.saveChunk(photoRepository, mediaChange.saveP, 100);
+    await this.saveChunk(videoRepository, mediaChange.saveV, 100);
+    await this.saveChunk(photoRepository, mediaChange.insertP, 100);
+    await this.saveChunk(videoRepository, mediaChange.insertV, 100);
+
+    indexedMedia = (await mediaRepository.createQueryBuilder('media')
+      .where('media.directory = :dir', {
+        dir: parentDirId
+      })
+      .select(['media.name', 'media.id'])
+      .getMany());
+
+    const faces: FaceRegionEntry[] = [];
+    facesPerPhoto.forEach(group => {
+      const mIndex = indexedMedia.findIndex(m => m.name === group.mediaName);
+      group.faces.forEach((sf: FaceRegionEntry) => sf.media = <any>{id: indexedMedia[mIndex].id});
+
+      faces.push(...group.faces);
+      indexedMedia.splice(mIndex, 1);
+    });
+
+    await this.saveFaces(connection, parentDirId, faces);
+    await mediaRepository.remove(indexedMedia);
+  }
+
+  protected async saveFaces(connection: Connection, parentDirId: number, scannedFaces: FaceRegion[]) {
+    const faceRepository = connection.getRepository(FaceRegionEntry);
+
+    const persons: string[] = [];
+
+    for (let i = 0; i < scannedFaces.length; i++) {
+      if (persons.indexOf(scannedFaces[i].name) === -1) {
+        persons.push(scannedFaces[i].name);
+      }
+    }
+    await ObjectManagerRepository.getInstance().PersonManager.saveAll(persons);
+
+
+    const indexedFaces = await faceRepository.createQueryBuilder('face')
+      .leftJoin('face.media', 'media')
+      .where('media.directory = :directory', {
+        directory: parentDirId
+      })
+      .leftJoinAndSelect('face.person', 'person')
+      .getMany();
+
+
+    const faceToInsert = [];
+    for (let i = 0; i < scannedFaces.length; i++) {
+      let face: FaceRegionEntry = null;
+      for (let j = 0; j < indexedFaces.length; j++) {
+        if (indexedFaces[j].box.height === scannedFaces[i].box.height &&
+          indexedFaces[j].box.width === scannedFaces[i].box.width &&
+          indexedFaces[j].box.x === scannedFaces[i].box.x &&
+          indexedFaces[j].box.y === scannedFaces[i].box.y &&
+          indexedFaces[j].person.name === scannedFaces[i].name) {
+          face = indexedFaces[j];
+          indexedFaces.splice(j, 1);
+          break;
+        }
+      }
+
+      if (face == null) {
+        (<FaceRegionEntry>scannedFaces[i]).person = await ObjectManagerRepository.getInstance().PersonManager.get(scannedFaces[i].name);
+        faceToInsert.push(scannedFaces[i]);
+      }
+    }
+    if (faceToInsert.length > 0) {
+      await this.insertChunk(faceRepository, faceToInsert, 100);
+    }
+    await faceRepository.remove(indexedFaces, {chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1)});
+
+  }
+
+  protected async saveToDB(scannedDirectory: DirectoryDTO): Promise<void> {
+    this.isSaving = true;
+    try {
+      const connection = await SQLConnection.getConnection();
+      const currentDirId: number = await this.saveParentDir(connection, scannedDirectory);
+      await this.saveChildDirs(connection, currentDirId, scannedDirectory);
+      await this.saveMedia(connection, currentDirId, scannedDirectory.media);
+      await this.saveMetaFiles(connection, currentDirId, scannedDirectory);
+    } catch (e) {
+      throw e;
+    } finally {
+      this.isSaving = false;
     }
   }
 
-  async reset(): Promise<void> {
+  private async saveChunk<T>(repository: Repository<any>, entities: T[], size: number): Promise<T[]> {
-    Logger.info(LOG_TAG, 'Resetting DB');
+    if (entities.length === 0) {
-    this.directoriesToIndex = [];
+      return [];
-    this.indexingProgress = null;
+    }
-    this.enabled = false;
+    if (entities.length < size) {
-    const connection = await SQLConnection.getConnection();
+      return await repository.save(entities);
-    return connection
+    }
-      .getRepository(DirectoryEntity)
+    let list: T[] = [];
-      .createQueryBuilder('directory')
+    for (let i = 0; i < entities.length / size; i++) {
-      .delete()
+      list = list.concat(await repository.save(entities.slice(i * size, (i + 1) * size)));
-      .execute().then(() => {
+    }
-      });
+    return list;
+  }
+
+  private async insertChunk<T>(repository: Repository<any>, entities: T[], size: number): Promise<number[]> {
+    if (entities.length === 0) {
+      return [];
+    }
+    if (entities.length < size) {
+      return (await repository.insert(entities)).identifiers.map((i: any) => i.id);
+    }
+    let list: number[] = [];
+    for (let i = 0; i < entities.length / size; i++) {
+      list = list.concat((await repository.insert(entities.slice(i * size, (i + 1) * size))).identifiers.map(ids => ids.id));
+    }
+    return list;
   }
 }

backend/model/sql/IndexingTaskManager.ts

@@ -0,0 +1,118 @@
+import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager';
+import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
+import {ObjectManagerRepository} from '../ObjectManagerRepository';
+import * as path from 'path';
+import * as fs from 'fs';
+import {SQLConnection} from './SQLConnection';
+import {DirectoryEntity} from './enitites/DirectoryEntity';
+import {Logger} from '../../Logger';
+import {RendererInput, ThumbnailSourceType, ThumbnailWorker} from '../threading/ThumbnailWorker';
+import {Config} from '../../../common/config/private/Config';
+import {MediaDTO} from '../../../common/entities/MediaDTO';
+import {ProjectPath} from '../../ProjectPath';
+import {ThumbnailGeneratorMWs} from '../../middlewares/thumbnail/ThumbnailGeneratorMWs';
+
+const LOG_TAG = '[IndexingTaskManager]';
+
+export class IndexingTaskManager implements IIndexingTaskManager {
+  directoriesToIndex: string[] = [];
+  indexingProgress: IndexingProgressDTO = null;
+  enabled = false;
+  private indexNewDirectory = async (createThumbnails: boolean = false) => {
+    if (this.directoriesToIndex.length === 0) {
+      this.indexingProgress = null;
+      if (global.gc) {
+        global.gc();
+      }
+      return;
+    }
+    const directory = this.directoriesToIndex.shift();
+    this.indexingProgress.current = directory;
+    this.indexingProgress.left = this.directoriesToIndex.length;
+    const scanned = await ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(directory);
+    if (this.enabled === false) {
+      return;
+    }
+    this.indexingProgress.indexed++;
+    this.indexingProgress.time.current = Date.now();
+    for (let i = 0; i < scanned.directories.length; i++) {
+      this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
+    }
+    if (createThumbnails) {
+      for (let i = 0; i < scanned.media.length; i++) {
+        try {
+          const media = scanned.media[i];
+          const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name);
+          const thPath = path.join(ProjectPath.ThumbnailFolder,
+            ThumbnailGeneratorMWs.generateThumbnailName(mPath, Config.Client.Thumbnail.thumbnailSizes[0]));
+          if (fs.existsSync(thPath)) { // skip existing thumbnails
+            continue;
+          }
+          await ThumbnailWorker.render(<RendererInput>{
+            type: MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Image,
+            mediaPath: mPath,
+            size: Config.Client.Thumbnail.thumbnailSizes[0],
+            thPath: thPath,
+            makeSquare: false,
+            qualityPriority: Config.Server.thumbnail.qualityPriority
+          }, Config.Server.thumbnail.processingLibrary);
+        } catch (e) {
+          console.error(e);
+          Logger.error(LOG_TAG, 'Error during indexing job: ' + e.toString());
+        }
+      }
+
+    }
+    process.nextTick(() => {
+      this.indexNewDirectory(createThumbnails).catch(console.error);
+    });
+  };
+
+  startIndexing(createThumbnails: boolean = false): void {
+    if (this.directoriesToIndex.length === 0 && this.enabled === false) {
+      Logger.info(LOG_TAG, 'Starting indexing');
+      this.indexingProgress = {
+        indexed: 0,
+        left: 0,
+        current: '',
+        time: {
+          start: Date.now(),
+          current: Date.now()
+        }
+      };
+      this.directoriesToIndex.push('/');
+      this.enabled = true;
+      this.indexNewDirectory(createThumbnails).catch(console.error);
+    } else {
+      Logger.info(LOG_TAG, 'Already indexing..');
+    }
+  }
+
+  getProgress(): IndexingProgressDTO {
+    return this.indexingProgress;
+  }
+
+  cancelIndexing(): void {
+    Logger.info(LOG_TAG, 'Canceling indexing');
+    this.directoriesToIndex = [];
+    this.indexingProgress = null;
+    this.enabled = false;
+    if (global.gc) {
+      global.gc();
+    }
+  }
+
+  async reset(): Promise<void> {
+    Logger.info(LOG_TAG, 'Resetting DB');
+    this.directoriesToIndex = [];
+    this.indexingProgress = null;
+    this.enabled = false;
+    const connection = await SQLConnection.getConnection();
+    return connection
+      .getRepository(DirectoryEntity)
+      .createQueryBuilder('directory')
+      .delete()
+      .execute().then(() => {
+      });
+  }
+}

backend/model/sql/PersonManager.ts

@@ -0,0 +1,50 @@
+import {IPersonManager} from '../interfaces/IPersonManager';
+import {PersonEntry} from './enitites/PersonEntry';
+import {SQLConnection} from './SQLConnection';
+
+const LOG_TAG = '[PersonManager]';
+
+export class PersonManager implements IPersonManager {
+
+  persons: PersonEntry[] = [];
+
+  async get(name: string): Promise<PersonEntry> {
+
+    let person = this.persons.find(p => p.name === name);
+    if (!person) {
+      const connection = await SQLConnection.getConnection();
+      const personRepository = connection.getRepository(PersonEntry);
+      person = await personRepository.findOne({name: name});
+      if (!person) {
+        person = await personRepository.save(<PersonEntry>{name: name});
+      }
+      this.persons.push(person);
+    }
+    return person;
+  }
+
+
+  async saveAll(names: string[]): Promise<void> {
+    const toSave: { name: string }[] = [];
+    const connection = await SQLConnection.getConnection();
+    const personRepository = connection.getRepository(PersonEntry);
+    this.persons = await personRepository.find();
+
+    for (let i = 0; i < names.length; i++) {
+
+      const person = this.persons.find(p => p.name === names[i]);
+      if (!person) {
+        toSave.push({name: names[i]});
+      }
+    }
+
+    if (toSave.length > 0) {
+      for (let i = 0; i < toSave.length / 200; i++) {
+        await personRepository.insert(toSave.slice(i * 200, (i + 1) * 200));
+      }
+       this.persons = await personRepository.find();
+    }
+
+  }
+
+}

backend/model/sql/SQLConnection.ts

@@ -44,7 +44,7 @@ export class SQLConnection {
         VersionEntity
       ];
       options.synchronize = false;
-      options.logging = 'all';
+      // options.logging = 'all';
       this.connection = await createConnection(options);
       await SQLConnection.schemeSync(this.connection);
     }

common/entities/PhotoDTO.ts

@@ -1,6 +1,6 @@
 import {DirectoryDTO} from './DirectoryDTO';
 import {OrientationTypes} from 'ts-exif-parser';
-import {MediaDTO, MediaMetadata, MediaDimension} from './MediaDTO';
+import {MediaDimension, MediaDTO, MediaMetadata} from './MediaDTO';
 
 export interface PhotoDTO extends MediaDTO {
   id: number;

package.json

@@ -42,7 +42,7 @@
     "ts-exif-parser": "0.1.4",
     "ts-node-iptc": "1.0.11",
     "typeconfig": "1.0.7",
-    "typeorm": "0.2.9",
+    "typeorm": "0.2.11",
     "winston": "2.4.2",
     "xmldom": "0.1.27"
   },

test/backend/unit/assets/test image öüóőúéáű-.,.json

@@ -0,0 +1,45 @@
+{
+  "cameraData": {
+    "ISO": 3200,
+    "exposure": 0.00125,
+    "fStop": 5.6,
+    "focalLength": 85,
+    "lens": "EF-S15-85mm f/3.5-5.6 IS USM",
+    "make": "Canon",
+    "model": "óüöúőűáé ÓÜÖÚŐŰÁÉ"
+  },
+  "caption": "Test caption",
+  "creationDate": 1434018566000,
+  "faces": [
+    {
+      "box": {
+        "height": 19,
+        "width": 20,
+        "x": 82,
+        "y": 38
+      },
+      "name": "squirrel"
+    }
+  ],
+  "fileSize": 59187,
+  "keywords": [
+    "Berkley",
+    "USA",
+    "űáéúőóüö ŰÁÉÚŐÓÜÖ"
+  ],
+  "orientation": 1,
+  "positionData": {
+    "GPSData": {
+      "altitude": 90,
+      "latitude": 37.871093333333334,
+      "longitude": -122.25678
+    },
+    "city": "test city őúéáűóöí-.,)(=",
+    "country": "test country őúéáűóöí-.,)(=/%!+\"'",
+    "state": "test state őúéáűóöí-.,)("
+  },
+  "size": {
+    "height": 93,
+    "width": 140
+  }
+}

test/backend/unit/model/sql/IndexingManager.ts

@@ -12,6 +12,9 @@ import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/Directo
 import {Utils} from '../../../../../common/Utils';
 import {MediaDTO} from '../../../../../common/entities/MediaDTO';
 import {FileDTO} from '../../../../../common/entities/FileDTO';
+import {IndexingManager} from '../../../../../backend/model/sql/IndexingManager';
+import {ObjectManagerRepository} from '../../../../../backend/model/ObjectManagerRepository';
+import {PersonManager} from '../../../../../backend/model/sql/PersonManager';
 
 class GalleryManagerTest extends GalleryManager {
 
@@ -24,16 +27,21 @@ class GalleryManagerTest extends GalleryManager {
     return super.fillParentDir(connection, dir);
   }
 
-  public async saveToDB(scannedDirectory: DirectoryDTO) {
+}
-    return super.saveToDB(scannedDirectory);
+
-  }
+class IndexingManagerTest extends IndexingManager {
+
 
   public async queueForSave(scannedDirectory: DirectoryDTO): Promise<void> {
     return super.queueForSave(scannedDirectory);
   }
+
+  public async saveToDB(scannedDirectory: DirectoryDTO): Promise<void> {
+    return super.saveToDB(scannedDirectory);
+  }
 }
 
-describe('GalleryManager', () => {
+describe('IndexingManager', () => {
 
 
   const tempDir = path.join(__dirname, '../../tmp');
@@ -50,6 +58,7 @@ describe('GalleryManager', () => {
 
     Config.Server.database.type = DatabaseType.sqlite;
     Config.Server.database.sqlite.storage = dbPath;
+    ObjectManagerRepository.getInstance().PersonManager = new PersonManager();
 
   };
 
@@ -94,18 +103,19 @@ describe('GalleryManager', () => {
 
   it('should save parent directory', async () => {
     const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
 
     const parent = TestHelper.getRandomizedDirectoryEntry();
     const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
     const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
     const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
     const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
-    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
+    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0);
-    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
+    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0);
 
 
     DirectoryDTO.removeReferences(parent);
-    await gm.saveToDB(Utils.clone(parent));
+    await im.saveToDB(Utils.clone(parent));
 
     const conn = await SQLConnection.getConnection();
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
@@ -122,13 +132,14 @@ describe('GalleryManager', () => {
 
   it('should skip meta files', async () => {
     const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
     const parent = TestHelper.getRandomizedDirectoryEntry();
     const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
     const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
     const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
     DirectoryDTO.removeReferences(parent);
     Config.Client.MetaFile.enabled = true;
-    await gm.saveToDB(Utils.clone(parent));
+    await im.saveToDB(Utils.clone(parent));
 
     Config.Client.MetaFile.enabled = false;
     const conn = await SQLConnection.getConnection();
@@ -144,6 +155,7 @@ describe('GalleryManager', () => {
 
   it('should update sub directory', async () => {
     const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
 
     const parent = TestHelper.getRandomizedDirectoryEntry();
     parent.name = 'parent';
@@ -153,13 +165,13 @@ describe('GalleryManager', () => {
     const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
 
     DirectoryDTO.removeReferences(parent);
-    await gm.saveToDB(Utils.clone(parent));
+    await im.saveToDB(Utils.clone(parent));
 
     const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
     const sp3 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto3');
 
     DirectoryDTO.removeReferences(subDir);
-    await gm.saveToDB(Utils.clone(subDir));
+    await im.saveToDB(Utils.clone(subDir));
 
     const conn = await SQLConnection.getConnection();
     const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
@@ -179,20 +191,21 @@ describe('GalleryManager', () => {
   it('should avoid race condition', async () => {
     const conn = await SQLConnection.getConnection();
     const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
     Config.Client.MetaFile.enabled = true;
     const parent = TestHelper.getRandomizedDirectoryEntry();
     const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
     const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
     const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
     const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
-    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
+    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 1);
-    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
+    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 1);
 
 
     DirectoryDTO.removeReferences(parent);
-    const s1 = gm.queueForSave(Utils.clone(parent));
+    const s1 = im.queueForSave(Utils.clone(parent));
-    const s2 = gm.queueForSave(Utils.clone(parent));
+    const s2 = im.queueForSave(Utils.clone(parent));
-    const s3 = gm.queueForSave(Utils.clone(parent));
+    const s3 = im.queueForSave(Utils.clone(parent));
 
     await Promise.all([s1, s2, s3]);
 
@@ -204,30 +217,33 @@ describe('GalleryManager', () => {
     subDir.isPartial = true;
     delete subDir.directories;
     delete subDir.metaFile;
+    delete sp1.metadata.faces;
+    delete sp2.metadata.faces;
     expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
       .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
   });
 
 
-  (<any>it('should save 1500 photos', async () => {
+  (it('should save 1500 photos', async () => {
     const conn = await SQLConnection.getConnection();
     const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
     Config.Client.MetaFile.enabled = true;
     const parent = TestHelper.getRandomizedDirectoryEntry();
     DirectoryDTO.removeReferences(parent);
-    await gm.saveToDB(Utils.clone(parent));
+    await im.saveToDB(Utils.clone(parent));
     const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
     for (let i = 0; i < 1500; i++) {
       TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i);
     }
 
     DirectoryDTO.removeReferences(parent);
-    await gm.saveToDB(subDir);
+    await im.saveToDB(subDir);
 
 
     const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
     expect(selected.media.length).to.deep.equal(subDir.media.length);
-  })).timeout(20000);
+  }) as any).timeout(40000);
 
   describe('Test listDirectory', () => {
     const statSync = fs.statSync;
@@ -239,6 +255,8 @@ describe('GalleryManager', () => {
 
     beforeEach(() => {
       dirTime = 0;
+
+      ObjectManagerRepository.getInstance().IndexingManager = new IndexingManagerTest();
       indexedTime.lastModified = 0;
       indexedTime.lastScanned = 0;
     });
@@ -261,7 +279,7 @@ describe('GalleryManager', () => {
         return Promise.resolve();
       };
 
-      gm.indexDirectory = (...args) => {
+      ObjectManagerRepository.getInstance().IndexingManager.indexDirectory = (...args) => {
         return <any>Promise.resolve('indexing');
       };
 

test/backend/unit/model/sql/TestHelper.ts

@@ -1,7 +1,8 @@
 import {MediaDimensionEntity} from '../../../../../backend/model/sql/enitites/MediaEntity';
 import {
   CameraMetadataEntity,
-  GPSMetadataEntity, PhotoEntity,
+  GPSMetadataEntity,
+  PhotoEntity,
   PhotoMetadataEntity,
   PositionMetaDataEntity
 } from '../../../../../backend/model/sql/enitites/PhotoEntity';
@@ -9,9 +10,8 @@ import * as path from 'path';
 import {OrientationTypes} from 'ts-exif-parser';
 import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity';
 import {VideoEntity, VideoMetadataEntity} from '../../../../../backend/model/sql/enitites/VideoEntity';
-import {FileEntity} from '../../../../../backend/model/sql/enitites/FileEntity';
 import {MediaDimension} from '../../../../../common/entities/MediaDTO';
-import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
+import {CameraMetadata, FaceRegion, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
 import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO';
 import {FileDTO} from '../../../../../common/entities/FileDTO';
 
@@ -157,14 +157,37 @@ export class TestHelper {
     return d;
   }
 
-  public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null) {
+
+  public static getRandomizedFace(media: PhotoDTO, forceStr: string = null) {
+    const rndStr = () => {
+      return forceStr + '_' + Math.random().toString(36).substring(7);
+    };
+
+    const rndInt = (max = 5000) => {
+      return Math.floor(Math.random() * max);
+    };
+
+    const f: FaceRegion = {
+      name: rndStr() + '.jpg',
+      box: {
+        x: rndInt(),
+        y: rndInt(),
+        width: rndInt(),
+        height: rndInt()
+      }
+    };
+    media.metadata.faces = (media.metadata.faces || []);
+    media.metadata.faces.push(f);
+    return f;
+  }
+
+  public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null, faces: number = 2): PhotoDTO {
 
 
     const rndStr = () => {
       return forceStr + '_' + Math.random().toString(36).substring(7);
     };
 
-
     const rndInt = (max = 5000) => {
       return Math.floor(Math.random() * max);
     };
@@ -215,6 +238,10 @@ export class TestHelper {
       readyIcon: false
     };
 
+    for (let i = 0; i < faces; i++) {
+      this.getRandomizedFace(d, 'Person ' + i);
+    }
+
     dir.media.push(d);
     return d;
   }

test/backend/unit/model/threading/DiskMangerWorker.spec.ts

@@ -3,7 +3,7 @@ import {DiskMangerWorker} from '../../../../../backend/model/threading/DiskMange
 import * as path from 'path';
 import {Config} from '../../../../../common/config/private/Config';
 import {ProjectPath} from '../../../../../backend/ProjectPath';
-import {PhotoDTO} from '../../../../../common/entities/PhotoDTO';
+import {Utils} from '../../../../../common/Utils';
 
 describe('DiskMangerWorker', () => {
 
@@ -12,33 +12,9 @@ describe('DiskMangerWorker', () => {
     ProjectPath.ImageFolder = path.join(__dirname, '/../../assets');
     const dir = await DiskMangerWorker.scanDirectory('/');
     expect(dir.media.length).to.be.equals(2);
-    expect(dir.media[0].name).to.be.equals('test image öüóőúéáű-.,.jpg');
+    const expected = require(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.json'));
-    expect((<PhotoDTO>dir.media[0]).metadata.keywords).to.deep.equals(['Berkley', 'USA', 'űáéúőóüö ŰÁÉÚŐÓÜÖ']);
+    expect(Utils.clone(dir.media[0].name)).to.be.deep.equal('test image öüóőúéáű-.,.jpg');
-    expect(dir.media[0].metadata.fileSize).to.deep.equals(62786);
+    expect(Utils.clone(dir.media[0].metadata)).to.be.deep.equal(expected);
-    expect(dir.media[0].metadata.size).to.deep.equals({width: 140, height: 93});
-    expect((<PhotoDTO>dir.media[0]).metadata.cameraData).to.deep.equals({
-      ISO: 3200,
-      model: 'óüöúőűáé ÓÜÖÚŐŰÁÉ',
-      make: 'Canon',
-      fStop: 5.6,
-      exposure: 0.00125,
-      focalLength: 85,
-      lens: 'EF-S15-85mm f/3.5-5.6 IS USM'
-    });
-
-    expect((<PhotoDTO>dir.media[0]).metadata.positionData).to.deep.equals({
-      GPSData: {
-        latitude: 37.871093333333334,
-        longitude: -122.25678,
-        altitude: 102.4498997995992
-      },
-      country: 'mmóüöúőűáé ÓÜÖÚŐŰÁÉmm-.,|\\mm',
-      state: 'óüöúőűáé ÓÜÖÚŐŰÁ',
-      city: 'óüöúőűáé ÓÜÖÚŐŰÁ'
-    });
-
-    expect(dir.media[0].metadata.creationDate).to.be.equals(1434018566000);
-
   });
 
 });

test/backend/unit/model/threading/MetaDataLoader.spec.ts

@@ -20,36 +20,8 @@ describe('MetadataLoader', () => {
 
   it('should load jpg', async () => {
     const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.jpg'));
-    expect(Utils.clone(data)).to.be.deep.equal(Utils.clone({
+    const expected = require(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.json'));
-      size: {width: 140, height: 93},
+    expect(Utils.clone(data)).to.be.deep.equal(expected);
-      orientation: 1,
-      caption: 'Test caption',
-      creationDate: 1434018566000,
-      fileSize: 62786,
-      cameraData:
-        {
-          ISO: 3200,
-          model: 'óüöúőűáé ÓÜÖÚŐŰÁÉ',
-          make: 'Canon',
-          fStop: 5.6,
-          exposure: 0.00125,
-          focalLength: 85,
-          lens: 'EF-S15-85mm f/3.5-5.6 IS USM'
-        },
-      positionData:
-        {
-          GPSData:
-            {
-              latitude: 37.871093333333334,
-              longitude: -122.25678,
-              altitude: 102.4498997995992
-            },
-          country: 'mmóüöúőűáé ÓÜÖÚŐŰÁÉmm-.,|\\mm',
-          state: 'óüöúőűáé ÓÜÖÚŐŰÁ',
-          city: 'óüöúőűáé ÓÜÖÚŐŰÁ'
-        },
-      keywords: ['Berkley', 'USA', 'űáéúőóüö ŰÁÉÚŐÓÜÖ']
-    }));
   });
 
 });