Implementing ContentWrapper packing.

This enables to extract common string into a map and only reference their values.
This is expected to bring a further 43% savings on search results. Altogether leading to a 50% reduction.
 #437

benchmark/index.ts

@@ -55,7 +55,7 @@ const printExperimentResult = (result: BenchmarkResult, isSubResult = false) =>
     if (result.contentWrapper.directory) {
       details = 'media: ' + result.contentWrapper.directory.media.length +
         ', directories: ' + result.contentWrapper.directory.directories.length +
-        ', size: ' + fileSize(JSON.stringify(DirectoryDTOUtils.packDirectory(result.contentWrapper.directory)).length);
+        ', size: ' + fileSize(JSON.stringify(DirectoryDTOUtils.removeReferences(result.contentWrapper.directory)).length);
     } else {
       details = 'media: ' + result.contentWrapper.searchResult.media.length +
         ', directories: ' + result.contentWrapper.searchResult.directories.length +

src/backend/middlewares/GalleryMWs.ts

@@ -1,31 +1,26 @@
 import * as path from 'path';
-import { promises as fsp } from 'fs';
+import {promises as fsp} from 'fs';
 import * as archiver from 'archiver';
-import { NextFunction, Request, Response } from 'express';
+import {NextFunction, Request, Response} from 'express';
-import { ErrorCodes, ErrorDTO } from '../../common/entities/Error';
+import {ErrorCodes, ErrorDTO} from '../../common/entities/Error';
 import {
-  DirectoryBaseDTO,
-  DirectoryDTOUtils,
   ParentDirectoryDTO,
 } from '../../common/entities/DirectoryDTO';
-import { ObjectManagers } from '../model/ObjectManagers';
+import {ObjectManagers} from '../model/ObjectManagers';
-import { ContentWrapper } from '../../common/entities/ConentWrapper';
+import {ContentWrapper} from '../../common/entities/ConentWrapper';
-import { PhotoDTO } from '../../common/entities/PhotoDTO';
+import {ProjectPath} from '../ProjectPath';
-import { ProjectPath } from '../ProjectPath';
+import {Config} from '../../common/config/private/Config';
-import { Config } from '../../common/config/private/Config';
+import {UserDTOUtils} from '../../common/entities/UserDTO';
-import { UserDTOUtils } from '../../common/entities/UserDTO';
+import {MediaDTO, MediaDTOUtils} from '../../common/entities/MediaDTO';
-import { MediaDTO, MediaDTOUtils } from '../../common/entities/MediaDTO';
+import {QueryParams} from '../../common/QueryParams';
-import { VideoDTO } from '../../common/entities/VideoDTO';
+import {VideoProcessing} from '../model/fileprocessing/VideoProcessing';
-import { Utils } from '../../common/Utils';
-import { QueryParams } from '../../common/QueryParams';
-import { VideoProcessing } from '../model/fileprocessing/VideoProcessing';
 import {
   SearchQueryDTO,
   SearchQueryTypes,
 } from '../../common/entities/SearchQueryDTO';
-import { LocationLookupException } from '../exceptions/LocationLookupException';
+import {LocationLookupException} from '../exceptions/LocationLookupException';
-import { SupportedFormats } from '../../common/SupportedFormats';
+import {SupportedFormats} from '../../common/SupportedFormats';
-import { ServerTime } from './ServerTimingMWs';
+import {ServerTime} from './ServerTimingMWs';
 
 export class GalleryMWs {
   @ServerTime('1.db', 'List Directory')
@@ -130,12 +125,12 @@ export class GalleryMWs {
       // append photos in absoluteDirectoryName
       // using case-insensitive glob of extensions
       for (const ext of SupportedFormats.WithDots.Photos) {
-        archive.glob(`*${ext}`, { cwd: absoluteDirectoryName, nocase: true });
+        archive.glob(`*${ext}`, {cwd: absoluteDirectoryName, nocase: true});
       }
       // append videos in absoluteDirectoryName
       // using case-insensitive glob of extensions
       for (const ext of SupportedFormats.WithDots.Videos) {
-        archive.glob(`*${ext}`, { cwd: absoluteDirectoryName, nocase: true });
+        archive.glob(`*${ext}`, {cwd: absoluteDirectoryName, nocase: true});
       }
 
       await archive.finalize();
@@ -147,7 +142,7 @@ export class GalleryMWs {
     }
   }
 
-  @ServerTime('3.cleanUp', 'Clean up')
+  @ServerTime('3.pack', 'pack result')
   public static cleanUpGalleryResults(
     req: Request,
     res: Response,
@@ -162,38 +157,6 @@ export class GalleryMWs {
       return next();
     }
 
-    const cleanUpMedia = (media: MediaDTO[]): void => {
-      for (const m of media) {
-        delete m.id;
-        if (MediaDTOUtils.isPhoto(m)) {
-          delete (m as VideoDTO).metadata.bitRate;
-          delete (m as VideoDTO).metadata.duration;
-        } else if (MediaDTOUtils.isVideo(m)) {
-          delete (m as PhotoDTO).metadata.rating;
-          delete (m as PhotoDTO).metadata.caption;
-          delete (m as PhotoDTO).metadata.cameraData;
-          delete (m as PhotoDTO).metadata.keywords;
-          delete (m as PhotoDTO).metadata.positionData;
-        }
-        if (m.directory) {
-          delete (m.directory as DirectoryBaseDTO).id;
-        }
-        Utils.removeNullOrEmptyObj(m);
-      }
-    };
-
-    if (cw.directory) {
-      DirectoryDTOUtils.packDirectory(cw.directory);
-      // TODO: remove when typeorm inheritance is fixed (and handles proper inheritance)
-      cleanUpMedia(cw.directory.media);
-    }
-    if (cw.searchResult) {
-      cw.searchResult.directories.forEach((d) =>
-        DirectoryDTOUtils.packDirectory(d)
-      );
-      cleanUpMedia(cw.searchResult.media);
-    }
-
     if (Config.Client.Media.Video.enabled === false) {
       if (cw.directory) {
         const removeVideos = (dir: ParentDirectoryDTO): void => {
@@ -210,6 +173,8 @@ export class GalleryMWs {
       }
     }
 
+    ContentWrapper.pack(cw);
+
     return next();
   }
 
@@ -236,7 +201,7 @@ export class GalleryMWs {
         new ErrorDTO(
           ErrorCodes.GENERAL_ERROR,
           'no such file:' + req.params['mediaPath'],
-          "can't find file: " + fullMediaPath
+          'can\'t find file: ' + fullMediaPath
         )
       );
     }
@@ -263,7 +228,8 @@ export class GalleryMWs {
       await fsp.access(convertedVideo);
       req.resultPipe = convertedVideo;
       // eslint-disable-next-line no-empty
-    } catch (e) {}
+    } catch (e) {
+    }
 
     return next();
   }
@@ -375,7 +341,7 @@ export class GalleryMWs {
       return next(
         new ErrorDTO(
           ErrorCodes.GENERAL_ERROR,
-          "Can't get random photo: " + e.toString()
+          'Can\'t get random photo: ' + e.toString()
         )
       );
     }

src/backend/model/DiskManger.ts

@@ -52,7 +52,6 @@ export class DiskManager {
         settings
       )) as ParentDirectoryDTO;
     }
-    DirectoryDTOUtils.unpackDirectory(directory);
     return directory;
   }
 }

src/backend/model/database/memory/GalleryManager.ts

@@ -1,4 +1,4 @@
-import { ParentDirectoryDTO } from '../../../../common/entities/DirectoryDTO';
+import {DirectoryDTOUtils, ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO';
 import { IGalleryManager } from '../interfaces/IGalleryManager';
 import * as path from 'path';
 import * as fs from 'fs';
@@ -32,6 +32,7 @@ export class GalleryManager implements IGalleryManager {
       }
     }
     const dir = await DiskManager.scanDirectory(relativeDirectoryName);
+    DirectoryDTOUtils.addReferences(dir);
     dir.metaFile = dir.metaFile.filter((m) => !ServerPG2ConfMap[m.name]);
     return dir;
   }

src/backend/model/database/sql/IndexingManager.ts

@@ -1,4 +1,4 @@
-import {ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO';
+import {DirectoryDTOUtils, DirectoryPathDTO, ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO';
 import {DirectoryEntity} from './enitites/DirectoryEntity';
 import {SQLConnection} from './SQLConnection';
 import {DiskManager} from '../../DiskManger';
@@ -43,14 +43,15 @@ export class IndexingManager implements IIndexingManager {
   }
 
   private static async processServerSidePG2Conf(
+    parent: DirectoryPathDTO,
     files: FileDTO[]
   ): Promise<void> {
     for (const f of files) {
       if (ServerPG2ConfMap[f.name] === ServerSidePG2ConfAction.SAVED_SEARCH) {
         const fullMediaPath = path.join(
           ProjectPath.ImageFolder,
-          f.directory.path,
+          parent.path,
-          f.directory.name,
+          parent.name,
           f.name
         );
 
@@ -94,12 +95,14 @@ export class IndexingManager implements IIndexingManager {
           relativeDirectoryName
         );
 
-        const dirClone = Utils.shallowClone(scannedDirectory);
+
+        const dirClone = Utils.clone(scannedDirectory);
         // filter server side only config from returning
         dirClone.metaFile = dirClone.metaFile.filter(
           (m) => !ServerPG2ConfMap[m.name]
         );
 
+        DirectoryDTOUtils.addReferences(dirClone);
         resolve(dirClone);
 
         // save directory to DB
@@ -142,7 +145,7 @@ export class IndexingManager implements IIndexingManager {
       await this.saveChildDirs(connection, currentDirId, scannedDirectory);
       await this.saveMedia(connection, currentDirId, scannedDirectory.media);
       await this.saveMetaFiles(connection, currentDirId, scannedDirectory);
-      await IndexingManager.processServerSidePG2Conf(serverSideConfigs);
+      await IndexingManager.processServerSidePG2Conf(scannedDirectory, serverSideConfigs);
       await ObjectManagers.getInstance().onDataChange(scannedDirectory);
     } finally {
       this.isSaving = false;

src/common/entities/ConentWrapper.ts

@@ -1,10 +1,434 @@
-import { ParentDirectoryDTO } from './DirectoryDTO';
+/* eslint-disable @typescript-eslint/ban-ts-comment */
-import { SearchResultDTO } from './SearchResultDTO';
+import {DirectoryBaseDTO, DirectoryPathDTO, ParentDirectoryDTO} from './DirectoryDTO';
+import {SearchResultDTO} from './SearchResultDTO';
+import {Utils} from '../Utils';
+import {MediaDTO, MediaDTOUtils} from './MediaDTO';
+import {FileDTO} from './FileDTO';
+import {VideoDTO} from './VideoDTO';
+import {PhotoDTO} from './PhotoDTO';
+
 
 export class ContentWrapper {
+  private map: {
+    faces: string[],
+    keywords: string[],
+    lens: string[],
+    camera: string[],
+    directories: DirectoryPathDTO[]
+  };
+  private reverseMap: {
+    faces: Map<string, number>,
+    keywords: Map<string, number>,
+    lens: Map<string, number>,
+    camera: Map<string, number>,
+    directories: Map<string, number>
+  };
+  public directory: ParentDirectoryDTO;
+  public searchResult: SearchResultDTO;
+  public notModified: boolean;
+
   constructor(
-    public directory: ParentDirectoryDTO = null,
+    directory: ParentDirectoryDTO = null,
-    public searchResult: SearchResultDTO = null,
+    searchResult: SearchResultDTO = null,
-    public notModified?: boolean
+    notModified?: boolean
-  ) {}
+  ) {
+    if (directory) {
+      this.directory = directory;
+    }
+    if (searchResult) {
+      this.searchResult = searchResult;
+    }
+    if (notModified) {
+      this.notModified = notModified;
+    }
+  }
+
+
+  private static mapify(cw: ContentWrapper, media: FileDTO, isSearchResult: boolean, isPhoto: boolean, isVideo: boolean): void {
+    if (isSearchResult) {
+      const k = JSON.stringify(media.directory);
+      if (!cw.reverseMap.directories.has(k)) {
+        cw.reverseMap.directories.set(k, cw.map.directories.length);
+        cw.map.directories.push(media.directory);
+      }
+      // @ts-ignore
+      media.d = cw.reverseMap.directories.get(k);
+      delete media.directory;
+    }
+
+    if ((media as MediaDTO).metadata) {
+      // @ts-ignore
+      (media as MediaDTO).metadata.s = [(media as MediaDTO).metadata.size.width, (media as MediaDTO).metadata.size.height];
+      delete (media as MediaDTO).metadata.size;
+    }
+    if (isPhoto) {
+      if ((media as PhotoDTO).metadata.faces) {
+        for (let i = 0; i < (media as PhotoDTO).metadata.faces.length; ++i) {
+          const name = (media as PhotoDTO).metadata.faces[i].name;
+          if (!cw.reverseMap.faces.has(name)) {
+            cw.reverseMap.faces.set(name, cw.map.faces.length);
+            cw.map.faces.push(name);
+          }
+          // @ts-ignore
+          (media as PhotoDTO).metadata.faces[i].n = cw.reverseMap.faces.get(name);
+          delete (media as PhotoDTO).metadata.faces[i].name;
+        }
+
+      }
+
+      if ((media as PhotoDTO).metadata.keywords) {
+        for (let i = 0; i < (media as PhotoDTO).metadata.keywords.length; ++i) {
+          const k = (media as PhotoDTO).metadata.keywords[i];
+          if (!cw.reverseMap.keywords.has(k)) {
+            cw.reverseMap.keywords.set(k, cw.map.keywords.length);
+            cw.map.keywords.push(k);
+          }
+          // @ts-ignore
+          (media as PhotoDTO).metadata.keywords[i] = cw.reverseMap.keywords.get(k);
+        }
+      }
+      const mapifyOne = <T>(map: string[], reverseMap: Map<string, number>,
+                            obj: T, key: keyof T, mappedKey: string) => {
+
+        // @ts-ignore
+        const k: string = obj[key];
+
+        if (!reverseMap.has(k)) {
+          reverseMap.set(k, map.length);
+          map.push(k);
+        }
+        // @ts-ignore
+        obj[mappedKey] = reverseMap.get(k);
+        delete obj[key];
+      };
+      if ((media as PhotoDTO).metadata.cameraData) {
+        if ((media as PhotoDTO).metadata.cameraData.lens) {
+          mapifyOne(cw.map.lens, cw.reverseMap.lens,
+            (media as PhotoDTO).metadata.cameraData, 'lens', 'l');
+        }
+        if ((media as PhotoDTO).metadata.cameraData.make) {
+          mapifyOne(cw.map.camera, cw.reverseMap.camera,
+            (media as PhotoDTO).metadata.cameraData, 'make', 'm');
+        }
+        if ((media as PhotoDTO).metadata.cameraData.model) {
+          mapifyOne(cw.map.camera, cw.reverseMap.camera,
+            (media as PhotoDTO).metadata.cameraData, 'model', 'ml');
+        }
+
+        // @ts-ignore
+        (media as PhotoDTO).metadata.c = (media as PhotoDTO).metadata.cameraData;
+        delete (media as PhotoDTO).metadata.cameraData;
+      }
+      if ((media as PhotoDTO).metadata.positionData) {
+        if ((media as PhotoDTO).metadata.positionData.country) {
+          mapifyOne(cw.map.keywords, cw.reverseMap.keywords,
+            (media as PhotoDTO).metadata.positionData, 'country', 'c');
+        }
+        if ((media as PhotoDTO).metadata.positionData.city) {
+          mapifyOne(cw.map.keywords, cw.reverseMap.keywords,
+            (media as PhotoDTO).metadata.positionData, 'city', 'cy');
+        }
+        if ((media as PhotoDTO).metadata.positionData.state) {
+          mapifyOne(cw.map.keywords, cw.reverseMap.keywords,
+            (media as PhotoDTO).metadata.positionData, 'state', 's');
+        }
+
+        if ((media as PhotoDTO).metadata.positionData.GPSData) {
+          // @ts-ignore
+          (media as PhotoDTO).metadata.positionData.g = [(media as PhotoDTO).metadata.positionData.GPSData.latitude, (media as PhotoDTO).metadata.positionData.GPSData.longitude];
+          delete (media as PhotoDTO).metadata.positionData.GPSData;
+        }
+        // @ts-ignore
+        (media as PhotoDTO).metadata.p = (media as PhotoDTO).metadata.positionData;
+        delete (media as PhotoDTO).metadata.positionData;
+      }
+    }
+
+  }
+
+  private static packMedia(cw: ContentWrapper, media: MediaDTO[], isSearchResult: boolean): void {
+
+    // clean up media
+    for (let i = 0; i < media.length; ++i) {
+      const m = media[i];
+      delete m.id;
+
+      if (m.directory) {
+        if (isSearchResult) {
+          // keep the directory for search result
+          delete (m.directory as DirectoryBaseDTO).id;
+        } else {
+          // for gallery listing, photos belong to one directory,
+          // this can be deleted as the directory know its child
+          delete m.directory;
+        }
+      }
+
+      if (MediaDTOUtils.isPhoto(m)) {
+        delete (m as VideoDTO).metadata.bitRate;
+        delete (m as VideoDTO).metadata.duration;
+
+        // compress faces
+        if ((m as PhotoDTO).metadata.faces) {
+          for (let j = 0; j < (m as PhotoDTO).metadata.faces.length; ++j) {
+            const f = (m as PhotoDTO).metadata.faces[j];
+            // @ts-ignore
+            f['b'] = [f.box.top, f.box.left, f.box.height, f.box.width];
+            delete f.box;
+          }
+        }
+        ContentWrapper.mapify(cw, m, isSearchResult, true, false);
+      } else if (MediaDTOUtils.isVideo(m)) {
+        delete (m as PhotoDTO).metadata.rating;
+        delete (m as PhotoDTO).metadata.caption;
+        delete (m as PhotoDTO).metadata.cameraData;
+        delete (m as PhotoDTO).metadata.keywords;
+        delete (m as PhotoDTO).metadata.faces;
+        delete (m as PhotoDTO).metadata.positionData;
+        ContentWrapper.mapify(cw, m, isSearchResult, false, true);
+      }
+      Utils.removeNullOrEmptyObj(m);
+    }
+  }
+
+  private static packDirectory(cw: ContentWrapper, dir: DirectoryBaseDTO | SearchResultDTO, isSearchResult = false): void {
+    if ((dir as DirectoryBaseDTO).preview) {
+      (dir as DirectoryBaseDTO).preview.directory = {
+        path: (dir as DirectoryBaseDTO).preview.directory.path,
+        name: (dir as DirectoryBaseDTO).preview.directory.name,
+      } as DirectoryPathDTO;
+
+      // make sure that it is not a same object as one of the photo in the media[]
+      // as the next foreach would remove the directory
+      (dir as DirectoryBaseDTO).preview = Utils.clone((dir as DirectoryBaseDTO).preview);
+    }
+
+    if (dir.media) {
+      ContentWrapper.packMedia(cw, dir.media, isSearchResult);
+    }
+
+    if (dir.metaFile) {
+      for (let i = 0; i < dir.metaFile.length; ++i) {
+        if (isSearchResult) {
+          delete (dir.metaFile[i].directory as DirectoryBaseDTO).id;
+        } else {
+          delete dir.metaFile[i].directory;
+        }
+        delete dir.metaFile[i].id;
+        ContentWrapper.mapify(cw, dir.metaFile[i], isSearchResult, false, false);
+      }
+    }
+    if (dir.directories) {
+      for (let i = 0; i < dir.directories.length; ++i) {
+        ContentWrapper.packDirectory(cw, dir.directories[i]);
+        delete dir.directories[i].parent;
+      }
+    }
+
+    delete (dir as DirectoryBaseDTO).validPreview; // should not go to the client side;
+  }
+
+  private static deMapify(cw: ContentWrapper, media: FileDTO, isSearchResult: boolean, isPhoto: boolean, isVideo: boolean): void {
+
+    const deMapifyOne = <T>(map: any[],
+                            obj: T, key: keyof T, mappedKey: string) => {
+      // @ts-ignore
+      obj[key] = map[obj[mappedKey]];
+      // @ts-ignore
+      delete obj[mappedKey];
+    };
+    if (isSearchResult) {
+      deMapifyOne(cw.map.directories, media, 'directory', 'd');
+    }
+
+    if ((media as MediaDTO).metadata) {
+      (media as MediaDTO).metadata.size = {
+        // @ts-ignore
+        width: (media as MediaDTO).metadata.s[0],
+        // @ts-ignore
+        height: (media as MediaDTO).metadata.s[1],
+      };
+      // @ts-ignore
+      delete (media as MediaDTO).metadata.s;
+    }
+    if (isPhoto) {
+      if ((media as PhotoDTO).metadata.faces) {
+        for (let i = 0; i < (media as PhotoDTO).metadata.faces.length; ++i) {
+          // @ts-ignore
+          (media as PhotoDTO).metadata.faces[i].name = cw.map.faces[(media as PhotoDTO).metadata.faces[i].n];
+          // @ts-ignore
+          delete (media as PhotoDTO).metadata.faces[i].n;
+        }
+
+      }
+
+      if ((media as PhotoDTO).metadata.keywords) {
+        for (let i = 0; i < (media as PhotoDTO).metadata.keywords.length; ++i) {
+          // @ts-ignore
+          (media as PhotoDTO).metadata.keywords[i] = cw.map.keywords[(media as PhotoDTO).metadata.keywords[i]];
+        }
+      }
+
+      // @ts-ignore
+      if ((media as PhotoDTO).metadata.c) {
+        // @ts-ignore
+        (media as PhotoDTO).metadata.cameraData = (media as PhotoDTO).metadata.c;
+        // @ts-ignore
+        delete (media as PhotoDTO).metadata.c;
+
+        // @ts-ignore
+        if (typeof (media as PhotoDTO).metadata.cameraData.l !== 'undefined') {
+          deMapifyOne(cw.map.lens,
+            (media as PhotoDTO).metadata.cameraData, 'lens', 'l');
+        }
+        // @ts-ignore
+        if (typeof (media as PhotoDTO).metadata.cameraData.m !== 'undefined') {
+          deMapifyOne(cw.map.camera,
+            (media as PhotoDTO).metadata.cameraData, 'make', 'm');
+        }
+        // @ts-ignore
+        if (typeof (media as PhotoDTO).metadata.cameraData.ml !== 'undefined') {
+          deMapifyOne(cw.map.camera,
+            (media as PhotoDTO).metadata.cameraData, 'model', 'ml');
+        }
+
+      }
+      // @ts-ignore
+      if ((media as PhotoDTO).metadata.p) {
+        // @ts-ignore
+        (media as PhotoDTO).metadata.positionData = (media as PhotoDTO).metadata.p;
+        // @ts-ignore
+        delete (media as PhotoDTO).metadata.p;
+        // @ts-ignore
+        if (typeof (media as PhotoDTO).metadata.positionData.c !== 'undefined') {
+          deMapifyOne(cw.map.keywords,
+            (media as PhotoDTO).metadata.positionData, 'country', 'c');
+        }
+        // @ts-ignore
+        if (typeof (media as PhotoDTO).metadata.positionData.cy !== 'undefined') {
+          deMapifyOne(cw.map.keywords,
+            (media as PhotoDTO).metadata.positionData, 'city', 'cy');
+        }
+        // @ts-ignore
+        if (typeof (media as PhotoDTO).metadata.positionData.s !== 'undefined') {
+          deMapifyOne(cw.map.keywords,
+            (media as PhotoDTO).metadata.positionData, 'state', 's');
+        }
+
+        // @ts-ignore
+        if ((media as PhotoDTO).metadata.positionData['g']) {
+          (media as PhotoDTO).metadata.positionData.GPSData =
+            {
+              // @ts-ignore
+              latitude: (media as PhotoDTO).metadata.positionData['g'][0],
+              // @ts-ignore
+              longitude: (media as PhotoDTO).metadata.positionData['g'][1]
+            };
+          // @ts-ignore
+          delete (media as PhotoDTO).metadata.positionData['g'];
+        }
+      }
+    }
+
+  }
+
+  private static unPackMedia(cw: ContentWrapper, dir: DirectoryBaseDTO, media: MediaDTO[], isSearchResult: boolean): void {
+// clean up media
+    for (let i = 0; i < media.length; ++i) {
+      const m = media[i];
+      if (MediaDTOUtils.isPhoto(m)) {
+        // compress faces
+        if ((m as PhotoDTO).metadata.faces) {
+          for (let j = 0; j < (m as PhotoDTO).metadata.faces.length; ++j) {
+            const f = (m as PhotoDTO).metadata.faces[j];
+            // @ts-ignore
+            const boxArr = f.b;
+            f.box = {
+              top: boxArr[0],
+              left: boxArr[1],
+              height: boxArr[2],
+              width: boxArr[3],
+            };
+            // @ts-ignore
+            delete f.b;
+          }
+        }
+        ContentWrapper.deMapify(cw, m, isSearchResult, true, false);
+      } else if (MediaDTOUtils.isVideo(m)) {
+        ContentWrapper.deMapify(cw, m, isSearchResult, false, true);
+      }
+      if (!isSearchResult) {
+        m.directory = dir;
+      }
+    }
+  }
+
+  private static unPackDirectory(cw: ContentWrapper, dir: DirectoryBaseDTO | SearchResultDTO, isSearchResult = false): void {
+    if (dir.media) {
+      ContentWrapper.unPackMedia(cw, dir as DirectoryBaseDTO, dir.media, isSearchResult);
+    }
+
+    if (dir.metaFile) {
+      for (let i = 0; i < dir.metaFile.length; ++i) {
+        if (!isSearchResult) {
+          dir.metaFile[i].directory = dir as DirectoryBaseDTO;
+        }
+        ContentWrapper.deMapify(cw, dir.metaFile[i], isSearchResult, false, false);
+      }
+    }
+    if (dir.directories) {
+      for (let i = 0; i < dir.directories.length; ++i) {
+        ContentWrapper.unPackDirectory(cw, dir.directories[i]);
+        if (!isSearchResult) {
+          dir.directories[i].parent = dir as DirectoryBaseDTO;
+        }
+      }
+    }
+
+  }
+
+  static pack(cw: ContentWrapper): ContentWrapper {
+
+    // init CW for packing
+    cw.map = {
+      faces: [], keywords: [], lens: [],
+      camera: [], directories: []
+    };
+    cw.reverseMap = {
+      faces: new Map(), keywords: new Map(),
+      lens: new Map(), camera: new Map(), directories: new Map()
+    };
+
+    if (cw.directory) {
+      ContentWrapper.packDirectory(cw, cw.directory);
+    } else if (cw.searchResult) {
+      ContentWrapper.packDirectory(cw, cw.searchResult, true);
+    }
+
+    // remove empty maps
+    for (const k in cw.map) {
+      // @ts-ignore
+      if (cw.map[k].length === 0) {
+        // @ts-ignore
+        delete cw.map[k];
+      }
+    }
+
+    delete cw.reverseMap;
+    return cw;
+
+  }
+
+  static unpack(cw: ContentWrapper): ContentWrapper {
+    if (!cw || cw.notModified) {
+      return cw;
+    }
+    if (cw.directory) {
+      ContentWrapper.unPackDirectory(cw, cw.directory);
+    } else if (cw.searchResult) {
+      ContentWrapper.unPackDirectory(cw, cw.searchResult, true);
+    }
+    delete cw.map;
+    return cw;
+  }
 }

src/common/entities/DirectoryDTO.ts

@@ -8,21 +8,7 @@ export interface DirectoryPathDTO {
   path: string;
 }
 
-//
+
-// export interface DirectoryDTO<S extends FileDTO = MediaDTO> extends DirectoryPathDTO {
-//   id: number;
-//   name: string;
-//   path: string;
-//   lastModified: number;
-//   lastScanned: number;
-//   isPartial?: boolean;
-//   parent: DirectoryDTO<S>;
-//   mediaCount: number;
-//   directories: DirectoryDTO<S>[];
-//   preview: PreviewPhotoDTO;
-//   media: S[];
-//   metaFile: FileDTO[];
-// }
 
 export interface DirectoryBaseDTO<S extends FileDTO = MediaDTO>
   extends DirectoryPathDTO {
@@ -72,7 +58,7 @@ export interface SubDirectoryDTO<S extends FileDTO = MediaDTO>
 }
 
 export const DirectoryDTOUtils = {
-  unpackDirectory: (dir: DirectoryBaseDTO): void => {
+  addReferences: (dir: DirectoryBaseDTO): void => {
     dir.media.forEach((media: MediaDTO) => {
       media.directory = dir;
     });
@@ -85,13 +71,13 @@ export const DirectoryDTOUtils = {
 
     if (dir.directories) {
       dir.directories.forEach((directory) => {
-        DirectoryDTOUtils.unpackDirectory(directory);
+        DirectoryDTOUtils.addReferences(directory);
         directory.parent = dir;
       });
     }
   },
 
-  packDirectory: (dir: DirectoryBaseDTO): DirectoryBaseDTO => {
+  removeReferences: (dir: DirectoryBaseDTO): DirectoryBaseDTO => {
     if (dir.preview) {
       dir.preview.directory = {
         path: dir.preview.directory.path,
@@ -116,7 +102,7 @@ export const DirectoryDTOUtils = {
     }
     if (dir.directories) {
       dir.directories.forEach((directory) => {
-        DirectoryDTOUtils.packDirectory(directory);
+        DirectoryDTOUtils.removeReferences(directory);
         directory.parent = null;
       });
     }
@@ -125,10 +111,4 @@ export const DirectoryDTOUtils = {
 
     return dir;
   },
-  filterPhotos: (dir: DirectoryBaseDTO): PhotoDTO[] => {
-    return dir.media.filter((m) => MediaDTOUtils.isPhoto(m)) as PhotoDTO[];
-  },
-  filterVideos: (dir: DirectoryBaseDTO): PhotoDTO[] => {
-    return dir.media.filter((m) => MediaDTOUtils.isPhoto(m)) as PhotoDTO[];
-  },
 };

src/frontend/app/ui/gallery/cache.gallery.service.ts

@@ -1,5 +1,5 @@
 import {Injectable} from '@angular/core';
-import {DirectoryDTOUtils, DirectoryPathDTO, ParentDirectoryDTO,} from '../../../../common/entities/DirectoryDTO';
+import { DirectoryPathDTO, ParentDirectoryDTO,} from '../../../../common/entities/DirectoryDTO';
 import {Utils} from '../../../../common/Utils';
 import {Config} from '../../../../common/config/public/Config';
 import {IAutoCompleteItem} from '../../../../common/entities/AutoCompleteItem';
@@ -8,6 +8,8 @@ import {MediaDTO} from '../../../../common/entities/MediaDTO';
 import {SortingMethods} from '../../../../common/entities/SortingMethods';
 import {VersionService} from '../../model/version.service';
 import {SearchQueryDTO, SearchQueryTypes,} from '../../../../common/entities/SearchQueryDTO';
+import {ContentWrapper} from '../../../../common/entities/ConentWrapper';
+import {ContentWrapperWithError} from './content.service';
 
 interface CacheItem<T> {
   timestamp: number;
@@ -49,10 +51,10 @@ export class GalleryCacheService {
     return perfEntries && perfEntries[0] && perfEntries[0].type === 'reload';
   }
 
-  private static loadCacheItem(key: string): SearchResultDTO {
+  private static loadCacheItem(key: string): ContentWrapperWithError {
     const tmp = localStorage.getItem(key);
     if (tmp != null) {
-      const value: CacheItem<SearchResultDTO> = JSON.parse(tmp);
+      const value: CacheItem<ContentWrapperWithError> = JSON.parse(tmp);
       if (
         value.timestamp <
         Date.now() - Config.Client.Search.searchCacheTimeout
@@ -177,34 +179,7 @@ export class GalleryCacheService {
     }
   }
 
-  public getInstantSearch(text: string): SearchResultDTO {
+  public getSearch(query: SearchQueryDTO): ContentWrapperWithError {
-    if (Config.Client.Other.enableCache === false) {
-      return null;
-    }
-    const key = GalleryCacheService.INSTANT_SEARCH_PREFIX + text;
-    return GalleryCacheService.loadCacheItem(key);
-  }
-
-  public setInstantSearch(text: string, searchResult: SearchResultDTO): void {
-    if (Config.Client.Other.enableCache === false) {
-      return;
-    }
-    const tmp: CacheItem<SearchResultDTO> = {
-      timestamp: Date.now(),
-      item: searchResult,
-    };
-    try {
-      localStorage.setItem(
-        GalleryCacheService.INSTANT_SEARCH_PREFIX + text,
-        JSON.stringify(tmp)
-      );
-    } catch (e) {
-      this.reset();
-      console.error(e);
-    }
-  }
-
-  public getSearch(query: SearchQueryDTO): SearchResultDTO {
     if (Config.Client.Other.enableCache === false) {
       return null;
     }
@@ -212,15 +187,15 @@ export class GalleryCacheService {
     return GalleryCacheService.loadCacheItem(key);
   }
 
-  public setSearch(query: SearchQueryDTO, searchResult: SearchResultDTO): void {
+  public setSearch(cw: ContentWrapperWithError): void {
     if (Config.Client.Other.enableCache === false) {
       return;
     }
-    const tmp: CacheItem<SearchResultDTO> = {
+    const tmp: CacheItem<ContentWrapperWithError> = {
       timestamp: Date.now(),
-      item: searchResult,
+      item: cw,
     };
-    const key = GalleryCacheService.SEARCH_PREFIX + JSON.stringify(query);
+    const key = GalleryCacheService.SEARCH_PREFIX + JSON.stringify(cw.searchResult.searchQuery);
     try {
       localStorage.setItem(key, JSON.stringify(tmp));
     } catch (e) {
@@ -229,7 +204,7 @@ export class GalleryCacheService {
     }
   }
 
-  public getDirectory(directoryName: string): ParentDirectoryDTO {
+  public getDirectory(directoryName: string): ContentWrapperWithError {
     if (Config.Client.Other.enableCache === false) {
       return null;
     }
@@ -238,41 +213,29 @@ export class GalleryCacheService {
         GalleryCacheService.CONTENT_PREFIX + Utils.concatUrls(directoryName)
       );
       if (value != null) {
-        const directory: ParentDirectoryDTO = JSON.parse(value);
+        return JSON.parse(value);
-
-        DirectoryDTOUtils.unpackDirectory(directory);
-        return directory;
       }
     } catch (e) {
       // ignoring errors
     }
-    return null;
+    return new ContentWrapperWithError();
   }
 
-  public setDirectory(directory: ParentDirectoryDTO): void {
+  public setDirectory(cw: ContentWrapper): void {
     if (Config.Client.Other.enableCache === false) {
       return;
     }
 
     const key =
       GalleryCacheService.CONTENT_PREFIX +
-      Utils.concatUrls(directory.path, directory.name);
+      Utils.concatUrls(cw.directory.path, cw.directory.name);
-    if (directory.isPartial === true && localStorage.getItem(key)) {
+    if (cw.directory.isPartial === true && localStorage.getItem(key)) {
       return;
     }
 
     try {
       // try to fit it
-      localStorage.setItem(key, JSON.stringify(directory));
+      localStorage.setItem(key, JSON.stringify(cw));
-      directory.directories.forEach((dir) => {
-        const subKey =
-          GalleryCacheService.CONTENT_PREFIX +
-          Utils.concatUrls(dir.path, dir.name);
-        if (localStorage.getItem(subKey) == null) {
-          // don't override existing
-          localStorage.setItem(subKey, JSON.stringify(dir));
-        }
-      });
     } catch (e) {
       this.reset();
       console.error(e);
@@ -295,7 +258,7 @@ export class GalleryCacheService {
       const value = localStorage.getItem(directoryKey);
       if (value != null) {
         const directory: ParentDirectoryDTO = JSON.parse(value);
-        directory.media.forEach((p) => {
+        directory?.media?.forEach((p) => {
           if (p.name === media.name) {
             // update data
             p.metadata = media.metadata;

src/frontend/app/ui/gallery/content.service.ts

@@ -1,22 +1,21 @@
-import { Injectable } from '@angular/core';
+import {Injectable} from '@angular/core';
-import { NetworkService } from '../../model/network/network.service';
+import {NetworkService} from '../../model/network/network.service';
-import { ContentWrapper } from '../../../../common/entities/ConentWrapper';
+import {ContentWrapper} from '../../../../common/entities/ConentWrapper';
 import {
-  DirectoryDTOUtils,
   ParentDirectoryDTO,
   SubDirectoryDTO,
 } from '../../../../common/entities/DirectoryDTO';
-import { GalleryCacheService } from './cache.gallery.service';
+import {GalleryCacheService} from './cache.gallery.service';
-import { BehaviorSubject, Observable } from 'rxjs';
+import {BehaviorSubject, Observable} from 'rxjs';
-import { Config } from '../../../../common/config/public/Config';
+import {Config} from '../../../../common/config/public/Config';
-import { ShareService } from './share.service';
+import {ShareService} from './share.service';
-import { NavigationService } from '../../model/navigation.service';
+import {NavigationService} from '../../model/navigation.service';
-import { QueryParams } from '../../../../common/QueryParams';
+import {QueryParams} from '../../../../common/QueryParams';
-import { SearchQueryDTO } from '../../../../common/entities/SearchQueryDTO';
+import {SearchQueryDTO} from '../../../../common/entities/SearchQueryDTO';
-import { ErrorCodes } from '../../../../common/entities/Error';
+import {ErrorCodes} from '../../../../common/entities/Error';
-import { map } from 'rxjs/operators';
+import {map} from 'rxjs/operators';
-import { MediaDTO } from '../../../../common/entities/MediaDTO';
+import {MediaDTO} from '../../../../common/entities/MediaDTO';
-import { FileDTO } from '../../../../common/entities/FileDTO';
+import {FileDTO} from '../../../../common/entities/FileDTO';
 
 @Injectable()
 export class ContentService {
@@ -48,14 +47,15 @@ export class ContentService {
   }
 
   public async loadDirectory(directoryName: string): Promise<void> {
-    const content = new ContentWrapperWithError();
 
-    content.directory = this.galleryCacheService.getDirectory(directoryName);
+    // load from cache
-    content.searchResult = null;
+    const cw = this.galleryCacheService.getDirectory(directoryName);
 
-    this.setContent(content);
+    ContentWrapper.unpack(cw);
+    this.setContent(cw);
     this.lastRequest.directory = directoryName;
 
+    // prepare server request
     const params: { [key: string]: any } = {};
     if (Config.Client.Sharing.enabled === true) {
       if (this.shareService.isSharing()) {
@@ -65,15 +65,15 @@ export class ContentService {
     }
 
     if (
-      content.directory &&
+      cw.directory &&
-      content.directory.lastModified &&
+      cw.directory.lastModified &&
-      content.directory.lastScanned &&
+      cw.directory.lastScanned &&
-      !content.directory.isPartial
+      !cw.directory.isPartial
     ) {
       params[QueryParams.gallery.knownLastModified] =
-        content.directory.lastModified;
+        cw.directory.lastModified;
       params[QueryParams.gallery.knownLastScanned] =
-        content.directory.lastScanned;
+        cw.directory.lastScanned;
     }
 
     try {
@@ -86,13 +86,13 @@ export class ContentService {
         return;
       }
 
-      this.galleryCacheService.setDirectory(cw.directory); // save it before adding references
+      this.galleryCacheService.setDirectory(cw); // save it before adding references
 
       if (this.lastRequest.directory !== directoryName) {
         return;
       }
 
-      DirectoryDTOUtils.unpackDirectory(cw.directory);
+      ContentWrapper.unpack(cw);
 
       this.lastDirectory = cw.directory;
       this.setContent(cw);
@@ -110,14 +110,11 @@ export class ContentService {
     this.ongoingSearch = query;
 
     this.setContent(new ContentWrapperWithError());
-    const cw = new ContentWrapperWithError();
+    let cw = this.galleryCacheService.getSearch(query);
-    cw.searchResult = this.galleryCacheService.getSearch(query);
+    if (!cw || cw.searchResult == null) {
-    if (cw.searchResult == null) {
       try {
-        cw.searchResult = (
+        cw = await this.networkService.getJson<ContentWrapperWithError>('/search/' + query);
-          await this.networkService.getJson<ContentWrapper>('/search/' + query)
+        this.galleryCacheService.setSearch(cw);
-        ).searchResult;
-        this.galleryCacheService.setSearch(query, cw.searchResult);
       } catch (e) {
         if (e.code === ErrorCodes.LocationLookUp_ERROR) {
           cw.error = 'Cannot find location: ' + e.message;
@@ -131,6 +128,7 @@ export class ContentService {
       return;
     }
 
+    ContentWrapper.unpack(cw);
     this.setContent(cw);
   }
 
@@ -140,7 +138,7 @@ export class ContentService {
 }
 
 export class ContentWrapperWithError extends ContentWrapper {
-  public error: string;
+  public error?: string;
 }
 
 export interface DirectoryContent {

test/TestHelper.ts

@@ -3,11 +3,11 @@ import {
   GPSMetadataEntity,
   MediaDimensionEntity,
   PositionMetaDataEntity
-} from '../../../../../src/backend/model/database/sql/enitites/MediaEntity';
+} from '../src/backend/model/database/sql/enitites/MediaEntity';
-import {PhotoEntity, PhotoMetadataEntity} from '../../../../../src/backend/model/database/sql/enitites/PhotoEntity';
+import {PhotoEntity, PhotoMetadataEntity} from '../src/backend/model/database/sql/enitites/PhotoEntity';
-import {DirectoryEntity} from '../../../../../src/backend/model/database/sql/enitites/DirectoryEntity';
+import {DirectoryEntity} from '../src/backend/model/database/sql/enitites/DirectoryEntity';
-import {VideoEntity, VideoMetadataEntity} from '../../../../../src/backend/model/database/sql/enitites/VideoEntity';
+import {VideoEntity, VideoMetadataEntity} from '../src/backend/model/database/sql/enitites/VideoEntity';
-import {MediaDimension, MediaDTO} from '../../../../../src/common/entities/MediaDTO';
+import {MediaDimension, MediaDTO} from '../src/common/entities/MediaDTO';
 import {
   CameraMetadata,
   FaceRegion,
@@ -16,10 +16,10 @@ import {
   PhotoMetadata,
   PositionMetaData,
   PreviewPhotoDTO
-} from '../../../../../src/common/entities/PhotoDTO';
+} from '../src/common/entities/PhotoDTO';
-import {DirectoryBaseDTO} from '../../../../../src/common/entities/DirectoryDTO';
+import {DirectoryBaseDTO, DirectoryPathDTO} from '../src/common/entities/DirectoryDTO';
-import {FileDTO} from '../../../../../src/common/entities/FileDTO';
+import {FileDTO} from '../src/common/entities/FileDTO';
-import {DiskMangerWorker} from '../../../../../src/backend/model/threading/DiskMangerWorker';
+import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker';
 
 export class TestHelper {
 
@@ -34,8 +34,8 @@ export class TestHelper {
     dir.directories = [];
     dir.metaFile = [];
     dir.media = [];
-    dir.lastModified = Date.now();
+    dir.lastModified = 1656069687773;
-    dir.lastScanned = Date.now();
+    dir.lastScanned = 1656069687773;
     // dir.parent = null;
     if (parent !== null) {
       dir.path = DiskMangerWorker.pathFromParent(parent);
@@ -44,9 +44,9 @@ export class TestHelper {
     return dir;
   }
 
-  public static getPhotoEntry(dir: DirectoryBaseDTO): PhotoEntity {
+  public static getPhotoEntry(dir: DirectoryPathDTO): PhotoEntity {
     const sd = new MediaDimensionEntity();
-    sd.height = 200;
+    sd.height = 400;
     sd.width = 200;
     const gps = new GPSMetadataEntity();
     gps.latitude = 1;
@@ -70,7 +70,7 @@ export class TestHelper {
     m.cameraData = cd;
     m.positionData = pd;
     m.size = sd;
-    m.creationDate = Date.now();
+    m.creationDate = 1656069387772;
     m.fileSize = 123456789;
     // m.rating = 0; no rating by default
 
@@ -82,23 +82,25 @@ export class TestHelper {
     const d = new PhotoEntity();
     d.name = 'test media.jpg';
     d.directory = (dir as any);
-    dir.media.push(d);
+    if ((dir as DirectoryBaseDTO).media) {
+      (dir as DirectoryBaseDTO).media.push(d);
+      (dir as DirectoryBaseDTO).mediaCount++;
+    }
     d.metadata = m;
-    dir.mediaCount++;
     return d;
   }
 
-  public static getVideoEntry(dir: DirectoryBaseDTO): VideoEntity {
+  public static getVideoEntry(dir: DirectoryPathDTO): VideoEntity {
     const sd = new MediaDimensionEntity();
     sd.height = 200;
-    sd.width = 200;
+    sd.width = 300;
 
     const m = new VideoMetadataEntity();
     m.caption = null;
     m.keywords = null;
     m.rating = null;
     m.size = sd;
-    m.creationDate = Date.now();
+    m.creationDate = 1656069387771;
     m.fileSize = 123456789;
 
     m.duration = 10000;
@@ -107,18 +109,34 @@ export class TestHelper {
 
     const d = new VideoEntity();
     d.name = 'test video.mp4';
-    dir.media.push(d);
+    d.directory = (dir as any);
+    if ((dir as DirectoryBaseDTO).media) {
+      (dir as DirectoryBaseDTO).media.push(d);
+      (dir as DirectoryBaseDTO).mediaCount++;
+    }
     d.metadata = m;
     return d;
   }
 
+  public static getGPXEntry(dir: DirectoryPathDTO): FileDTO {
+    const d: FileDTO = {
+      id: null,
+      name: 'saturdayRun.gpx',
+      directory: dir
+    };
+    if ((dir as DirectoryBaseDTO).metaFile) {
+      (dir as DirectoryBaseDTO).metaFile.push(d);
+    }
+    return d;
+  }
+
   public static getVideoEntry1(dir: DirectoryBaseDTO): VideoEntity {
     const p = TestHelper.getVideoEntry(dir);
     p.name = 'swVideo.mp4';
     return p;
   }
 
-  public static getPhotoEntry1(dir: DirectoryBaseDTO): PhotoEntity {
+  public static getPhotoEntry1(dir: DirectoryPathDTO): PhotoEntity {
     const p = TestHelper.getPhotoEntry(dir);
 
     p.metadata.caption = 'Han Solo\'s dice';
@@ -128,7 +146,7 @@ export class TestHelper {
     p.name = 'sw1.jpg';
     p.metadata.positionData.GPSData.latitude = 10;
     p.metadata.positionData.GPSData.longitude = 10;
-    p.metadata.creationDate = Date.now() - 1000;
+    p.metadata.creationDate = 1656069387772 - 1000;
     p.metadata.rating = 1;
     p.metadata.size.height = 1000;
     p.metadata.size.width = 1000;
@@ -155,7 +173,7 @@ export class TestHelper {
     return p;
   }
 
-  public static getPhotoEntry2(dir: DirectoryBaseDTO): PhotoEntity {
+  public static getPhotoEntry2(dir: DirectoryPathDTO): PhotoEntity {
     const p = TestHelper.getPhotoEntry(dir);
 
     p.metadata.caption = 'Light saber';
@@ -166,7 +184,7 @@ export class TestHelper {
     p.name = 'sw2.jpg';
     p.metadata.positionData.GPSData.latitude = -10;
     p.metadata.positionData.GPSData.longitude = -10;
-    p.metadata.creationDate = Date.now() - 2000;
+    p.metadata.creationDate = 1656069387772 - 2000;
     p.metadata.rating = 2;
     p.metadata.size.height = 2000;
     p.metadata.size.width = 1000;
@@ -187,7 +205,7 @@ export class TestHelper {
     return p;
   }
 
-  public static getPhotoEntry3(dir: DirectoryBaseDTO): PhotoEntity {
+  public static getPhotoEntry3(dir: DirectoryPathDTO): PhotoEntity {
     const p = TestHelper.getPhotoEntry(dir);
 
     p.metadata.caption = 'Amber stone';
@@ -198,7 +216,7 @@ export class TestHelper {
     p.name = 'sw3.jpg';
     p.metadata.positionData.GPSData.latitude = 10;
     p.metadata.positionData.GPSData.longitude = 15;
-    p.metadata.creationDate = Date.now() - 3000;
+    p.metadata.creationDate = 1656069387772 - 3000;
     p.metadata.rating = 3;
     p.metadata.size.height = 1000;
     p.metadata.size.width = 2000;
@@ -215,7 +233,7 @@ export class TestHelper {
     return p;
   }
 
-  public static getPhotoEntry4(dir: DirectoryBaseDTO): PhotoEntity {
+  public static getPhotoEntry4(dir: DirectoryPathDTO): PhotoEntity {
     const p = TestHelper.getPhotoEntry(dir);
 
     p.metadata.caption = 'Millennium falcon';
@@ -226,7 +244,7 @@ export class TestHelper {
     p.name = 'sw4.jpg';
     p.metadata.positionData.GPSData.latitude = 15;
     p.metadata.positionData.GPSData.longitude = 10;
-    p.metadata.creationDate = Date.now() - 4000;
+    p.metadata.creationDate = 1656069387772 - 4000;
     p.metadata.size.height = 3000;
     p.metadata.size.width = 2000;
 
@@ -303,7 +321,7 @@ export class TestHelper {
     return f;
   }
 
-  public static getRandomizedPhotoEntry(dir: DirectoryBaseDTO, forceStr: string = null, faces: number = 2): PhotoDTO {
+  public static getRandomizedPhotoEntry(dir: DirectoryBaseDTO, forceStr: string = null, faces = 2): PhotoDTO {
 
 
     const rndStr = (): string => {

test/backend/DBTestHelper.ts

@@ -11,7 +11,7 @@ import {IndexingManager} from '../../src/backend/model/database/sql/IndexingMana
 import {GalleryManager} from '../../src/backend/model/database/sql/GalleryManager';
 import {Connection} from 'typeorm';
 import {Utils} from '../../src/common/Utils';
-import {TestHelper} from './unit/model/sql/TestHelper';
+import {TestHelper} from '../TestHelper';
 import {VideoDTO} from '../../src/common/entities/VideoDTO';
 import {PhotoDTO} from '../../src/common/entities/PhotoDTO';
 

test/backend/unit/model/sql/AlbumManager.spec.ts

@@ -1,6 +1,6 @@
 import {DBTestHelper} from '../../../DBTestHelper';
 import {ParentDirectoryDTO, SubDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
-import {TestHelper} from './TestHelper';
+import {TestHelper} from '../../../../TestHelper';
 import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers';
 import {PhotoDTO} from '../../../../../src/common/entities/PhotoDTO';
 import {VideoDTO} from '../../../../../src/common/entities/VideoDTO';

test/backend/unit/model/sql/IndexingManager.spec.ts

@@ -3,7 +3,7 @@ import {Config} from '../../../../../src/common/config/private/Config';
 import {SQLConnection} from '../../../../../src/backend/model/database/sql/SQLConnection';
 import {GalleryManager} from '../../../../../src/backend/model/database/sql/GalleryManager';
 import {DirectoryBaseDTO, DirectoryDTOUtils, ParentDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
-import {TestHelper} from './TestHelper';
+import {TestHelper} from '../../../../TestHelper';
 import {Connection} from 'typeorm';
 import {DirectoryEntity} from '../../../../../src/backend/model/database/sql/enitites/DirectoryEntity';
 import {Utils} from '../../../../../src/common/Utils';
@@ -145,14 +145,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     p1.name = 'test.jpg';
     p2.name = 'Test.jpg';
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
 
     const conn = await SQLConnection.getConnection();
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
     await gm.fillParentDir(conn, selected);
 
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
 
     expect(Utils.clone(Utils.removeNullOrEmptyObj(removeIds(selected))))
       .to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
@@ -210,14 +210,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     const p2 = TestHelper.getRandomizedPhotoEntry(subDir2, 'subPhoto2', 0);
 
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
 
     const conn = await SQLConnection.getConnection();
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
     await gm.fillParentDir(conn, selected);
 
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
     removeIds(selected);
     setPartial(subDir1);
     setPartial(subDir2);
@@ -237,9 +237,9 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     const p2 = TestHelper.getRandomizedPhotoEntry(subDir2, 'subPhoto2', 0);
 
 
-    DirectoryDTOUtils.packDirectory(parent1);
+    DirectoryDTOUtils.removeReferences(parent1);
     await im.saveToDB(Utils.clone(parent1) as ParentDirectoryDTO);
-    DirectoryDTOUtils.packDirectory(parent2);
+    DirectoryDTOUtils.removeReferences(parent2);
     await im.saveToDB(Utils.clone(parent2) as ParentDirectoryDTO);
 
     const conn = await SQLConnection.getConnection();
@@ -247,7 +247,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
       const selected = await gm.selectParentDir(conn, parent1.name, parent1.path);
       await gm.fillParentDir(conn, selected);
 
-      DirectoryDTOUtils.packDirectory(selected);
+      DirectoryDTOUtils.removeReferences(selected);
       removeIds(selected);
       setPartial(subDir1);
       expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
@@ -257,7 +257,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
       const selected = await gm.selectParentDir(conn, parent2.name, parent2.path);
       await gm.fillParentDir(conn, selected);
 
-      DirectoryDTOUtils.packDirectory(selected);
+      DirectoryDTOUtils.removeReferences(selected);
       removeIds(selected);
       setPartial(subDir2);
       expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
@@ -273,14 +273,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
     p1.name = 'test 😀.jpg';
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
 
     const conn = await SQLConnection.getConnection();
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
     await gm.fillParentDir(conn, selected);
 
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
 
     expect(Utils.clone(Utils.removeNullOrEmptyObj(removeIds(selected))))
       .to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
@@ -293,7 +293,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
       const selected = await gmTest.selectParentDir(conn, dir.name, dir.path);
       await gmTest.fillParentDir(conn, selected);
 
-      DirectoryDTOUtils.packDirectory(selected);
+      DirectoryDTOUtils.removeReferences(selected);
       removeIds(selected);
       return selected;
     };
@@ -316,10 +316,10 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     };
 
     const saveToDBAndCheck = async (dir: DirectoryBaseDTO) => {
-      DirectoryDTOUtils.packDirectory(parent);
+      DirectoryDTOUtils.removeReferences(parent);
       await im.saveToDB(Utils.clone(dir) as ParentDirectoryDTO);
       await checkParent();
-      DirectoryDTOUtils.unpackDirectory(parent);
+      DirectoryDTOUtils.addReferences(parent);
     };
 
     await saveToDBAndCheck(parent);
@@ -359,20 +359,20 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     subDir.preview = sp1;
     Config.Server.Preview.Sorting = [SortingMethods.descRating];
 
-    DirectoryDTOUtils.packDirectory(subDir);
+    DirectoryDTOUtils.removeReferences(subDir);
     await im.saveToDB(Utils.clone(subDir) as ParentDirectoryDTO);
 
     parent.directories.push(subDir);
 
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
 
     const conn = await SQLConnection.getConnection();
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
     await gm.fillParentDir(conn, selected);
 
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
     removeIds(selected);
     setPartial(subDir);
     expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
@@ -397,20 +397,20 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     Config.Server.Preview.Sorting = [SortingMethods.descRating];
 
 
-    DirectoryDTOUtils.packDirectory(subDir);
+    DirectoryDTOUtils.removeReferences(subDir);
     await im.saveToDB(Utils.clone(subDir) as ParentDirectoryDTO);
 
     parent.directories.push(subDir);
 
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
 
     const conn = await SQLConnection.getConnection();
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
     await gm.fillParentDir(conn, selected);
 
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
     removeIds(selected);
     setPartial(subDir);
     expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
@@ -433,14 +433,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     subDir.preview = sp1;
     Config.Server.Preview.Sorting = [SortingMethods.descRating];
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
 
     const conn = await SQLConnection.getConnection();
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
     await gm.fillParentDir(conn, selected);
 
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
     removeIds(selected);
     setPartial(subDir);
     expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
@@ -469,14 +469,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     p2.metadata.positionData.GPSData.longitude = minFloat;
 
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
 
     const conn = await SQLConnection.getConnection();
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
     await gm.fillParentDir(conn, selected);
 
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
     removeIds(selected);
     expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
       .to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
@@ -489,7 +489,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
     const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
     const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     Config.Client.MetaFile.gpx = true;
     Config.Client.MetaFile.markdown = true;
     Config.Client.MetaFile.pg2conf = true;
@@ -503,7 +503,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     await gm.fillParentDir(conn, selected);
 
     delete parent.metaFile;
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
     removeIds(selected);
     expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
       .to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
@@ -520,13 +520,13 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     subDir.name = 'subDir';
     const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
 
     const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
     const sp3 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto3');
 
-    DirectoryDTOUtils.packDirectory(subDir);
+    DirectoryDTOUtils.removeReferences(subDir);
     await im.saveToDB(Utils.clone(subDir) as ParentDirectoryDTO);
 
     const conn = await SQLConnection.getConnection();
@@ -535,7 +535,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
 
     // subDir.isPartial = true;
     //  delete subDir.directories;
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
     delete subDir.parent;
     delete subDir.metaFile;
     removeIds(selected);
@@ -563,7 +563,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     subDir.preview = sp1;
     Config.Server.Preview.Sorting = [SortingMethods.descRating];
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     const s1 = im.queueForSave(Utils.clone(parent) as ParentDirectoryDTO);
     const s2 = im.queueForSave(Utils.clone(parent) as ParentDirectoryDTO);
     const s3 = im.queueForSave(Utils.clone(parent) as ParentDirectoryDTO);
@@ -573,7 +573,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
     await gm.fillParentDir(conn, selected);
 
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
     removeIds(selected);
     setPartial(subDir);
     parent.directories.forEach(d => delete (d.preview.metadata as any).faces);
@@ -591,14 +591,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
     const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
 
     const conn = await SQLConnection.getConnection();
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
     await gm.fillParentDir(conn, selected);
 
-    DirectoryDTOUtils.packDirectory(selected);
+    DirectoryDTOUtils.removeReferences(selected);
     removeIds(selected);
     expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
       .to.deep.equal(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
@@ -617,14 +617,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     Config.Client.MetaFile.markdown = true;
     Config.Client.MetaFile.pg2conf = true;
     const parent = TestHelper.getRandomizedDirectoryEntry();
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
     const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
     for (let i = 0; i < 1500; i++) {
       TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i);
     }
 
-    DirectoryDTOUtils.packDirectory(parent);
+    DirectoryDTOUtils.removeReferences(parent);
     await im.saveToDB(subDir as ParentDirectoryDTO);
 
 

test/backend/unit/model/sql/PersonManager.spec.ts

@@ -1,7 +1,7 @@
 import {expect} from 'chai';
 import {PersonManager} from '../../../../../src/backend/model/database/sql/PersonManager';
 import {DBTestHelper} from '../../../DBTestHelper';
-import {TestHelper} from './TestHelper';
+import {TestHelper} from '../../../../TestHelper';
 import {PhotoDTO} from '../../../../../src/common/entities/PhotoDTO';
 import {Utils} from '../../../../../src/common/Utils';
 import {PersonWithSampleRegion} from '../../../../../src/common/entities/PersonDTO';

test/backend/unit/model/sql/PreviewManager.spec.ts

@@ -3,7 +3,7 @@ import {DBTestHelper} from '../../../DBTestHelper';
 import {SearchQueryDTO, SearchQueryTypes, TextSearch} from '../../../../../src/common/entities/SearchQueryDTO';
 import {IndexingManager} from '../../../../../src/backend/model/database/sql/IndexingManager';
 import {DirectoryBaseDTO, ParentDirectoryDTO, SubDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
-import {TestHelper} from './TestHelper';
+import {TestHelper} from '../../../../TestHelper';
 import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers';
 import {GalleryManager} from '../../../../../src/backend/model/database/sql/GalleryManager';
 import {Connection} from 'typeorm';

test/backend/unit/model/sql/SearchManager.spec.ts

@@ -23,7 +23,7 @@ import {
 } from '../../../../../src/common/entities/SearchQueryDTO';
 import {IndexingManager} from '../../../../../src/backend/model/database/sql/IndexingManager';
 import {DirectoryBaseDTO, ParentDirectoryDTO, SubDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
-import {TestHelper} from './TestHelper';
+import {TestHelper} from '../../../../TestHelper';
 import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers';
 import {GalleryManager} from '../../../../../src/backend/model/database/sql/GalleryManager';
 import {Connection} from 'typeorm';
@@ -1039,7 +1039,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
         .to.deep.equalInAnyOrder(removeDir({
         searchQuery: query,
         directories: [],
-        media: [p, p2, p4, v],
+        media: [p, p2, p4],
         metaFile: [],
         resultOverflow: false
       } as SearchResultDTO));

test/common/unit/ContentWrapper.spec.ts

@@ -0,0 +1,89 @@
+import {expect} from 'chai';
+import {ContentWrapper} from '../../../src/common/entities/ConentWrapper';
+import {TestHelper} from '../../TestHelper';
+import {DirectoryPathDTO, ParentDirectoryDTO} from '../../../src/common/entities/DirectoryDTO';
+import {SearchResultDTO} from '../../../src/common/entities/SearchResultDTO';
+import {SearchQueryTypes, TextSearch} from '../../../src/common/entities/SearchQueryDTO';
+import {Utils} from '../../../src/common/Utils';
+import {MediaDTOUtils} from '../../../src/common/entities/MediaDTO';
+import {VideoDTO} from '../../../src/common/entities/VideoDTO';
+import {PhotoDTO} from '../../../src/common/entities/PhotoDTO';
+
+
+describe('ContentWrapper', () => {
+
+  const cleanUpCW = (cw: ContentWrapper): ContentWrapper => {
+    if (typeof cw.notModified === 'undefined') {
+      delete cw.notModified;
+    }
+
+    const content = (cw.directory ? cw.directory : cw.searchResult);
+    for (let i = 0; i < content.media.length; ++i) {
+      const m = content.media[i];
+      if (MediaDTOUtils.isPhoto(m)) {
+        delete (m as VideoDTO).metadata.bitRate;
+        delete (m as VideoDTO).metadata.duration;
+        if (!(m as PhotoDTO).metadata.caption) {
+          delete (m as PhotoDTO).metadata.caption;
+        }
+      } else if (MediaDTOUtils.isVideo(m)) {
+        delete (m as PhotoDTO).metadata.rating;
+        delete (m as PhotoDTO).metadata.caption;
+        delete (m as PhotoDTO).metadata.cameraData;
+        delete (m as PhotoDTO).metadata.keywords;
+        delete (m as PhotoDTO).metadata.faces;
+        delete (m as PhotoDTO).metadata.positionData;
+      }
+    }
+    for (let i = 0; i < content.metaFile.length; ++i) {
+      delete content.metaFile[i].id;
+    }
+    return cw;
+  };
+
+  it('pack and unpack directory', () => {
+    const parent = TestHelper.getDirectoryEntry();
+    TestHelper.getPhotoEntry(parent);
+    TestHelper.getPhotoEntry1(parent);
+    TestHelper.getPhotoEntry2(parent);
+    TestHelper.getVideoEntry(parent);
+    TestHelper.getGPXEntry(parent);
+    const parentOrig = TestHelper.getDirectoryEntry();
+    TestHelper.getPhotoEntry(parentOrig);
+    TestHelper.getPhotoEntry1(parentOrig);
+    TestHelper.getPhotoEntry2(parentOrig);
+    TestHelper.getVideoEntry(parentOrig);
+    TestHelper.getGPXEntry(parentOrig);
+    const cwOrig = new ContentWrapper(parentOrig as ParentDirectoryDTO, null);
+    const cw = new ContentWrapper(parent as ParentDirectoryDTO, null);
+    expect(ContentWrapper.unpack(ContentWrapper.pack(cw))).to.deep.equals(cleanUpCW(cwOrig));
+  });
+  it('pack and unpack search result', () => {
+
+    const parent: DirectoryPathDTO = {
+      name: 'parent',
+      path: ''
+    };
+
+    const subDir: DirectoryPathDTO = {
+      name: 'subDir',
+      path: 'parent/'
+    };
+
+    const sr: SearchResultDTO = {
+      directories: [subDir as any],
+      media: [TestHelper.getPhotoEntry(parent),
+        TestHelper.getPhotoEntry1(parent),
+        TestHelper.getPhotoEntry2(subDir),
+        TestHelper.getVideoEntry(parent)
+      ],
+      metaFile: [
+        TestHelper.getGPXEntry(parent)],
+      resultOverflow: false,
+      searchQuery: {type: SearchQueryTypes.any_text, text: ''} as TextSearch
+    };
+
+    const cw = new ContentWrapper(null, sr);
+    expect(ContentWrapper.unpack(ContentWrapper.pack(Utils.clone(cw)))).to.deep.equals(cleanUpCW(cw));
+  });
+});