self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2024-11-24 08:42:24 +02:00
Code Issues Releases Activity

Implementing ContentWrapper packing.

Browse Source 
This enables to extract common string into a map and only reference their values.
This is expected to bring a further 43% savings on search results. Altogether leading to a 50% reduction.
 #437
This commit is contained in:
Patrik J. Braun 2022-06-24 18:05:45 +02:00
parent 9f1b8b95ad
commit b6b576ba2f
17 changed files with 697 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
benchmark/index.ts
View File

@ -55,7 +55,7 @@ const printExperimentResult = (result: BenchmarkResult, isSubResult = false) =>
if (result.contentWrapper.directory) {
details = 'media: ' + result.contentWrapper.directory.media.length +
', directories: ' + result.contentWrapper.directory.directories.length +
', size: ' + fileSize(JSON.stringify(DirectoryDTOUtils.packDirectory(result.contentWrapper.directory)).length);
', size: ' + fileSize(JSON.stringify(DirectoryDTOUtils.removeReferences(result.contentWrapper.directory)).length);
} else {
details = 'media: ' + result.contentWrapper.searchResult.media.length +
', directories: ' + result.contentWrapper.searchResult.directories.length +

80
src/backend/middlewares/GalleryMWs.ts
View File

@ -1,31 +1,26 @@
import * as path from 'path';
import { promises as fsp } from 'fs';
import {promises as fsp} from 'fs';
import * as archiver from 'archiver';
import { NextFunction, Request, Response } from 'express';
import { ErrorCodes, ErrorDTO } from '../../common/entities/Error';
import {NextFunction, Request, Response} from 'express';
import {ErrorCodes, ErrorDTO} from '../../common/entities/Error';
import {
DirectoryBaseDTO,
DirectoryDTOUtils,
ParentDirectoryDTO,
} from '../../common/entities/DirectoryDTO';
import { ObjectManagers } from '../model/ObjectManagers';
import { ContentWrapper } from '../../common/entities/ConentWrapper';
import { PhotoDTO } from '../../common/entities/PhotoDTO';
import { ProjectPath } from '../ProjectPath';
import { Config } from '../../common/config/private/Config';
import { UserDTOUtils } from '../../common/entities/UserDTO';
import { MediaDTO, MediaDTOUtils } from '../../common/entities/MediaDTO';
import { VideoDTO } from '../../common/entities/VideoDTO';
import { Utils } from '../../common/Utils';
import { QueryParams } from '../../common/QueryParams';
import { VideoProcessing } from '../model/fileprocessing/VideoProcessing';
import {ObjectManagers} from '../model/ObjectManagers';
import {ContentWrapper} from '../../common/entities/ConentWrapper';
import {ProjectPath} from '../ProjectPath';
import {Config} from '../../common/config/private/Config';
import {UserDTOUtils} from '../../common/entities/UserDTO';
import {MediaDTO, MediaDTOUtils} from '../../common/entities/MediaDTO';
import {QueryParams} from '../../common/QueryParams';
import {VideoProcessing} from '../model/fileprocessing/VideoProcessing';
import {
SearchQueryDTO,
SearchQueryTypes,
} from '../../common/entities/SearchQueryDTO';
import { LocationLookupException } from '../exceptions/LocationLookupException';
import { SupportedFormats } from '../../common/SupportedFormats';
import { ServerTime } from './ServerTimingMWs';
import {LocationLookupException} from '../exceptions/LocationLookupException';
import {SupportedFormats} from '../../common/SupportedFormats';
import {ServerTime} from './ServerTimingMWs';
export class GalleryMWs {
@ServerTime('1.db', 'List Directory')
@ -130,12 +125,12 @@ export class GalleryMWs {
// append photos in absoluteDirectoryName
// using case-insensitive glob of extensions
for (const ext of SupportedFormats.WithDots.Photos) {
archive.glob(`*${ext}`, { cwd: absoluteDirectoryName, nocase: true });
archive.glob(`*${ext}`, {cwd: absoluteDirectoryName, nocase: true});
}
// append videos in absoluteDirectoryName
// using case-insensitive glob of extensions
for (const ext of SupportedFormats.WithDots.Videos) {
archive.glob(`*${ext}`, { cwd: absoluteDirectoryName, nocase: true });
archive.glob(`*${ext}`, {cwd: absoluteDirectoryName, nocase: true});
}
await archive.finalize();
@ -147,7 +142,7 @@ export class GalleryMWs {
}
}
@ServerTime('3.cleanUp', 'Clean up')
@ServerTime('3.pack', 'pack result')
public static cleanUpGalleryResults(
req: Request,
res: Response,
@ -162,38 +157,6 @@ export class GalleryMWs {
return next();
}
const cleanUpMedia = (media: MediaDTO[]): void => {
for (const m of media) {
delete m.id;
if (MediaDTOUtils.isPhoto(m)) {
delete (m as VideoDTO).metadata.bitRate;
delete (m as VideoDTO).metadata.duration;
} else if (MediaDTOUtils.isVideo(m)) {
delete (m as PhotoDTO).metadata.rating;
delete (m as PhotoDTO).metadata.caption;
delete (m as PhotoDTO).metadata.cameraData;
delete (m as PhotoDTO).metadata.keywords;
delete (m as PhotoDTO).metadata.positionData;
}
if (m.directory) {
delete (m.directory as DirectoryBaseDTO).id;
}
Utils.removeNullOrEmptyObj(m);
}
};
if (cw.directory) {
DirectoryDTOUtils.packDirectory(cw.directory);
// TODO: remove when typeorm inheritance is fixed (and handles proper inheritance)
cleanUpMedia(cw.directory.media);
}
if (cw.searchResult) {
cw.searchResult.directories.forEach((d) =>
DirectoryDTOUtils.packDirectory(d)
);
cleanUpMedia(cw.searchResult.media);
}
if (Config.Client.Media.Video.enabled === false) {
if (cw.directory) {
const removeVideos = (dir: ParentDirectoryDTO): void => {
@ -210,6 +173,8 @@ export class GalleryMWs {
}
}
ContentWrapper.pack(cw);
return next();
}
@ -236,7 +201,7 @@ export class GalleryMWs {
new ErrorDTO(
ErrorCodes.GENERAL_ERROR,
'no such file:' + req.params['mediaPath'],
"can't find file: " + fullMediaPath
'can\'t find file: ' + fullMediaPath
)
);
}
@ -263,7 +228,8 @@ export class GalleryMWs {
await fsp.access(convertedVideo);
req.resultPipe = convertedVideo;
// eslint-disable-next-line no-empty
} catch (e) {}
} catch (e) {
}
return next();
}
@ -375,7 +341,7 @@ export class GalleryMWs {
return next(
new ErrorDTO(
ErrorCodes.GENERAL_ERROR,
"Can't get random photo: " + e.toString()
'Can\'t get random photo: ' + e.toString()
)
);
}

1
src/backend/model/DiskManger.ts
View File

@ -52,7 +52,6 @@ export class DiskManager {
settings
)) as ParentDirectoryDTO;
}
DirectoryDTOUtils.unpackDirectory(directory);
return directory;
}
}

3
src/backend/model/database/memory/GalleryManager.ts
View File

@ -1,4 +1,4 @@
import { ParentDirectoryDTO } from '../../../../common/entities/DirectoryDTO';
import {DirectoryDTOUtils, ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO';
import { IGalleryManager } from '../interfaces/IGalleryManager';
import * as path from 'path';
import * as fs from 'fs';
@ -32,6 +32,7 @@ export class GalleryManager implements IGalleryManager {
}
}
const dir = await DiskManager.scanDirectory(relativeDirectoryName);
DirectoryDTOUtils.addReferences(dir);
dir.metaFile = dir.metaFile.filter((m) => !ServerPG2ConfMap[m.name]);
return dir;
}

13
src/backend/model/database/sql/IndexingManager.ts
View File

@ -1,4 +1,4 @@
import {ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO';
import {DirectoryDTOUtils, DirectoryPathDTO, ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO';
import {DirectoryEntity} from './enitites/DirectoryEntity';
import {SQLConnection} from './SQLConnection';
import {DiskManager} from '../../DiskManger';
@ -43,14 +43,15 @@ export class IndexingManager implements IIndexingManager {
}
private static async processServerSidePG2Conf(
parent: DirectoryPathDTO,
files: FileDTO[]
): Promise<void> {
for (const f of files) {
if (ServerPG2ConfMap[f.name] === ServerSidePG2ConfAction.SAVED_SEARCH) {
const fullMediaPath = path.join(
ProjectPath.ImageFolder,
f.directory.path,
f.directory.name,
parent.path,
parent.name,
f.name
);
@ -94,12 +95,14 @@ export class IndexingManager implements IIndexingManager {
relativeDirectoryName
);
const dirClone = Utils.shallowClone(scannedDirectory);
const dirClone = Utils.clone(scannedDirectory);
// filter server side only config from returning
dirClone.metaFile = dirClone.metaFile.filter(
(m) => !ServerPG2ConfMap[m.name]
);
DirectoryDTOUtils.addReferences(dirClone);
resolve(dirClone);
// save directory to DB
@ -142,7 +145,7 @@ export class IndexingManager implements IIndexingManager {
await this.saveChildDirs(connection, currentDirId, scannedDirectory);
await this.saveMedia(connection, currentDirId, scannedDirectory.media);
await this.saveMetaFiles(connection, currentDirId, scannedDirectory);
await IndexingManager.processServerSidePG2Conf(serverSideConfigs);
await IndexingManager.processServerSidePG2Conf(scannedDirectory, serverSideConfigs);
await ObjectManagers.getInstance().onDataChange(scannedDirectory);
} finally {
this.isSaving = false;

436
src/common/entities/ConentWrapper.ts
View File

@ -1,10 +1,434 @@
import { ParentDirectoryDTO } from './DirectoryDTO';
import { SearchResultDTO } from './SearchResultDTO';
/* eslint-disable @typescript-eslint/ban-ts-comment */
import {DirectoryBaseDTO, DirectoryPathDTO, ParentDirectoryDTO} from './DirectoryDTO';
import {SearchResultDTO} from './SearchResultDTO';
import {Utils} from '../Utils';
import {MediaDTO, MediaDTOUtils} from './MediaDTO';
import {FileDTO} from './FileDTO';
import {VideoDTO} from './VideoDTO';
import {PhotoDTO} from './PhotoDTO';
export class ContentWrapper {
private map: {
faces: string[],
keywords: string[],
lens: string[],
camera: string[],
directories: DirectoryPathDTO[]
};
private reverseMap: {
faces: Map<string, number>,
keywords: Map<string, number>,
lens: Map<string, number>,
camera: Map<string, number>,
directories: Map<string, number>
};
public directory: ParentDirectoryDTO;
public searchResult: SearchResultDTO;
public notModified: boolean;
constructor(
public directory: ParentDirectoryDTO = null,
public searchResult: SearchResultDTO = null,
public notModified?: boolean
) {}
directory: ParentDirectoryDTO = null,
searchResult: SearchResultDTO = null,
notModified?: boolean
) {
if (directory) {
this.directory = directory;
}
if (searchResult) {
this.searchResult = searchResult;
}
if (notModified) {
this.notModified = notModified;
}
}
private static mapify(cw: ContentWrapper, media: FileDTO, isSearchResult: boolean, isPhoto: boolean, isVideo: boolean): void {
if (isSearchResult) {
const k = JSON.stringify(media.directory);
if (!cw.reverseMap.directories.has(k)) {
cw.reverseMap.directories.set(k, cw.map.directories.length);
cw.map.directories.push(media.directory);
}
// @ts-ignore
media.d = cw.reverseMap.directories.get(k);
delete media.directory;
}
if ((media as MediaDTO).metadata) {
// @ts-ignore
(media as MediaDTO).metadata.s = [(media as MediaDTO).metadata.size.width, (media as MediaDTO).metadata.size.height];
delete (media as MediaDTO).metadata.size;
}
if (isPhoto) {
if ((media as PhotoDTO).metadata.faces) {
for (let i = 0; i < (media as PhotoDTO).metadata.faces.length; ++i) {
const name = (media as PhotoDTO).metadata.faces[i].name;
if (!cw.reverseMap.faces.has(name)) {
cw.reverseMap.faces.set(name, cw.map.faces.length);
cw.map.faces.push(name);
}
// @ts-ignore
(media as PhotoDTO).metadata.faces[i].n = cw.reverseMap.faces.get(name);
delete (media as PhotoDTO).metadata.faces[i].name;
}
}
if ((media as PhotoDTO).metadata.keywords) {
for (let i = 0; i < (media as PhotoDTO).metadata.keywords.length; ++i) {
const k = (media as PhotoDTO).metadata.keywords[i];
if (!cw.reverseMap.keywords.has(k)) {
cw.reverseMap.keywords.set(k, cw.map.keywords.length);
cw.map.keywords.push(k);
}
// @ts-ignore
(media as PhotoDTO).metadata.keywords[i] = cw.reverseMap.keywords.get(k);
}
}
const mapifyOne = <T>(map: string[], reverseMap: Map<string, number>,
obj: T, key: keyof T, mappedKey: string) => {
// @ts-ignore
const k: string = obj[key];
if (!reverseMap.has(k)) {
reverseMap.set(k, map.length);
map.push(k);
}
// @ts-ignore
obj[mappedKey] = reverseMap.get(k);
delete obj[key];
};
if ((media as PhotoDTO).metadata.cameraData) {
if ((media as PhotoDTO).metadata.cameraData.lens) {
mapifyOne(cw.map.lens, cw.reverseMap.lens,
(media as PhotoDTO).metadata.cameraData, 'lens', 'l');
}
if ((media as PhotoDTO).metadata.cameraData.make) {
mapifyOne(cw.map.camera, cw.reverseMap.camera,
(media as PhotoDTO).metadata.cameraData, 'make', 'm');
}
if ((media as PhotoDTO).metadata.cameraData.model) {
mapifyOne(cw.map.camera, cw.reverseMap.camera,
(media as PhotoDTO).metadata.cameraData, 'model', 'ml');
}
// @ts-ignore
(media as PhotoDTO).metadata.c = (media as PhotoDTO).metadata.cameraData;
delete (media as PhotoDTO).metadata.cameraData;
}
if ((media as PhotoDTO).metadata.positionData) {
if ((media as PhotoDTO).metadata.positionData.country) {
mapifyOne(cw.map.keywords, cw.reverseMap.keywords,
(media as PhotoDTO).metadata.positionData, 'country', 'c');
}
if ((media as PhotoDTO).metadata.positionData.city) {
mapifyOne(cw.map.keywords, cw.reverseMap.keywords,
(media as PhotoDTO).metadata.positionData, 'city', 'cy');
}
if ((media as PhotoDTO).metadata.positionData.state) {
mapifyOne(cw.map.keywords, cw.reverseMap.keywords,
(media as PhotoDTO).metadata.positionData, 'state', 's');
}
if ((media as PhotoDTO).metadata.positionData.GPSData) {
// @ts-ignore
(media as PhotoDTO).metadata.positionData.g = [(media as PhotoDTO).metadata.positionData.GPSData.latitude, (media as PhotoDTO).metadata.positionData.GPSData.longitude];
delete (media as PhotoDTO).metadata.positionData.GPSData;
}
// @ts-ignore
(media as PhotoDTO).metadata.p = (media as PhotoDTO).metadata.positionData;
delete (media as PhotoDTO).metadata.positionData;
}
}
}
private static packMedia(cw: ContentWrapper, media: MediaDTO[], isSearchResult: boolean): void {
// clean up media
for (let i = 0; i < media.length; ++i) {
const m = media[i];
delete m.id;
if (m.directory) {
if (isSearchResult) {
// keep the directory for search result
delete (m.directory as DirectoryBaseDTO).id;
} else {
// for gallery listing, photos belong to one directory,
// this can be deleted as the directory know its child
delete m.directory;
}
}
if (MediaDTOUtils.isPhoto(m)) {
delete (m as VideoDTO).metadata.bitRate;
delete (m as VideoDTO).metadata.duration;
// compress faces
if ((m as PhotoDTO).metadata.faces) {
for (let j = 0; j < (m as PhotoDTO).metadata.faces.length; ++j) {
const f = (m as PhotoDTO).metadata.faces[j];
// @ts-ignore
f['b'] = [f.box.top, f.box.left, f.box.height, f.box.width];
delete f.box;
}
}
ContentWrapper.mapify(cw, m, isSearchResult, true, false);
} else if (MediaDTOUtils.isVideo(m)) {
delete (m as PhotoDTO).metadata.rating;
delete (m as PhotoDTO).metadata.caption;
delete (m as PhotoDTO).metadata.cameraData;
delete (m as PhotoDTO).metadata.keywords;
delete (m as PhotoDTO).metadata.faces;
delete (m as PhotoDTO).metadata.positionData;
ContentWrapper.mapify(cw, m, isSearchResult, false, true);
}
Utils.removeNullOrEmptyObj(m);
}
}
private static packDirectory(cw: ContentWrapper, dir: DirectoryBaseDTO | SearchResultDTO, isSearchResult = false): void {
if ((dir as DirectoryBaseDTO).preview) {
(dir as DirectoryBaseDTO).preview.directory = {
path: (dir as DirectoryBaseDTO).preview.directory.path,
name: (dir as DirectoryBaseDTO).preview.directory.name,
} as DirectoryPathDTO;
// make sure that it is not a same object as one of the photo in the media[]
// as the next foreach would remove the directory
(dir as DirectoryBaseDTO).preview = Utils.clone((dir as DirectoryBaseDTO).preview);
}
if (dir.media) {
ContentWrapper.packMedia(cw, dir.media, isSearchResult);
}
if (dir.metaFile) {
for (let i = 0; i < dir.metaFile.length; ++i) {
if (isSearchResult) {
delete (dir.metaFile[i].directory as DirectoryBaseDTO).id;
} else {
delete dir.metaFile[i].directory;
}
delete dir.metaFile[i].id;
ContentWrapper.mapify(cw, dir.metaFile[i], isSearchResult, false, false);
}
}
if (dir.directories) {
for (let i = 0; i < dir.directories.length; ++i) {
ContentWrapper.packDirectory(cw, dir.directories[i]);
delete dir.directories[i].parent;
}
}
delete (dir as DirectoryBaseDTO).validPreview; // should not go to the client side;
}
private static deMapify(cw: ContentWrapper, media: FileDTO, isSearchResult: boolean, isPhoto: boolean, isVideo: boolean): void {
const deMapifyOne = <T>(map: any[],
obj: T, key: keyof T, mappedKey: string) => {
// @ts-ignore
obj[key] = map[obj[mappedKey]];
// @ts-ignore
delete obj[mappedKey];
};
if (isSearchResult) {
deMapifyOne(cw.map.directories, media, 'directory', 'd');
}
if ((media as MediaDTO).metadata) {
(media as MediaDTO).metadata.size = {
// @ts-ignore
width: (media as MediaDTO).metadata.s[0],
// @ts-ignore
height: (media as MediaDTO).metadata.s[1],
};
// @ts-ignore
delete (media as MediaDTO).metadata.s;
}
if (isPhoto) {
if ((media as PhotoDTO).metadata.faces) {
for (let i = 0; i < (media as PhotoDTO).metadata.faces.length; ++i) {
// @ts-ignore
(media as PhotoDTO).metadata.faces[i].name = cw.map.faces[(media as PhotoDTO).metadata.faces[i].n];
// @ts-ignore
delete (media as PhotoDTO).metadata.faces[i].n;
}
}
if ((media as PhotoDTO).metadata.keywords) {
for (let i = 0; i < (media as PhotoDTO).metadata.keywords.length; ++i) {
// @ts-ignore
(media as PhotoDTO).metadata.keywords[i] = cw.map.keywords[(media as PhotoDTO).metadata.keywords[i]];
}
}
// @ts-ignore
if ((media as PhotoDTO).metadata.c) {
// @ts-ignore
(media as PhotoDTO).metadata.cameraData = (media as PhotoDTO).metadata.c;
// @ts-ignore
delete (media as PhotoDTO).metadata.c;
// @ts-ignore
if (typeof (media as PhotoDTO).metadata.cameraData.l !== 'undefined') {
deMapifyOne(cw.map.lens,
(media as PhotoDTO).metadata.cameraData, 'lens', 'l');
}
// @ts-ignore
if (typeof (media as PhotoDTO).metadata.cameraData.m !== 'undefined') {
deMapifyOne(cw.map.camera,
(media as PhotoDTO).metadata.cameraData, 'make', 'm');
}
// @ts-ignore
if (typeof (media as PhotoDTO).metadata.cameraData.ml !== 'undefined') {
deMapifyOne(cw.map.camera,
(media as PhotoDTO).metadata.cameraData, 'model', 'ml');
}
}
// @ts-ignore
if ((media as PhotoDTO).metadata.p) {
// @ts-ignore
(media as PhotoDTO).metadata.positionData = (media as PhotoDTO).metadata.p;
// @ts-ignore
delete (media as PhotoDTO).metadata.p;
// @ts-ignore
if (typeof (media as PhotoDTO).metadata.positionData.c !== 'undefined') {
deMapifyOne(cw.map.keywords,
(media as PhotoDTO).metadata.positionData, 'country', 'c');
}
// @ts-ignore
if (typeof (media as PhotoDTO).metadata.positionData.cy !== 'undefined') {
deMapifyOne(cw.map.keywords,
(media as PhotoDTO).metadata.positionData, 'city', 'cy');
}
// @ts-ignore
if (typeof (media as PhotoDTO).metadata.positionData.s !== 'undefined') {
deMapifyOne(cw.map.keywords,
(media as PhotoDTO).metadata.positionData, 'state', 's');
}
// @ts-ignore
if ((media as PhotoDTO).metadata.positionData['g']) {
(media as PhotoDTO).metadata.positionData.GPSData =
{
// @ts-ignore
latitude: (media as PhotoDTO).metadata.positionData['g'][0],
// @ts-ignore
longitude: (media as PhotoDTO).metadata.positionData['g'][1]
};
// @ts-ignore
delete (media as PhotoDTO).metadata.positionData['g'];
}
}
}
}
private static unPackMedia(cw: ContentWrapper, dir: DirectoryBaseDTO, media: MediaDTO[], isSearchResult: boolean): void {
// clean up media
for (let i = 0; i < media.length; ++i) {
const m = media[i];
if (MediaDTOUtils.isPhoto(m)) {
// compress faces
if ((m as PhotoDTO).metadata.faces) {
for (let j = 0; j < (m as PhotoDTO).metadata.faces.length; ++j) {
const f = (m as PhotoDTO).metadata.faces[j];
// @ts-ignore
const boxArr = f.b;
f.box = {
top: boxArr[0],
left: boxArr[1],
height: boxArr[2],
width: boxArr[3],
};
// @ts-ignore
delete f.b;
}
}
ContentWrapper.deMapify(cw, m, isSearchResult, true, false);
} else if (MediaDTOUtils.isVideo(m)) {
ContentWrapper.deMapify(cw, m, isSearchResult, false, true);
}
if (!isSearchResult) {
m.directory = dir;
}
}
}
private static unPackDirectory(cw: ContentWrapper, dir: DirectoryBaseDTO | SearchResultDTO, isSearchResult = false): void {
if (dir.media) {
ContentWrapper.unPackMedia(cw, dir as DirectoryBaseDTO, dir.media, isSearchResult);
}
if (dir.metaFile) {
for (let i = 0; i < dir.metaFile.length; ++i) {
if (!isSearchResult) {
dir.metaFile[i].directory = dir as DirectoryBaseDTO;
}
ContentWrapper.deMapify(cw, dir.metaFile[i], isSearchResult, false, false);
}
}
if (dir.directories) {
for (let i = 0; i < dir.directories.length; ++i) {
ContentWrapper.unPackDirectory(cw, dir.directories[i]);
if (!isSearchResult) {
dir.directories[i].parent = dir as DirectoryBaseDTO;
}
}
}
}
static pack(cw: ContentWrapper): ContentWrapper {
// init CW for packing
cw.map = {
faces: [], keywords: [], lens: [],
camera: [], directories: []
};
cw.reverseMap = {
faces: new Map(), keywords: new Map(),
lens: new Map(), camera: new Map(), directories: new Map()
};
if (cw.directory) {
ContentWrapper.packDirectory(cw, cw.directory);
} else if (cw.searchResult) {
ContentWrapper.packDirectory(cw, cw.searchResult, true);
}
// remove empty maps
for (const k in cw.map) {
// @ts-ignore
if (cw.map[k].length === 0) {
// @ts-ignore
delete cw.map[k];
}
}
delete cw.reverseMap;
return cw;
}
static unpack(cw: ContentWrapper): ContentWrapper {
if (!cw || cw.notModified) {
return cw;
}
if (cw.directory) {
ContentWrapper.unPackDirectory(cw, cw.directory);
} else if (cw.searchResult) {
ContentWrapper.unPackDirectory(cw, cw.searchResult, true);
}
delete cw.map;
return cw;
}
}

30
src/common/entities/DirectoryDTO.ts
View File

@ -8,21 +8,7 @@ export interface DirectoryPathDTO {
path: string;
}
//
// export interface DirectoryDTO<S extends FileDTO = MediaDTO> extends DirectoryPathDTO {
// id: number;
// name: string;
// path: string;
// lastModified: number;
// lastScanned: number;
// isPartial?: boolean;
// parent: DirectoryDTO<S>;
// mediaCount: number;
// directories: DirectoryDTO<S>[];
// preview: PreviewPhotoDTO;
// media: S[];
// metaFile: FileDTO[];
// }
export interface DirectoryBaseDTO<S extends FileDTO = MediaDTO>
extends DirectoryPathDTO {
@ -72,7 +58,7 @@ export interface SubDirectoryDTO<S extends FileDTO = MediaDTO>
}
export const DirectoryDTOUtils = {
unpackDirectory: (dir: DirectoryBaseDTO): void => {
addReferences: (dir: DirectoryBaseDTO): void => {
dir.media.forEach((media: MediaDTO) => {
media.directory = dir;
});
@ -85,13 +71,13 @@ export const DirectoryDTOUtils = {
if (dir.directories) {
dir.directories.forEach((directory) => {
DirectoryDTOUtils.unpackDirectory(directory);
DirectoryDTOUtils.addReferences(directory);
directory.parent = dir;
});
}
},
packDirectory: (dir: DirectoryBaseDTO): DirectoryBaseDTO => {
removeReferences: (dir: DirectoryBaseDTO): DirectoryBaseDTO => {
if (dir.preview) {
dir.preview.directory = {
path: dir.preview.directory.path,
@ -116,7 +102,7 @@ export const DirectoryDTOUtils = {
}
if (dir.directories) {
dir.directories.forEach((directory) => {
DirectoryDTOUtils.packDirectory(directory);
DirectoryDTOUtils.removeReferences(directory);
directory.parent = null;
});
}
@ -125,10 +111,4 @@ export const DirectoryDTOUtils = {
return dir;
},
filterPhotos: (dir: DirectoryBaseDTO): PhotoDTO[] => {
return dir.media.filter((m) => MediaDTOUtils.isPhoto(m)) as PhotoDTO[];
},
filterVideos: (dir: DirectoryBaseDTO): PhotoDTO[] => {
return dir.media.filter((m) => MediaDTOUtils.isPhoto(m)) as PhotoDTO[];
},
};

73
src/frontend/app/ui/gallery/cache.gallery.service.ts
View File

@ -1,5 +1,5 @@
import {Injectable} from '@angular/core';
import {DirectoryDTOUtils, DirectoryPathDTO, ParentDirectoryDTO,} from '../../../../common/entities/DirectoryDTO';
import { DirectoryPathDTO, ParentDirectoryDTO,} from '../../../../common/entities/DirectoryDTO';
import {Utils} from '../../../../common/Utils';
import {Config} from '../../../../common/config/public/Config';
import {IAutoCompleteItem} from '../../../../common/entities/AutoCompleteItem';
@ -8,6 +8,8 @@ import {MediaDTO} from '../../../../common/entities/MediaDTO';
import {SortingMethods} from '../../../../common/entities/SortingMethods';
import {VersionService} from '../../model/version.service';
import {SearchQueryDTO, SearchQueryTypes,} from '../../../../common/entities/SearchQueryDTO';
import {ContentWrapper} from '../../../../common/entities/ConentWrapper';
import {ContentWrapperWithError} from './content.service';
interface CacheItem<T> {
timestamp: number;
@ -49,10 +51,10 @@ export class GalleryCacheService {
return perfEntries && perfEntries[0] && perfEntries[0].type === 'reload';
}
private static loadCacheItem(key: string): SearchResultDTO {
private static loadCacheItem(key: string): ContentWrapperWithError {
const tmp = localStorage.getItem(key);
if (tmp != null) {
const value: CacheItem<SearchResultDTO> = JSON.parse(tmp);
const value: CacheItem<ContentWrapperWithError> = JSON.parse(tmp);
if (
value.timestamp <
Date.now() - Config.Client.Search.searchCacheTimeout
@ -177,34 +179,7 @@ export class GalleryCacheService {
}
}
public getInstantSearch(text: string): SearchResultDTO {
if (Config.Client.Other.enableCache === false) {
return null;
}
const key = GalleryCacheService.INSTANT_SEARCH_PREFIX + text;
return GalleryCacheService.loadCacheItem(key);
}
public setInstantSearch(text: string, searchResult: SearchResultDTO): void {
if (Config.Client.Other.enableCache === false) {
return;
}
const tmp: CacheItem<SearchResultDTO> = {
timestamp: Date.now(),
item: searchResult,
};
try {
localStorage.setItem(
GalleryCacheService.INSTANT_SEARCH_PREFIX + text,
JSON.stringify(tmp)
);
} catch (e) {
this.reset();
console.error(e);
}
}
public getSearch(query: SearchQueryDTO): SearchResultDTO {
public getSearch(query: SearchQueryDTO): ContentWrapperWithError {
if (Config.Client.Other.enableCache === false) {
return null;
}
@ -212,15 +187,15 @@ export class GalleryCacheService {
return GalleryCacheService.loadCacheItem(key);
}
public setSearch(query: SearchQueryDTO, searchResult: SearchResultDTO): void {
public setSearch(cw: ContentWrapperWithError): void {
if (Config.Client.Other.enableCache === false) {
return;
}
const tmp: CacheItem<SearchResultDTO> = {
const tmp: CacheItem<ContentWrapperWithError> = {
timestamp: Date.now(),
item: searchResult,
item: cw,
};
const key = GalleryCacheService.SEARCH_PREFIX + JSON.stringify(query);
const key = GalleryCacheService.SEARCH_PREFIX + JSON.stringify(cw.searchResult.searchQuery);
try {
localStorage.setItem(key, JSON.stringify(tmp));
} catch (e) {
@ -229,7 +204,7 @@ export class GalleryCacheService {
}
}
public getDirectory(directoryName: string): ParentDirectoryDTO {
public getDirectory(directoryName: string): ContentWrapperWithError {
if (Config.Client.Other.enableCache === false) {
return null;
}
@ -238,41 +213,29 @@ export class GalleryCacheService {
GalleryCacheService.CONTENT_PREFIX + Utils.concatUrls(directoryName)
);
if (value != null) {
const directory: ParentDirectoryDTO = JSON.parse(value);
DirectoryDTOUtils.unpackDirectory(directory);
return directory;
return JSON.parse(value);
}
} catch (e) {
// ignoring errors
}
return null;
return new ContentWrapperWithError();
}
public setDirectory(directory: ParentDirectoryDTO): void {
public setDirectory(cw: ContentWrapper): void {
if (Config.Client.Other.enableCache === false) {
return;
}
const key =
GalleryCacheService.CONTENT_PREFIX +
Utils.concatUrls(directory.path, directory.name);
if (directory.isPartial === true && localStorage.getItem(key)) {
Utils.concatUrls(cw.directory.path, cw.directory.name);
if (cw.directory.isPartial === true && localStorage.getItem(key)) {
return;
}
try {
// try to fit it
localStorage.setItem(key, JSON.stringify(directory));
directory.directories.forEach((dir) => {
const subKey =
GalleryCacheService.CONTENT_PREFIX +
Utils.concatUrls(dir.path, dir.name);
if (localStorage.getItem(subKey) == null) {
// don't override existing
localStorage.setItem(subKey, JSON.stringify(dir));
}
});
localStorage.setItem(key, JSON.stringify(cw));
} catch (e) {
this.reset();
console.error(e);
@ -295,7 +258,7 @@ export class GalleryCacheService {
const value = localStorage.getItem(directoryKey);
if (value != null) {
const directory: ParentDirectoryDTO = JSON.parse(value);
directory.media.forEach((p) => {
directory?.media?.forEach((p) => {
if (p.name === media.name) {
// update data
p.metadata = media.metadata;

68
src/frontend/app/ui/gallery/content.service.ts
View File

@ -1,22 +1,21 @@
import { Injectable } from '@angular/core';
import { NetworkService } from '../../model/network/network.service';
import { ContentWrapper } from '../../../../common/entities/ConentWrapper';
import {Injectable} from '@angular/core';
import {NetworkService} from '../../model/network/network.service';
import {ContentWrapper} from '../../../../common/entities/ConentWrapper';
import {
DirectoryDTOUtils,
ParentDirectoryDTO,
SubDirectoryDTO,
} from '../../../../common/entities/DirectoryDTO';
import { GalleryCacheService } from './cache.gallery.service';
import { BehaviorSubject, Observable } from 'rxjs';
import { Config } from '../../../../common/config/public/Config';
import { ShareService } from './share.service';
import { NavigationService } from '../../model/navigation.service';
import { QueryParams } from '../../../../common/QueryParams';
import { SearchQueryDTO } from '../../../../common/entities/SearchQueryDTO';
import { ErrorCodes } from '../../../../common/entities/Error';
import { map } from 'rxjs/operators';
import { MediaDTO } from '../../../../common/entities/MediaDTO';
import { FileDTO } from '../../../../common/entities/FileDTO';
import {GalleryCacheService} from './cache.gallery.service';
import {BehaviorSubject, Observable} from 'rxjs';
import {Config} from '../../../../common/config/public/Config';
import {ShareService} from './share.service';
import {NavigationService} from '../../model/navigation.service';
import {QueryParams} from '../../../../common/QueryParams';
import {SearchQueryDTO} from '../../../../common/entities/SearchQueryDTO';
import {ErrorCodes} from '../../../../common/entities/Error';
import {map} from 'rxjs/operators';
import {MediaDTO} from '../../../../common/entities/MediaDTO';
import {FileDTO} from '../../../../common/entities/FileDTO';
@Injectable()
export class ContentService {
@ -48,14 +47,15 @@ export class ContentService {
}
public async loadDirectory(directoryName: string): Promise<void> {
const content = new ContentWrapperWithError();
content.directory = this.galleryCacheService.getDirectory(directoryName);
content.searchResult = null;
// load from cache
const cw = this.galleryCacheService.getDirectory(directoryName);
this.setContent(content);
ContentWrapper.unpack(cw);
this.setContent(cw);
this.lastRequest.directory = directoryName;
// prepare server request
const params: { [key: string]: any } = {};
if (Config.Client.Sharing.enabled === true) {
if (this.shareService.isSharing()) {
@ -65,15 +65,15 @@ export class ContentService {
}
if (
content.directory &&
content.directory.lastModified &&
content.directory.lastScanned &&
!content.directory.isPartial
cw.directory &&
cw.directory.lastModified &&
cw.directory.lastScanned &&
!cw.directory.isPartial
) {
params[QueryParams.gallery.knownLastModified] =
content.directory.lastModified;
cw.directory.lastModified;
params[QueryParams.gallery.knownLastScanned] =
content.directory.lastScanned;
cw.directory.lastScanned;
}
try {
@ -86,13 +86,13 @@ export class ContentService {
return;
}
this.galleryCacheService.setDirectory(cw.directory); // save it before adding references
this.galleryCacheService.setDirectory(cw); // save it before adding references
if (this.lastRequest.directory !== directoryName) {
return;
}
DirectoryDTOUtils.unpackDirectory(cw.directory);
ContentWrapper.unpack(cw);
this.lastDirectory = cw.directory;
this.setContent(cw);
@ -110,14 +110,11 @@ export class ContentService {
this.ongoingSearch = query;
this.setContent(new ContentWrapperWithError());
const cw = new ContentWrapperWithError();
cw.searchResult = this.galleryCacheService.getSearch(query);
if (cw.searchResult == null) {
let cw = this.galleryCacheService.getSearch(query);
if (!cw || cw.searchResult == null) {
try {
cw.searchResult = (
await this.networkService.getJson<ContentWrapper>('/search/' + query)
).searchResult;
this.galleryCacheService.setSearch(query, cw.searchResult);
cw = await this.networkService.getJson<ContentWrapperWithError>('/search/' + query);
this.galleryCacheService.setSearch(cw);
} catch (e) {
if (e.code === ErrorCodes.LocationLookUp_ERROR) {
cw.error = 'Cannot find location: ' + e.message;
@ -131,6 +128,7 @@ export class ContentService {
return;
}
ContentWrapper.unpack(cw);
this.setContent(cw);
}
@ -140,7 +138,7 @@ export class ContentService {
}
export class ContentWrapperWithError extends ContentWrapper {
public error: string;
public error?: string;
}
export interface DirectoryContent {

76
test/backend/unit/model/sql/TestHelper.ts → test/TestHelper.ts
View File

@ -3,11 +3,11 @@ import {
GPSMetadataEntity,
MediaDimensionEntity,
PositionMetaDataEntity
} from '../../../../../src/backend/model/database/sql/enitites/MediaEntity';
import {PhotoEntity, PhotoMetadataEntity} from '../../../../../src/backend/model/database/sql/enitites/PhotoEntity';
import {DirectoryEntity} from '../../../../../src/backend/model/database/sql/enitites/DirectoryEntity';
import {VideoEntity, VideoMetadataEntity} from '../../../../../src/backend/model/database/sql/enitites/VideoEntity';
import {MediaDimension, MediaDTO} from '../../../../../src/common/entities/MediaDTO';
} from '../src/backend/model/database/sql/enitites/MediaEntity';
import {PhotoEntity, PhotoMetadataEntity} from '../src/backend/model/database/sql/enitites/PhotoEntity';
import {DirectoryEntity} from '../src/backend/model/database/sql/enitites/DirectoryEntity';
import {VideoEntity, VideoMetadataEntity} from '../src/backend/model/database/sql/enitites/VideoEntity';
import {MediaDimension, MediaDTO} from '../src/common/entities/MediaDTO';
import {
CameraMetadata,
FaceRegion,
@ -16,10 +16,10 @@ import {
PhotoMetadata,
PositionMetaData,
PreviewPhotoDTO
} from '../../../../../src/common/entities/PhotoDTO';
import {DirectoryBaseDTO} from '../../../../../src/common/entities/DirectoryDTO';
import {FileDTO} from '../../../../../src/common/entities/FileDTO';
import {DiskMangerWorker} from '../../../../../src/backend/model/threading/DiskMangerWorker';
} from '../src/common/entities/PhotoDTO';
import {DirectoryBaseDTO, DirectoryPathDTO} from '../src/common/entities/DirectoryDTO';
import {FileDTO} from '../src/common/entities/FileDTO';
import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker';
export class TestHelper {
@ -34,8 +34,8 @@ export class TestHelper {
dir.directories = [];
dir.metaFile = [];
dir.media = [];
dir.lastModified = Date.now();
dir.lastScanned = Date.now();
dir.lastModified = 1656069687773;
dir.lastScanned = 1656069687773;
// dir.parent = null;
if (parent !== null) {
dir.path = DiskMangerWorker.pathFromParent(parent);
@ -44,9 +44,9 @@ export class TestHelper {
return dir;
}
public static getPhotoEntry(dir: DirectoryBaseDTO): PhotoEntity {
public static getPhotoEntry(dir: DirectoryPathDTO): PhotoEntity {
const sd = new MediaDimensionEntity();
sd.height = 200;
sd.height = 400;
sd.width = 200;
const gps = new GPSMetadataEntity();
gps.latitude = 1;
@ -70,7 +70,7 @@ export class TestHelper {
m.cameraData = cd;
m.positionData = pd;
m.size = sd;
m.creationDate = Date.now();
m.creationDate = 1656069387772;
m.fileSize = 123456789;
// m.rating = 0; no rating by default
@ -82,23 +82,25 @@ export class TestHelper {
const d = new PhotoEntity();
d.name = 'test media.jpg';
d.directory = (dir as any);
dir.media.push(d);
if ((dir as DirectoryBaseDTO).media) {
(dir as DirectoryBaseDTO).media.push(d);
(dir as DirectoryBaseDTO).mediaCount++;
}
d.metadata = m;
dir.mediaCount++;
return d;
}
public static getVideoEntry(dir: DirectoryBaseDTO): VideoEntity {
public static getVideoEntry(dir: DirectoryPathDTO): VideoEntity {
const sd = new MediaDimensionEntity();
sd.height = 200;
sd.width = 200;
sd.width = 300;
const m = new VideoMetadataEntity();
m.caption = null;
m.keywords = null;
m.rating = null;
m.size = sd;
m.creationDate = Date.now();
m.creationDate = 1656069387771;
m.fileSize = 123456789;
m.duration = 10000;
@ -107,18 +109,34 @@ export class TestHelper {
const d = new VideoEntity();
d.name = 'test video.mp4';
dir.media.push(d);
d.directory = (dir as any);
if ((dir as DirectoryBaseDTO).media) {
(dir as DirectoryBaseDTO).media.push(d);
(dir as DirectoryBaseDTO).mediaCount++;
}
d.metadata = m;
return d;
}
public static getGPXEntry(dir: DirectoryPathDTO): FileDTO {
const d: FileDTO = {
id: null,
name: 'saturdayRun.gpx',
directory: dir
};
if ((dir as DirectoryBaseDTO).metaFile) {
(dir as DirectoryBaseDTO).metaFile.push(d);
}
return d;
}
public static getVideoEntry1(dir: DirectoryBaseDTO): VideoEntity {
const p = TestHelper.getVideoEntry(dir);
p.name = 'swVideo.mp4';
return p;
}
public static getPhotoEntry1(dir: DirectoryBaseDTO): PhotoEntity {
public static getPhotoEntry1(dir: DirectoryPathDTO): PhotoEntity {
const p = TestHelper.getPhotoEntry(dir);
p.metadata.caption = 'Han Solo\'s dice';
@ -128,7 +146,7 @@ export class TestHelper {
p.name = 'sw1.jpg';
p.metadata.positionData.GPSData.latitude = 10;
p.metadata.positionData.GPSData.longitude = 10;
p.metadata.creationDate = Date.now() - 1000;
p.metadata.creationDate = 1656069387772 - 1000;
p.metadata.rating = 1;
p.metadata.size.height = 1000;
p.metadata.size.width = 1000;
@ -155,7 +173,7 @@ export class TestHelper {
return p;
}
public static getPhotoEntry2(dir: DirectoryBaseDTO): PhotoEntity {
public static getPhotoEntry2(dir: DirectoryPathDTO): PhotoEntity {
const p = TestHelper.getPhotoEntry(dir);
p.metadata.caption = 'Light saber';
@ -166,7 +184,7 @@ export class TestHelper {
p.name = 'sw2.jpg';
p.metadata.positionData.GPSData.latitude = -10;
p.metadata.positionData.GPSData.longitude = -10;
p.metadata.creationDate = Date.now() - 2000;
p.metadata.creationDate = 1656069387772 - 2000;
p.metadata.rating = 2;
p.metadata.size.height = 2000;
p.metadata.size.width = 1000;
@ -187,7 +205,7 @@ export class TestHelper {
return p;
}
public static getPhotoEntry3(dir: DirectoryBaseDTO): PhotoEntity {
public static getPhotoEntry3(dir: DirectoryPathDTO): PhotoEntity {
const p = TestHelper.getPhotoEntry(dir);
p.metadata.caption = 'Amber stone';
@ -198,7 +216,7 @@ export class TestHelper {
p.name = 'sw3.jpg';
p.metadata.positionData.GPSData.latitude = 10;
p.metadata.positionData.GPSData.longitude = 15;
p.metadata.creationDate = Date.now() - 3000;
p.metadata.creationDate = 1656069387772 - 3000;
p.metadata.rating = 3;
p.metadata.size.height = 1000;
p.metadata.size.width = 2000;
@ -215,7 +233,7 @@ export class TestHelper {
return p;
}
public static getPhotoEntry4(dir: DirectoryBaseDTO): PhotoEntity {
public static getPhotoEntry4(dir: DirectoryPathDTO): PhotoEntity {
const p = TestHelper.getPhotoEntry(dir);
p.metadata.caption = 'Millennium falcon';
@ -226,7 +244,7 @@ export class TestHelper {
p.name = 'sw4.jpg';
p.metadata.positionData.GPSData.latitude = 15;
p.metadata.positionData.GPSData.longitude = 10;
p.metadata.creationDate = Date.now() - 4000;
p.metadata.creationDate = 1656069387772 - 4000;
p.metadata.size.height = 3000;
p.metadata.size.width = 2000;
@ -303,7 +321,7 @@ export class TestHelper {
return f;
}
public static getRandomizedPhotoEntry(dir: DirectoryBaseDTO, forceStr: string = null, faces: number = 2): PhotoDTO {
public static getRandomizedPhotoEntry(dir: DirectoryBaseDTO, forceStr: string = null, faces = 2): PhotoDTO {
const rndStr = (): string => {

2
test/backend/DBTestHelper.ts
View File

@ -11,7 +11,7 @@ import {IndexingManager} from '../../src/backend/model/database/sql/IndexingMana
import {GalleryManager} from '../../src/backend/model/database/sql/GalleryManager';
import {Connection} from 'typeorm';
import {Utils} from '../../src/common/Utils';
import {TestHelper} from './unit/model/sql/TestHelper';
import {TestHelper} from '../TestHelper';
import {VideoDTO} from '../../src/common/entities/VideoDTO';
import {PhotoDTO} from '../../src/common/entities/PhotoDTO';

2
test/backend/unit/model/sql/AlbumManager.spec.ts
View File

@ -1,6 +1,6 @@
import {DBTestHelper} from '../../../DBTestHelper';
import {ParentDirectoryDTO, SubDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
import {TestHelper} from './TestHelper';
import {TestHelper} from '../../../../TestHelper';
import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers';
import {PhotoDTO} from '../../../../../src/common/entities/PhotoDTO';
import {VideoDTO} from '../../../../../src/common/entities/VideoDTO';

70
test/backend/unit/model/sql/IndexingManager.spec.ts
View File

@ -3,7 +3,7 @@ import {Config} from '../../../../../src/common/config/private/Config';
import {SQLConnection} from '../../../../../src/backend/model/database/sql/SQLConnection';
import {GalleryManager} from '../../../../../src/backend/model/database/sql/GalleryManager';
import {DirectoryBaseDTO, DirectoryDTOUtils, ParentDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
import {TestHelper} from './TestHelper';
import {TestHelper} from '../../../../TestHelper';
import {Connection} from 'typeorm';
import {DirectoryEntity} from '../../../../../src/backend/model/database/sql/enitites/DirectoryEntity';
import {Utils} from '../../../../../src/common/Utils';
@ -145,14 +145,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
p1.name = 'test.jpg';
p2.name = 'Test.jpg';
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
expect(Utils.clone(Utils.removeNullOrEmptyObj(removeIds(selected))))
.to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
@ -210,14 +210,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const p2 = TestHelper.getRandomizedPhotoEntry(subDir2, 'subPhoto2', 0);
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
setPartial(subDir1);
setPartial(subDir2);
@ -237,9 +237,9 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const p2 = TestHelper.getRandomizedPhotoEntry(subDir2, 'subPhoto2', 0);
DirectoryDTOUtils.packDirectory(parent1);
DirectoryDTOUtils.removeReferences(parent1);
await im.saveToDB(Utils.clone(parent1) as ParentDirectoryDTO);
DirectoryDTOUtils.packDirectory(parent2);
DirectoryDTOUtils.removeReferences(parent2);
await im.saveToDB(Utils.clone(parent2) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
@ -247,7 +247,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const selected = await gm.selectParentDir(conn, parent1.name, parent1.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
setPartial(subDir1);
expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
@ -257,7 +257,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const selected = await gm.selectParentDir(conn, parent2.name, parent2.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
setPartial(subDir2);
expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
@ -273,14 +273,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
p1.name = 'test 😀.jpg';
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
expect(Utils.clone(Utils.removeNullOrEmptyObj(removeIds(selected))))
.to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
@ -293,7 +293,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const selected = await gmTest.selectParentDir(conn, dir.name, dir.path);
await gmTest.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
return selected;
};
@ -316,10 +316,10 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
};
const saveToDBAndCheck = async (dir: DirectoryBaseDTO) => {
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(dir) as ParentDirectoryDTO);
await checkParent();
DirectoryDTOUtils.unpackDirectory(parent);
DirectoryDTOUtils.addReferences(parent);
};
await saveToDBAndCheck(parent);
@ -359,20 +359,20 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
subDir.preview = sp1;
Config.Server.Preview.Sorting = [SortingMethods.descRating];
DirectoryDTOUtils.packDirectory(subDir);
DirectoryDTOUtils.removeReferences(subDir);
await im.saveToDB(Utils.clone(subDir) as ParentDirectoryDTO);
parent.directories.push(subDir);
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
setPartial(subDir);
expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
@ -397,20 +397,20 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
Config.Server.Preview.Sorting = [SortingMethods.descRating];
DirectoryDTOUtils.packDirectory(subDir);
DirectoryDTOUtils.removeReferences(subDir);
await im.saveToDB(Utils.clone(subDir) as ParentDirectoryDTO);
parent.directories.push(subDir);
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
setPartial(subDir);
expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
@ -433,14 +433,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
subDir.preview = sp1;
Config.Server.Preview.Sorting = [SortingMethods.descRating];
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
setPartial(subDir);
expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
@ -469,14 +469,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
p2.metadata.positionData.GPSData.longitude = minFloat;
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
.to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
@ -489,7 +489,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
Config.Client.MetaFile.gpx = true;
Config.Client.MetaFile.markdown = true;
Config.Client.MetaFile.pg2conf = true;
@ -503,7 +503,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await gm.fillParentDir(conn, selected);
delete parent.metaFile;
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
.to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
@ -520,13 +520,13 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
subDir.name = 'subDir';
const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
const sp3 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto3');
DirectoryDTOUtils.packDirectory(subDir);
DirectoryDTOUtils.removeReferences(subDir);
await im.saveToDB(Utils.clone(subDir) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
@ -535,7 +535,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
// subDir.isPartial = true;
// delete subDir.directories;
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
delete subDir.parent;
delete subDir.metaFile;
removeIds(selected);
@ -563,7 +563,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
subDir.preview = sp1;
Config.Server.Preview.Sorting = [SortingMethods.descRating];
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
const s1 = im.queueForSave(Utils.clone(parent) as ParentDirectoryDTO);
const s2 = im.queueForSave(Utils.clone(parent) as ParentDirectoryDTO);
const s3 = im.queueForSave(Utils.clone(parent) as ParentDirectoryDTO);
@ -573,7 +573,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
setPartial(subDir);
parent.directories.forEach(d => delete (d.preview.metadata as any).faces);
@ -591,14 +591,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
.to.deep.equal(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
@ -617,14 +617,14 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
Config.Client.MetaFile.markdown = true;
Config.Client.MetaFile.pg2conf = true;
const parent = TestHelper.getRandomizedDirectoryEntry();
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
for (let i = 0; i < 1500; i++) {
TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i);
}
DirectoryDTOUtils.packDirectory(parent);
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(subDir as ParentDirectoryDTO);

2
test/backend/unit/model/sql/PersonManager.spec.ts
View File

@ -1,7 +1,7 @@
import {expect} from 'chai';
import {PersonManager} from '../../../../../src/backend/model/database/sql/PersonManager';
import {DBTestHelper} from '../../../DBTestHelper';
import {TestHelper} from './TestHelper';
import {TestHelper} from '../../../../TestHelper';
import {PhotoDTO} from '../../../../../src/common/entities/PhotoDTO';
import {Utils} from '../../../../../src/common/Utils';
import {PersonWithSampleRegion} from '../../../../../src/common/entities/PersonDTO';

2
test/backend/unit/model/sql/PreviewManager.spec.ts
View File

@ -3,7 +3,7 @@ import {DBTestHelper} from '../../../DBTestHelper';
import {SearchQueryDTO, SearchQueryTypes, TextSearch} from '../../../../../src/common/entities/SearchQueryDTO';
import {IndexingManager} from '../../../../../src/backend/model/database/sql/IndexingManager';
import {DirectoryBaseDTO, ParentDirectoryDTO, SubDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
import {TestHelper} from './TestHelper';
import {TestHelper} from '../../../../TestHelper';
import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers';
import {GalleryManager} from '../../../../../src/backend/model/database/sql/GalleryManager';
import {Connection} from 'typeorm';

4
test/backend/unit/model/sql/SearchManager.spec.ts
View File

@ -23,7 +23,7 @@ import {
} from '../../../../../src/common/entities/SearchQueryDTO';
import {IndexingManager} from '../../../../../src/backend/model/database/sql/IndexingManager';
import {DirectoryBaseDTO, ParentDirectoryDTO, SubDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
import {TestHelper} from './TestHelper';
import {TestHelper} from '../../../../TestHelper';
import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers';
import {GalleryManager} from '../../../../../src/backend/model/database/sql/GalleryManager';
import {Connection} from 'typeorm';
@ -1039,7 +1039,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
.to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [],
media: [p, p2, p4, v],
media: [p, p2, p4],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));

89
test/common/unit/ContentWrapper.spec.ts Normal file
View File

@ -0,0 +1,89 @@
import {expect} from 'chai';
import {ContentWrapper} from '../../../src/common/entities/ConentWrapper';
import {TestHelper} from '../../TestHelper';
import {DirectoryPathDTO, ParentDirectoryDTO} from '../../../src/common/entities/DirectoryDTO';
import {SearchResultDTO} from '../../../src/common/entities/SearchResultDTO';
import {SearchQueryTypes, TextSearch} from '../../../src/common/entities/SearchQueryDTO';
import {Utils} from '../../../src/common/Utils';
import {MediaDTOUtils} from '../../../src/common/entities/MediaDTO';
import {VideoDTO} from '../../../src/common/entities/VideoDTO';
import {PhotoDTO} from '../../../src/common/entities/PhotoDTO';
describe('ContentWrapper', () => {
const cleanUpCW = (cw: ContentWrapper): ContentWrapper => {
if (typeof cw.notModified === 'undefined') {
delete cw.notModified;
}
const content = (cw.directory ? cw.directory : cw.searchResult);
for (let i = 0; i < content.media.length; ++i) {
const m = content.media[i];
if (MediaDTOUtils.isPhoto(m)) {
delete (m as VideoDTO).metadata.bitRate;
delete (m as VideoDTO).metadata.duration;
if (!(m as PhotoDTO).metadata.caption) {
delete (m as PhotoDTO).metadata.caption;
}
} else if (MediaDTOUtils.isVideo(m)) {
delete (m as PhotoDTO).metadata.rating;
delete (m as PhotoDTO).metadata.caption;
delete (m as PhotoDTO).metadata.cameraData;
delete (m as PhotoDTO).metadata.keywords;
delete (m as PhotoDTO).metadata.faces;
delete (m as PhotoDTO).metadata.positionData;
}
}
for (let i = 0; i < content.metaFile.length; ++i) {
delete content.metaFile[i].id;
}
return cw;
};
it('pack and unpack directory', () => {
const parent = TestHelper.getDirectoryEntry();
TestHelper.getPhotoEntry(parent);
TestHelper.getPhotoEntry1(parent);
TestHelper.getPhotoEntry2(parent);
TestHelper.getVideoEntry(parent);
TestHelper.getGPXEntry(parent);
const parentOrig = TestHelper.getDirectoryEntry();
TestHelper.getPhotoEntry(parentOrig);
TestHelper.getPhotoEntry1(parentOrig);
TestHelper.getPhotoEntry2(parentOrig);
TestHelper.getVideoEntry(parentOrig);
TestHelper.getGPXEntry(parentOrig);
const cwOrig = new ContentWrapper(parentOrig as ParentDirectoryDTO, null);
const cw = new ContentWrapper(parent as ParentDirectoryDTO, null);
expect(ContentWrapper.unpack(ContentWrapper.pack(cw))).to.deep.equals(cleanUpCW(cwOrig));
});
it('pack and unpack search result', () => {
const parent: DirectoryPathDTO = {
name: 'parent',
path: ''
};
const subDir: DirectoryPathDTO = {
name: 'subDir',
path: 'parent/'
};
const sr: SearchResultDTO = {
directories: [subDir as any],
media: [TestHelper.getPhotoEntry(parent),
TestHelper.getPhotoEntry1(parent),
TestHelper.getPhotoEntry2(subDir),
TestHelper.getVideoEntry(parent)
],
metaFile: [
TestHelper.getGPXEntry(parent)],
resultOverflow: false,
searchQuery: {type: SearchQueryTypes.any_text, text: ''} as TextSearch
};
const cw = new ContentWrapper(null, sr);
expect(ContentWrapper.unpack(ContentWrapper.pack(Utils.clone(cw)))).to.deep.equals(cleanUpCW(cw));
});
});