self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2025-02-03 13:22:05 +02:00
Code Issues Releases Activity

improving database usage

Browse Source
This commit is contained in:
Patrik J. Braun 2018-12-18 00:05:12 +01:00
parent 298a6600d3
commit ada2c007df
7 changed files with 116 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
backend/model/sql/GalleryManager.ts
View File

@ -275,7 +275,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
delete directory.media;
await directoryRepository.save(directory);
}
} else {
} else { //dir does not exists yet
scannedDirectory.directories[i].parent = currentDir;
(<DirectoryEntity>scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet
const d = await directoryRepository.save(<DirectoryEntity>scannedDirectory.directories[i]);
@ -288,7 +288,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
}
// Remove child Dirs that are not anymore in the parent dir
await directoryRepository.remove(childDirectories);
await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)});
// save media
const indexedMedia = await mediaRepository.createQueryBuilder('media')
@ -321,6 +321,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
await this.saveMedia(connection, mediaToSave);
await mediaRepository.remove(indexedMedia);
// save files
const indexedMetaFiles = await fileRepository.createQueryBuilder('file')
.where('file.directory = :dir', {
@ -346,8 +347,8 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
metaFilesToSave.push(metaFile);
}
}
await fileRepository.save(metaFilesToSave);
await fileRepository.remove(indexedMetaFiles);
await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)});
await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)});
} catch (e) {
throw e;
} finally {

23
backend/model/sql/enitites/PhotoEntity.ts
View File

@ -12,7 +12,7 @@ export class CameraMetadataEntity implements CameraMetadata {
model: string;
@Column('text', {nullable: true})
maker: string;
make: string;
@Column('int', {nullable: true})
fStop: number;
@ -55,19 +55,19 @@ export class PositionMetaDataEntity implements PositionMetaData {
export class PhotoMetadataEntity extends MediaMetadataEntity implements PhotoMetadata {
/*
@Column('simple-array')
keywords: string[];
/*
@Column('simple-array')
keywords: string[];
@Column(type => CameraMetadataEntity)
cameraData: CameraMetadataEntity;
@Column(type => CameraMetadataEntity)
cameraData: CameraMetadataEntity;
@Column(type => PositionMetaDataEntity)
positionData: PositionMetaDataEntity;
@Column(type => PositionMetaDataEntity)
positionData: PositionMetaDataEntity;
@Column('tinyint', {default: OrientationTypes.TOP_LEFT})
orientation: OrientationTypes;
*/
@Column('tinyint', {default: OrientationTypes.TOP_LEFT})
orientation: OrientationTypes;
*/
}
@ -75,4 +75,5 @@ export class PhotoMetadataEntity extends MediaMetadataEntity implements PhotoMet
export class PhotoEntity extends MediaEntity implements PhotoDTO {
@Column(type => PhotoMetadataEntity)
metadata: PhotoMetadataEntity;
}

12
backend/model/threading/DiskMangerWorker.ts
View File

@ -209,7 +209,7 @@ export class DiskMangerWorker {
try {
const exif = ExifParserFactory.create(data).parse();
metadata.cameraData = <CameraMetadata>{
metadata.cameraData = {
ISO: exif.tags.ISO,
model: exif.tags.Model,
make: exif.tags.Make,
@ -220,7 +220,7 @@ export class DiskMangerWorker {
};
if (!isNaN(exif.tags.GPSLatitude) || exif.tags.GPSLongitude || exif.tags.GPSAltitude) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.GPSData = <GPSMetadata>{
metadata.positionData.GPSData = {
latitude: exif.tags.GPSLatitude,
longitude: exif.tags.GPSLongitude,
altitude: exif.tags.GPSAltitude
@ -236,15 +236,15 @@ export class DiskMangerWorker {
}
if (exif.imageSize) {
metadata.size = <MediaDimension>{width: exif.imageSize.width, height: exif.imageSize.height};
metadata.size = {width: exif.imageSize.width, height: exif.imageSize.height};
} else if (exif.tags.RelatedImageWidth && exif.tags.RelatedImageHeight) {
metadata.size = <MediaDimension>{width: exif.tags.RelatedImageWidth, height: exif.tags.RelatedImageHeight};
metadata.size = {width: exif.tags.RelatedImageWidth, height: exif.tags.RelatedImageHeight};
} else {
metadata.size = <MediaDimension>{width: 1, height: 1};
metadata.size = {width: 1, height: 1};
}
} catch (err) {
Logger.debug(LOG_TAG, 'Error parsing exif', fullPath, err);
metadata.size = <MediaDimension>{width: 1, height: 1};
metadata.size = {width: 1, height: 1};
}
try {

8
common/entities/DirectoryDTO.ts
View File

@ -36,9 +36,11 @@ export module DirectoryDTO {
};
export const removeReferences = (dir: DirectoryDTO): void => {
dir.media.forEach((media: MediaDTO) => {
media.directory = null;
});
if (dir.media) {
dir.media.forEach((media: MediaDTO) => {
media.directory = null;
});
}
if (dir.metaFile) {
dir.metaFile.forEach((file: FileDTO) => {
file.directory = null;

2
test/backend/integration/model/sql/typeorm.ts
View File

@ -83,7 +83,7 @@ describe('Typeorm integration', () => {
const cd = new CameraMetadataEntity();
cd.ISO = 100;
cd.model = '60D';
cd.maker = 'Canon';
cd.make = 'Canon';
cd.fStop = 1;
cd.exposure = 1;
cd.focalLength = 1;

27
test/backend/unit/model/sql/GalleryManager.ts
View File

@ -12,9 +12,6 @@ import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/Directo
import {Utils} from '../../../../../common/Utils';
import {MediaDTO} from '../../../../../common/entities/MediaDTO';
import {FileDTO} from '../../../../../common/entities/FileDTO';
import {PhotoEntity} from '../../../../../backend/model/sql/enitites/PhotoEntity';
import {FileEntity} from '../../../../../backend/model/sql/enitites/FileEntity';
class GalleryManagerTest extends GalleryManager {
@ -202,9 +199,6 @@ describe('GalleryManager', () => {
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const query = conn.getRepository(FileEntity).createQueryBuilder('photo');
query.innerJoinAndSelect('photo.directory', 'directory');
console.log((await query.getMany()));
DirectoryDTO.removeReferences(selected);
removeIds(selected);
subDir.isPartial = true;
@ -214,4 +208,25 @@ describe('GalleryManager', () => {
.to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
});
(<any>it('should save 1500 photos', async () => {
const conn = await SQLConnection.getConnection();
const gm = new GalleryManagerTest();
Config.Client.MetaFile.enabled = true;
const parent = TestHelper.getRandomizedDirectoryEntry();
DirectoryDTO.removeReferences(parent);
await gm.saveToDB(Utils.clone(parent));
const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
for (let i = 0; i < 1500; i++) {
TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i);
}
DirectoryDTO.removeReferences(parent);
await gm.saveToDB(subDir);
const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
expect(selected.media.length).to.deep.equal(subDir.media.length);
})).timeout(20000);
});

118
test/backend/unit/model/sql/TestHelper.ts
View File

@ -10,6 +10,10 @@ import {OrientationTypes} from 'ts-exif-parser';
import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity';
import {VideoEntity, VideoMetadataEntity} from '../../../../../backend/model/sql/enitites/VideoEntity';
import {FileEntity} from '../../../../../backend/model/sql/enitites/FileEntity';
import {MediaDimension} from '../../../../../common/entities/MediaDTO';
import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO';
import {FileDTO} from '../../../../../common/entities/FileDTO';
export class TestHelper {
@ -40,7 +44,7 @@ export class TestHelper {
const cd = new CameraMetadataEntity();
cd.ISO = 100;
cd.model = '60D';
cd.maker = 'Canon';
cd.make = 'Canon';
cd.fStop = 1;
cd.exposure = 1;
cd.focalLength = 1;
@ -119,35 +123,38 @@ export class TestHelper {
}
public static getRandomizedDirectoryEntry(parent: DirectoryEntity = null, forceStr: string = null) {
public static getRandomizedDirectoryEntry(parent: DirectoryDTO = null, forceStr: string = null) {
const dir = new DirectoryEntity();
dir.name = forceStr || Math.random().toString(36).substring(7);
dir.path = '.';
const dir: DirectoryDTO = {
id: null,
name: forceStr || Math.random().toString(36).substring(7),
path: '.',
directories: [],
metaFile: [],
media: [],
lastModified: Date.now(),
lastScanned: null,
parent: null
};
if (parent !== null) {
dir.path = path.join(parent.path, parent.name);
parent.directories.push(dir);
}
dir.directories = [];
dir.metaFile = [];
dir.media = [];
dir.lastModified = Date.now();
dir.lastScanned = null;
return dir;
}
public static getRandomizedGPXEntry(dir: DirectoryEntity, forceStr: string = null): FileEntity {
const d = new FileEntity();
d.name = forceStr + '_' + Math.random().toString(36).substring(7) + '.gpx';
d.directory = dir;
public static getRandomizedGPXEntry(dir: DirectoryDTO, forceStr: string = null): FileDTO {
const d: FileDTO = {
id: null,
name: forceStr + '_' + Math.random().toString(36).substring(7) + '.gpx',
directory: dir
};
dir.metaFile.push(d);
return d;
}
public static getRandomizedPhotoEntry(dir: DirectoryEntity, forceStr: string = null) {
public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null) {
const rndStr = () => {
@ -159,44 +166,51 @@ export class TestHelper {
return Math.floor(Math.random() * max);
};
const sd = new MediaDimensionEntity();
sd.height = rndInt();
sd.width = rndInt();
const gps = new GPSMetadataEntity();
gps.altitude = rndInt(1000);
gps.latitude = rndInt(1000);
gps.longitude = rndInt(1000);
const pd = new PositionMetaDataEntity();
pd.city = rndStr();
pd.country = rndStr();
pd.state = rndStr();
pd.GPSData = gps;
const cd = new CameraMetadataEntity();
cd.ISO = rndInt(500);
cd.model = rndStr();
cd.maker = rndStr();
cd.fStop = rndInt(10);
cd.exposure = rndInt(10);
cd.focalLength = rndInt(10);
cd.lens = rndStr();
const m = new PhotoMetadataEntity();
m.keywords = [rndStr(), rndStr()];
m.cameraData = cd;
m.positionData = pd;
m.size = sd;
m.creationDate = Date.now();
m.fileSize = rndInt(10000);
m.orientation = OrientationTypes.TOP_LEFT;
const sd: MediaDimension = {
height: rndInt(),
width: rndInt(),
};
// TODO: remove when typeorm is fixed
m.duration = null;
m.bitRate = null;
const gps: GPSMetadata = {
altitude: rndInt(1000),
latitude: rndInt(1000),
longitude: rndInt(1000)
};
const pd: PositionMetaData = {
city: rndStr(),
country: rndStr(),
state: rndStr(),
GPSData: gps
};
const cd: CameraMetadata = {
ISO: rndInt(500),
model: rndStr(),
make: rndStr(),
fStop: rndInt(10),
exposure: rndInt(10),
focalLength: rndInt(10),
lens: rndStr()
};
const m: PhotoMetadata = {
keywords: [rndStr(), rndStr()],
cameraData: cd,
positionData: pd,
size: sd,
creationDate: Date.now(),
fileSize: rndInt(10000),
orientation: OrientationTypes.TOP_LEFT,
caption: rndStr()
};
const d = new PhotoEntity();
d.name = rndStr() + '.jpg';
d.directory = dir;
d.metadata = m;
const d: PhotoDTO = {
id: null,
name: rndStr() + '.jpg',
directory: dir,
metadata: m,
readyThumbnails: null,
readyIcon: false
};
dir.media.push(d);
return d;