1
0
mirror of https://github.com/bpatrik/pigallery2.git synced 2024-11-30 09:07:06 +02:00

10x performance improvement for listing faces

This commit is contained in:
Patrik J. Braun 2021-01-01 17:58:41 +01:00
parent 402bf0e134
commit bd60900f7c
12 changed files with 118 additions and 216 deletions

View File

@ -1,8 +1,8 @@
import {NextFunction, Request, Response} from 'express';
import {ErrorCodes, ErrorDTO} from '../../common/entities/Error';
import {ObjectManagers} from '../model/ObjectManagers';
import {PersonDTO} from '../../common/entities/PersonDTO';
import {PhotoDTO} from '../../common/entities/PhotoDTO';
import {PersonDTO, PersonWithSampleRegion} from '../../common/entities/PersonDTO';
import {Utils} from '../../common/Utils';
export class PersonMWs {
@ -24,6 +24,22 @@ export class PersonMWs {
}
}
public static async getPerson(req: Request, res: Response, next: NextFunction) {
if (!req.params.name) {
return next();
}
try {
req.resultPipe = await ObjectManagers.getInstance()
.PersonManager.get(req.params.name as string);
return next();
} catch (err) {
return next(new ErrorDTO(ErrorCodes.PERSON_ERROR, 'Error during updating a person', err));
}
}
public static async listPersons(req: Request, res: Response, next: NextFunction) {
try {
req.resultPipe = await ObjectManagers.getInstance()
@ -37,33 +53,14 @@ export class PersonMWs {
}
public static async addSamplePhotoForAll(req: Request, res: Response, next: NextFunction) {
public static async cleanUpPersonResults(req: Request, res: Response, next: NextFunction) {
if (!req.resultPipe) {
return next();
}
try {
const persons = (req.resultPipe as PersonWithPhoto[]);
const photoMap = await ObjectManagers.getInstance()
.PersonManager.getSamplePhotos(persons.map(p => p.name));
persons.forEach(p => p.samplePhoto = photoMap[p.name]);
req.resultPipe = persons;
return next();
} catch (err) {
return next(new ErrorDTO(ErrorCodes.PERSON_ERROR, 'Error during adding sample photo for all persons', err));
}
}
public static async removeSamplePhotoForAll(req: Request, res: Response, next: NextFunction) {
if (!req.resultPipe) {
return next();
}
try {
const persons = (req.resultPipe as PersonWithPhoto[]);
const persons = Utils.clone(req.resultPipe as PersonWithSampleRegion[]);
for (let i = 0; i < persons.length; i++) {
delete persons[i].samplePhoto;
delete persons[i].sampleRegion;
}
req.resultPipe = persons;
return next();
@ -74,29 +71,6 @@ export class PersonMWs {
}
public static async getSamplePhoto(req: Request, res: Response, next: NextFunction) {
if (!req.params.name) {
return next();
}
const name = req.params.name;
try {
const photo = await ObjectManagers.getInstance()
.PersonManager.getSamplePhoto(name);
if (photo === null) {
return next();
}
req.resultPipe = photo;
return next();
} catch (err) {
return next(new ErrorDTO(ErrorCodes.PERSON_ERROR, 'Error during getting sample photo for a person', err));
}
}
}
export interface PersonWithPhoto extends PersonDTO {
samplePhoto: PhotoDTO;
}

View File

@ -8,9 +8,8 @@ import {ProjectPath} from '../../ProjectPath';
import {Config} from '../../../common/config/private/Config';
import {ThumbnailSourceType} from '../../model/threading/PhotoWorker';
import {MediaDTO} from '../../../common/entities/MediaDTO';
import {PersonWithPhoto} from '../PersonMWs';
import {PhotoProcessing} from '../../model/fileprocessing/PhotoProcessing';
import {PhotoDTO} from '../../../common/entities/PhotoDTO';
import {PersonWithSampleRegion} from '../../../common/entities/PersonDTO';
export class ThumbnailGeneratorMWs {
@ -49,15 +48,15 @@ export class ThumbnailGeneratorMWs {
try {
const size: number = Config.Client.Media.Thumbnail.personThumbnailSize;
const persons: PersonWithPhoto[] = req.resultPipe;
const persons: PersonWithSampleRegion[] = req.resultPipe;
for (let i = 0; i < persons.length; i++) {
// load parameters
const mediaPath = path.join(ProjectPath.ImageFolder,
persons[i].samplePhoto.directory.path,
persons[i].samplePhoto.directory.name, persons[i].samplePhoto.name);
persons[i].sampleRegion.media.directory.path,
persons[i].sampleRegion.media.directory.name, persons[i].sampleRegion.media.name);
// generate thumbnail path
const thPath = PhotoProcessing.generatePersonThumbnailPath(mediaPath, persons[i].samplePhoto.metadata.faces[0], size);
const thPath = PhotoProcessing.generatePersonThumbnailPath(mediaPath, persons[i].sampleRegion, size);
persons[i].readyThumbnail = fs.existsSync(thPath);
}
@ -76,14 +75,14 @@ export class ThumbnailGeneratorMWs {
if (!req.resultPipe) {
return next();
}
const photo: PhotoDTO = req.resultPipe;
const person: PersonWithSampleRegion = req.resultPipe;
try {
req.resultPipe = await PhotoProcessing.generatePersonThumbnail(photo);
req.resultPipe = await PhotoProcessing.generatePersonThumbnail(person);
return next();
} catch (error) {
console.error(error);
return next(new ErrorDTO(ErrorCodes.THUMBNAIL_GENERATION_ERROR,
'Error during generating face thumbnail: ' + photo.name, error.toString()));
'Error during generating face thumbnail: ' + person.name, error.toString()));
}
}

View File

@ -1,14 +1,9 @@
import {PersonEntry} from '../sql/enitites/PersonEntry';
import {PhotoDTO} from '../../../../common/entities/PhotoDTO';
import {PersonDTO} from '../../../../common/entities/PersonDTO';
export interface IPersonManager {
getAll(): Promise<PersonEntry[]>;
getSamplePhoto(name: string): Promise<PhotoDTO>;
getSamplePhotos(names: string[]): Promise<{ [key: string]: PhotoDTO }>;
get(name: string): Promise<PersonEntry>;
saveAll(names: string[]): Promise<void>;

View File

@ -1,5 +1,4 @@
import {IPersonManager} from '../interfaces/IPersonManager';
import {PhotoDTO} from '../../../../common/entities/PhotoDTO';
import {PersonDTO} from '../../../../common/entities/PersonDTO';
export class PersonManager implements IPersonManager {
@ -8,14 +7,6 @@ export class PersonManager implements IPersonManager {
throw new Error('not supported by memory DB');
}
getSamplePhoto(name: string): Promise<PhotoDTO> {
throw new Error('not supported by memory DB');
}
getSamplePhotos(names: string[]): Promise<{ [key: string]: PhotoDTO }> {
throw new Error('not supported by memory DB');
}
get(name: string): Promise<any> {
throw new Error('not supported by memory DB');
}

View File

@ -1,17 +1,13 @@
import {SQLConnection} from './SQLConnection';
import {PersonEntry} from './enitites/PersonEntry';
import {PhotoDTO} from '../../../../common/entities/PhotoDTO';
import {MediaEntity} from './enitites/MediaEntity';
import {FaceRegionEntry} from './enitites/FaceRegionEntry';
import {PersonDTO} from '../../../../common/entities/PersonDTO';
import {Utils} from '../../../../common/Utils';
import {SelectQueryBuilder} from 'typeorm';
import {ISQLPersonManager} from './IPersonManager';
export class PersonManager implements ISQLPersonManager {
samplePhotos: { [key: string]: PhotoDTO } = {};
persons: PersonEntry[] = [];
// samplePhotos: { [key: string]: PhotoDTO } = {};
persons: PersonEntry[] = null;
async updatePerson(name: string, partialPerson: PersonDTO): Promise<PersonEntry> {
const connection = await SQLConnection.getConnection();
@ -34,92 +30,44 @@ export class PersonManager implements ISQLPersonManager {
return person;
}
async getSamplePhoto(name: string): Promise<PhotoDTO> {
return (await this.getSamplePhotos([name]))[name];
}
async getSamplePhotos(names: string[]): Promise<{ [key: string]: PhotoDTO }> {
const hasAll = names.reduce((prev, name) => prev && !!this.samplePhotos[name], true);
if (!hasAll) {
const connection = await SQLConnection.getConnection();
const namesObj: any = {};
let queryStr = '';
names.forEach((n, i) => {
if (i > 0) {
queryStr += ', ';
}
queryStr += ':n' + i + ' COLLATE utf8_general_ci';
namesObj['n' + i] = n;
});
const query: SelectQueryBuilder<MediaEntity> = await (connection
.getRepository(MediaEntity)
.createQueryBuilder('media') as SelectQueryBuilder<MediaEntity>)
.select(['media.name', 'media.id', 'person.name', 'directory.name',
'directory.path', 'media.metadata.size.width', 'media.metadata.size.height'])
.leftJoin('media.directory', 'directory')
.leftJoinAndSelect('media.metadata.faces', 'faces')
.leftJoin('faces.person', 'person')
.groupBy('person.name, media.name, media.id, directory.name, faces.id');
// TODO: improve it. SQLITE does not support case-insensitive special characters like ÁÉÚŐ
for (let i = 0; i < names.length; ++i) {
const opt: any = {};
opt['n' + i] = names[i];
query.orWhere(`person.name LIKE :n${i} COLLATE utf8_general_ci`, opt);
}
const rawAndEntities = await query.getRawAndEntities();
for (let i = 0; i < rawAndEntities.raw.length; ++i) {
this.samplePhotos[rawAndEntities.raw[i].person_name.toLowerCase()] =
Utils.clone(rawAndEntities.entities.find(m => m.name === rawAndEntities.raw[i].media_name));
this.samplePhotos[rawAndEntities.raw[i].person_name.toLowerCase()].metadata.faces =
[FaceRegionEntry.fromRawToDTO(rawAndEntities.raw[i])];
}
}
const photoMap: { [key: string]: PhotoDTO } = {};
names.forEach(n => photoMap[n] = this.samplePhotos[n.toLowerCase()]);
return photoMap;
}
async loadAll(): Promise<void> {
private async loadAll(): Promise<void> {
const connection = await SQLConnection.getConnection();
const personRepository = connection.getRepository(PersonEntry);
this.persons = await personRepository.find();
this.persons = await personRepository.find({
relations: ['sampleRegion',
'sampleRegion.media',
'sampleRegion.media.directory']
});
}
async getAll(): Promise<PersonEntry[]> {
await this.loadAll();
public async getAll(): Promise<PersonEntry[]> {
if (this.persons === null) {
await this.loadAll();
}
return this.persons;
}
async countFaces(): Promise<number> {
/**
* Used for statistic
*/
public async countFaces(): Promise<number> {
const connection = await SQLConnection.getConnection();
return await connection.getRepository(FaceRegionEntry)
.createQueryBuilder('faceRegion')
.getCount();
}
async get(name: string): Promise<PersonEntry> {
let person = this.persons.find(p => p.name === name);
if (!person) {
const connection = await SQLConnection.getConnection();
const personRepository = connection.getRepository(PersonEntry);
person = await personRepository.findOne({name: name});
if (!person) {
person = await personRepository.save(<PersonEntry>{name: name});
}
this.persons.push(person);
public async get(name: string): Promise<PersonEntry> {
if (this.persons === null) {
await this.loadAll();
}
return person;
return this.persons.find(p => p.name === name);
}
async saveAll(names: string[]): Promise<void> {
public async saveAll(names: string[]): Promise<void> {
const toSave: { name: string }[] = [];
const connection = await SQLConnection.getConnection();
const personRepository = connection.getRepository(PersonEntry);
@ -137,7 +85,7 @@ export class PersonManager implements ISQLPersonManager {
for (let i = 0; i < toSave.length / 200; i++) {
await personRepository.insert(toSave.slice(i * 200, (i + 1) * 200));
}
this.persons = await personRepository.find();
await this.loadAll();
}
}
@ -145,10 +93,11 @@ export class PersonManager implements ISQLPersonManager {
public async onGalleryIndexUpdate() {
await this.updateCounts();
this.samplePhotos = {};
await this.updateSamplePhotos();
}
public async updateCounts() {
private async updateCounts() {
const connection = await SQLConnection.getConnection();
await connection.query('update person_entry set count = ' +
' (select COUNT(1) from face_region_entry where face_region_entry.personId = person_entry.id)');
@ -161,4 +110,15 @@ export class PersonManager implements ISQLPersonManager {
.execute();
}
private async updateSamplePhotos() {
const connection = await SQLConnection.getConnection();
await connection.query('update person_entry set sampleRegionId = ' +
'(Select face_region_entry.id from media_entity ' +
'left join face_region_entry on media_entity.id = face_region_entry.mediaId ' +
'where face_region_entry.personId=person_entry.id ' +
'order by media_entity.metadataCreationdate desc ' +
'limit 1)');
}
}

View File

@ -26,11 +26,9 @@ export class FaceRegionEntry {
@Column(type => FaceRegionBoxEntry)
box: FaceRegionBoxEntry;
// @PrimaryColumn('int')
@ManyToOne(type => MediaEntity, media => media.metadata.faces, {onDelete: 'CASCADE', nullable: false})
media: MediaEntity;
// @PrimaryColumn('int')
@ManyToOne(type => PersonEntry, person => person.faces, {onDelete: 'CASCADE', nullable: false})
person: PersonEntry;

View File

@ -1,12 +1,12 @@
import {Column, Entity, Index, OneToMany, PrimaryGeneratedColumn, Unique} from 'typeorm';
import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, Unique} from 'typeorm';
import {FaceRegionEntry} from './FaceRegionEntry';
import {PersonDTO} from '../../../../../common/entities/PersonDTO';
import {columnCharsetCS} from './EntityUtils';
import {PersonWithSampleRegion} from '../../../../../common/entities/PersonDTO';
@Entity()
@Unique(['name'])
export class PersonEntry implements PersonDTO {
export class PersonEntry implements PersonWithSampleRegion {
@Index()
@PrimaryGeneratedColumn({unsigned: true})
@ -24,5 +24,8 @@ export class PersonEntry implements PersonDTO {
@OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.person)
public faces: FaceRegionEntry[];
@ManyToOne(type => FaceRegionEntry, {onDelete: 'SET NULL', nullable: true})
sampleRegion: FaceRegionEntry;
}

View File

@ -10,6 +10,7 @@ import {ITaskExecuter, TaskExecuter} from '../threading/TaskExecuter';
import {FaceRegion, PhotoDTO} from '../../../common/entities/PhotoDTO';
import {SupportedFormats} from '../../../common/SupportedFormats';
import {ServerConfig} from '../../../common/config/private/PrivateConfig';
import {PersonWithSampleRegion} from '../../../common/entities/PersonDTO';
export class PhotoProcessing {
@ -45,19 +46,14 @@ export class PhotoProcessing {
}
public static async generatePersonThumbnail(photo: PhotoDTO) {
// load parameters
if (!photo.metadata.faces || photo.metadata.faces.length !== 1) {
throw new Error('Photo does not contain a face');
}
public static async generatePersonThumbnail(person: PersonWithSampleRegion) {
// load parameters
const photo: PhotoDTO = person.sampleRegion.media;
const mediaPath = path.join(ProjectPath.ImageFolder, photo.directory.path, photo.directory.name, photo.name);
const size: number = Config.Client.Media.Thumbnail.personThumbnailSize;
// generate thumbnail path
const thPath = PhotoProcessing.generatePersonThumbnailPath(mediaPath, photo.metadata.faces[0], size);
const thPath = PhotoProcessing.generatePersonThumbnailPath(mediaPath, person.sampleRegion, size);
// check if thumbnail already exist
@ -69,8 +65,8 @@ export class PhotoProcessing {
const margin = {
x: Math.round(photo.metadata.faces[0].box.width * (Config.Server.Media.Thumbnail.personFaceMargin)),
y: Math.round(photo.metadata.faces[0].box.height * (Config.Server.Media.Thumbnail.personFaceMargin))
x: Math.round(person.sampleRegion.box.width * (Config.Server.Media.Thumbnail.personFaceMargin)),
y: Math.round(person.sampleRegion.box.height * (Config.Server.Media.Thumbnail.personFaceMargin))
};
@ -82,10 +78,10 @@ export class PhotoProcessing {
outPath: thPath,
makeSquare: false,
cut: {
left: Math.round(Math.max(0, photo.metadata.faces[0].box.left - margin.x / 2)),
top: Math.round(Math.max(0, photo.metadata.faces[0].box.top - margin.y / 2)),
width: photo.metadata.faces[0].box.width + margin.x,
height: photo.metadata.faces[0].box.height + margin.y
left: Math.round(Math.max(0, person.sampleRegion.box.left - margin.x / 2)),
top: Math.round(Math.max(0, person.sampleRegion.box.top - margin.y / 2)),
width: person.sampleRegion.box.width + margin.x,
height: person.sampleRegion.box.height + margin.y
},
qualityPriority: Config.Server.Media.Thumbnail.qualityPriority
};

View File

@ -38,9 +38,9 @@ export class PersonRouter {
// specific part
PersonMWs.listPersons,
PersonMWs.addSamplePhotoForAll,
// PersonMWs.addSamplePhotoForAll,
ThumbnailGeneratorMWs.addThumbnailInfoForPersons,
PersonMWs.removeSamplePhotoForAll,
PersonMWs.cleanUpPersonResults,
RenderingMWs.renderResult
);
}
@ -53,7 +53,7 @@ export class PersonRouter {
VersionMWs.injectGalleryVersion,
// specific part
PersonMWs.getSamplePhoto,
PersonMWs.getPerson,
ThumbnailGeneratorMWs.generatePersonThumbnail,
RenderingMWs.renderFile
);

View File

@ -1 +1 @@
export const DataStructureVersion = 16;
export const DataStructureVersion = 17;

View File

@ -1,3 +1,9 @@
import {FaceRegionEntry} from '../../backend/model/database/sql/enitites/FaceRegionEntry';
export interface PersonWithSampleRegion extends PersonDTO {
sampleRegion: FaceRegionEntry;
}
export interface PersonDTO {
id: number;
name: string;

View File

@ -9,6 +9,8 @@ import {SQLConnection} from '../../../../../src/backend/model/database/sql/SQLCo
import {PhotoEntity} from '../../../../../src/backend/model/database/sql/enitites/PhotoEntity';
import {DirectoryEntity} from '../../../../../src/backend/model/database/sql/enitites/DirectoryEntity';
import {VideoEntity} from '../../../../../src/backend/model/database/sql/enitites/VideoEntity';
import {Utils} from '../../../../../src/common/Utils';
import {PersonWithSampleRegion} from '../../../../../src/common/entities/PersonDTO';
// to help WebStorm to handle the test cases
@ -23,12 +25,13 @@ describe('PersonManager', (sqlHelper: SQLTestHelper) => {
const dir = TestHelper.getDirectoryEntry();
const p = TestHelper.getPhotoEntry1(dir);
const p2 = TestHelper.getPhotoEntry2(dir);
const p_faceLess = TestHelper.getPhotoEntry2(dir);
let p = TestHelper.getPhotoEntry1(dir);
let p2 = TestHelper.getPhotoEntry2(dir);
let p_faceLess = TestHelper.getPhotoEntry2(dir);
delete p_faceLess.metadata.faces;
p_faceLess.name = 'fl';
const v = TestHelper.getVideoEntry1(dir);
const savedPerson: PersonWithSampleRegion[] = [];
const setUpSqlDB = async () => {
await sqlHelper.initDB();
@ -36,25 +39,28 @@ describe('PersonManager', (sqlHelper: SQLTestHelper) => {
const savePhoto = async (photo: PhotoDTO) => {
const savedPhoto = await pr.save(photo);
if (!photo.metadata.faces) {
return;
return savedPhoto;
}
for (let i = 0; i < photo.metadata.faces.length; i++) {
const face = photo.metadata.faces[i];
const person = await conn.getRepository(PersonEntry).save({name: face.name});
await conn.getRepository(FaceRegionEntry).save({box: face.box, person: person, media: savedPhoto});
savedPhoto.metadata.faces[i] = await conn.getRepository(FaceRegionEntry).save({box: face.box, person: person, media: savedPhoto});
savedPerson.push(person);
}
return savedPhoto;
};
const conn = await SQLConnection.getConnection();
const pr = conn.getRepository(PhotoEntity);
await conn.getRepository(DirectoryEntity).save(p.directory);
await savePhoto(p);
await savePhoto(p2);
await savePhoto(p_faceLess);
p = await savePhoto(p);
console.log(p.id);
p2 = await savePhoto(p2);
p_faceLess = await savePhoto(p_faceLess);
await conn.getRepository(VideoEntity).save(v);
await (new PersonManager()).onGalleryIndexUpdate();
await SQLConnection.close();
};
@ -68,46 +74,20 @@ describe('PersonManager', (sqlHelper: SQLTestHelper) => {
});
const mapPhoto = (photo: PhotoDTO) => {
const map: { [key: string]: PhotoDTO } = {};
photo.metadata.faces.forEach(face => {
map[face.name] = <any>{
id: photo.id,
name: photo.name,
directory: {
path: photo.directory.path,
name: photo.directory.name,
},
metadata: {
size: photo.metadata.size,
faces: [photo.metadata.faces.find(f => f.name === face.name)]
},
readyIcon: false,
readyThumbnails: []
};
});
return map;
};
it('should get sample photos', async () => {
it('should get person', async () => {
const pm = new PersonManager();
const map = mapPhoto(p);
expect(await pm.getSamplePhotos(p.metadata.faces.map(f => f.name))).to.deep.equal(map);
const person = Utils.clone(savedPerson[0]);
person.sampleRegion = <any>{
id: p.metadata.faces[0].id,
box: p.metadata.faces[0].box
};
const tmp = p.metadata.faces;
delete p.metadata.faces;
person.sampleRegion.media = Utils.clone(p);
p.metadata.faces = tmp;
person.count = 1;
expect(await pm.get(person.name)).to.deep.equal(person);
});
it('should get sample photos case insensitive', async () => {
const pm = new PersonManager();
const map = mapPhoto(p);
for (const k of Object.keys(map)) {
if (k.toLowerCase() !== k) {
map[k.toLowerCase()] = map[k];
delete map[k];
}
}
expect(await pm.getSamplePhotos(p.metadata.faces.map(f => f.name.toLowerCase()))).to.deep.equal(map);
});
});