You've already forked pigallery2
mirror of
https://github.com/bpatrik/pigallery2.git
synced 2025-11-23 22:24:44 +02:00
Implementing ProjectedPersonCacheEntity #1015
This commit is contained in:
@@ -28,8 +28,9 @@ import {
|
|||||||
TextSearchQueryMatchTypes,
|
TextSearchQueryMatchTypes,
|
||||||
TextSearchQueryTypes
|
TextSearchQueryTypes
|
||||||
} from '../src/common/entities/SearchQueryDTO';
|
} from '../src/common/entities/SearchQueryDTO';
|
||||||
import {defaultQueryKeywords, QueryKeywords, SearchQueryParser} from '../src/common/SearchQueryParser';
|
import {defaultQueryKeywords, SearchQueryParser} from '../src/common/SearchQueryParser';
|
||||||
import {ParentDirectoryDTO} from '../src/common/entities/DirectoryDTO';
|
import {ParentDirectoryDTO} from '../src/common/entities/DirectoryDTO';
|
||||||
|
import {SessionManager} from '../src/backend/model/database/SessionManager';
|
||||||
import {SessionContext} from '../src/backend/model/SessionContext';
|
import {SessionContext} from '../src/backend/model/SessionContext';
|
||||||
|
|
||||||
|
|
||||||
@@ -79,9 +80,11 @@ export class BenchmarkRunner {
|
|||||||
query: {},
|
query: {},
|
||||||
session: {}
|
session: {}
|
||||||
};
|
};
|
||||||
|
private session: SessionContext;
|
||||||
|
|
||||||
constructor(public RUNS: number) {
|
constructor(public RUNS: number) {
|
||||||
Config.Users.authenticationRequired = false;
|
Config.Users.authenticationRequired = false;
|
||||||
|
this.session = {user: {projectionKey: SessionManager.NO_PROJECTION_KEY}} as SessionContext;
|
||||||
}
|
}
|
||||||
|
|
||||||
async bmSaveDirectory(): Promise<BenchmarkResult[]> {
|
async bmSaveDirectory(): Promise<BenchmarkResult[]> {
|
||||||
@@ -150,7 +153,7 @@ export class BenchmarkRunner {
|
|||||||
await this.setupDB();
|
await this.setupDB();
|
||||||
|
|
||||||
const queryParser = new SearchQueryParser(defaultQueryKeywords);
|
const queryParser = new SearchQueryParser(defaultQueryKeywords);
|
||||||
const names = (await ObjectManagers.getInstance().PersonManager.getAll()).sort((a, b) => b.count - a.count);
|
const names = (await ObjectManagers.getInstance().PersonManager.getAll(this.session)).sort((a, b) => b.count - a.count);
|
||||||
const queries: { query: SearchQueryDTO, description: string }[] = TextSearchQueryTypes.map(t => {
|
const queries: { query: SearchQueryDTO, description: string }[] = TextSearchQueryTypes.map(t => {
|
||||||
const q = {
|
const q = {
|
||||||
type: t, text: 'a'
|
type: t, text: 'a'
|
||||||
@@ -255,7 +258,7 @@ export class BenchmarkRunner {
|
|||||||
', videos: ' + await gm.countVideos() +
|
', videos: ' + await gm.countVideos() +
|
||||||
', diskUsage : ' + Utils.renderDataSize(await gm.countMediaSize()) +
|
', diskUsage : ' + Utils.renderDataSize(await gm.countMediaSize()) +
|
||||||
', persons : ' + await pm.countFaces() +
|
', persons : ' + await pm.countFaces() +
|
||||||
', unique persons (faces): ' + (await pm.getAll()).length;
|
', unique persons (faces): ' + (await pm.getAll(this.session)).length;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -269,8 +272,7 @@ export class BenchmarkRunner {
|
|||||||
const queue = ['/'];
|
const queue = ['/'];
|
||||||
while (queue.length > 0) {
|
while (queue.length > 0) {
|
||||||
const dirPath = queue.shift();
|
const dirPath = queue.shift();
|
||||||
const session = new SessionContext();
|
const dir = await gm.listDirectory(this.session, dirPath);
|
||||||
const dir = await gm.listDirectory(session,dirPath);
|
|
||||||
dir.directories.forEach((d): number => queue.push(path.join(d.path + d.name)));
|
dir.directories.forEach((d): number => queue.push(path.join(d.path + d.name)));
|
||||||
if (biggest < dir.media.length) {
|
if (biggest < dir.media.length) {
|
||||||
biggestPath = path.join(dir.path + dir.name);
|
biggestPath = path.join(dir.path + dir.name);
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ Author: Junie (JetBrains autonomous programmer)
|
|||||||
Date: 2025-08-13
|
Date: 2025-08-13
|
||||||
Status: Updated after maintainer clarifications
|
Status: Updated after maintainer clarifications
|
||||||
|
|
||||||
Terminology: The project uses the term projection. Earlier drafts and filenames may still say “scoped”; whenever you see “scoped,” read it as “projection.” Benchmark file names keep “scoped” for historical reasons.
|
Terminology: The project uses the term projection. Earlier drafts and filenames may still say “projected”; whenever you see “projected,” read it as “projection.” Benchmark file names keep “projected” for historical reasons.
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
This document proposes a design to support:
|
This document proposes a design to support:
|
||||||
@@ -302,11 +302,11 @@ Purpose
|
|||||||
- Variant B (alternative): separate ProjectionKey table (unique key hash), projection tables reference it via projectionId (FK ON DELETE CASCADE).
|
- Variant B (alternative): separate ProjectionKey table (unique key hash), projection tables reference it via projectionId (FK ON DELETE CASCADE).
|
||||||
|
|
||||||
Benchmark harness
|
Benchmark harness
|
||||||
- Location: benchmark\\scoped-cache-bench.ts (historical filename)
|
- Location: benchmark\\projected-cache-bench.ts (historical filename)
|
||||||
- Run: npm run bench:scoped-cache
|
- Run: npm run bench:projected-cache
|
||||||
- Params (env vars): BENCH_SCOPES, BENCH_DIRS, BENCH_PERSONS, BENCH_LOOKUPS. Example (Windows cmd):
|
- Params (env vars): BENCH_SCOPES, BENCH_DIRS, BENCH_PERSONS, BENCH_LOOKUPS. Example (Windows cmd):
|
||||||
- set BENCH_SCOPES=20&&set BENCH_DIRS=120&&set BENCH_PERSONS=90&&set BENCH_LOOKUPS=1000&&npm run bench:scoped-cache
|
- set BENCH_SCOPES=20&&set BENCH_DIRS=120&&set BENCH_PERSONS=90&&set BENCH_LOOKUPS=1000&&npm run bench:projected-cache
|
||||||
- DB: temporary SQLite file at db\\scoped_bench.sqlite (does not touch app DB)
|
- DB: temporary SQLite file at db\\projected_bench.sqlite (does not touch app DB)
|
||||||
- Measures: upsert throughput for dir/person tables, lookup latency by (projection, directory), cascade delete performance, file size delta.
|
- Measures: upsert throughput for dir/person tables, lookup latency by (projection, directory), cascade delete performance, file size delta.
|
||||||
|
|
||||||
Sample results (SQLite, N=20, D=120, P=90, lookups=1000)
|
Sample results (SQLite, N=20, D=120, P=90, lookups=1000)
|
||||||
|
|||||||
@@ -7,9 +7,9 @@ import {PersonEntry} from '../model/database/enitites/PersonEntry';
|
|||||||
|
|
||||||
export class PersonMWs {
|
export class PersonMWs {
|
||||||
public static async updatePerson(
|
public static async updatePerson(
|
||||||
req: Request,
|
req: Request,
|
||||||
res: Response,
|
res: Response,
|
||||||
next: NextFunction
|
next: NextFunction
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (!req.params['name']) {
|
if (!req.params['name']) {
|
||||||
return next();
|
return next();
|
||||||
@@ -17,72 +17,72 @@ export class PersonMWs {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
req.resultPipe =
|
req.resultPipe =
|
||||||
await ObjectManagers.getInstance().PersonManager.updatePerson(
|
await ObjectManagers.getInstance().PersonManager.updatePerson(
|
||||||
req.params['name'] as string,
|
req.params['name'] as string,
|
||||||
req.body as PersonDTO
|
req.body as PersonDTO
|
||||||
);
|
);
|
||||||
return next();
|
return next();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return next(
|
return next(
|
||||||
new ErrorDTO(
|
new ErrorDTO(
|
||||||
ErrorCodes.PERSON_ERROR,
|
ErrorCodes.PERSON_ERROR,
|
||||||
'Error during updating a person',
|
'Error during updating a person',
|
||||||
err
|
err
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async getPerson(
|
public static async getPerson(
|
||||||
req: Request,
|
req: Request,
|
||||||
res: Response,
|
res: Response,
|
||||||
next: NextFunction
|
next: NextFunction
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (!req.params['name']) {
|
if (!req.params['name']) {
|
||||||
return next();
|
return next();
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
req.resultPipe = await ObjectManagers.getInstance().PersonManager.get(
|
req.resultPipe = await ObjectManagers.getInstance().PersonManager.get(req.session.context,
|
||||||
req.params['name'] as string
|
req.params['name'] as string
|
||||||
);
|
);
|
||||||
return next();
|
return next();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return next(
|
return next(
|
||||||
new ErrorDTO(
|
new ErrorDTO(
|
||||||
ErrorCodes.PERSON_ERROR,
|
ErrorCodes.PERSON_ERROR,
|
||||||
'Error during updating a person',
|
'Error during updating a person',
|
||||||
err
|
err
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async listPersons(
|
public static async listPersons(
|
||||||
req: Request,
|
req: Request,
|
||||||
res: Response,
|
res: Response,
|
||||||
next: NextFunction
|
next: NextFunction
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
try {
|
try {
|
||||||
req.resultPipe =
|
req.resultPipe =
|
||||||
await ObjectManagers.getInstance().PersonManager.getAll();
|
await ObjectManagers.getInstance().PersonManager.getAll(req.session.context);
|
||||||
|
|
||||||
return next();
|
return next();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return next(
|
return next(
|
||||||
new ErrorDTO(
|
new ErrorDTO(
|
||||||
ErrorCodes.PERSON_ERROR,
|
ErrorCodes.PERSON_ERROR,
|
||||||
'Error during listing persons',
|
'Error during listing persons',
|
||||||
err
|
err
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async cleanUpPersonResults(
|
public static async cleanUpPersonResults(
|
||||||
req: Request,
|
req: Request,
|
||||||
res: Response,
|
res: Response,
|
||||||
next: NextFunction
|
next: NextFunction
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (!req.resultPipe) {
|
if (!req.resultPipe) {
|
||||||
return next();
|
return next();
|
||||||
@@ -96,11 +96,11 @@ export class PersonMWs {
|
|||||||
return next();
|
return next();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return next(
|
return next(
|
||||||
new ErrorDTO(
|
new ErrorDTO(
|
||||||
ErrorCodes.PERSON_ERROR,
|
ErrorCodes.PERSON_ERROR,
|
||||||
'Error during removing sample photo from all persons',
|
'Error during removing sample photo from all persons',
|
||||||
err
|
err
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,57 +5,181 @@ import {Logger} from '../../Logger';
|
|||||||
import {SQL_COLLATE} from './enitites/EntityUtils';
|
import {SQL_COLLATE} from './enitites/EntityUtils';
|
||||||
import {PersonJunctionTable} from './enitites/PersonJunctionTable';
|
import {PersonJunctionTable} from './enitites/PersonJunctionTable';
|
||||||
import {IObjectManager} from './IObjectManager';
|
import {IObjectManager} from './IObjectManager';
|
||||||
|
import {ParentDirectoryDTO} from '../../../common/entities/DirectoryDTO';
|
||||||
|
import {ProjectedPersonCacheEntity} from './enitites/ProjectedPersonCacheEntity';
|
||||||
|
import {SessionContext} from '../SessionContext';
|
||||||
|
|
||||||
const LOG_TAG = '[PersonManager]';
|
const LOG_TAG = '[PersonManager]';
|
||||||
|
|
||||||
export class PersonManager implements IObjectManager {
|
export class PersonManager implements IObjectManager {
|
||||||
persons: PersonEntry[] = null;
|
personsCache: Record<string, PersonEntry[]> = null;
|
||||||
/**
|
|
||||||
* Person table contains denormalized data that needs to update when isDBValid = false
|
|
||||||
*/
|
|
||||||
private isDBValid = false;
|
|
||||||
|
|
||||||
private static async updateCounts(): Promise<void> {
|
private static async updateCounts(): Promise<void> {
|
||||||
const connection = await SQLConnection.getConnection();
|
const connection = await SQLConnection.getConnection();
|
||||||
await connection.query(
|
await connection.query(
|
||||||
'UPDATE person_entry SET count = ' +
|
'UPDATE person_entry SET count = ' +
|
||||||
' (SELECT COUNT(1) FROM person_junction_table WHERE person_junction_table.personId = person_entry.id)'
|
' (SELECT COUNT(1) FROM person_junction_table WHERE person_junction_table.personId = person_entry.id)'
|
||||||
);
|
);
|
||||||
|
|
||||||
// remove persons without photo
|
// remove persons without photo
|
||||||
await connection
|
await connection
|
||||||
.createQueryBuilder()
|
.createQueryBuilder()
|
||||||
.delete()
|
.delete()
|
||||||
.from(PersonEntry)
|
.from(PersonEntry)
|
||||||
.where('count = 0')
|
.where('count = 0')
|
||||||
.execute();
|
.execute();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async updateSamplePhotos(): Promise<void> {
|
private static async updateSamplePhotos(): Promise<void> {
|
||||||
const connection = await SQLConnection.getConnection();
|
const connection = await SQLConnection.getConnection();
|
||||||
await connection.query(
|
await connection.query(
|
||||||
'update person_entry set sampleRegionId = ' +
|
'update person_entry set sampleRegionId = ' +
|
||||||
'(Select person_junction_table.id from media_entity ' +
|
'(Select person_junction_table.id from media_entity ' +
|
||||||
'left join person_junction_table on media_entity.id = person_junction_table.mediaId ' +
|
'left join person_junction_table on media_entity.id = person_junction_table.mediaId ' +
|
||||||
'where person_junction_table.personId=person_entry.id ' +
|
'where person_junction_table.personId=person_entry.id ' +
|
||||||
'order by media_entity.metadataRating desc, ' +
|
'order by media_entity.metadataRating desc, ' +
|
||||||
'media_entity.metadataCreationdate desc ' +
|
'media_entity.metadataCreationdate desc ' +
|
||||||
'limit 1)'
|
'limit 1)'
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async updateCacheForAll(session: SessionContext): Promise<void> {
|
||||||
|
const connection = await SQLConnection.getConnection();
|
||||||
|
const projectionKey = session.user.projectionKey;
|
||||||
|
|
||||||
|
// Get all persons that need cache updates (either missing or invalid)
|
||||||
|
const personsNeedingUpdate = await connection
|
||||||
|
.getRepository(PersonEntry)
|
||||||
|
.createQueryBuilder('person')
|
||||||
|
.leftJoin(
|
||||||
|
ProjectedPersonCacheEntity,
|
||||||
|
'cache',
|
||||||
|
'cache.person = person.id AND cache.projectionKey = :projectionKey',
|
||||||
|
{projectionKey}
|
||||||
|
)
|
||||||
|
.where('cache.id IS NULL OR cache.valid = false')
|
||||||
|
.getMany();
|
||||||
|
|
||||||
|
if (personsNeedingUpdate.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process persons in batches to avoid memory issues
|
||||||
|
const batchSize = 200;
|
||||||
|
for (let i = 0; i < personsNeedingUpdate.length; i += batchSize) {
|
||||||
|
const batch = personsNeedingUpdate.slice(i, i + batchSize);
|
||||||
|
const personIds = batch.map(p => p.id);
|
||||||
|
|
||||||
|
// Build base query for person junction table with projection constraints
|
||||||
|
const baseQb = connection
|
||||||
|
.getRepository(PersonJunctionTable)
|
||||||
|
.createQueryBuilder('pjt')
|
||||||
|
.innerJoin('pjt.media', 'media')
|
||||||
|
.where('pjt.person IN (:...personIds)', {personIds});
|
||||||
|
|
||||||
|
// Apply projection query if it exists
|
||||||
|
if (session.projectionQuery) {
|
||||||
|
baseQb.andWhere(session.projectionQuery);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute counts per person
|
||||||
|
const countResults = await baseQb
|
||||||
|
.clone()
|
||||||
|
.select(['pjt.person as personId', 'COUNT(*) as count'])
|
||||||
|
.groupBy('pjt.person')
|
||||||
|
.getRawMany();
|
||||||
|
|
||||||
|
// Compute sample regions per person (best rated/newest photo)
|
||||||
|
// Use individual queries per person to ensure compatibility with older SQLite versions
|
||||||
|
const topSamples: Record<number, number> = {};
|
||||||
|
for (const personId of personIds) {
|
||||||
|
const sampleQb = connection
|
||||||
|
.getRepository(PersonJunctionTable)
|
||||||
|
.createQueryBuilder('pjt')
|
||||||
|
.innerJoin('pjt.media', 'media')
|
||||||
|
.where('pjt.person = :personId', {personId});
|
||||||
|
|
||||||
|
// Apply projection query if it exists
|
||||||
|
if (session.projectionQuery) {
|
||||||
|
sampleQb.andWhere(session.projectionQuery);
|
||||||
|
}
|
||||||
|
|
||||||
|
const sampleResult = await sampleQb
|
||||||
|
.select('pjt.id')
|
||||||
|
.orderBy('media.metadataRating', 'DESC')
|
||||||
|
.addOrderBy('media.metadataCreationdate', 'DESC')
|
||||||
|
.limit(1)
|
||||||
|
.getOne();
|
||||||
|
|
||||||
|
if (sampleResult) {
|
||||||
|
topSamples[personId] = sampleResult.id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build count lookup
|
||||||
|
const counts = countResults.reduce((acc: Record<number, number>, r: any) => {
|
||||||
|
acc[parseInt(r.personId, 10)] = parseInt(r.count, 10);
|
||||||
|
return acc;
|
||||||
|
}, {});
|
||||||
|
|
||||||
|
// Batch upsert cache entries to minimize DB transactions
|
||||||
|
const cacheRepo = connection.getRepository(ProjectedPersonCacheEntity);
|
||||||
|
const cacheEntriesToSave: ProjectedPersonCacheEntity[] = [];
|
||||||
|
|
||||||
|
// Get existing cache entries for this batch
|
||||||
|
const existingEntries = await cacheRepo
|
||||||
|
.createQueryBuilder('cache')
|
||||||
|
.where('cache.projectionKey = :projectionKey', {projectionKey})
|
||||||
|
.andWhere('cache.person IN (:...personIds)', {personIds})
|
||||||
|
.getMany();
|
||||||
|
|
||||||
|
const existingByPersonId = existingEntries.reduce((acc, entry) => {
|
||||||
|
acc[entry.person.id] = entry;
|
||||||
|
return acc;
|
||||||
|
}, {} as Record<number, ProjectedPersonCacheEntity>);
|
||||||
|
|
||||||
|
for (const person of batch) {
|
||||||
|
const count = counts[person.id] || 0;
|
||||||
|
const sampleRegionId = topSamples[person.id] || null;
|
||||||
|
|
||||||
|
let cacheEntry = existingByPersonId[person.id];
|
||||||
|
if (cacheEntry) {
|
||||||
|
// Update existing entry
|
||||||
|
cacheEntry.count = count;
|
||||||
|
cacheEntry.sampleRegion = sampleRegionId ? {id: sampleRegionId} as any : null;
|
||||||
|
cacheEntry.valid = true;
|
||||||
|
} else {
|
||||||
|
// Create new entry
|
||||||
|
cacheEntry = new ProjectedPersonCacheEntity();
|
||||||
|
cacheEntry.projectionKey = projectionKey;
|
||||||
|
cacheEntry.person = {id: person.id} as any;
|
||||||
|
cacheEntry.count = count;
|
||||||
|
cacheEntry.sampleRegion = sampleRegionId ? {id: sampleRegionId} as any : null;
|
||||||
|
cacheEntry.valid = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheEntriesToSave.push(cacheEntry);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch save all cache entries for this batch
|
||||||
|
if (cacheEntriesToSave.length > 0) {
|
||||||
|
await cacheRepo.save(cacheEntriesToSave);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
async updatePerson(
|
async updatePerson(
|
||||||
name: string,
|
name: string,
|
||||||
partialPerson: PersonDTO
|
partialPerson: PersonDTO
|
||||||
): Promise<PersonEntry> {
|
): Promise<PersonEntry> {
|
||||||
this.isDBValid = false;
|
|
||||||
const connection = await SQLConnection.getConnection();
|
const connection = await SQLConnection.getConnection();
|
||||||
const repository = connection.getRepository(PersonEntry);
|
const repository = connection.getRepository(PersonEntry);
|
||||||
const person = await repository
|
const person = await repository
|
||||||
.createQueryBuilder('person')
|
.createQueryBuilder('person')
|
||||||
.limit(1)
|
.limit(1)
|
||||||
.where('person.name LIKE :name COLLATE ' + SQL_COLLATE, {name})
|
.where('person.name LIKE :name COLLATE ' + SQL_COLLATE, {name})
|
||||||
.getOne();
|
.getOne();
|
||||||
|
|
||||||
if (typeof partialPerson.name !== 'undefined') {
|
if (typeof partialPerson.name !== 'undefined') {
|
||||||
person.name = partialPerson.name;
|
person.name = partialPerson.name;
|
||||||
@@ -65,16 +189,16 @@ export class PersonManager implements IObjectManager {
|
|||||||
}
|
}
|
||||||
await repository.save(person);
|
await repository.save(person);
|
||||||
|
|
||||||
await this.loadAll();
|
await this.resetMemoryCache();
|
||||||
|
|
||||||
return person;
|
return person;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getAll(): Promise<PersonEntry[]> {
|
public async getAll(session: SessionContext): Promise<PersonEntry[]> {
|
||||||
if (this.persons === null) {
|
if (!this.personsCache?.[session.user.projectionKey]) {
|
||||||
await this.loadAll();
|
await this.loadAll(session);
|
||||||
}
|
}
|
||||||
return this.persons;
|
return this.personsCache[session.user.projectionKey];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -83,20 +207,20 @@ export class PersonManager implements IObjectManager {
|
|||||||
public async countFaces(): Promise<number> {
|
public async countFaces(): Promise<number> {
|
||||||
const connection = await SQLConnection.getConnection();
|
const connection = await SQLConnection.getConnection();
|
||||||
return await connection
|
return await connection
|
||||||
.getRepository(PersonJunctionTable)
|
.getRepository(PersonJunctionTable)
|
||||||
.createQueryBuilder('personJunction')
|
.createQueryBuilder('personJunction')
|
||||||
.getCount();
|
.getCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
public async get(name: string): Promise<PersonEntry> {
|
public async get(session: SessionContext, name: string): Promise<PersonEntry> {
|
||||||
if (this.persons === null) {
|
if (!this.personsCache?.[session.user.projectionKey]) {
|
||||||
await this.loadAll();
|
await this.loadAll(session);
|
||||||
}
|
}
|
||||||
return this.persons.find((p): boolean => p.name === name);
|
return this.personsCache[session.user.projectionKey].find((p): boolean => p.name === name);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async saveAll(
|
public async saveAll(
|
||||||
persons: { name: string; mediaId: number }[]
|
persons: { name: string; mediaId: number }[]
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const toSave: { name: string; mediaId: number }[] = [];
|
const toSave: { name: string; mediaId: number }[] = [];
|
||||||
const connection = await SQLConnection.getConnection();
|
const connection = await SQLConnection.getConnection();
|
||||||
@@ -107,7 +231,7 @@ export class PersonManager implements IObjectManager {
|
|||||||
// filter already existing persons
|
// filter already existing persons
|
||||||
for (const personToSave of persons) {
|
for (const personToSave of persons) {
|
||||||
const person = savedPersons.find(
|
const person = savedPersons.find(
|
||||||
(p): boolean => p.name === personToSave.name
|
(p): boolean => p.name === personToSave.name
|
||||||
);
|
);
|
||||||
if (!person) {
|
if (!person) {
|
||||||
toSave.push(personToSave);
|
toSave.push(personToSave);
|
||||||
@@ -119,50 +243,91 @@ export class PersonManager implements IObjectManager {
|
|||||||
const saving = toSave.slice(i * 200, (i + 1) * 200);
|
const saving = toSave.slice(i * 200, (i + 1) * 200);
|
||||||
// saving person
|
// saving person
|
||||||
const inserted = await personRepository.insert(
|
const inserted = await personRepository.insert(
|
||||||
saving.map((p) => ({name: p.name}))
|
saving.map((p) => ({name: p.name}))
|
||||||
);
|
);
|
||||||
// saving junction table
|
// saving junction table
|
||||||
const junctionTable = inserted.identifiers.map((idObj, j) => ({person: idObj, media: {id: saving[j].mediaId}}));
|
const junctionTable = inserted.identifiers.map((idObj, j) => ({person: idObj, media: {id: saving[j].mediaId}}));
|
||||||
await personJunction.insert(junctionTable);
|
await personJunction.insert(junctionTable);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.isDBValid = false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async onNewDataVersion(): Promise<void> {
|
public async onNewDataVersion(changedDir?: ParentDirectoryDTO): Promise<void> {
|
||||||
await this.resetPreviews();
|
await this.invalidateCacheForDir(changedDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected async invalidateCacheForDir(changedDir?: ParentDirectoryDTO): Promise<void> {
|
||||||
|
if (!changedDir || !changedDir.id) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const connection = await SQLConnection.getConnection();
|
||||||
|
|
||||||
|
// Collect affected person ids from this directory (non-recursive)
|
||||||
|
const rows = await connection.getRepository(PersonJunctionTable)
|
||||||
|
.createQueryBuilder('pjt')
|
||||||
|
.innerJoin('pjt.media', 'm')
|
||||||
|
.innerJoin('m.directory', 'd')
|
||||||
|
.innerJoin('pjt.person', 'person')
|
||||||
|
.where('d.id = :dirId', {dirId: changedDir.id})
|
||||||
|
.select('DISTINCT person.id', 'pid')
|
||||||
|
.getRawMany();
|
||||||
|
|
||||||
|
const pids = rows.map((r: any) => parseInt(r.pid, 10)).filter((n: number) => !isNaN(n));
|
||||||
|
if (pids.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark projection-aware person cache entries invalid for these persons
|
||||||
|
await connection.getRepository(ProjectedPersonCacheEntity)
|
||||||
|
.createQueryBuilder()
|
||||||
|
.update()
|
||||||
|
.set({valid: false})
|
||||||
|
.where('personId IN (:...pids)', {pids})
|
||||||
|
.execute();
|
||||||
|
this.personsCache = null;
|
||||||
|
|
||||||
|
} catch (err) {
|
||||||
|
Logger.warn(LOG_TAG, 'Failed to invalidate projected person cache on data change', err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private resetMemoryCache(): void {
|
||||||
|
this.personsCache = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async resetPreviews(): Promise<void> {
|
public async resetPreviews(): Promise<void> {
|
||||||
this.persons = null;
|
const connection = await SQLConnection.getConnection();
|
||||||
this.isDBValid = false;
|
await connection.getRepository(ProjectedPersonCacheEntity)
|
||||||
|
.createQueryBuilder()
|
||||||
|
.update()
|
||||||
|
.set({valid: false})
|
||||||
|
.execute();
|
||||||
|
this.resetMemoryCache();
|
||||||
}
|
}
|
||||||
|
|
||||||
private async loadAll(): Promise<void> {
|
private async loadAll(session: SessionContext): Promise<void> {
|
||||||
await this.updateDerivedValues();
|
await this.updateCacheForAll(session);
|
||||||
const connection = await SQLConnection.getConnection();
|
const connection = await SQLConnection.getConnection();
|
||||||
const personRepository = connection.getRepository(PersonEntry);
|
const personRepository = connection.getRepository(PersonEntry);
|
||||||
this.persons = await personRepository.find({
|
this.personsCache = this.personsCache || {};
|
||||||
relations: [
|
this.personsCache[session.user.projectionKey] = await personRepository
|
||||||
|
.createQueryBuilder('person')
|
||||||
|
.leftJoin('person.cache', 'cache', 'cache.projectionKey = :pk AND cache.valid = 1', {pk: session.user.projectionKey})
|
||||||
|
.leftJoin('cache.sampleRegion', 'sampleRegion')
|
||||||
|
.leftJoin('sampleRegion.media', 'media')
|
||||||
|
.leftJoin('media.directory', 'directory')
|
||||||
|
.select([
|
||||||
|
'person.id',
|
||||||
|
'person.name',
|
||||||
|
'person.isFavourite',
|
||||||
|
'cache.count',
|
||||||
'sampleRegion',
|
'sampleRegion',
|
||||||
'sampleRegion.media',
|
'media.name',
|
||||||
'sampleRegion.media.directory',
|
'directory.path',
|
||||||
'sampleRegion.media.metadata',
|
'directory.name'
|
||||||
],
|
])
|
||||||
});
|
.getMany();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Person table contains derived, denormalized data for faster select, this needs to be updated after data change
|
|
||||||
* @private
|
|
||||||
*/
|
|
||||||
private async updateDerivedValues(): Promise<void> {
|
|
||||||
if (this.isDBValid === true) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
Logger.debug(LOG_TAG, 'Updating derived persons data');
|
|
||||||
await PersonManager.updateCounts();
|
|
||||||
await PersonManager.updateSamplePhotos();
|
|
||||||
this.isDBValid = false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ import {NotificationManager} from '../NotifocationManager';
|
|||||||
import {PersonJunctionTable} from './enitites/PersonJunctionTable';
|
import {PersonJunctionTable} from './enitites/PersonJunctionTable';
|
||||||
import {MDFileEntity} from './enitites/MDFileEntity';
|
import {MDFileEntity} from './enitites/MDFileEntity';
|
||||||
import { ProjectedDirectoryCacheEntity } from './enitites/ProjectedDirectoryCacheEntity';
|
import { ProjectedDirectoryCacheEntity } from './enitites/ProjectedDirectoryCacheEntity';
|
||||||
|
import { ProjectedPersonCacheEntity } from './enitites/ProjectedPersonCacheEntity';
|
||||||
|
|
||||||
const LOG_TAG = '[SQLConnection]';
|
const LOG_TAG = '[SQLConnection]';
|
||||||
|
|
||||||
@@ -61,7 +62,8 @@ export class SQLConnection {
|
|||||||
SavedSearchEntity,
|
SavedSearchEntity,
|
||||||
VersionEntity,
|
VersionEntity,
|
||||||
// projection-aware cache entries
|
// projection-aware cache entries
|
||||||
ProjectedDirectoryCacheEntity
|
ProjectedDirectoryCacheEntity,
|
||||||
|
ProjectedPersonCacheEntity
|
||||||
];
|
];
|
||||||
|
|
||||||
private static connection: Connection = null;
|
private static connection: Connection = null;
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, Unique,} from 'typeorm';
|
import {Column, Entity, Index, OneToMany, PrimaryGeneratedColumn, Unique,} from 'typeorm';
|
||||||
import {PersonJunctionTable} from './PersonJunctionTable';
|
import {PersonJunctionTable} from './PersonJunctionTable';
|
||||||
import {columnCharsetCS} from './EntityUtils';
|
import {columnCharsetCS} from './EntityUtils';
|
||||||
import {PersonDTO} from '../../../../common/entities/PersonDTO';
|
import {PersonDTO} from '../../../../common/entities/PersonDTO';
|
||||||
|
import {ProjectedPersonCacheEntity} from './ProjectedPersonCacheEntity';
|
||||||
|
|
||||||
@Entity()
|
@Entity()
|
||||||
@Unique(['name'])
|
@Unique(['name'])
|
||||||
@@ -16,20 +17,16 @@ export class PersonEntry implements PersonDTO {
|
|||||||
})
|
})
|
||||||
name: string;
|
name: string;
|
||||||
|
|
||||||
@Column('int', {unsigned: true, default: 0})
|
|
||||||
count: number;
|
|
||||||
|
|
||||||
@Column({default: false})
|
@Column({default: false})
|
||||||
isFavourite: boolean;
|
isFavourite: boolean;
|
||||||
|
|
||||||
@OneToMany(() => PersonJunctionTable, (junctionTable) => junctionTable.person)
|
@OneToMany(() => PersonJunctionTable, (junctionTable) => junctionTable.person)
|
||||||
public faces: PersonJunctionTable[];
|
public faces: PersonJunctionTable[];
|
||||||
|
|
||||||
@ManyToOne(() => PersonJunctionTable, {
|
|
||||||
onDelete: 'SET NULL',
|
@OneToMany(() => ProjectedPersonCacheEntity, (ppc) => ppc.person)
|
||||||
nullable: true,
|
public cache: ProjectedPersonCacheEntity;
|
||||||
})
|
|
||||||
sampleRegion: PersonJunctionTable;
|
|
||||||
|
|
||||||
// does not store in the DB, temporal field
|
// does not store in the DB, temporal field
|
||||||
missingThumbnail?: boolean;
|
missingThumbnail?: boolean;
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { Column, Entity, Index, ManyToOne, PrimaryGeneratedColumn, Unique } from 'typeorm';
|
||||||
|
import { PersonEntry } from './PersonEntry';
|
||||||
|
import { PersonJunctionTable } from './PersonJunctionTable';
|
||||||
|
import {PersonCacheDTO} from '../../../../common/entities/PersonDTO';
|
||||||
|
|
||||||
|
// Projection-aware cache for persons, analogous to ProjectedDirectoryCacheEntity
|
||||||
|
// Stores per-projection aggregates for a given person
|
||||||
|
@Entity()
|
||||||
|
@Unique(['projectionKey', 'person'])
|
||||||
|
export class ProjectedPersonCacheEntity implements PersonCacheDTO{
|
||||||
|
@PrimaryGeneratedColumn({ unsigned: true })
|
||||||
|
id: number;
|
||||||
|
|
||||||
|
// hash key of the projection (built by SessionManager)
|
||||||
|
@Index()
|
||||||
|
@Column({ type: 'text', select: false }) // not needed in payloads; used to select the right cache per session
|
||||||
|
projectionKey: string;
|
||||||
|
|
||||||
|
// the person this cache row is about
|
||||||
|
@Index()
|
||||||
|
@ManyToOne(() => PersonEntry, {
|
||||||
|
onDelete: 'CASCADE',
|
||||||
|
nullable: false,
|
||||||
|
})
|
||||||
|
person: PersonEntry;
|
||||||
|
|
||||||
|
// number of visible face regions for this person under the projection
|
||||||
|
@Column('int', { unsigned: true, default: 0 })
|
||||||
|
count: number;
|
||||||
|
|
||||||
|
// a PersonJunctionTable row id under the projection chosen by ranking
|
||||||
|
@ManyToOne(() => PersonJunctionTable, {
|
||||||
|
onDelete: 'SET NULL',
|
||||||
|
nullable: true,
|
||||||
|
})
|
||||||
|
sampleRegion: PersonJunctionTable;
|
||||||
|
|
||||||
|
// if false, sample (or other aggregates) need recomputation
|
||||||
|
@Column({ type: 'boolean', default: true })
|
||||||
|
valid: boolean;
|
||||||
|
}
|
||||||
@@ -53,7 +53,9 @@ export class AlbumCoverFillingJob extends Job {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async stepPersonsPreview(): Promise<boolean> {
|
private async stepPersonsPreview(): Promise<boolean> {
|
||||||
await ObjectManagers.getInstance().PersonManager.getAll();
|
for (const session of this.availableSessions) {
|
||||||
|
await ObjectManagers.getInstance().PersonManager.getAll(session);
|
||||||
|
}
|
||||||
this.Progress.log('Updating Persons preview');
|
this.Progress.log('Updating Persons preview');
|
||||||
this.Progress.Processed++;
|
this.Progress.Processed++;
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
export interface PersonDTO {
|
export interface PersonDTO {
|
||||||
id: number;
|
id: number;
|
||||||
name: string;
|
name: string;
|
||||||
count: number;
|
|
||||||
missingThumbnail?: boolean;
|
missingThumbnail?: boolean;
|
||||||
isFavourite: boolean;
|
isFavourite: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface PersonCacheDTO {
|
||||||
|
count: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -46,11 +46,22 @@ describe('PersonManager', (sqlHelper: DBTestHelper) => {
|
|||||||
p2 = (dir.media.filter(m => m.name === p2.name)[0] as any);
|
p2 = (dir.media.filter(m => m.name === p2.name)[0] as any);
|
||||||
pFaceLess = (dir.media[2] as any);
|
pFaceLess = (dir.media[2] as any);
|
||||||
v = (dir.media.filter(m => m.name === v.name)[0] as any);
|
v = (dir.media.filter(m => m.name === v.name)[0] as any);
|
||||||
savedPerson = await (await SQLConnection.getConnection()).getRepository(PersonEntry).find({
|
savedPerson = await (await SQLConnection.getConnection()).getRepository(PersonEntry).createQueryBuilder('person')
|
||||||
relations: ['sampleRegion',
|
.leftJoin('person.cache', 'cache', 'cache.projectionKey = :pk AND cache.valid = 1', {pk: DBTestHelper.defaultSession.user.projectionKey})
|
||||||
'sampleRegion.media',
|
.leftJoin('cache.sampleRegion', 'sampleRegion')
|
||||||
'sampleRegion.media.directory']
|
.leftJoin('sampleRegion.media', 'media')
|
||||||
});
|
.leftJoin('media.directory', 'directory')
|
||||||
|
.select([
|
||||||
|
'person.id',
|
||||||
|
'person.name',
|
||||||
|
'person.isFavourite',
|
||||||
|
'cache.count',
|
||||||
|
'sampleRegion',
|
||||||
|
'media.name',
|
||||||
|
'directory.path',
|
||||||
|
'directory.name'
|
||||||
|
])
|
||||||
|
.getMany();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
@@ -67,13 +78,12 @@ describe('PersonManager', (sqlHelper: DBTestHelper) => {
|
|||||||
const pm = new PersonManager();
|
const pm = new PersonManager();
|
||||||
const person = Utils.clone(savedPerson[0]);
|
const person = Utils.clone(savedPerson[0]);
|
||||||
|
|
||||||
const selected = Utils.clone(await pm.get('Boba Fett'));
|
const selected = Utils.clone(await pm.get(DBTestHelper.defaultSession, 'Boba Fett'));
|
||||||
delete selected.sampleRegion;
|
expect(selected.cache).to.be.not.undefined;
|
||||||
delete person.sampleRegion;
|
delete selected.cache;
|
||||||
person.count = 1;
|
delete person.cache;
|
||||||
expect(selected).to.deep.equal(person);
|
expect(selected).to.deep.equal(person);
|
||||||
|
|
||||||
expect((await pm.get('Boba Fett') as PersonEntry).sampleRegion.media.name).to.deep.equal(p.name);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user