self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2025-01-06 03:54:12 +02:00
Code Issues Releases Activity

Improving SOME_OF search to support expressions with more parameters

Browse Source
This commit is contained in:
Patrik J. Braun 2021-05-30 17:26:07 +02:00
parent 404b82e12b
commit b126022454
6 changed files with 140 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/backend/model/database/sql/AlbumManager.ts
View File

@ -47,6 +47,7 @@ export class AlbumManager implements IAlbumManager {
public async getAlbums(): Promise<AlbumBaseDTO[]> { public async getAlbums(): Promise<AlbumBaseDTO[]> {
const connection = await SQLConnection.getConnection(); const connection = await SQLConnection.getConnection();
const albums = await connection.getRepository(AlbumBaseEntity).find(); const albums = await connection.getRepository(AlbumBaseEntity).find();
for (const a of albums) { for (const a of albums) {
await AlbumManager.fillPreviewToAlbum(a); await AlbumManager.fillPreviewToAlbum(a);
} }

176
src/backend/model/database/sql/SearchManager.ts
View File

@ -27,12 +27,12 @@ import {
} from '../../../../common/entities/SearchQueryDTO'; } from '../../../../common/entities/SearchQueryDTO';
import {GalleryManager} from './GalleryManager'; import {GalleryManager} from './GalleryManager';
import {ObjectManagers} from '../../ObjectManagers'; import {ObjectManagers} from '../../ObjectManagers';
import {Utils} from '../../../../common/Utils';
import {PhotoDTO} from '../../../../common/entities/PhotoDTO'; import {PhotoDTO} from '../../../../common/entities/PhotoDTO';
import {DatabaseType} from '../../../../common/config/private/PrivateConfig'; import {DatabaseType} from '../../../../common/config/private/PrivateConfig';
import {ISQLGalleryManager} from './IGalleryManager'; import {ISQLGalleryManager} from './IGalleryManager';
import {ISQLSearchManager} from './ISearchManager'; import {ISQLSearchManager} from './ISearchManager';
import {MediaDTO} from '../../../../common/entities/MediaDTO'; import {MediaDTO} from '../../../../common/entities/MediaDTO';
import {Utils} from '../../../../common/Utils';
export class SearchManager implements ISQLSearchManager { export class SearchManager implements ISQLSearchManager {
@ -139,8 +139,7 @@ export class SearchManager implements ISQLSearchManager {
} }
async search(queryIN: SearchQueryDTO): Promise<SearchResultDTO> { async search(queryIN: SearchQueryDTO): Promise<SearchResultDTO> {
let query = this.flattenSameOfQueries(queryIN); const query = await this.prepareQuery(queryIN);
query = await this.getGPSData(query);
const connection = await SQLConnection.getConnection(); const connection = await SQLConnection.getConnection();
const result: SearchResultDTO = { const result: SearchResultDTO = {
@ -225,8 +224,7 @@ export class SearchManager implements ISQLSearchManager {
} }
public async getPreview(queryIN: SearchQueryDTO): Promise<MediaDTO> { public async getPreview(queryIN: SearchQueryDTO): Promise<MediaDTO> {
let query = this.flattenSameOfQueries(queryIN); const query = await this.prepareQuery(queryIN);
query = await this.getGPSData(query);
const connection = await SQLConnection.getConnection(); const connection = await SQLConnection.getConnection();
return await connection return await connection
@ -239,6 +237,28 @@ export class SearchManager implements ISQLSearchManager {
.getOne(); .getOne();
} }
private async prepareQuery(queryIN: SearchQueryDTO): Promise<SearchQueryDTO> {
let query: SearchQueryDTO = this.assignQueryIDs(Utils.clone(queryIN)); // assign local ids before flattening SOME_OF queries
query = this.flattenSameOfQueries(query);
query = await this.getGPSData(query);
return query;
}
/**
* Assigning IDs to search queries. It is a help / simplification to typeorm,
* so less parameters are needed to pass down to SQL.
* Witch SOME_OF query the number of WHERE constrains have O(N!) complexity
*/
private assignQueryIDs(queryIN: SearchQueryDTO, id = {value: 1}): SearchQueryDTO {
if ((queryIN as SearchListQuery).list) {
(queryIN as SearchListQuery).list.forEach(q => this.assignQueryIDs(q, id));
return queryIN;
}
(queryIN as SearchQueryDTOWithID).queryId = id.value;
id.value++;
return queryIN;
}
/** /**
* Returns only those part of a query tree that only contains directory related search queries * Returns only those part of a query tree that only contains directory related search queries
*/ */
@ -297,16 +317,17 @@ export class SearchManager implements ISQLSearchManager {
* @param directoryOnly Only builds directory related queries * @param directoryOnly Only builds directory related queries
* @private * @private
*/ */
private buildWhereQuery(query: SearchQueryDTO, directoryOnly = false, paramCounter = {value: 0}): Brackets { private buildWhereQuery(query: SearchQueryDTO, directoryOnly = false): Brackets {
const queryId = (query as SearchQueryDTOWithID).queryId;
switch (query.type) { switch (query.type) {
case SearchQueryTypes.AND: case SearchQueryTypes.AND:
return new Brackets((q): any => { return new Brackets((q): any => {
(query as ANDSearchQuery).list.forEach((sq): any => q.andWhere(this.buildWhereQuery(sq, directoryOnly, paramCounter))); (query as ANDSearchQuery).list.forEach((sq): any => q.andWhere(this.buildWhereQuery(sq, directoryOnly)));
return q; return q;
}); });
case SearchQueryTypes.OR: case SearchQueryTypes.OR:
return new Brackets((q): any => { return new Brackets((q): any => {
(query as ANDSearchQuery).list.forEach((sq): any => q.orWhere(this.buildWhereQuery(sq, directoryOnly, paramCounter))); (query as ANDSearchQuery).list.forEach((sq): any => q.orWhere(this.buildWhereQuery(sq, directoryOnly)));
return q; return q;
}); });
@ -340,21 +361,20 @@ export class SearchManager implements ISQLSearchManager {
return new Brackets((q): any => { return new Brackets((q): any => {
const textParam: any = {}; const textParam: any = {};
paramCounter.value++; textParam['maxLat' + queryId] = maxLat;
textParam['maxLat' + paramCounter.value] = maxLat; textParam['minLat' + queryId] = minLat;
textParam['minLat' + paramCounter.value] = minLat; textParam['maxLon' + queryId] = maxLon;
textParam['maxLon' + paramCounter.value] = maxLon; textParam['minLon' + queryId] = minLon;
textParam['minLon' + paramCounter.value] = minLon;
if (!(query as DistanceSearch).negate) { if (!(query as DistanceSearch).negate) {
q.where(`media.metadata.positionData.GPSData.latitude < :maxLat${paramCounter.value}`, textParam); q.where(`media.metadata.positionData.GPSData.latitude < :maxLat${queryId}`, textParam);
q.andWhere(`media.metadata.positionData.GPSData.latitude > :minLat${paramCounter.value}`, textParam); q.andWhere(`media.metadata.positionData.GPSData.latitude > :minLat${queryId}`, textParam);
q.andWhere(`media.metadata.positionData.GPSData.longitude < :maxLon${paramCounter.value}`, textParam); q.andWhere(`media.metadata.positionData.GPSData.longitude < :maxLon${queryId}`, textParam);
q.andWhere(`media.metadata.positionData.GPSData.longitude > :minLon${paramCounter.value}`, textParam); q.andWhere(`media.metadata.positionData.GPSData.longitude > :minLon${queryId}`, textParam);
} else { } else {
q.where(`media.metadata.positionData.GPSData.latitude > :maxLat${paramCounter.value}`, textParam); q.where(`media.metadata.positionData.GPSData.latitude > :maxLat${queryId}`, textParam);
q.orWhere(`media.metadata.positionData.GPSData.latitude < :minLat${paramCounter.value}`, textParam); q.orWhere(`media.metadata.positionData.GPSData.latitude < :minLat${queryId}`, textParam);
q.orWhere(`media.metadata.positionData.GPSData.longitude > :maxLon${paramCounter.value}`, textParam); q.orWhere(`media.metadata.positionData.GPSData.longitude > :maxLon${queryId}`, textParam);
q.orWhere(`media.metadata.positionData.GPSData.longitude < :minLon${paramCounter.value}`, textParam); q.orWhere(`media.metadata.positionData.GPSData.longitude < :minLon${queryId}`, textParam);
} }
return q; return q;
}); });
@ -370,11 +390,9 @@ export class SearchManager implements ISQLSearchManager {
const relation = (query as TextSearch).negate ? '<' : '>='; const relation = (query as TextSearch).negate ? '<' : '>=';
const textParam: any = {}; const textParam: any = {};
textParam['from' + paramCounter.value] = (query as FromDateSearch).value; textParam['from' + queryId] = (query as FromDateSearch).value;
q.where(`media.metadata.creationDate ${relation} :from${paramCounter.value}`, textParam); q.where(`media.metadata.creationDate ${relation} :from${queryId}`, textParam);
paramCounter.value++;
return q; return q;
}); });
@ -389,10 +407,9 @@ export class SearchManager implements ISQLSearchManager {
const relation = (query as TextSearch).negate ? '>' : '<='; const relation = (query as TextSearch).negate ? '>' : '<=';
const textParam: any = {}; const textParam: any = {};
textParam['to' + paramCounter.value] = (query as ToDateSearch).value; textParam['to' + queryId] = (query as ToDateSearch).value;
q.where(`media.metadata.creationDate ${relation} :to${paramCounter.value}`, textParam); q.where(`media.metadata.creationDate ${relation} :to${queryId}`, textParam);
paramCounter.value++;
return q; return q;
}); });
@ -408,10 +425,9 @@ export class SearchManager implements ISQLSearchManager {
const relation = (query as TextSearch).negate ? '<' : '>='; const relation = (query as TextSearch).negate ? '<' : '>=';
const textParam: any = {}; const textParam: any = {};
textParam['min' + paramCounter.value] = (query as MinRatingSearch).value; textParam['min' + queryId] = (query as MinRatingSearch).value;
q.where(`media.metadata.rating ${relation} :min${paramCounter.value}`, textParam); q.where(`media.metadata.rating ${relation} :min${queryId}`, textParam);
paramCounter.value++;
return q; return q;
}); });
case SearchQueryTypes.max_rating: case SearchQueryTypes.max_rating:
@ -427,10 +443,9 @@ export class SearchManager implements ISQLSearchManager {
if (typeof (query as MaxRatingSearch).value !== 'undefined') { if (typeof (query as MaxRatingSearch).value !== 'undefined') {
const textParam: any = {}; const textParam: any = {};
textParam['max' + paramCounter.value] = (query as MaxRatingSearch).value; textParam['max' + queryId] = (query as MaxRatingSearch).value;
q.where(`media.metadata.rating ${relation} :max${paramCounter.value}`, textParam); q.where(`media.metadata.rating ${relation} :max${queryId}`, textParam);
} }
paramCounter.value++;
return q; return q;
}); });
@ -446,11 +461,10 @@ export class SearchManager implements ISQLSearchManager {
const relation = (query as TextSearch).negate ? '<' : '>='; const relation = (query as TextSearch).negate ? '<' : '>=';
const textParam: any = {}; const textParam: any = {};
textParam['min' + paramCounter.value] = (query as MinResolutionSearch).value * 1000 * 1000; textParam['min' + queryId] = (query as MinResolutionSearch).value * 1000 * 1000;
q.where(`media.metadata.size.width * media.metadata.size.height ${relation} :min${paramCounter.value}`, textParam); q.where(`media.metadata.size.width * media.metadata.size.height ${relation} :min${queryId}`, textParam);
paramCounter.value++;
return q; return q;
}); });
@ -466,10 +480,9 @@ export class SearchManager implements ISQLSearchManager {
const relation = (query as TextSearch).negate ? '>' : '<='; const relation = (query as TextSearch).negate ? '>' : '<=';
const textParam: any = {}; const textParam: any = {};
textParam['max' + paramCounter.value] = (query as MaxResolutionSearch).value * 1000 * 1000; textParam['max' + queryId] = (query as MaxResolutionSearch).value * 1000 * 1000;
q.where(`media.metadata.size.width * media.metadata.size.height ${relation} :max${paramCounter.value}`, textParam); q.where(`media.metadata.size.width * media.metadata.size.height ${relation} :max${queryId}`, textParam);
paramCounter.value++;
return q; return q;
}); });
@ -483,7 +496,6 @@ export class SearchManager implements ISQLSearchManager {
} else { } else {
q.where('media.metadata.size.width <= media.metadata.size.height'); q.where('media.metadata.size.width <= media.metadata.size.height');
} }
paramCounter.value++;
return q; return q;
}); });
@ -514,26 +526,25 @@ export class SearchManager implements ISQLSearchManager {
const whereFNRev = (query as TextSearch).negate ? 'orWhere' : 'andWhere'; const whereFNRev = (query as TextSearch).negate ? 'orWhere' : 'andWhere';
const textParam: any = {}; const textParam: any = {};
paramCounter.value++; textParam['text' + queryId] = createMatchString((query as TextSearch).text);
textParam['text' + paramCounter.value] = createMatchString((query as TextSearch).text);
if (query.type === SearchQueryTypes.any_text || if (query.type === SearchQueryTypes.any_text ||
query.type === SearchQueryTypes.directory) { query.type === SearchQueryTypes.directory) {
const dirPathStr = ((query as TextSearch).text).replace(new RegExp('\\\\', 'g'), '/'); const dirPathStr = ((query as TextSearch).text).replace(new RegExp('\\\\', 'g'), '/');
textParam['fullPath' + paramCounter.value] = createMatchString(dirPathStr); textParam['fullPath' + queryId] = createMatchString(dirPathStr);
q[whereFN](`directory.path ${LIKE} :fullPath${paramCounter.value} COLLATE utf8_general_ci`, q[whereFN](`directory.path ${LIKE} :fullPath${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
const directoryPath = GalleryManager.parseRelativeDirePath(dirPathStr); const directoryPath = GalleryManager.parseRelativeDirePath(dirPathStr);
q[whereFN](new Brackets((dq): any => { q[whereFN](new Brackets((dq): any => {
textParam['dirName' + paramCounter.value] = createMatchString(directoryPath.name); textParam['dirName' + queryId] = createMatchString(directoryPath.name);
dq[whereFNRev](`directory.name ${LIKE} :dirName${paramCounter.value} COLLATE utf8_general_ci`, dq[whereFNRev](`directory.name ${LIKE} :dirName${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
if (dirPathStr.includes('/')) { if (dirPathStr.includes('/')) {
textParam['parentName' + paramCounter.value] = createMatchString(directoryPath.parent); textParam['parentName' + queryId] = createMatchString(directoryPath.parent);
dq[whereFNRev](`directory.path ${LIKE} :parentName${paramCounter.value} COLLATE utf8_general_ci`, dq[whereFNRev](`directory.path ${LIKE} :parentName${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
} }
return dq; return dq;
@ -541,21 +552,21 @@ export class SearchManager implements ISQLSearchManager {
} }
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.file_name) { if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.file_name) {
q[whereFN](`media.name ${LIKE} :text${paramCounter.value} COLLATE utf8_general_ci`, q[whereFN](`media.name ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
} }
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.caption) { if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.caption) {
q[whereFN](`media.metadata.caption ${LIKE} :text${paramCounter.value} COLLATE utf8_general_ci`, q[whereFN](`media.metadata.caption ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
} }
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.position) { if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.position) {
q[whereFN](`media.metadata.positionData.country ${LIKE} :text${paramCounter.value} COLLATE utf8_general_ci`, q[whereFN](`media.metadata.positionData.country ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
textParam) textParam)
[whereFN](`media.metadata.positionData.state ${LIKE} :text${paramCounter.value} COLLATE utf8_general_ci`, [whereFN](`media.metadata.positionData.state ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
textParam) textParam)
[whereFN](`media.metadata.positionData.city ${LIKE} :text${paramCounter.value} COLLATE utf8_general_ci`, [whereFN](`media.metadata.positionData.city ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
} }
@ -563,22 +574,22 @@ export class SearchManager implements ISQLSearchManager {
const matchArrayField = (fieldName: string): void => { const matchArrayField = (fieldName: string): void => {
q[whereFN](new Brackets((qbr): void => { q[whereFN](new Brackets((qbr): void => {
if ((query as TextSearch).matchType !== TextSearchQueryMatchTypes.exact_match) { if ((query as TextSearch).matchType !== TextSearchQueryMatchTypes.exact_match) {
qbr[whereFN](`${fieldName} ${LIKE} :text${paramCounter.value} COLLATE utf8_general_ci`, qbr[whereFN](`${fieldName} ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
} else { } else {
qbr[whereFN](new Brackets((qb): void => { qbr[whereFN](new Brackets((qb): void => {
textParam['CtextC' + paramCounter.value] = `%,${(query as TextSearch).text},%`; textParam['CtextC' + queryId] = `%,${(query as TextSearch).text},%`;
textParam['Ctext' + paramCounter.value] = `%,${(query as TextSearch).text}`; textParam['Ctext' + queryId] = `%,${(query as TextSearch).text}`;
textParam['textC' + paramCounter.value] = `${(query as TextSearch).text},%`; textParam['textC' + queryId] = `${(query as TextSearch).text},%`;
textParam['text_exact' + paramCounter.value] = `${(query as TextSearch).text}`; textParam['text_exact' + queryId] = `${(query as TextSearch).text}`;
qb[whereFN](`${fieldName} ${LIKE} :CtextC${paramCounter.value} COLLATE utf8_general_ci`, qb[whereFN](`${fieldName} ${LIKE} :CtextC${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
qb[whereFN](`${fieldName} ${LIKE} :Ctext${paramCounter.value} COLLATE utf8_general_ci`, qb[whereFN](`${fieldName} ${LIKE} :Ctext${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
qb[whereFN](`${fieldName} ${LIKE} :textC${paramCounter.value} COLLATE utf8_general_ci`, qb[whereFN](`${fieldName} ${LIKE} :textC${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
qb[whereFN](`${fieldName} ${LIKE} :text_exact${paramCounter.value} COLLATE utf8_general_ci`, qb[whereFN](`${fieldName} ${LIKE} :text_exact${queryId} COLLATE utf8_general_ci`,
textParam); textParam);
})); }));
} }
@ -626,14 +637,41 @@ export class SearchManager implements ISQLSearchManager {
} as ANDSearchQuery); } as ANDSearchQuery);
} }
const combinations: SearchQueryDTO[][] = Utils.getAnyX(someOfQ.min, (query as SearchListQuery).list); const getAllCombinations = (num: number, arr: SearchQueryDTO[], start = 0): SearchQueryDTO[] => {
if (num <= 0 || num > arr.length || start >= arr.length) {
return [];
}
if (num <= 1) {
return arr.slice(start);
}
if (num === arr.length - start) {
return arr.slice(start);
}
const ret: ANDSearchQuery[] = [];
for (let i = start; i < arr.length - num + 1; ++i) {
const subRes = getAllCombinations(num - 1, arr, i + 1);
const and: ANDSearchQuery = {
type: SearchQueryTypes.AND,
list: [
arr[i],
subRes.length === 1 ? subRes[0] : (
{
type: SearchQueryTypes.OR,
list: subRes
} as ORSearchQuery)
]
};
ret.push(and);
}
return ret;
};
return this.flattenSameOfQueries({ return this.flattenSameOfQueries({
type: SearchQueryTypes.OR, type: SearchQueryTypes.OR,
list: combinations.map((c): ANDSearchQuery => ({ list: getAllCombinations(someOfQ.min, (query as SearchListQuery).list)
type: SearchQueryTypes.AND, list: c
} as ANDSearchQuery))
} as ORSearchQuery); } as ORSearchQuery);
} }
@ -650,3 +688,7 @@ export class SearchManager implements ISQLSearchManager {
} }
export interface SearchQueryDTOWithID extends SearchQueryDTO {
queryId: number;
}

2
src/backend/model/database/sql/enitites/MediaEntity.ts
View File

@ -29,7 +29,7 @@ export class MediaMetadataEntity implements MediaMetadata {
* you do not want to see 2AM next to a photo that was taken during lunch * you do not want to see 2AM next to a photo that was taken during lunch
*/ */
@Column('bigint', { @Column('bigint', {
unsigned: true, transformer: { transformer: {
from: v => parseInt(v, 10), from: v => parseInt(v, 10),
to: v => v to: v => v
} }

5
src/common/DataStructureVersion.ts
View File

@ -1 +1,4 @@
export const DataStructureVersion = 23; /**
* This version indicates that the SQL sql/entities/*Entity.ts files got changed and the db needs to be recreated
*/
export const DataStructureVersion = 24;

2
src/common/Utils.ts
View File

@ -49,7 +49,7 @@ export class Utils {
static shallowClone<T>(object: T): T { static shallowClone<T>(object: T): T {
const c: any = {}; const c: any = {};
for (const e of Object.entries(object)) { for (const e of Object.entries(object)) {
c[e[0]] = [1]; c[e[0]] = e[1];
} }
return c; return c;
} }

28
test/backend/unit/model/sql/SearchManager.spec.ts
View File

@ -736,7 +736,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
query = ({ query = ({
text: '/wars dir/Return of the Jedi', text: '/wars dir/Return of the Jedi',
// matchType: TextSearchQueryMatchTypes.like, // matchType: TextSearchQueryMatchTypes.like,
type: SearchQueryTypes.directory type: SearchQueryTypes.directory
} as TextSearch); } as TextSearch);
@ -1114,10 +1114,32 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
}); });
(it('should execute complex SOME_OF querry', async () => {
const sm = new SearchManager();
const query: SomeOfSearchQuery = {
type: SearchQueryTypes.SOME_OF,
min: 5,
//
list: 'abcdefghijklmnopqrstu'.split('').map(t => ({
type: SearchQueryTypes.file_name,
text: t
} as TextSearch))
};
expect(removeDir(await sm.search(query)))
.to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [],
media: [v],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
}) as any).timeout(40000);
it('search result should return directory', async () => { it('search result should return directory', async () => {
const sm = new SearchManager(); const sm = new SearchManager();
let query = { const query = {
text: subDir.name, text: subDir.name,
type: SearchQueryTypes.any_text type: SearchQueryTypes.any_text
} as TextSearch; } as TextSearch;
@ -1129,8 +1151,6 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
metaFile: [], metaFile: [],
resultOverflow: false resultOverflow: false
} as SearchResultDTO)); } as SearchResultDTO));
}); });
}); });