self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2025-11-29 22:48:28 +02:00
Code Issues Releases Activity

Adding more indices to fields and improving SQL queries #437

Browse Source
This commit is contained in:
Patrik J. Braun
2022-12-04 22:23:51 +01:00
parent eea77a5bea
commit d75a307be6
21 changed files with 306 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
test/backend/unit/model/sql/IndexingManager.spec.ts
View File

@@ -5,7 +5,6 @@ import {GalleryManager} from '../../../../../src/backend/model/database/sql/Gall
import {DirectoryBaseDTO, DirectoryDTOUtils, ParentDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
import {TestHelper} from '../../../../TestHelper';
import {Connection} from 'typeorm';
import {DirectoryEntity} from '../../../../../src/backend/model/database/sql/enitites/DirectoryEntity';
import {Utils} from '../../../../../src/common/Utils';
import {MediaDTO} from '../../../../../src/common/entities/MediaDTO';
import {FileDTO} from '../../../../../src/common/entities/FileDTO';
@@ -13,7 +12,7 @@ import {IndexingManager} from '../../../../../src/backend/model/database/sql/Ind
import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers';
import {DBTestHelper} from '../../../DBTestHelper';
import {DiskMangerWorker} from '../../../../../src/backend/model/threading/DiskMangerWorker';
import {ReIndexingSensitivity} from '../../../../../src/common/config/private/PrivateConfig';
import {ReIndexingSensitivity, SQLLogLevel} from '../../../../../src/common/config/private/PrivateConfig';
import {SearchQueryTypes, TextSearch, TextSearchQueryMatchTypes} from '../../../../../src/common/entities/SearchQueryDTO';
import {ProjectPath} from '../../../../../src/backend/ProjectPath';
import * as path from 'path';
@@ -31,15 +30,13 @@ const {expect} = chai;
class GalleryManagerTest extends GalleryManager {
public async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<ParentDirectoryDTO> {
return super.selectParentDir(connection, directoryName, directoryParent);
public async getDirIdAndTime(connection: Connection, directoryName: string, directoryParent: string) {
return super.getDirIdAndTime(connection, directoryName, directoryParent);
}
public async fillParentDir(connection: Connection, dir: ParentDirectoryDTO): Promise<void> {
return super.fillParentDir(connection, dir);
public async getParentDirFromId(connection: Connection, dir: number): Promise<ParentDirectoryDTO> {
return super.getParentDirFromId(connection, dir);
}
}
class IndexingManagerTest extends IndexingManager {
@@ -149,8 +146,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent.name, parent.path)).id);
DirectoryDTOUtils.removeReferences(selected);
@@ -175,14 +172,16 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
'.'
);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, directoryPath.name,
directoryPath.parent);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, directoryPath.name, directoryPath.parent)).id);
expect(selected?.media?.length)
.to.be.greaterThan(0);
const tmpDir = path.join(__dirname, '/../../../tmp/rnd5sdf_emptyDir');
if (!fs.existsSync(sqlHelper.tempDir)) {
fs.mkdirSync(sqlHelper.tempDir);
}
const tmpDir = path.join(sqlHelper.tempDir, '/rnd5sdf_emptyDir');
fs.mkdirSync(tmpDir);
ProjectPath.ImageFolder = tmpDir;
let notFailed = false;
@@ -214,8 +213,9 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent.name, parent.path)).id);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
@@ -244,8 +244,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
const conn = await SQLConnection.getConnection();
{
const selected = await gm.selectParentDir(conn, parent1.name, parent1.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent1.name, parent1.path)).id);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
@@ -254,8 +254,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
.to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent1)));
}
{
const selected = await gm.selectParentDir(conn, parent2.name, parent2.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent2.name, parent2.path)).id);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
@@ -277,8 +277,9 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent.name, parent.path)).id);
DirectoryDTOUtils.removeReferences(selected);
@@ -290,8 +291,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
it('should select preview', async () => {
const selectDirectory = async (gmTest: GalleryManagerTest, dir: DirectoryBaseDTO): Promise<ParentDirectoryDTO> => {
const conn = await SQLConnection.getConnection();
const selected = await gmTest.selectParentDir(conn, dir.name, dir.path);
await gmTest.fillParentDir(conn, selected);
const selected = await gmTest.getParentDirFromId(conn,
(await gmTest.getDirIdAndTime(conn, dir.name, dir.path)).id);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
@@ -369,8 +370,9 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent.name, parent.path)).id);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
@@ -407,8 +409,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent.name, parent.path)).id);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
@@ -437,8 +439,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent.name, parent.path)).id);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
@@ -473,8 +475,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent.name, parent.path)).id);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
@@ -499,8 +501,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
Config.Client.MetaFile.markdown = false;
Config.Client.MetaFile.pg2conf = false;
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent.name, parent.path)).id);
delete parent.metaFile;
DirectoryDTOUtils.removeReferences(selected);
@@ -530,8 +532,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await im.saveToDB(Utils.clone(subDir) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, subDir.name, subDir.path)).id);
// subDir.isPartial = true;
// delete subDir.directories;
@@ -570,8 +572,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await Promise.all([s1, s2, s3]);
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent.name, parent.path)).id);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
@@ -595,8 +597,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, parent.name, parent.path)).id);
DirectoryDTOUtils.removeReferences(selected);
removeIds(selected);
@@ -604,7 +606,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
.to.deep.equal(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
await im.resetDB();
const selectReset = await gm.selectParentDir(conn, parent.name, parent.path);
const selectReset = await gm.getDirIdAndTime(conn, parent.name, parent.path);
expect(selectReset).to.deep.equal(null);
});
@@ -627,8 +629,8 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
DirectoryDTOUtils.removeReferences(parent);
await im.saveToDB(subDir as ParentDirectoryDTO);
const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
const selected = await gm.getParentDirFromId(conn,
(await gm.getDirIdAndTime(conn, subDir.name, subDir.path)).id);
expect(selected.media.length).to.equal(subDir.media.length);
}) as any).timeout(40000);
@@ -661,11 +663,11 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
// @ts-ignore
fs.statSync = () => ({ctime: new Date(dirTime), mtime: new Date(dirTime)});
const gm = new GalleryManagerTest();
gm.selectParentDir = (connection: Connection, directoryName: string, directoryParent: string) => {
gm.getDirIdAndTime = () => {
return Promise.resolve(indexedTime as any);
};
gm.fillParentDir = (connection: Connection, dir: DirectoryEntity) => {
return Promise.resolve();
gm.getParentDirFromId = ():Promise<ParentDirectoryDTO> => {
return Promise.resolve(indexedTime) as unknown as Promise<ParentDirectoryDTO>;
};
ObjectManagers.getInstance().IndexingManager.indexDirectory = (...args) => {