self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2025-02-11 13:53:28 +02:00
Code Issues Releases Activity

Forcing mysql to use utf8mb4 fixes #399

Browse Source
This commit is contained in:
Patrik J. Braun 2022-01-14 12:14:28 +01:00
parent 07d8261034
commit e607ae810a
6 changed files with 51 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/backend/model/database/sql/PersonManager.ts
View File

@ -5,6 +5,7 @@ import {PersonDTO} from '../../../../common/entities/PersonDTO';
import {ISQLPersonManager} from './IPersonManager'; import {ISQLPersonManager} from './IPersonManager';
import {Logger} from '../../../Logger'; import {Logger} from '../../../Logger';
import {FaceRegion} from '../../../../common/entities/PhotoDTO'; import {FaceRegion} from '../../../../common/entities/PhotoDTO';
import {SQL_COLLATE} from './enitites/EntityUtils';
const LOG_TAG = '[PersonManager]'; const LOG_TAG = '[PersonManager]';
@ -47,7 +48,7 @@ export class PersonManager implements ISQLPersonManager {
const repository = connection.getRepository(PersonEntry); const repository = connection.getRepository(PersonEntry);
const person = await repository.createQueryBuilder('person') const person = await repository.createQueryBuilder('person')
.limit(1) .limit(1)
.where('person.name LIKE :name COLLATE utf8_general_ci', {name}).getOne(); .where('person.name LIKE :name COLLATE ' + SQL_COLLATE, {name}).getOne();
if (typeof partialPerson.name !== 'undefined') { if (typeof partialPerson.name !== 'undefined') {
@ -93,7 +94,7 @@ export class PersonManager implements ISQLPersonManager {
const personRepository = connection.getRepository(PersonEntry); const personRepository = connection.getRepository(PersonEntry);
const faceRegionRepository = connection.getRepository(FaceRegionEntry); const faceRegionRepository = connection.getRepository(FaceRegionEntry);
const savedPersons = await personRepository.find(); const savedPersons = await personRepository.find();
// filter already existing persons // filter already existing persons
for (const personToSave of persons) { for (const personToSave of persons) {

2
src/backend/model/database/sql/SQLConnection.ts
View File

@ -206,7 +206,7 @@ export class SQLConnection {
username: config.mysql.username, username: config.mysql.username,
password: config.mysql.password, password: config.mysql.password,
database: config.mysql.database, database: config.mysql.database,
charset: 'utf8' charset: 'utf8mb4'
}; };
} else if (config.type === DatabaseType.sqlite) { } else if (config.type === DatabaseType.sqlite) {
driver = { driver = {

43
src/backend/model/database/sql/SearchManager.ts
View File

@ -33,6 +33,7 @@ import {ISQLGalleryManager} from './IGalleryManager';
import {ISQLSearchManager} from './ISearchManager'; import {ISQLSearchManager} from './ISearchManager';
import {Utils} from '../../../../common/Utils'; import {Utils} from '../../../../common/Utils';
import {FileEntity} from './enitites/FileEntity'; import {FileEntity} from './enitites/FileEntity';
import {SQL_COLLATE} from './enitites/EntityUtils';
export class SearchManager implements ISQLSearchManager { export class SearchManager implements ISQLSearchManager {
@ -72,7 +73,7 @@ export class SearchManager implements ISQLSearchManager {
(await photoRepository (await photoRepository
.createQueryBuilder('photo') .createQueryBuilder('photo')
.select('DISTINCT(photo.metadata.keywords)') .select('DISTINCT(photo.metadata.keywords)')
.where('photo.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) .where('photo.metadata.keywords LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany()) .getRawMany())
.map((r): Array<string> => (r.metadataKeywords as string).split(',') as Array<string>) .map((r): Array<string> => (r.metadataKeywords as string).split(',') as Array<string>)
@ -86,7 +87,7 @@ export class SearchManager implements ISQLSearchManager {
result = result.concat(this.encapsulateAutoComplete((await personRepository result = result.concat(this.encapsulateAutoComplete((await personRepository
.createQueryBuilder('person') .createQueryBuilder('person')
.select('DISTINCT(person.name)') .select('DISTINCT(person.name)')
.where('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) .where('person.name LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.orderBy('person.name') .orderBy('person.name')
.getRawMany()) .getRawMany())
@ -98,9 +99,9 @@ export class SearchManager implements ISQLSearchManager {
.createQueryBuilder('photo') .createQueryBuilder('photo')
.select('photo.metadata.positionData.country as country, ' + .select('photo.metadata.positionData.country as country, ' +
'photo.metadata.positionData.state as state, photo.metadata.positionData.city as city') 'photo.metadata.positionData.state as state, photo.metadata.positionData.city as city')
.where('photo.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) .where('photo.metadata.positionData.country LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) .orWhere('photo.metadata.positionData.state LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) .orWhere('photo.metadata.positionData.city LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.groupBy('photo.metadata.positionData.country, photo.metadata.positionData.state, photo.metadata.positionData.city') .groupBy('photo.metadata.positionData.country, photo.metadata.positionData.state, photo.metadata.positionData.city')
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany()) .getRawMany())
@ -117,7 +118,7 @@ export class SearchManager implements ISQLSearchManager {
result = result.concat(this.encapsulateAutoComplete((await mediaRepository result = result.concat(this.encapsulateAutoComplete((await mediaRepository
.createQueryBuilder('media') .createQueryBuilder('media')
.select('DISTINCT(media.name)') .select('DISTINCT(media.name)')
.where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) .where('media.name LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany()) .getRawMany())
.map(r => r.name), SearchQueryTypes.file_name)); .map(r => r.name), SearchQueryTypes.file_name));
@ -127,7 +128,7 @@ export class SearchManager implements ISQLSearchManager {
result = result.concat(this.encapsulateAutoComplete((await photoRepository result = result.concat(this.encapsulateAutoComplete((await photoRepository
.createQueryBuilder('media') .createQueryBuilder('media')
.select('DISTINCT(media.metadata.caption) as caption') .select('DISTINCT(media.metadata.caption) as caption')
.where('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) .where('media.metadata.caption LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany()) .getRawMany())
.map(r => r.caption), SearchQueryTypes.caption)); .map(r => r.caption), SearchQueryTypes.caption));
@ -137,7 +138,7 @@ export class SearchManager implements ISQLSearchManager {
result = result.concat(this.encapsulateAutoComplete((await directoryRepository result = result.concat(this.encapsulateAutoComplete((await directoryRepository
.createQueryBuilder('dir') .createQueryBuilder('dir')
.select('DISTINCT(dir.name)') .select('DISTINCT(dir.name)')
.where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) .where('dir.name LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany()) .getRawMany())
.map(r => r.name), SearchQueryTypes.directory)); .map(r => r.name), SearchQueryTypes.directory));
@ -488,17 +489,17 @@ export class SearchManager implements ISQLSearchManager {
textParam['fullPath' + queryId] = createMatchString(dirPathStr); textParam['fullPath' + queryId] = createMatchString(dirPathStr);
q[whereFN](`directory.path ${LIKE} :fullPath${queryId} COLLATE utf8_general_ci`, q[whereFN](`directory.path ${LIKE} :fullPath${queryId} COLLATE ` + SQL_COLLATE,
textParam); textParam);
const directoryPath = GalleryManager.parseRelativeDirePath(dirPathStr); const directoryPath = GalleryManager.parseRelativeDirePath(dirPathStr);
q[whereFN](new Brackets((dq): any => { q[whereFN](new Brackets((dq): any => {
textParam['dirName' + queryId] = createMatchString(directoryPath.name); textParam['dirName' + queryId] = createMatchString(directoryPath.name);
dq[whereFNRev](`directory.name ${LIKE} :dirName${queryId} COLLATE utf8_general_ci`, dq[whereFNRev](`directory.name ${LIKE} :dirName${queryId} COLLATE ${SQL_COLLATE}`,
textParam); textParam);
if (dirPathStr.includes('/')) { if (dirPathStr.includes('/')) {
textParam['parentName' + queryId] = createMatchString(directoryPath.parent); textParam['parentName' + queryId] = createMatchString(directoryPath.parent);
dq[whereFNRev](`directory.path ${LIKE} :parentName${queryId} COLLATE utf8_general_ci`, dq[whereFNRev](`directory.path ${LIKE} :parentName${queryId} COLLATE ${SQL_COLLATE}`,
textParam); textParam);
} }
return dq; return dq;
@ -506,21 +507,21 @@ export class SearchManager implements ISQLSearchManager {
} }
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.file_name) { if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.file_name) {
q[whereFN](`media.name ${LIKE} :text${queryId} COLLATE utf8_general_ci`, q[whereFN](`media.name ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam); textParam);
} }
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.caption) { if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.caption) {
q[whereFN](`media.metadata.caption ${LIKE} :text${queryId} COLLATE utf8_general_ci`, q[whereFN](`media.metadata.caption ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam); textParam);
} }
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.position) { if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.position) {
q[whereFN](`media.metadata.positionData.country ${LIKE} :text${queryId} COLLATE utf8_general_ci`, q[whereFN](`media.metadata.positionData.country ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam) textParam)
[whereFN](`media.metadata.positionData.state ${LIKE} :text${queryId} COLLATE utf8_general_ci`, [whereFN](`media.metadata.positionData.state ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam) textParam)
[whereFN](`media.metadata.positionData.city ${LIKE} :text${queryId} COLLATE utf8_general_ci`, [whereFN](`media.metadata.positionData.city ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam); textParam);
} }
@ -528,7 +529,7 @@ export class SearchManager implements ISQLSearchManager {
const matchArrayField = (fieldName: string): void => { const matchArrayField = (fieldName: string): void => {
q[whereFN](new Brackets((qbr): void => { q[whereFN](new Brackets((qbr): void => {
if ((query as TextSearch).matchType !== TextSearchQueryMatchTypes.exact_match) { if ((query as TextSearch).matchType !== TextSearchQueryMatchTypes.exact_match) {
qbr[whereFN](`${fieldName} ${LIKE} :text${queryId} COLLATE utf8_general_ci`, qbr[whereFN](`${fieldName} ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam); textParam);
} else { } else {
qbr[whereFN](new Brackets((qb): void => { qbr[whereFN](new Brackets((qb): void => {
@ -537,13 +538,13 @@ export class SearchManager implements ISQLSearchManager {
textParam['textC' + queryId] = `${(query as TextSearch).text},%`; textParam['textC' + queryId] = `${(query as TextSearch).text},%`;
textParam['text_exact' + queryId] = `${(query as TextSearch).text}`; textParam['text_exact' + queryId] = `${(query as TextSearch).text}`;
qb[whereFN](`${fieldName} ${LIKE} :CtextC${queryId} COLLATE utf8_general_ci`, qb[whereFN](`${fieldName} ${LIKE} :CtextC${queryId} COLLATE ${SQL_COLLATE}`,
textParam); textParam);
qb[whereFN](`${fieldName} ${LIKE} :Ctext${queryId} COLLATE utf8_general_ci`, qb[whereFN](`${fieldName} ${LIKE} :Ctext${queryId} COLLATE ${SQL_COLLATE}`,
textParam); textParam);
qb[whereFN](`${fieldName} ${LIKE} :textC${queryId} COLLATE utf8_general_ci`, qb[whereFN](`${fieldName} ${LIKE} :textC${queryId} COLLATE ${SQL_COLLATE}`,
textParam); textParam);
qb[whereFN](`${fieldName} ${LIKE} :text_exact${queryId} COLLATE utf8_general_ci`, qb[whereFN](`${fieldName} ${LIKE} :text_exact${queryId} COLLATE ${SQL_COLLATE}`,
textParam); textParam);
})); }));
} }

5
src/backend/model/database/sql/enitites/EntityUtils.ts
View File

@ -5,13 +5,14 @@ import {DatabaseType} from '../../../../../common/config/private/PrivateConfig';
export class ColumnCharsetCS implements ColumnOptions { export class ColumnCharsetCS implements ColumnOptions {
public get charset(): string { public get charset(): string {
return Config.Server.Database.type === DatabaseType.mysql ? 'utf8' : 'utf8'; return Config.Server.Database.type === DatabaseType.mysql ? 'utf8mb4' : 'utf8';
} }
public get collation(): string { public get collation(): string {
return Config.Server.Database.type === DatabaseType.mysql ? 'utf8_bin' : null; return Config.Server.Database.type === DatabaseType.mysql ? 'utf8mb4_bin' : null;
} }
} }
export const columnCharsetCS = new ColumnCharsetCS(); export const columnCharsetCS = new ColumnCharsetCS();
export const SQL_COLLATE = 'utf8mb4_general_ci';

2
src/frontend/app/ui/frame/frame.component.html
View File

@ -51,7 +51,7 @@
<li role="menuitem" *ngIf="isAdmin()"> <li role="menuitem" *ngIf="isAdmin()">
<a class="dropdown-item" [routerLink]="['/duplicates']"> <a class="dropdown-item" [routerLink]="['/duplicates']">
<span class="oi oi-layers"></span> <span class="oi oi-layers"></span>
<ng-container i18n>duplicates</ng-container> <ng-container i18n>Duplicates</ng-container>
</a> </a>
</li> </li>
<li role="menuitem" *ngIf="isAdmin()"> <li role="menuitem" *ngIf="isAdmin()">

21
test/backend/unit/model/sql/IndexingManager.spec.ts
View File

@ -221,6 +221,27 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
} }
}); });
it('should support emoji in names', async () => {
const gm = new GalleryManagerTest();
const im = new IndexingManagerTest();
const parent = TestHelper.getRandomizedDirectoryEntry(null, 'parent dir 😀');
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
p1.name = 'test.jpg';
DirectoryDTOUtils.packDirectory(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
expect(Utils.clone(Utils.removeNullOrEmptyObj(removeIds(selected))))
.to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
});
it('should select preview', async () => { it('should select preview', async () => {
const selectDirectory = async (gmTest: GalleryManagerTest, dir: DirectoryBaseDTO): Promise<ParentDirectoryDTO> => { const selectDirectory = async (gmTest: GalleryManagerTest, dir: DirectoryBaseDTO): Promise<ParentDirectoryDTO> => {