self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2025-07-15 01:24:25 +02:00
Code Issues Releases Activity

Forcing mysql to use utf8mb4 fixes #399

Browse Source
This commit is contained in:
Patrik J. Braun
2022-01-14 12:14:28 +01:00
parent 07d8261034
commit e607ae810a
6 changed files with 51 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/backend/model/database/sql/PersonManager.ts
View File

@ -5,6 +5,7 @@ import {PersonDTO} from '../../../../common/entities/PersonDTO';
import {ISQLPersonManager} from './IPersonManager';
import {Logger} from '../../../Logger';
import {FaceRegion} from '../../../../common/entities/PhotoDTO';
import {SQL_COLLATE} from './enitites/EntityUtils';
const LOG_TAG = '[PersonManager]';
@ -47,7 +48,7 @@ export class PersonManager implements ISQLPersonManager {
const repository = connection.getRepository(PersonEntry);
const person = await repository.createQueryBuilder('person')
.limit(1)
.where('person.name LIKE :name COLLATE utf8_general_ci', {name}).getOne();
.where('person.name LIKE :name COLLATE ' + SQL_COLLATE, {name}).getOne();
if (typeof partialPerson.name !== 'undefined') {
@ -93,7 +94,7 @@ export class PersonManager implements ISQLPersonManager {
const personRepository = connection.getRepository(PersonEntry);
const faceRegionRepository = connection.getRepository(FaceRegionEntry);
const savedPersons = await personRepository.find();
const savedPersons = await personRepository.find();
// filter already existing persons
for (const personToSave of persons) {

2
src/backend/model/database/sql/SQLConnection.ts
View File

@ -206,7 +206,7 @@ export class SQLConnection {
username: config.mysql.username,
password: config.mysql.password,
database: config.mysql.database,
charset: 'utf8'
charset: 'utf8mb4'
};
} else if (config.type === DatabaseType.sqlite) {
driver = {

43
src/backend/model/database/sql/SearchManager.ts
View File

@ -33,6 +33,7 @@ import {ISQLGalleryManager} from './IGalleryManager';
import {ISQLSearchManager} from './ISearchManager';
import {Utils} from '../../../../common/Utils';
import {FileEntity} from './enitites/FileEntity';
import {SQL_COLLATE} from './enitites/EntityUtils';
export class SearchManager implements ISQLSearchManager {
@ -72,7 +73,7 @@ export class SearchManager implements ISQLSearchManager {
(await photoRepository
.createQueryBuilder('photo')
.select('DISTINCT(photo.metadata.keywords)')
.where('photo.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('photo.metadata.keywords LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
.map((r): Array<string> => (r.metadataKeywords as string).split(',') as Array<string>)
@ -86,7 +87,7 @@ export class SearchManager implements ISQLSearchManager {
result = result.concat(this.encapsulateAutoComplete((await personRepository
.createQueryBuilder('person')
.select('DISTINCT(person.name)')
.where('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('person.name LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.orderBy('person.name')
.getRawMany())
@ -98,9 +99,9 @@ export class SearchManager implements ISQLSearchManager {
.createQueryBuilder('photo')
.select('photo.metadata.positionData.country as country, ' +
'photo.metadata.positionData.state as state, photo.metadata.positionData.city as city')
.where('photo.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('photo.metadata.positionData.country LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.orWhere('photo.metadata.positionData.state LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.orWhere('photo.metadata.positionData.city LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.groupBy('photo.metadata.positionData.country, photo.metadata.positionData.state, photo.metadata.positionData.city')
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
@ -117,7 +118,7 @@ export class SearchManager implements ISQLSearchManager {
result = result.concat(this.encapsulateAutoComplete((await mediaRepository
.createQueryBuilder('media')
.select('DISTINCT(media.name)')
.where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('media.name LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
.map(r => r.name), SearchQueryTypes.file_name));
@ -127,7 +128,7 @@ export class SearchManager implements ISQLSearchManager {
result = result.concat(this.encapsulateAutoComplete((await photoRepository
.createQueryBuilder('media')
.select('DISTINCT(media.metadata.caption) as caption')
.where('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('media.metadata.caption LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
.map(r => r.caption), SearchQueryTypes.caption));
@ -137,7 +138,7 @@ export class SearchManager implements ISQLSearchManager {
result = result.concat(this.encapsulateAutoComplete((await directoryRepository
.createQueryBuilder('dir')
.select('DISTINCT(dir.name)')
.where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('dir.name LIKE :text COLLATE ' + SQL_COLLATE, {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
.map(r => r.name), SearchQueryTypes.directory));
@ -488,17 +489,17 @@ export class SearchManager implements ISQLSearchManager {
textParam['fullPath' + queryId] = createMatchString(dirPathStr);
q[whereFN](`directory.path ${LIKE} :fullPath${queryId} COLLATE utf8_general_ci`,
q[whereFN](`directory.path ${LIKE} :fullPath${queryId} COLLATE ` + SQL_COLLATE,
textParam);
const directoryPath = GalleryManager.parseRelativeDirePath(dirPathStr);
q[whereFN](new Brackets((dq): any => {
textParam['dirName' + queryId] = createMatchString(directoryPath.name);
dq[whereFNRev](`directory.name ${LIKE} :dirName${queryId} COLLATE utf8_general_ci`,
dq[whereFNRev](`directory.name ${LIKE} :dirName${queryId} COLLATE ${SQL_COLLATE}`,
textParam);
if (dirPathStr.includes('/')) {
textParam['parentName' + queryId] = createMatchString(directoryPath.parent);
dq[whereFNRev](`directory.path ${LIKE} :parentName${queryId} COLLATE utf8_general_ci`,
dq[whereFNRev](`directory.path ${LIKE} :parentName${queryId} COLLATE ${SQL_COLLATE}`,
textParam);
}
return dq;
@ -506,21 +507,21 @@ export class SearchManager implements ISQLSearchManager {
}
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.file_name) {
q[whereFN](`media.name ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
q[whereFN](`media.name ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam);
}
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.caption) {
q[whereFN](`media.metadata.caption ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
q[whereFN](`media.metadata.caption ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam);
}
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.position) {
q[whereFN](`media.metadata.positionData.country ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
q[whereFN](`media.metadata.positionData.country ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam)
[whereFN](`media.metadata.positionData.state ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
[whereFN](`media.metadata.positionData.state ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam)
[whereFN](`media.metadata.positionData.city ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
[whereFN](`media.metadata.positionData.city ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam);
}
@ -528,7 +529,7 @@ export class SearchManager implements ISQLSearchManager {
const matchArrayField = (fieldName: string): void => {
q[whereFN](new Brackets((qbr): void => {
if ((query as TextSearch).matchType !== TextSearchQueryMatchTypes.exact_match) {
qbr[whereFN](`${fieldName} ${LIKE} :text${queryId} COLLATE utf8_general_ci`,
qbr[whereFN](`${fieldName} ${LIKE} :text${queryId} COLLATE ${SQL_COLLATE}`,
textParam);
} else {
qbr[whereFN](new Brackets((qb): void => {
@ -537,13 +538,13 @@ export class SearchManager implements ISQLSearchManager {
textParam['textC' + queryId] = `${(query as TextSearch).text},%`;
textParam['text_exact' + queryId] = `${(query as TextSearch).text}`;
qb[whereFN](`${fieldName} ${LIKE} :CtextC${queryId} COLLATE utf8_general_ci`,
qb[whereFN](`${fieldName} ${LIKE} :CtextC${queryId} COLLATE ${SQL_COLLATE}`,
textParam);
qb[whereFN](`${fieldName} ${LIKE} :Ctext${queryId} COLLATE utf8_general_ci`,
qb[whereFN](`${fieldName} ${LIKE} :Ctext${queryId} COLLATE ${SQL_COLLATE}`,
textParam);
qb[whereFN](`${fieldName} ${LIKE} :textC${queryId} COLLATE utf8_general_ci`,
qb[whereFN](`${fieldName} ${LIKE} :textC${queryId} COLLATE ${SQL_COLLATE}`,
textParam);
qb[whereFN](`${fieldName} ${LIKE} :text_exact${queryId} COLLATE utf8_general_ci`,
qb[whereFN](`${fieldName} ${LIKE} :text_exact${queryId} COLLATE ${SQL_COLLATE}`,
textParam);
}));
}

5
src/backend/model/database/sql/enitites/EntityUtils.ts
View File

@ -5,13 +5,14 @@ import {DatabaseType} from '../../../../../common/config/private/PrivateConfig';
export class ColumnCharsetCS implements ColumnOptions {
public get charset(): string {
return Config.Server.Database.type === DatabaseType.mysql ? 'utf8' : 'utf8';
return Config.Server.Database.type === DatabaseType.mysql ? 'utf8mb4' : 'utf8';
}
public get collation(): string {
return Config.Server.Database.type === DatabaseType.mysql ? 'utf8_bin' : null;
return Config.Server.Database.type === DatabaseType.mysql ? 'utf8mb4_bin' : null;
}
}
export const columnCharsetCS = new ColumnCharsetCS();
export const SQL_COLLATE = 'utf8mb4_general_ci';

2
src/frontend/app/ui/frame/frame.component.html
View File

@ -51,7 +51,7 @@
<li role="menuitem" *ngIf="isAdmin()">
<a class="dropdown-item" [routerLink]="['/duplicates']">
<span class="oi oi-layers"></span>
<ng-container i18n>duplicates</ng-container>
<ng-container i18n>Duplicates</ng-container>
</a>
</li>
<li role="menuitem" *ngIf="isAdmin()">

21
test/backend/unit/model/sql/IndexingManager.spec.ts
View File

@ -221,6 +221,27 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
}
});
it('should support emoji in names', async () => {
const gm = new GalleryManagerTest();
const im = new IndexingManagerTest();
const parent = TestHelper.getRandomizedDirectoryEntry(null, 'parent dir 😀');
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
p1.name = 'test.jpg';
DirectoryDTOUtils.packDirectory(parent);
await im.saveToDB(Utils.clone(parent) as ParentDirectoryDTO);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTOUtils.packDirectory(selected);
expect(Utils.clone(Utils.removeNullOrEmptyObj(removeIds(selected))))
.to.deep.equalInAnyOrder(Utils.removeNullOrEmptyObj(indexifyReturn(parent)));
});
it('should select preview', async () => {
const selectDirectory = async (gmTest: GalleryManagerTest, dir: DirectoryBaseDTO): Promise<ParentDirectoryDTO> => {