changing mysql collate and charset to be case insensitive

backend/model/sql/GalleryManager.ts

@@ -100,7 +100,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
         });
 
         if (queryFilter.recursive) {
-          qb.orWhere('directory.name LIKE :text COLLATE utf8_general_ci', {text: '%' + queryFilter.directory + '%'});
+          qb.orWhere('directory.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + queryFilter.directory + '%'});
         }
       }));
     }

backend/model/sql/SQLConnection.ts

@@ -170,7 +170,8 @@ export class SQLConnection {
         port: 3306,
         username: config.mysql.username,
         password: config.mysql.password,
-        database: config.mysql.database
+        database: config.mysql.database,
+        charset: 'utf8mb4'
       };
     } else if (config.type === DatabaseType.sqlite) {
       driver = {

backend/model/sql/SearchManager.ts

@@ -40,7 +40,7 @@ export class SearchManager implements ISearchManager {
     (await photoRepository
       .createQueryBuilder('photo')
       .select('DISTINCT(photo.metadata.keywords)')
-      .where('photo.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .where('photo.metadata.keywords LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
       .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .map(r => <Array<string>>(<string>r.metadataKeywords).split(','))
@@ -52,7 +52,7 @@ export class SearchManager implements ISearchManager {
     result = result.concat(this.encapsulateAutoComplete((await personRepository
       .createQueryBuilder('person')
       .select('DISTINCT(person.name)')
-      .where('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .where('person.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
       .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .orderBy('person.name')
       .getRawMany())
@@ -62,9 +62,9 @@ export class SearchManager implements ISearchManager {
       .createQueryBuilder('photo')
       .select('photo.metadata.positionData.country as country, ' +
         'photo.metadata.positionData.state as state, photo.metadata.positionData.city as city')
-      .where('photo.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .where('photo.metadata.positionData.country LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
-      .orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
-      .orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
       .groupBy('photo.metadata.positionData.country, photo.metadata.positionData.state, photo.metadata.positionData.city')
       .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
@@ -78,7 +78,7 @@ export class SearchManager implements ISearchManager {
     result = result.concat(this.encapsulateAutoComplete((await photoRepository
       .createQueryBuilder('media')
       .select('DISTINCT(media.name)')
-      .where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .where('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
       .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .map(r => r.name), SearchTypes.photo));
@@ -87,7 +87,7 @@ export class SearchManager implements ISearchManager {
     result = result.concat(this.encapsulateAutoComplete((await photoRepository
       .createQueryBuilder('media')
       .select('DISTINCT(media.metadata.caption) as caption')
-      .where('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .where('media.metadata.caption LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
       .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .map(r => r.caption), SearchTypes.photo));
@@ -96,7 +96,7 @@ export class SearchManager implements ISearchManager {
     result = result.concat(this.encapsulateAutoComplete((await videoRepository
       .createQueryBuilder('media')
       .select('DISTINCT(media.name)')
-      .where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .where('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
       .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .map(r => r.name), SearchTypes.video));
@@ -104,7 +104,7 @@ export class SearchManager implements ISearchManager {
     result = result.concat(this.encapsulateAutoComplete((await directoryRepository
       .createQueryBuilder('dir')
       .select('DISTINCT(dir.name)')
-      .where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .where('dir.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
       .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
       .getRawMany())
       .map(r => r.name), SearchTypes.directory));
@@ -142,31 +142,31 @@ export class SearchManager implements ISearchManager {
 
           if (!searchType || searchType === SearchTypes.directory) {
             subQuery.leftJoin('media.directory', 'directory')
-              .orWhere('directory.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+              .orWhere('directory.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
           }
 
           if (!searchType || searchType === SearchTypes.photo || searchType === SearchTypes.video) {
-            subQuery.orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+            subQuery.orWhere('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
           }
 
           if (!searchType || searchType === SearchTypes.photo) {
-            subQuery.orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+            subQuery.orWhere('media.metadata.caption LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
           }
           if (!searchType || searchType === SearchTypes.person) {
             subQuery
               .leftJoin('media.metadata.faces', 'faces')
               .leftJoin('faces.person', 'person')
-              .orWhere('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+              .orWhere('person.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
           }
 
           if (!searchType || searchType === SearchTypes.position) {
-            subQuery.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+            subQuery.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
-              .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+              .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
-              .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+              .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
 
           }
           if (!searchType || searchType === SearchTypes.keyword) {
-            subQuery.orWhere('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
+            subQuery.orWhere('media.metadata.keywords LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
           }
 
           return subQuery;
@@ -187,7 +187,7 @@ export class SearchManager implements ISearchManager {
     result.directories = await connection
       .getRepository(DirectoryEntity)
       .createQueryBuilder('dir')
-      .where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .where('dir.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
       .limit(201)
       .getMany();
 
@@ -217,13 +217,13 @@ export class SearchManager implements ISearchManager {
           .leftJoin('media.directory', 'directory')
           .leftJoin('media.metadata.faces', 'faces')
           .leftJoin('faces.person', 'person')
-          .where('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+          .where('media.metadata.keywords LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
-          .orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+          .orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
-          .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+          .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
-          .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+          .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
-          .orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+          .orWhere('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
-          .orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+          .orWhere('media.metadata.caption LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
-          .orWhere('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+          .orWhere('person.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
         ,
         'innerMedia',
         'media.id=innerMedia.id')
@@ -238,7 +238,7 @@ export class SearchManager implements ISearchManager {
     result.directories = await connection
       .getRepository(DirectoryEntity)
       .createQueryBuilder('dir')
-      .where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
+      .where('dir.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
       .limit(10)
       .getMany();
 

backend/model/sql/enitites/DirectoryEntity.ts

@@ -2,6 +2,7 @@ import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, Uni
 import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO';
 import {MediaEntity} from './MediaEntity';
 import {FileEntity} from './FileEntity';
+import {columnCharsetCS} from './EntityUtils';
 
 @Entity()
 @Unique(['name', 'path'])
@@ -12,7 +13,7 @@ export class DirectoryEntity implements DirectoryDTO {
   id: number;
 
   @Index()
-  @Column()
+  @Column(columnCharsetCS)
   name: string;
 
   @Index()

backend/model/sql/enitites/EntityUtils.ts

@@ -0,0 +1,17 @@
+import {Config} from '../../../../common/config/private/Config';
+import {DatabaseType} from '../../../../common/config/private/IPrivateConfig';
+import {ColumnOptions} from 'typeorm/decorator/options/ColumnOptions';
+
+export class ColumnCharsetCS implements ColumnOptions {
+
+  public get charset(): string {
+    return Config.Server.database.type === DatabaseType.mysql ? 'utf8mb4' : null;
+  }
+
+  public get collation(): string {
+    return Config.Server.database.type === DatabaseType.mysql ? 'utf8mb4_bin' : null;
+
+  }
+}
+
+export const columnCharsetCS = new ColumnCharsetCS();

backend/model/sql/enitites/FileEntity.ts

@@ -1,6 +1,7 @@
 import {Column, Entity, ManyToOne, PrimaryGeneratedColumn, Index} from 'typeorm';
 import {DirectoryEntity} from './DirectoryEntity';
 import {FileDTO} from '../../../../common/entities/FileDTO';
+import {columnCharsetCS} from './EntityUtils';
 
 
 @Entity()
@@ -10,7 +11,7 @@ export class FileEntity implements FileDTO {
   @PrimaryGeneratedColumn({unsigned: true})
   id: number;
 
-  @Column('text')
+  @Column(columnCharsetCS)
   name: string;
 
   @Index()

backend/model/sql/enitites/MediaEntity.ts

@@ -4,6 +4,9 @@ import {MediaDimension, MediaDTO, MediaMetadata} from '../../../../common/entiti
 import {OrientationTypes} from 'ts-exif-parser';
 import {CameraMetadataEntity, PositionMetaDataEntity} from './PhotoEntity';
 import {FaceRegionEntry} from './FaceRegionEntry';
+import {Config} from '../../../../common/config/private/Config';
+import {DatabaseType} from '../../../../common/config/private/IPrivateConfig';
+import {columnCharsetCS} from './EntityUtils';
 
 export class MediaDimensionEntity implements MediaDimension {
 
@@ -55,6 +58,8 @@ export class MediaMetadataEntity implements MediaMetadata {
   duration: number;
 }
 
+
+
 // TODO: fix inheritance once its working in typeorm
 @Entity()
 @Unique(['name', 'directory'])
@@ -65,7 +70,7 @@ export abstract class MediaEntity implements MediaDTO {
   @PrimaryGeneratedColumn({unsigned: true})
   id: number;
 
-  @Column()
+  @Column(columnCharsetCS)
   name: string;
 
   @Index()

common/entities/TaskDTO.ts

@@ -0,0 +1,28 @@
+export interface TaskType {
+  name: string;
+  parameter: any;
+}
+
+export enum TaskTriggerType {
+  scheduled, periodic
+}
+
+export interface TaskTrigger {
+  type: TaskTriggerType;
+}
+
+export interface ScheduledTaskTrigger extends TaskTrigger {
+  type: TaskTriggerType.scheduled;
+  time: number;
+}
+
+export interface PeriodicTaskTrigger extends TaskTrigger {
+  type: TaskTriggerType.periodic;
+
+}
+
+export interface TaskDTO {
+  priority: number;
+  type: TaskType;
+  trigger: TaskTrigger;
+}

test/backend/integration/model/sql/typeorm.ts

@@ -184,7 +184,7 @@ describe('Typeorm integration', () => {
     const photos = await pr
       .createQueryBuilder('media')
       .orderBy('media.metadata.creationDate', 'ASC')
-      .where('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + photo.metadata.positionData.city + '%'})
+      .where('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + photo.metadata.positionData.city + '%'})
       .innerJoinAndSelect('media.directory', 'directory')
       .limit(10)
       .getMany();
@@ -206,7 +206,7 @@ describe('Typeorm integration', () => {
     const photos = await pr
       .createQueryBuilder('media')
       .orderBy('media.metadata.creationDate', 'ASC')
-      .where('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + city + '%'})
+      .where('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + city + '%'})
       .innerJoinAndSelect('media.directory', 'directory')
       .limit(10)
       .getMany();

test/backend/unit/model/sql/IndexingManager.ts

@@ -44,6 +44,7 @@ class IndexingManagerTest extends IndexingManager {
 // to help WebStorm to handle the test cases
 declare let describe: any;
 declare const after: any;
+declare const it: any;
 describe = SQLTestHelper.describe;
 
 describe('IndexingManager', (sqlHelper: SQLTestHelper) => {
@@ -80,6 +81,29 @@ describe('IndexingManager', (sqlHelper: SQLTestHelper) => {
     }
   };
 
+  it('should support case sensitive file names', async () => {
+    const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
+
+    const parent = TestHelper.getRandomizedDirectoryEntry();
+    const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
+    const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
+    p1.name = 'test.jpg';
+    p2.name = 'Test.jpg';
+
+    DirectoryDTO.removeReferences(parent);
+    await im.saveToDB(Utils.clone(parent));
+
+    const conn = await SQLConnection.getConnection();
+    const selected = await gm.selectParentDir(conn, parent.name, parent.path);
+    await gm.fillParentDir(conn, selected);
+
+    DirectoryDTO.removeReferences(selected);
+    removeIds(selected);
+    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
+      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
+  });
+
   it('should save parent directory', async () => {
     const gm = new GalleryManagerTest();
     const im = new IndexingManagerTest();