improving database usage

backend/model/sql/GalleryManager.ts

@@ -275,7 +275,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
             delete directory.media;
             await directoryRepository.save(directory);
           }
-        } else {
+        } else { //dir does not exists yet
           scannedDirectory.directories[i].parent = currentDir;
           (<DirectoryEntity>scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet
           const d = await directoryRepository.save(<DirectoryEntity>scannedDirectory.directories[i]);
@@ -288,7 +288,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
       }
 
       // Remove child Dirs that are not anymore in the parent dir
-      await directoryRepository.remove(childDirectories);
+      await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)});
 
       // save media
       const indexedMedia = await mediaRepository.createQueryBuilder('media')
@@ -321,6 +321,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
       await this.saveMedia(connection, mediaToSave);
       await mediaRepository.remove(indexedMedia);
 
+
       // save files
       const indexedMetaFiles = await fileRepository.createQueryBuilder('file')
         .where('file.directory = :dir', {
@@ -346,8 +347,8 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
           metaFilesToSave.push(metaFile);
         }
       }
-      await fileRepository.save(metaFilesToSave);
+      await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)});
-      await fileRepository.remove(indexedMetaFiles);
+      await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)});
     } catch (e) {
       throw e;
     } finally {

backend/model/sql/enitites/PhotoEntity.ts

@@ -12,7 +12,7 @@ export class CameraMetadataEntity implements CameraMetadata {
   model: string;
 
   @Column('text', {nullable: true})
-  maker: string;
+  make: string;
 
   @Column('int', {nullable: true})
   fStop: number;
@@ -75,4 +75,5 @@ export class PhotoMetadataEntity extends MediaMetadataEntity implements PhotoMet
 export class PhotoEntity extends MediaEntity implements PhotoDTO {
   @Column(type => PhotoMetadataEntity)
   metadata: PhotoMetadataEntity;
+
 }

backend/model/threading/DiskMangerWorker.ts

@@ -209,7 +209,7 @@ export class DiskMangerWorker {
 
             try {
               const exif = ExifParserFactory.create(data).parse();
-              metadata.cameraData = <CameraMetadata>{
+              metadata.cameraData = {
                 ISO: exif.tags.ISO,
                 model: exif.tags.Model,
                 make: exif.tags.Make,
@@ -220,7 +220,7 @@ export class DiskMangerWorker {
               };
               if (!isNaN(exif.tags.GPSLatitude) || exif.tags.GPSLongitude || exif.tags.GPSAltitude) {
                 metadata.positionData = metadata.positionData || {};
-                metadata.positionData.GPSData = <GPSMetadata>{
+                metadata.positionData.GPSData = {
                   latitude: exif.tags.GPSLatitude,
                   longitude: exif.tags.GPSLongitude,
                   altitude: exif.tags.GPSAltitude
@@ -236,15 +236,15 @@ export class DiskMangerWorker {
               }
 
               if (exif.imageSize) {
-                metadata.size = <MediaDimension>{width: exif.imageSize.width, height: exif.imageSize.height};
+                metadata.size = {width: exif.imageSize.width, height: exif.imageSize.height};
               } else if (exif.tags.RelatedImageWidth && exif.tags.RelatedImageHeight) {
-                metadata.size = <MediaDimension>{width: exif.tags.RelatedImageWidth, height: exif.tags.RelatedImageHeight};
+                metadata.size = {width: exif.tags.RelatedImageWidth, height: exif.tags.RelatedImageHeight};
               } else {
-                metadata.size = <MediaDimension>{width: 1, height: 1};
+                metadata.size = {width: 1, height: 1};
               }
             } catch (err) {
               Logger.debug(LOG_TAG, 'Error parsing exif', fullPath, err);
-              metadata.size = <MediaDimension>{width: 1, height: 1};
+              metadata.size = {width: 1, height: 1};
             }
 
             try {

common/entities/DirectoryDTO.ts

@@ -36,9 +36,11 @@ export module DirectoryDTO {
   };
 
   export const removeReferences = (dir: DirectoryDTO): void => {
+    if (dir.media) {
       dir.media.forEach((media: MediaDTO) => {
         media.directory = null;
       });
+    }
     if (dir.metaFile) {
       dir.metaFile.forEach((file: FileDTO) => {
         file.directory = null;

test/backend/integration/model/sql/typeorm.ts

@@ -83,7 +83,7 @@ describe('Typeorm integration', () => {
     const cd = new CameraMetadataEntity();
     cd.ISO = 100;
     cd.model = '60D';
-    cd.maker = 'Canon';
+    cd.make = 'Canon';
     cd.fStop = 1;
     cd.exposure = 1;
     cd.focalLength = 1;

test/backend/unit/model/sql/GalleryManager.ts

@@ -12,9 +12,6 @@ import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/Directo
 import {Utils} from '../../../../../common/Utils';
 import {MediaDTO} from '../../../../../common/entities/MediaDTO';
 import {FileDTO} from '../../../../../common/entities/FileDTO';
-import {PhotoEntity} from '../../../../../backend/model/sql/enitites/PhotoEntity';
-import {FileEntity} from '../../../../../backend/model/sql/enitites/FileEntity';
-
 
 class GalleryManagerTest extends GalleryManager {
 
@@ -202,9 +199,6 @@ describe('GalleryManager', () => {
     const selected = await gm.selectParentDir(conn, parent.name, parent.path);
     await gm.fillParentDir(conn, selected);
 
-    const query = conn.getRepository(FileEntity).createQueryBuilder('photo');
-    query.innerJoinAndSelect('photo.directory', 'directory');
-    console.log((await query.getMany()));
     DirectoryDTO.removeReferences(selected);
     removeIds(selected);
     subDir.isPartial = true;
@@ -214,4 +208,25 @@ describe('GalleryManager', () => {
       .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
   });
 
+
+  (<any>it('should save 1500 photos', async () => {
+    const conn = await SQLConnection.getConnection();
+    const gm = new GalleryManagerTest();
+    Config.Client.MetaFile.enabled = true;
+    const parent = TestHelper.getRandomizedDirectoryEntry();
+    DirectoryDTO.removeReferences(parent);
+    await gm.saveToDB(Utils.clone(parent));
+    const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
+    for (let i = 0; i < 1500; i++) {
+      TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i);
+    }
+
+    DirectoryDTO.removeReferences(parent);
+    await gm.saveToDB(subDir);
+
+
+    const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
+    expect(selected.media.length).to.deep.equal(subDir.media.length);
+  })).timeout(20000);
+
 });

test/backend/unit/model/sql/TestHelper.ts

@@ -10,6 +10,10 @@ import {OrientationTypes} from 'ts-exif-parser';
 import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity';
 import {VideoEntity, VideoMetadataEntity} from '../../../../../backend/model/sql/enitites/VideoEntity';
 import {FileEntity} from '../../../../../backend/model/sql/enitites/FileEntity';
+import {MediaDimension} from '../../../../../common/entities/MediaDTO';
+import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
+import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO';
+import {FileDTO} from '../../../../../common/entities/FileDTO';
 
 export class TestHelper {
 
@@ -40,7 +44,7 @@ export class TestHelper {
     const cd = new CameraMetadataEntity();
     cd.ISO = 100;
     cd.model = '60D';
-    cd.maker = 'Canon';
+    cd.make = 'Canon';
     cd.fStop = 1;
     cd.exposure = 1;
     cd.focalLength = 1;
@@ -119,35 +123,38 @@ export class TestHelper {
   }
 
 
-  public static getRandomizedDirectoryEntry(parent: DirectoryEntity = null, forceStr: string = null) {
+  public static getRandomizedDirectoryEntry(parent: DirectoryDTO = null, forceStr: string = null) {
 
-    const dir = new DirectoryEntity();
+    const dir: DirectoryDTO = {
-    dir.name = forceStr || Math.random().toString(36).substring(7);
+      id: null,
-    dir.path = '.';
+      name: forceStr || Math.random().toString(36).substring(7),
+      path: '.',
+      directories: [],
+      metaFile: [],
+      media: [],
+      lastModified: Date.now(),
+      lastScanned: null,
+      parent: null
+    };
     if (parent !== null) {
       dir.path = path.join(parent.path, parent.name);
       parent.directories.push(dir);
     }
-    dir.directories = [];
-    dir.metaFile = [];
-    dir.media = [];
-    dir.lastModified = Date.now();
-    dir.lastScanned = null;
-
     return dir;
   }
 
 
-  public static getRandomizedGPXEntry(dir: DirectoryEntity, forceStr: string = null): FileEntity {
+  public static getRandomizedGPXEntry(dir: DirectoryDTO, forceStr: string = null): FileDTO {
-    const d = new FileEntity();
+    const d: FileDTO = {
-    d.name = forceStr + '_' + Math.random().toString(36).substring(7) + '.gpx';
+      id: null,
-    d.directory = dir;
+      name: forceStr + '_' + Math.random().toString(36).substring(7) + '.gpx',
-
+      directory: dir
+    };
     dir.metaFile.push(d);
     return d;
   }
 
-  public static getRandomizedPhotoEntry(dir: DirectoryEntity, forceStr: string = null) {
+  public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null) {
 
 
     const rndStr = () => {
@@ -159,44 +166,51 @@ export class TestHelper {
       return Math.floor(Math.random() * max);
     };
 
-    const sd = new MediaDimensionEntity();
+    const sd: MediaDimension = {
-    sd.height = rndInt();
+      height: rndInt(),
-    sd.width = rndInt();
+      width: rndInt(),
-    const gps = new GPSMetadataEntity();
+    };
-    gps.altitude = rndInt(1000);
-    gps.latitude = rndInt(1000);
-    gps.longitude = rndInt(1000);
-    const pd = new PositionMetaDataEntity();
-    pd.city = rndStr();
-    pd.country = rndStr();
-    pd.state = rndStr();
-    pd.GPSData = gps;
-    const cd = new CameraMetadataEntity();
-    cd.ISO = rndInt(500);
-    cd.model = rndStr();
-    cd.maker = rndStr();
-    cd.fStop = rndInt(10);
-    cd.exposure = rndInt(10);
-    cd.focalLength = rndInt(10);
-    cd.lens = rndStr();
-    const m = new PhotoMetadataEntity();
-    m.keywords = [rndStr(), rndStr()];
-    m.cameraData = cd;
-    m.positionData = pd;
-    m.size = sd;
-    m.creationDate = Date.now();
-    m.fileSize = rndInt(10000);
-    m.orientation = OrientationTypes.TOP_LEFT;
 
-    // TODO: remove when typeorm is fixed
+    const gps: GPSMetadata = {
-    m.duration = null;
+      altitude: rndInt(1000),
-    m.bitRate = null;
+      latitude: rndInt(1000),
+      longitude: rndInt(1000)
+    };
+    const pd: PositionMetaData = {
+      city: rndStr(),
+      country: rndStr(),
+      state: rndStr(),
+      GPSData: gps
+    };
+    const cd: CameraMetadata = {
+      ISO: rndInt(500),
+      model: rndStr(),
+      make: rndStr(),
+      fStop: rndInt(10),
+      exposure: rndInt(10),
+      focalLength: rndInt(10),
+      lens: rndStr()
+    };
+    const m: PhotoMetadata = {
+      keywords: [rndStr(), rndStr()],
+      cameraData: cd,
+      positionData: pd,
+      size: sd,
+      creationDate: Date.now(),
+      fileSize: rndInt(10000),
+      orientation: OrientationTypes.TOP_LEFT,
+      caption: rndStr()
+    };
 
 
-    const d = new PhotoEntity();
+    const d: PhotoDTO = {
-    d.name = rndStr() + '.jpg';
+      id: null,
-    d.directory = dir;
+      name: rndStr() + '.jpg',
-    d.metadata = m;
+      directory: dir,
+      metadata: m,
+      readyThumbnails: null,
+      readyIcon: false
+    };
 
     dir.media.push(d);
     return d;