Creating thumbnail filling job #381

The job fills, directory, albums and persons' thumbnails

src/backend/model/database/interfaces/IPreviewManager.ts

@@ -3,9 +3,11 @@ import {IObjectManager} from './IObjectManager';
 import {SearchQueryDTO} from '../../../../common/entities/SearchQueryDTO';
 
 export interface IPreviewManager extends IObjectManager {
-  getPreviewForDirectory(dir: { id: number, name: string, path: string }): Promise<PreviewPhotoDTOWithID>;
+  setAndGetPreviewForDirectory(dir: { id: number, name: string, path: string }): Promise<PreviewPhotoDTOWithID>;
 
   getAlbumPreview(album: { searchQuery: SearchQueryDTO }): Promise<PreviewPhotoDTOWithID>;
+
+  getPartialDirsWithoutPreviews(): Promise<{ id: number; name: string; path: string }[]>;
 }
 
 // ID is need within the backend so it can be saved to DB (ID is the external key)

src/backend/model/database/memory/PreviewManager.ts

@@ -4,10 +4,13 @@ import {MediaDTO} from '../../../../common/entities/MediaDTO';
 import {SavedSearchDTO} from '../../../../common/entities/album/SavedSearchDTO';
 
 export class PreviewManager implements IPreviewManager {
+  getPartialDirsWithoutPreviews(): Promise<{ id: number; name: string; path: string }[]> {
+    throw new Error('not implemented');
+  }
   getAlbumPreview(album: SavedSearchDTO): Promise<MediaDTO> {
     throw new Error('not implemented');
   }
-  getPreviewForDirectory(dir: DirectoryPathDTO): Promise<MediaDTO> {
+  setAndGetPreviewForDirectory(dir: DirectoryPathDTO): Promise<MediaDTO> {
     throw new Error('not implemented');
   }
 }

src/backend/model/database/sql/GalleryManager.ts

@@ -216,19 +216,14 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
     return await query.getOne();
   }
 
+
   /**
    * Sets preview for the directory and caches it in the DB
    */
   public async fillPreviewForSubDir(connection: Connection, dir: SubDirectoryDTO): Promise<void> {
 
-    if (!dir.preview || !dir.validPreview) {
-
-      dir.preview = await ObjectManagers.getInstance().PreviewManager.getPreviewForDirectory(dir);
-      // write preview back to db
-      await connection.createQueryBuilder()
-        .update(DirectoryEntity).set({preview: dir.preview, validPreview: true}).where('id = :dir', {
-          dir: dir.id
-        }).execute();
+    if (!dir.validPreview) {
+      dir.preview = await ObjectManagers.getInstance().PreviewManager.setAndGetPreviewForDirectory(dir);
     }
 
 

src/backend/model/database/sql/PreviewManager.ts

@@ -8,8 +8,8 @@ import {SortingMethods} from '../../../../common/entities/SortingMethods';
 import {ISQLSearchManager} from './ISearchManager';
 import {IPreviewManager, PreviewPhotoDTOWithID} from '../interfaces/IPreviewManager';
 import {SQLConnection} from './SQLConnection';
-import {SavedSearchDTO} from '../../../../common/entities/album/SavedSearchDTO';
 import {SearchQueryDTO} from '../../../../common/entities/SearchQueryDTO';
+import {DirectoryEntity} from './enitites/DirectoryEntity';
 
 
 const LOG_TAG = '[PreviewManager]';
@@ -79,7 +79,17 @@ export class PreviewManager implements IPreviewManager {
     return previewMedia || null;
   }
 
-  public async getPreviewForDirectory(dir: { id: number, name: string, path: string }): Promise<PreviewPhotoDTOWithID> {
+
+  public async getPartialDirsWithoutPreviews(): Promise<{ id: number, name: string, path: string }[]> {
+    const connection = await SQLConnection.getConnection();
+    return await connection
+      .getRepository(DirectoryEntity)
+      .createQueryBuilder('directory')
+      .where('directory.validPreview = :validPreview', {validPreview: 0}) // 0 === false
+      .select(['name', 'id', 'path']).getRawMany();
+  }
+
+  public async setAndGetPreviewForDirectory(dir: { id: number, name: string, path: string }): Promise<PreviewPhotoDTOWithID> {
     const connection = await SQLConnection.getConnection();
     const previewQuery = (): SelectQueryBuilder<MediaEntity> => {
       const query = connection
@@ -112,7 +122,8 @@ export class PreviewManager implements IPreviewManager {
       return query;
     };
 
-    let previewMedia = null;
+
+    let previewMedia: PreviewPhotoDTOWithID = null;
     if (Config.Server.Preview.SearchQuery) {
       previewMedia = await previewQuery()
         .andWhere(await (ObjectManagers.getInstance().SearchManager as ISQLSearchManager)
@@ -126,6 +137,13 @@ export class PreviewManager implements IPreviewManager {
         .limit(1)
         .getOne();
     }
+
+    // set validPreview bit to true even if there is no preview (to prevent future updates)
+    await connection.createQueryBuilder()
+      .update(DirectoryEntity).set({preview: previewMedia, validPreview: true}).where('id = :dir', {
+        dir: dir.id
+      }).execute();
+
     return previewMedia || null;
   }
 

src/backend/model/database/sql/SearchManager.ts

@@ -268,7 +268,6 @@ export class SearchManager implements ISQLSearchManager {
   /**
    * Builds the SQL Where query from search query
    * @param query input search query
-   * @param paramCounter Helper counter for generating parameterized query
    * @param directoryOnly Only builds directory related queries
    * @private
    */

src/backend/model/database/sql/enitites/DirectoryEntity.ts

@@ -60,7 +60,7 @@ export class DirectoryEntity implements ParentDirectoryDTO<MediaDTO>, SubDirecto
   public preview: MediaEntity;
 
   // On galley change, preview will be invalid
-  @Column({default: false})
+  @Column({type: 'boolean', default: false})
   validPreview: boolean;
 
   @OneToMany(type => MediaEntity, media => media.directory)

src/backend/model/jobs/JobRepository.ts

@@ -5,6 +5,7 @@ import {VideoConvertingJob} from './jobs/VideoConvertingJob';
 import {PhotoConvertingJob} from './jobs/PhotoConvertingJob';
 import {ThumbnailGenerationJob} from './jobs/ThumbnailGenerationJob';
 import {TempFolderCleaningJob} from './jobs/TempFolderCleaningJob';
+import {PreviewFillingJob} from './jobs/PreviewFillingJob';
 
 export class JobRepository {
 
@@ -32,6 +33,7 @@ export class JobRepository {
 
 
 JobRepository.Instance.register(new IndexingJob());
+JobRepository.Instance.register(new PreviewFillingJob());
 JobRepository.Instance.register(new DBRestJob());
 JobRepository.Instance.register(new VideoConvertingJob());
 JobRepository.Instance.register(new PhotoConvertingJob());

src/backend/model/jobs/jobs/FileJob.ts

@@ -19,7 +19,9 @@ declare var global: NodeJS.Global;
 
 const LOG_TAG = '[FileJob]';
 
-
+/**
+ * Abstract class for thumbnail creation, file deleting etc.
+ */
 export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly: boolean }> extends Job<S> {
   public readonly ConfigTemplate: ConfigTemplateEntry[] = [];
   directoryQueue: string[] = [];

src/backend/model/jobs/jobs/PreviewFillingJob.ts

@@ -0,0 +1,78 @@
+import {ObjectManagers} from '../../ObjectManagers';
+import {ConfigTemplateEntry, DefaultsJobs} from '../../../../common/entities/job/JobDTO';
+import {Job} from './Job';
+import {Config} from '../../../../common/config/private/Config';
+import {DatabaseType} from '../../../../common/config/private/PrivateConfig';
+
+
+export class PreviewFillingJob extends Job {
+  public readonly Name = DefaultsJobs[DefaultsJobs['Preview Filling']];
+  public readonly ConfigTemplate: ConfigTemplateEntry[] = null;
+  directoryToSetPreview: { id: number, name: string, path: string }[] = null;
+  status: 'Persons' | 'Albums' | 'Directory' = 'Persons';
+
+  public get Supported(): boolean {
+    return Config.Server.Database.type !== DatabaseType.memory;
+  }
+
+  protected async init(): Promise<void> {
+  }
+
+  protected async step(): Promise<boolean> {
+    if (!this.directoryToSetPreview) {
+      this.directoryToSetPreview = await ObjectManagers.getInstance().PreviewManager.getPartialDirsWithoutPreviews();
+      this.Progress.Left = this.directoryToSetPreview.length + 2;
+      return true;
+    }
+
+    switch (this.status) {
+      case 'Persons':
+        await this.stepPersonsPreview();
+        this.status = 'Albums';
+        return true;
+      case 'Albums':
+        await this.stepAlbumPreview();
+        this.status = 'Directory';
+        return true;
+      case 'Directory':
+        return await this.stepDirectoryPreview();
+    }
+    return false;
+  }
+
+  private async stepAlbumPreview(): Promise<boolean> {
+    await ObjectManagers.getInstance().AlbumManager.getAlbums();
+    this.Progress.log('Updating Albums preview');
+    this.Progress.Processed++;
+    return false;
+  }
+
+  private async stepPersonsPreview(): Promise<boolean> {
+    await ObjectManagers.getInstance().PersonManager.getAll();
+    this.Progress.log('Updating Persons preview');
+    this.Progress.Processed++;
+    return false;
+  }
+
+  private async stepDirectoryPreview(): Promise<boolean> {
+    if (this.directoryToSetPreview.length === 0) {
+      this.directoryToSetPreview = await ObjectManagers.getInstance().PreviewManager.getPartialDirsWithoutPreviews();
+      // double check if there is really no more
+      if (this.directoryToSetPreview.length > 0) {
+        return true; // continue
+      }
+      this.Progress.Left = 0;
+      return false;
+    }
+    const directory = this.directoryToSetPreview.shift();
+    this.Progress.log('Setting preview: ' + directory.path + '/' + directory.name);
+    this.Progress.Left = this.directoryToSetPreview.length;
+
+    await ObjectManagers.getInstance().PreviewManager.setAndGetPreviewForDirectory(directory);
+    this.Progress.Processed++;
+    return true;
+
+  }
+
+
+}

src/common/config/private/PrivateConfig.ts

@@ -255,10 +255,15 @@ export class ServerJobConfig {
       false,
       new NeverJobTrigger(), {indexChangesOnly: true}
     ),
+    new JobScheduleConfig(DefaultsJobs[DefaultsJobs['Preview Filling']],
+      DefaultsJobs[DefaultsJobs['Preview Filling']],
+      false,
+      new NeverJobTrigger(), {}
+    ),
     new JobScheduleConfig(DefaultsJobs[DefaultsJobs['Thumbnail Generation']],
       DefaultsJobs[DefaultsJobs['Thumbnail Generation']],
       false,
-      new AfterJobTrigger(DefaultsJobs[DefaultsJobs.Indexing]), {sizes: [240], indexedOnly: true}
+      new AfterJobTrigger(DefaultsJobs[DefaultsJobs['Preview Filling']]), {sizes: [240], indexedOnly: true}
     ),
     new JobScheduleConfig(DefaultsJobs[DefaultsJobs['Photo Converting']],
       DefaultsJobs[DefaultsJobs['Photo Converting']],

src/common/entities/job/JobDTO.ts

@@ -9,7 +9,8 @@ export enum DefaultsJobs {
   'Video Converting' = 3,
   'Photo Converting' = 4,
   'Thumbnail Generation' = 5,
-  'Temp Folder Cleaning' = 6
+  'Temp Folder Cleaning' = 6,
+  'Preview Filling' = 7
 }
 
 export interface ConfigTemplateEntry {

src/frontend/app/model/backendtext.service.ts

@@ -35,6 +35,8 @@ export class BackendtextService {
     switch (job as DefaultsJobs) {
       case DefaultsJobs.Indexing:
         return $localize`Indexing`;
+      case DefaultsJobs['Preview Filling']:
+        return $localize`Preview Filling`;
       case DefaultsJobs['Database Reset']:
         return $localize`Database Reset`;
       case DefaultsJobs['Thumbnail Generation']:

test/backend/unit/model/sql/PreviewManager.spec.ts

@@ -14,6 +14,8 @@ import {PreviewManager} from '../../../../../src/backend/model/database/sql/Prev
 import {Config} from '../../../../../src/common/config/private/Config';
 import {SortingMethods} from '../../../../../src/common/entities/SortingMethods';
 import {Utils} from '../../../../../src/common/Utils';
+import {SQLConnection} from '../../../../../src/backend/model/database/sql/SQLConnection';
+import {DirectoryEntity} from '../../../../../src/backend/model/database/sql/enitites/DirectoryEntity';
 
 const deepEqualInAnyOrder = require('deep-equal-in-any-order');
 const chai = require('chai');
@@ -156,25 +158,40 @@ describe('PreviewManager', (sqlHelper: DBTestHelper) => {
     Config.Server.Preview.Sorting = [SortingMethods.descRating, SortingMethods.descDate];
   });
 
+
+  it('should list directories without preview', async () => {
+    const pm = new PreviewManager();
+    const partialDir = (d: DirectoryBaseDTO) => {
+      return {id: d.id, name: d.name, path: d.path};
+    };
+    expect(await pm.getPartialDirsWithoutPreviews()).to.deep.equalInAnyOrder([partialDir(dir)]);
+    const conn = await SQLConnection.getConnection();
+
+    await conn.createQueryBuilder()
+      .update(DirectoryEntity).set({validPreview: false}).execute();
+
+    expect(await pm.getPartialDirsWithoutPreviews()).to.deep.equalInAnyOrder([dir, subDir, subDir2].map(d => partialDir(d)));
+  });
+
   it('should sort directory preview', async () => {
     const pm = new PreviewManager();
     Config.Server.Preview.Sorting = [SortingMethods.descRating, SortingMethods.descDate];
-    expect(Utils.clone(await pm.getPreviewForDirectory(subDir))).to.deep.equalInAnyOrder(previewifyMedia(p2));
+    expect(Utils.clone(await pm.setAndGetPreviewForDirectory(subDir))).to.deep.equalInAnyOrder(previewifyMedia(p2));
     Config.Server.Preview.Sorting = [SortingMethods.descDate];
-    expect(Utils.clone(await pm.getPreviewForDirectory(subDir))).to.deep.equalInAnyOrder(previewifyMedia(pFaceLess));
+    expect(Utils.clone(await pm.setAndGetPreviewForDirectory(subDir))).to.deep.equalInAnyOrder(previewifyMedia(pFaceLess));
     Config.Server.Preview.Sorting = [SortingMethods.descRating];
-    expect(Utils.clone(await pm.getPreviewForDirectory(dir))).to.deep.equalInAnyOrder(previewifyMedia(p4));
+    expect(Utils.clone(await pm.setAndGetPreviewForDirectory(dir))).to.deep.equalInAnyOrder(previewifyMedia(p4));
   });
 
   it('should get preview for directory', async () => {
     const pm = new PreviewManager();
 
     Config.Server.Preview.SearchQuery = {type: SearchQueryTypes.any_text, text: 'Boba'} as TextSearch;
-    expect(Utils.clone(await pm.getPreviewForDirectory(subDir))).to.deep.equalInAnyOrder(previewifyMedia(p));
+    expect(Utils.clone(await pm.setAndGetPreviewForDirectory(subDir))).to.deep.equalInAnyOrder(previewifyMedia(p));
     Config.Server.Preview.SearchQuery = {type: SearchQueryTypes.any_text, text: 'Derem'} as TextSearch;
-    expect(Utils.clone(await pm.getPreviewForDirectory(subDir))).to.deep.equalInAnyOrder(previewifyMedia(p2));
-    expect(Utils.clone(await pm.getPreviewForDirectory(dir))).to.deep.equalInAnyOrder(previewifyMedia(p2));
-    expect(Utils.clone(await pm.getPreviewForDirectory(subDir2))).to.deep.equalInAnyOrder(previewifyMedia(p4));
+    expect(Utils.clone(await pm.setAndGetPreviewForDirectory(subDir))).to.deep.equalInAnyOrder(previewifyMedia(p2));
+    expect(Utils.clone(await pm.setAndGetPreviewForDirectory(dir))).to.deep.equalInAnyOrder(previewifyMedia(p2));
+    expect(Utils.clone(await pm.setAndGetPreviewForDirectory(subDir2))).to.deep.equalInAnyOrder(previewifyMedia(p4));
 
   });