filter raw indexed content when the projection is set #1015

src/backend/model/database/GalleryManager.ts

@@ -50,6 +50,8 @@ export class GalleryManager {
       // Return as soon as possible without touching the original data source (hdd)
       // See https://github.com/bpatrik/pigallery2/issues/613
       if (
+        knownLastModified &&
+        knownLastScanned &&
         Config.Indexing.reIndexingSensitivity ===
         ReIndexingSensitivity.never
       ) {
@@ -92,17 +94,26 @@ export class GalleryManager {
           ', current:' +
           lastModified
         );
-        const ret =
+        if (session?.projectionQuery) {
+          // Need to wait for save, then return a DB-based result with projection
           await ObjectManagers.getInstance().IndexingManager.indexDirectory(
-            relativeDirectoryName
+            relativeDirectoryName,
+            true
           );
-        for (const subDir of ret.directories) {
-          if (!subDir.cover) {
-            // if subdirectories do not have photos, so cannot show a cover, try getting one from DB
-            await this.fillCoverForSubDir(connection, subDir);
+          return await this.getParentDirFromId(connection, session, dir.id);
+        } else {
+          const ret =
+            await ObjectManagers.getInstance().IndexingManager.indexDirectory(
+              relativeDirectoryName
+            );
+          for (const subDir of ret.directories) {
+            if (!subDir.cover) {
+              // if subdirectories do not have photos, so cannot show a cover, try getting one from DB
+              await this.fillCoverForSubDir(connection, subDir);
+            }
           }
+          return ret;
         }
-        return ret;
       }
 
       // not indexed since a while, index it lazily
@@ -132,6 +143,16 @@ export class GalleryManager {
 
     // never scanned (deep indexed), do it and return with it
     Logger.silly(LOG_TAG, 'Reindexing reason: never scanned');
+    if (session?.projectionQuery) {
+      // Save must be completed to query with projection
+      await ObjectManagers.getInstance().IndexingManager.indexDirectory(
+        relativeDirectoryName,
+        true
+      );
+      const connection = await SQLConnection.getConnection();
+      const dir = await this.getDirIdAndTime(connection, directoryPath.name, directoryPath.parent);
+      return await this.getParentDirFromId(connection, session, dir.id);
+    }
     return ObjectManagers.getInstance().IndexingManager.indexDirectory(
       relativeDirectoryName
     );

src/backend/model/database/IndexingManager.ts

@@ -29,7 +29,7 @@ const LOG_TAG = '[IndexingManager]';
 export class IndexingManager {
   SavingReady: Promise<void> = null;
   private SavingReadyPR: () => void = null;
-  private savingQueue: ParentDirectoryDTO[] = [];
+  private savingQueue: { dir: ParentDirectoryDTO; promise: Promise<void>; resolve: () => void; reject: (e: any) => void }[] = [];
   private isSaving = false;
 
   get IsSavingInProgress(): boolean {
@@ -72,13 +72,14 @@ export class IndexingManager {
    * does not wait for the DB to be saved
    */
   public indexDirectory(
-    relativeDirectoryName: string
+    relativeDirectoryName: string,
+    waitForSave = false
   ): Promise<ParentDirectoryDTO> {
     // eslint-disable-next-line no-async-promise-executor
     return new Promise(async (resolve, reject): Promise<void> => {
       try {
         // Check if root is still a valid (non-empty) folder
-        // With weak devices it is possible that the media that stores
+        // With weak devices, it is possible that the media that stores
         // the galley gets unmounted that triggers a full gallery wipe.
         // Prevent it by stopping indexing on an empty folder.
         if (fs.readdirSync(ProjectPath.ImageFolder).length === 0) {
@@ -97,9 +98,20 @@ export class IndexingManager {
         );
 
         DirectoryDTOUtils.addReferences(dirClone);
-        resolve(dirClone);
 
-        // save directory to DB
+        if (waitForSave === true) {
+          // save directory to DB and wait until saving finishes
+          try {
+            await this.queueForSave(scannedDirectory);
+          } catch (e) {
+            // bubble up save error
+            return reject(e);
+          }
+          return resolve(dirClone);
+        }
+
+        // save directory to DB in the background
+        resolve(dirClone);
         this.queueForSave(scannedDirectory).catch(console.error);
       } catch (error) {
         NotificationManager.warning(
@@ -149,27 +161,44 @@ export class IndexingManager {
   // Todo fix it, once typeorm support connection pools for sqlite
   /**
    * Queues up a directory to save to the DB.
+   * Returns a promise that resolves when the directory is saved.
    */
   protected async queueForSave(
     scannedDirectory: ParentDirectoryDTO
   ): Promise<void> {
-    // Is this dir  already queued for saving?
-    if (
-      this.savingQueue.findIndex(
-        (dir): boolean =>
-          dir.name === scannedDirectory.name &&
-          dir.path === scannedDirectory.path &&
-          dir.lastModified === scannedDirectory.lastModified &&
-          dir.lastScanned === scannedDirectory.lastScanned &&
-          (dir.media || dir.media.length) ===
-          (scannedDirectory.media || scannedDirectory.media.length) &&
-          (dir.metaFile || dir.metaFile.length) ===
-          (scannedDirectory.metaFile || scannedDirectory.metaFile.length)
-      ) !== -1
-    ) {
-      return;
+    // Is this dir already queued for saving?
+    const existingIndex = this.savingQueue.findIndex(
+      (entry): boolean =>
+        entry.dir.name === scannedDirectory.name &&
+        entry.dir.path === scannedDirectory.path &&
+        entry.dir.lastModified === scannedDirectory.lastModified &&
+        entry.dir.lastScanned === scannedDirectory.lastScanned &&
+        (entry.dir.media || entry.dir.media.length) ===
+        (scannedDirectory.media || scannedDirectory.media.length) &&
+        (entry.dir.metaFile || entry.dir.metaFile.length) ===
+        (scannedDirectory.metaFile || scannedDirectory.metaFile.length)
+    );
+    if (existingIndex !== -1) {
+      return this.savingQueue[existingIndex].promise;
     }
-    this.savingQueue.push(scannedDirectory);
+
+    // queue for saving
+    let resolveFn: () => void;
+    let rejectFn: (e: any) => void;
+    const promise = new Promise<void>((resolve, reject): void => {
+      resolveFn = resolve;
+      rejectFn = reject;
+    });
+
+    this.savingQueue.push({dir: scannedDirectory, promise, resolve: resolveFn, reject: rejectFn});
+    this.runSavingLoop().catch(console.error);
+
+    return promise;
+  }
+
+  protected async runSavingLoop(): Promise<void> {
+
+    // start saving if not already started
     if (!this.SavingReady) {
       this.SavingReady = new Promise<void>((resolve): void => {
         this.SavingReadyPR = resolve;
@@ -177,20 +206,34 @@ export class IndexingManager {
     }
     try {
       while (this.isSaving === false && this.savingQueue.length > 0) {
-        await this.saveToDB(this.savingQueue[0]);
+        const item = this.savingQueue[0];
+        try {
+          await this.saveToDB(item.dir);
+          item.resolve();
+        } catch (e) {
+          // reject current and remaining queued items to avoid hanging promises
+          item.reject(e);
+          this.savingQueue.shift();
+          for (const remaining of this.savingQueue) {
+            remaining.reject(e);
+          }
+          this.savingQueue = [];
+          throw e;
+        }
         this.savingQueue.shift();
       }
-    } catch (e) {
-      this.savingQueue = [];
-      throw e;
     } finally {
-      if (this.savingQueue.length === 0) {
+      if (this.savingQueue.length === 0 && this.SavingReady) {
+        const pr = this.SavingReadyPR;
         this.SavingReady = null;
-        this.SavingReadyPR();
+        if (pr) {
+          pr();
+        }
       }
     }
   }
 
+
   protected async saveParentDir(
     connection: Connection,
     scannedDirectory: ParentDirectoryDTO

test/backend/unit/model/sql/GalleryManager.spec.ts

@@ -1,3 +1,4 @@
+import * as fs from 'fs';
 import {DBTestHelper} from '../../../DBTestHelper';
 import {GalleryManager} from '../../../../../src/backend/model/database/GalleryManager';
 import {ParentDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
@@ -6,6 +7,9 @@ import {SessionContext} from '../../../../../src/backend/model/SessionContext';
 import {SearchQueryTypes, TextSearchQueryMatchTypes} from '../../../../../src/common/entities/SearchQueryDTO';
 import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers';
 import {SQLConnection} from '../../../../../src/backend/model/database/SQLConnection';
+import {Config} from '../../../../../src/common/config/private/Config';
+import {ReIndexingSensitivity} from '../../../../../src/common/config/private/PrivateConfig';
+import {IndexingManager} from '../../../../../src/backend/model/database/IndexingManager';
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const deepEqualInAnyOrder = require('deep-equal-in-any-order');
@@ -132,4 +136,202 @@ describe('GalleryManager', (sqlHelper: DBTestHelper) => {
     });
 
   });
+  describe('GalleryManager.listDirectory - reindexing severities and projection behavior', () => {
+    const origStatSync = fs.statSync;
+
+    let gm: GalleryManagerTest;
+    let sessionNoProj: SessionContext;
+    let sessionProj: SessionContext;
+
+    const indexed: any = {id: 1, lastScanned: 0, lastModified: 0};
+    let calledArgs: any[] = [];
+    let bgCalls = 0;
+
+    beforeEach(() => {
+      // Reset config defaults that matter for tests
+      Config.loadSync();
+
+      gm = new GalleryManagerTest();
+      sessionNoProj = new SessionContext();
+      sessionProj = new SessionContext();
+      // Make projectionQuery truthy without relying on SearchManager
+      (sessionProj as any).projectionQuery = {} as any;
+
+      // Stub fs.statSync to control directory mtime/ctime -> lastModified
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      fs.statSync = ((): any => ({ctime: new Date(0), mtime: new Date(0)})) as any;
+
+      // Stub getDirIdAndTime and getParentDirFromId to avoid DB
+      (gm as any).getDirIdAndTime = () => Promise.resolve(indexed);
+      (gm as any).getParentDirFromId = () => Promise.resolve('DB_RESULT' as any);
+
+      // Stub IndexingManager.indexDirectory to capture calls
+      calledArgs = [];
+      bgCalls = 0;
+      ObjectManagers.getInstance().IndexingManager = new IndexingManager();
+      ObjectManagers.getInstance().IndexingManager.indexDirectory = ((...args: any[]) => {
+        calledArgs = args;
+        bgCalls++;
+        const retObj = {directories: [], media: [], metaFile: [], name: 'INDEX_RESULT'} as any;
+        return Promise.resolve(retObj);
+      }) as any;
+    });
+
+    afterEach(() => {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      fs.statSync = origStatSync;
+    });
+
+    const setStatTime = (t: number) => {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      fs.statSync = ((): any => ({ctime: new Date(t), mtime: new Date(t)})) as any;
+    };
+
+    it('never: returns DB result when already scanned (no projection) and known times are missing', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.never;
+      indexed.lastScanned = 123;
+      indexed.lastModified = 1;
+      setStatTime(1);
+
+      const res = await gm.listDirectory(sessionNoProj, './');
+      expect(res).to.equal('DB_RESULT');
+    });
+
+    it('never: returns DB result when already scanned (with projection) and known times are missing', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.never;
+      indexed.lastScanned = 123;
+      indexed.lastModified = 1;
+      setStatTime(1);
+
+      const res = await gm.listDirectory(sessionProj, './');
+      expect(res).to.equal('DB_RESULT');
+    });
+
+    it('low + mismatch: returns scanned result when no projection', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.low;
+      indexed.lastScanned = 10;
+      indexed.lastModified = 0; // DB says 0
+      setStatTime(1); // FS says 1 -> mismatch
+
+      const res = await gm.listDirectory(sessionNoProj, './');
+      expect(res).to.be.an('object');
+      expect((res as any).name).to.equal('INDEX_RESULT');
+      expect(calledArgs[0]).to.equal('./');
+      expect(calledArgs[1]).to.be.undefined; // no waitForSave
+    });
+
+    it('low + mismatch: waits for save and returns DB result when projection set', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.low;
+      indexed.lastScanned = 10;
+      indexed.lastModified = 0;
+      setStatTime(1);
+
+      const res = await gm.listDirectory(sessionProj, './');
+      expect(res).to.equal('DB_RESULT');
+      expect(calledArgs[0]).to.equal('./');
+      expect(calledArgs[1]).to.equal(true); // waitForSave
+    });
+
+    it('low + unchanged with known times: returns null (no projection)', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.low;
+      indexed.lastScanned = 10;
+      indexed.lastModified = 1;
+      setStatTime(1);
+
+      const res = await gm.listDirectory(sessionNoProj, './', 1, 10);
+      expect(res).to.equal(null);
+    });
+
+    it('low + unchanged with known times: returns null (with projection)', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.low;
+      indexed.lastScanned = 10;
+      indexed.lastModified = 1;
+      setStatTime(1);
+
+      const res = await gm.listDirectory(sessionProj, './', 1, 10);
+      expect(res).to.equal(null);
+    });
+
+    it('medium + unchanged within cache (known times): returns null', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.medium;
+      // Set lastScanned close to now so within cachedFolderTimeout
+      indexed.lastScanned = Date.now();
+      indexed.lastModified = 1;
+      setStatTime(1);
+
+      const res = await gm.listDirectory(sessionNoProj, './', 1, indexed.lastScanned);
+      expect(res).to.equal(null);
+    });
+
+    it('medium + cache expired (no known times): background reindex and DB result (no projection)', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.medium;
+      indexed.lastScanned = Date.now() - (Config.Indexing.cachedFolderTimeout + 1000);
+      indexed.lastModified = 1;
+      setStatTime(1);
+
+      const res = await gm.listDirectory(sessionNoProj, './');
+      expect(res).to.equal('DB_RESULT');
+      expect(bgCalls).to.be.greaterThan(0);
+    });
+
+    it('medium + cache expired (no known times): background reindex and DB result (with projection)', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.medium;
+      indexed.lastScanned = Date.now() - (Config.Indexing.cachedFolderTimeout + 1000);
+      indexed.lastModified = 1;
+      setStatTime(1);
+
+      const res = await gm.listDirectory(sessionProj, './');
+      expect(res).to.equal('DB_RESULT');
+      expect(bgCalls).to.be.greaterThan(0);
+    });
+
+    it('high + unchanged: background reindex and DB result (no projection)', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.high;
+      indexed.lastScanned = Date.now();
+      indexed.lastModified = 1;
+      setStatTime(1);
+
+      const res = await gm.listDirectory(sessionNoProj, './');
+      expect(res).to.equal('DB_RESULT');
+      expect(bgCalls).to.be.greaterThan(0);
+    });
+
+    it('high + unchanged: background reindex and DB result (with projection)', async () => {
+      Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.high;
+      indexed.lastScanned = Date.now();
+      indexed.lastModified = 1;
+      setStatTime(1);
+
+      const res = await gm.listDirectory(sessionProj, './');
+      expect(res).to.equal('DB_RESULT');
+      expect(bgCalls).to.be.greaterThan(0);
+    });
+
+    it('never scanned (lastScanned=null): without projection returns scanned result', async () => {
+      // Simulate never scanned dir
+      indexed.lastScanned = null;
+      indexed.lastModified = 0;
+      setStatTime(0);
+
+      const res = await gm.listDirectory(sessionNoProj, './');
+      expect(res).to.be.an('object');
+      expect((res as any).name).to.equal('INDEX_RESULT');
+    });
+
+    it('never scanned (lastScanned=null): with projection waits for save and returns DB result', async () => {
+      // Simulate never scanned dir
+      indexed.lastScanned = null;
+      indexed.lastModified = 0;
+      setStatTime(0);
+
+      const res = await gm.listDirectory(sessionProj, './');
+      expect(res).to.equal('DB_RESULT');
+      expect(calledArgs[1]).to.equal(true);
+    });
+  });
+
 });
+

test/backend/unit/model/sql/IndexingManager.spec.ts

@@ -734,6 +734,101 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     });
   });
 
+
+  DBTestHelper.savedDescribe('indexDirectory(waitForSave)', () => {
+    beforeEach(async () => {
+      await sqlHelper.initDB();
+    });
+
+    afterEach(async () => {
+      Config.loadSync();
+      await sqlHelper.clearDB();
+    });
+
+    it('resolves after the specific directory is saved and does not wait for the whole queue (even with duplicates)', async () => {
+      // Ensure ImageFolder is non-empty for the empty-root guard
+      Config.Album.enabled = true;
+      Config.Faces.enabled = true;
+      ProjectPath.reset();
+      Config.Media.folder = path.join(__dirname, '/../../../assets');
+      ProjectPath.ImageFolder = path.join(__dirname, '/../../../assets');
+
+      const completedSaves: string[] = [];
+
+      class IMWithDelay extends IndexingManager {
+        public async saveToDB(scannedDirectory: ParentDirectoryDTO): Promise<void> {
+          // mimic original concurrency guard
+          (this as any).isSaving = true;
+          try {
+            if (scannedDirectory.name === 'B') {
+              await new Promise((res) => setTimeout(res, 30));
+              completedSaves.push('B');
+              return;
+            }
+            if (scannedDirectory.name === 'C') {
+              await new Promise((res) => setTimeout(res, 80));
+              completedSaves.push('C');
+              return;
+            }
+            await new Promise((res) => setTimeout(res, 10));
+            completedSaves.push(scannedDirectory.name);
+          } finally {
+            (this as any).isSaving = false;
+          }
+        }
+      }
+
+      // Stub DiskManager.scanDirectory to return consistent DTOs
+      const originalScan = DiskManager.scanDirectory;
+      const makeDir = (name: string): ParentDirectoryDTO => ({
+        name,
+        path: path.join(path.sep),
+        mediaCount: 0,
+        lastModified: 1,
+        lastScanned: 1,
+        youngestMedia: 0,
+        oldestMedia: 0,
+        directories: [],
+        media: [],
+        metaFile: []
+      } as unknown as ParentDirectoryDTO);
+
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      DiskManager.scanDirectory = async (rel: string): Promise<ParentDirectoryDTO> => makeDir(rel);
+
+      const im = new IMWithDelay();
+
+      try {
+        // Queue two saves for the same directory ('B'), then another different directory ('C')
+        const pB1 = im.indexDirectory('B', true);
+        const pB2 = im.indexDirectory('B', true);
+        const pC = im.indexDirectory('C', true); // queued after Bs and slower
+
+        // Wait for first 'B' save only; should not wait for 'C'
+        await pB1;
+        expect(completedSaves[0]).to.equal('B');
+        expect(completedSaves.filter(n => n === 'B').length).to.be.equal(1);
+        expect(completedSaves.includes('C')).to.equal(false);
+
+        // Wait for second 'B' save; regardless of dedupe, it should finish before 'C'
+        await pB2;
+        expect(completedSaves.filter(n => n === 'B').length).to.be.equal(2);
+        expect(completedSaves.includes('C')).to.equal(false);
+
+        // Now ensure 'C' completes afterwards
+        await pC;
+        expect(completedSaves[completedSaves.length - 1]).to.equal('C');
+      } finally {
+        // restore stubs
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore
+        DiskManager.scanDirectory = originalScan;
+        ProjectPath.reset();
+      }
+    });
+  });
+
   DBTestHelper.savedDescribe('should index .pg2conf', () => {