add basic extension support #753

benchmark/BenchmarkRunner.ts

@@ -121,7 +121,7 @@ export class BenchmarkRunner {
     const bm = new Benchmark('List directory', req,
       async (): Promise<void> => {
         await ObjectManagers.reset();
-        await ObjectManagers.InitSQLManagers();
+        await ObjectManagers.InitManagers();
       }, null,
       async (): Promise<void> => {
         Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.low;
@@ -135,7 +135,7 @@ export class BenchmarkRunner {
   async bmListPersons(): Promise<BenchmarkResult[]> {
     const bm = new Benchmark('Listing Faces', Utils.clone(this.requestTemplate), async (): Promise<void> => {
         await ObjectManagers.reset();
-        await ObjectManagers.InitSQLManagers();
+        await ObjectManagers.InitManagers();
       }, null,
       async (): Promise<void> => {
         Config.Indexing.reIndexingSensitivity = ReIndexingSensitivity.low;
@@ -289,7 +289,7 @@ export class BenchmarkRunner {
     await fs.promises.rm(ProjectPath.DBFolder, {recursive: true, force: true});
     Config.Database.type = DatabaseType.sqlite;
     Config.Jobs.scheduled = [];
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
   };
 
   private async setupDB(): Promise<void> {

src/backend/ProjectPath.ts

@@ -2,13 +2,14 @@ import * as path from 'path';
 import * as fs from 'fs';
 import {Config} from '../common/config/private/Config';
 
-class ProjectPathClass {
+export class ProjectPathClass {
   public Root: string;
   public ImageFolder: string;
   public TempFolder: string;
   public TranscodedFolder: string;
   public FacesFolder: string;
   public FrontendFolder: string;
+  public ExtensionFolder: string;
   public DBFolder: string;
 
   constructor() {
@@ -35,6 +36,7 @@ class ProjectPathClass {
     this.TranscodedFolder = path.join(this.TempFolder, 'tc');
     this.FacesFolder = path.join(this.TempFolder, 'f');
     this.DBFolder = this.getAbsolutePath(Config.Database.dbFolder);
+    this.ExtensionFolder = path.join(this.Root, 'extension');
 
     // create thumbnail folder if not exist
     if (!fs.existsSync(this.TempFolder)) {

src/backend/index.ts

@@ -11,5 +11,5 @@ if ((process.argv || []).includes('--run-diagnostics')) {
     process.exit(0);
   });
 } else {
-  new Server();
+  Server.getInstance();
 }

src/backend/model/ObjectManagers.ts

@@ -14,6 +14,7 @@ import {AlbumManager} from './database/AlbumManager';
 import {PersonManager} from './database/PersonManager';
 import {SharingManager} from './database/SharingManager';
 import {IObjectManager} from './database/IObjectManager';
+import {ExtensionManager} from './extension/ExtensionManager';
 
 const LOG_TAG = '[ObjectManagers]';
 
@@ -32,6 +33,7 @@ export class ObjectManagers {
   private jobManager: JobManager;
   private locationManager: LocationManager;
   private albumManager: AlbumManager;
+  private extensionManager: ExtensionManager;
 
   constructor() {
     this.managers = [];
@@ -169,6 +171,18 @@ export class ObjectManagers {
     this.managers.push(this.jobManager as IObjectManager);
   }
 
+  get ExtensionManager(): ExtensionManager {
+    return this.extensionManager;
+  }
+
+  set ExtensionManager(value: ExtensionManager) {
+    if (this.extensionManager) {
+      this.managers.splice(this.managers.indexOf(this.extensionManager as IObjectManager), 1);
+    }
+    this.extensionManager = value;
+    this.managers.push(this.extensionManager as IObjectManager);
+  }
+
   public static getInstance(): ObjectManagers {
     if (this.instance === null) {
       this.instance = new ObjectManagers();
@@ -179,27 +193,33 @@ export class ObjectManagers {
   public static async reset(): Promise<void> {
     Logger.silly(LOG_TAG, 'Object manager reset begin');
     if (
-        ObjectManagers.getInstance().IndexingManager &&
+      ObjectManagers.getInstance().IndexingManager &&
-        ObjectManagers.getInstance().IndexingManager.IsSavingInProgress
+      ObjectManagers.getInstance().IndexingManager.IsSavingInProgress
     ) {
       await ObjectManagers.getInstance().IndexingManager.SavingReady;
     }
-    if (ObjectManagers.getInstance().JobManager) {
+    for (const manager of ObjectManagers.getInstance().managers) {
-      ObjectManagers.getInstance().JobManager.stopSchedules();
+      if (manager === ObjectManagers.getInstance().versionManager) {
+        continue;
+      }
+      if (manager.cleanUp) {
+        await manager.cleanUp();
+      }
     }
+
     await SQLConnection.close();
     this.instance = null;
-    Logger.debug(LOG_TAG, 'Object manager reset');
+    Logger.debug(LOG_TAG, 'Object manager reset done');
   }
 
-  public static async InitSQLManagers(): Promise<void> {
+  public static async InitManagers(): Promise<void> {
     await ObjectManagers.reset();
     await SQLConnection.init();
-    this.initManagers();
+    await this.initManagers();
     Logger.debug(LOG_TAG, 'SQL DB inited');
   }
 
-  private static initManagers(): void {
+  private static async initManagers(): Promise<void> {
     ObjectManagers.getInstance().AlbumManager = new AlbumManager();
     ObjectManagers.getInstance().GalleryManager = new GalleryManager();
     ObjectManagers.getInstance().IndexingManager = new IndexingManager();
@@ -211,10 +231,20 @@ export class ObjectManagers {
     ObjectManagers.getInstance().VersionManager = new VersionManager();
     ObjectManagers.getInstance().JobManager = new JobManager();
     ObjectManagers.getInstance().LocationManager = new LocationManager();
+    ObjectManagers.getInstance().ExtensionManager = new ExtensionManager();
+
+    for (const manager of ObjectManagers.getInstance().managers) {
+      if (manager === ObjectManagers.getInstance().versionManager) {
+        continue;
+      }
+      if (manager.init) {
+        await manager.init();
+      }
+    }
   }
 
   public async onDataChange(
-      changedDir: ParentDirectoryDTO = null
+    changedDir: ParentDirectoryDTO = null
   ): Promise<void> {
     await this.VersionManager.onNewDataVersion();
 

src/backend/model/database/IObjectManager.ts

@@ -2,4 +2,6 @@ import {ParentDirectoryDTO} from '../../../common/entities/DirectoryDTO';
 
 export interface IObjectManager {
   onNewDataVersion?: (changedDir?: ParentDirectoryDTO) => Promise<void>;
+  cleanUp?: () => Promise<void>;
+  init?: () => Promise<void>;
 }

src/backend/model/extension/ExtensionDecorator.ts

@@ -0,0 +1,27 @@
+import {IExtensionEvent, IExtensionEvents} from './IExtension';
+import {ObjectManagers} from '../ObjectManagers';
+import {ExtensionEvent} from './ExtensionEvent';
+
+export const ExtensionDecorator = <I extends [], O>(fn: (ee: IExtensionEvents) => IExtensionEvent<I, O>) => {
+  return (
+    target: unknown,
+    propertyName: string,
+    descriptor: PropertyDescriptor
+  ) => {
+    const targetMethod = descriptor.value;
+    descriptor.value = async function(...args: I) {
+      const event = fn(ObjectManagers.getInstance().ExtensionManager.events) as ExtensionEvent<I, O>;
+      const eventObj = {stopPropagation: false};
+      const input = await event.triggerBefore({inputs: args}, eventObj);
+
+      // skip the rest of the execution if the before handler asked for stop propagation
+      if (eventObj.stopPropagation) {
+        return input as O;
+      }
+      const out = await targetMethod.apply(this, args);
+      return await event.triggerAfter(out);
+    };
+
+    return descriptor;
+  };
+};

src/backend/model/extension/ExtensionEvent.ts

@@ -0,0 +1,57 @@
+import {IExtensionAfterEventHandler, IExtensionBeforeEventHandler, IExtensionEvent} from './IExtension';
+
+export class ExtensionEvent<I, O> implements IExtensionEvent<I, O> {
+  protected beforeHandlers: IExtensionBeforeEventHandler<I, O>[] = [];
+  protected afterHandlers: IExtensionAfterEventHandler<O>[] = [];
+
+  public before(handler: IExtensionBeforeEventHandler<I, O>): void {
+    if (typeof handler !== 'function') {
+      throw new Error('ExtensionEvent::before: Handler is not a function');
+    }
+    this.beforeHandlers.push(handler);
+  }
+
+  public after(handler: IExtensionAfterEventHandler<O>): void {
+    if (typeof handler !== 'function') {
+      throw new Error('ExtensionEvent::after: Handler is not a function');
+    }
+    this.afterHandlers.push(handler);
+  }
+
+  public offAfter(handler: IExtensionAfterEventHandler<O>): void {
+    this.afterHandlers = this.afterHandlers.filter((h) => h !== handler);
+  }
+
+  public offBefore(handler: IExtensionBeforeEventHandler<I, O>): void {
+    this.beforeHandlers = this.beforeHandlers.filter((h) => h !== handler);
+  }
+
+
+  public async triggerBefore(input: { inputs: I }, event: { stopPropagation: boolean }): Promise<{ inputs: I } | O> {
+    let pipe: { inputs: I } | O = input;
+    if (this.beforeHandlers && this.beforeHandlers.length > 0) {
+      const s = this.beforeHandlers.slice(0);
+      for (let i = 0; i < s.length; ++i) {
+        if (event.stopPropagation) {
+          break;
+        }
+        pipe = await s[i](pipe as { inputs: I }, event);
+      }
+    }
+    return pipe;
+  }
+
+  public async triggerAfter(output: O): Promise<O> {
+    if (this.afterHandlers && this.afterHandlers.length > 0) {
+      const s = this.afterHandlers.slice(0);
+      for (let i = 0; i < s.length; ++i) {
+        output = await s[i](output);
+      }
+    }
+    return output;
+  }
+
+}
+
+
+

src/backend/model/extension/ExtensionManager.ts

@@ -0,0 +1,89 @@
+import {ProjectPath} from '../../ProjectPath';
+import {Config} from '../../../common/config/private/Config';
+import * as fs from 'fs';
+import * as path from 'path';
+import {IObjectManager} from '../database/IObjectManager';
+import {Logger} from '../../Logger';
+import {IExtensionEvents, IExtensionObject, IServerExtension} from './IExtension';
+import {ObjectManagers} from '../ObjectManagers';
+import {Server} from '../../server';
+import {ExtensionEvent} from './ExtensionEvent';
+
+const LOG_TAG = '[ExtensionManager]';
+
+export class ExtensionManager implements IObjectManager {
+
+  events: IExtensionEvents = {
+    gallery: {
+      MetadataLoader: {
+        loadPhotoMetadata: new ExtensionEvent()
+      }
+    }
+  };
+
+  public async init() {
+    this.loadExtensionsList();
+    await this.initExtensions();
+  }
+
+  public loadExtensionsList() {
+    if (!fs.existsSync(ProjectPath.ExtensionFolder)) {
+      return;
+    }
+    Config.Extensions.list = fs
+      .readdirSync(ProjectPath.ExtensionFolder)
+      .filter((f): boolean =>
+        fs.statSync(path.join(ProjectPath.ExtensionFolder, f)).isDirectory()
+      );
+    Config.Extensions.list.sort();
+  }
+
+  private async callServerFN(fn: (ext: IServerExtension, extName: string) => Promise<void>) {
+    for (let i = 0; i < Config.Extensions.list.length; ++i) {
+      const extName = Config.Extensions.list[i];
+      const extPath = path.join(ProjectPath.ExtensionFolder, extName);
+      const serverExt = path.join(extPath, 'server.js');
+      if (!fs.existsSync(serverExt)) {
+        continue;
+      }
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const ext = require(serverExt);
+      await fn(ext, extName);
+    }
+  }
+
+  private createExtensionObject(): IExtensionObject {
+    return {
+      app: {
+        objectManagers: ObjectManagers.getInstance(),
+        config: Config,
+        paths: ProjectPath,
+        expressApp: Server.getInstance().app
+      },
+      events: null
+    };
+  }
+
+  private async initExtensions() {
+    await this.callServerFN(async (ext, extName) => {
+      if (typeof ext?.init === 'function') {
+        Logger.debug(LOG_TAG, 'Running Init on extension:' + extName);
+        await ext?.init(this.createExtensionObject());
+      }
+    });
+  }
+
+  private async cleanUpExtensions() {
+    await this.callServerFN(async (ext, extName) => {
+      if (typeof ext?.cleanUp === 'function') {
+        Logger.debug(LOG_TAG, 'Running Init on extension:' + extName);
+        await ext?.cleanUp();
+      }
+    });
+  }
+
+
+  public async cleanUp() {
+    await this.cleanUpExtensions();
+  }
+}

src/backend/model/extension/IExtension.ts

@@ -0,0 +1,46 @@
+import * as express from 'express';
+import {PrivateConfigClass} from '../../../common/config/private/Config';
+import {ObjectManagers} from '../ObjectManagers';
+import {ProjectPathClass} from '../../ProjectPath';
+import {ExtensionEvent} from './ExtensionEvent';
+
+
+export type IExtensionBeforeEventHandler<I, O> = (input: { inputs: I }, event: { stopPropagation: boolean }) => Promise<{ inputs: I } | O>;
+export type IExtensionAfterEventHandler<O> = (output: O) => Promise<O>;
+
+
+export interface IExtensionEvent<I, O> {
+  before: (handler: IExtensionBeforeEventHandler<I, O>) => void;
+  after: (handler: IExtensionAfterEventHandler<O>) => void
+}
+
+export interface IExtensionEvents {
+  gallery: {
+  //  indexing: IExtensionEvent<any, any>;
+   // scanningDirectory: IExtensionEvent<any, any>;
+    MetadataLoader: {
+      loadPhotoMetadata: IExtensionEvent<any, any>
+    }
+
+    //listingDirectory: IExtensionEvent<any, any>;
+    //searching: IExtensionEvent<any, any>;
+  };
+}
+
+export interface IExtensionApp {
+  expressApp: express.Express;
+  config: PrivateConfigClass;
+  objectManagers: ObjectManagers;
+  paths: ProjectPathClass;
+}
+
+export interface IExtensionObject {
+  app: IExtensionApp;
+  events: IExtensionEvents;
+}
+
+export interface IServerExtension {
+  init(app: IExtensionObject): Promise<void>;
+
+  cleanUp?: () => Promise<void>;
+}

src/backend/model/fileaccess/MetadataLoader.ts

@@ -12,6 +12,7 @@ import {IptcParser} from 'ts-node-iptc';
 import {FFmpegFactory} from '../FFmpegFactory';
 import {FfprobeData} from 'fluent-ffmpeg';
 import {Utils} from '../../../common/Utils';
+import { ExtensionDecorator } from '../extension/ExtensionDecorator';
 
 const LOG_TAG = '[MetadataLoader]';
 const ffmpeg = FFmpegFactory.get();
@@ -128,6 +129,7 @@ export class MetadataLoader {
     fileSize: 0,
   };
 
+  @ExtensionDecorator(e=>e.gallery.MetadataLoader.loadPhotoMetadata)
   public static loadPhotoMetadata(fullPath: string): Promise<PhotoMetadata> {
     return new Promise<PhotoMetadata>((resolve, reject) => {
       try {

src/backend/model/jobs/JobManager.ts

@@ -10,14 +10,15 @@ import {JobProgress} from './jobs/JobProgress';
 import {JobProgressManager} from './JobProgressManager';
 import {JobDTOUtils} from '../../../common/entities/job/JobDTO';
 import {Utils} from '../../../common/Utils';
+import {IObjectManager} from '../database/IObjectManager';
 
 const LOG_TAG = '[JobManager]';
 
-export class JobManager implements IJobListener {
+export class JobManager implements IJobListener, IObjectManager {
   protected timers: { schedule: JobScheduleDTO; timer: NodeJS.Timeout }[] = [];
   protected progressManager: JobProgressManager = null;
 
-  constructor() {
+  async init(){
     this.progressManager = new JobProgressManager();
     this.runSchedules();
   }
@@ -124,7 +125,12 @@ export class JobManager implements IJobListener {
     return JobRepository.Instance.getAvailableJobs();
   }
 
+  public async cleanUp() {
+    this.stopSchedules();
+  }
+
   public stopSchedules(): void {
+    Logger.silly(LOG_TAG, 'Stopping all schedules');
     this.timers.forEach((t): void => clearTimeout(t.timer));
     this.timers = [];
   }

src/backend/server.ts

@@ -32,9 +32,18 @@ const LOG_TAG = '[server]';
 
 export class Server {
   public onStarted = new Event<void>();
-  private app: express.Express;
+  public app: express.Express;
   private server: HttpServer;
 
+  static instance: Server;
+
+  public static getInstance(): Server {
+    if (this.instance === null) {
+      this.instance = new Server();
+    }
+    return this.instance;
+  }
+
   constructor() {
     if (!(process.env.NODE_ENV === 'production')) {
       Logger.info(
@@ -115,7 +124,7 @@ export class Server {
     Localizations.init();
 
     this.app.use(locale(Config.Server.languages, 'en'));
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
 
     Router.route(this.app);
 

src/common/config/private/MessagingConfig.ts

@@ -96,7 +96,6 @@ export class EmailMessagingConfig {
   smtp?: EmailSMTPMessagingConfig = new EmailSMTPMessagingConfig();
 
 }
-
 @SubConfigClass<TAGS>({softReadonly: true})
 export class MessagingConfig {
   @ConfigProperty({

src/common/config/private/PrivateConfig.ts

@@ -1013,6 +1013,14 @@ export class ServerServiceConfig extends ClientServiceConfig {
   Log: ServerLogConfig = new ServerLogConfig();
 }
 
+
+
+@SubConfigClass<TAGS>({softReadonly: true})
+export class ServerExtensionsConfig {
+  @ConfigProperty({volatile: true})
+  list: string[] = [];
+}
+
 @SubConfigClass({softReadonly: true})
 export class ServerEnvironmentConfig {
   @ConfigProperty({volatile: true})
@@ -1133,6 +1141,15 @@ export class ServerConfig extends ClientConfig {
   })
   Messaging: MessagingConfig = new MessagingConfig();
 
+
+  @ConfigProperty({
+    tags: {
+      name: $localize`Extensions`,
+      uiIcon: 'ionCloudOutline'
+    } as TAGS,
+  })
+  Extensions: ServerExtensionsConfig = new ServerExtensionsConfig();
+
   @ConfigProperty({
     tags: {
       name: $localize`Jobs`,

test/backend/DBTestHelper.ts

@@ -111,7 +111,7 @@ export class DBTestHelper {
   }
 
   public static async persistTestDir(directory: DirectoryBaseDTO): Promise<ParentDirectoryDTO> {
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
     const connection = await SQLConnection.getConnection();
     ObjectManagers.getInstance().IndexingManager.indexDirectory = () => Promise.resolve(null);
 
@@ -197,7 +197,7 @@ export class DBTestHelper {
     const conn = await SQLConnection.getConnection();
     await conn.query('CREATE DATABASE IF NOT EXISTS ' + conn.options.database);
     await SQLConnection.close();
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
   }
 
   private async clearUpMysql(): Promise<void> {
@@ -218,7 +218,7 @@ export class DBTestHelper {
   private async resetSQLite(): Promise<void> {
     Logger.debug(LOG_TAG, 'resetting sqlite');
     await this.clearUpSQLite();
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
   }
 
   private async clearUpSQLite(): Promise<void> {

test/backend/integration/routers/PublicRouter.ts

@@ -41,7 +41,7 @@ describe('PublicRouter', () => {
     server = new Server();
     await server.onStarted.wait();
 
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
     await ObjectManagers.getInstance().UserManager.createUser(Utils.clone(testUser));
     await SQLConnection.close();
   };

test/backend/integration/routers/SharingRouter.ts

@@ -42,7 +42,7 @@ describe('SharingRouter', () => {
     server = new Server();
     await server.onStarted.wait();
 
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
     await ObjectManagers.getInstance().UserManager.createUser(Utils.clone(testUser));
     await SQLConnection.close();
   };

test/backend/integration/routers/UserRouter.ts

@@ -42,7 +42,7 @@ describe('UserRouter', () => {
 
     server = new Server();
     await server.onStarted.wait();
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
     await ObjectManagers.getInstance().UserManager.createUser(Utils.clone(testUser));
     await SQLConnection.close();
   };

test/backend/unit/middlewares/admin/SettingsMWs.ts

@@ -21,7 +21,7 @@ describe('Settings middleware', () => {
   beforeEach(async () => {
     await ObjectManagers.reset();
     await fs.promises.rm(tempDir, {recursive: true, force: true});
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
   });
 
   it('should save empty enforced users settings', (done: (err?: any) => void) => {

test/backend/unit/model/sql/AlbumManager.spec.ts

@@ -33,7 +33,7 @@ describe('AlbumManager', (sqlHelper: DBTestHelper) => {
   const setUpSqlDB = async () => {
     await sqlHelper.initDB();
     await sqlHelper.setUpTestGallery();
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
   };
 
 

test/backend/unit/model/sql/CoverManager.spec.ts

@@ -122,7 +122,7 @@ describe('CoverManager', (sqlHelper: DBTestHelper) => {
   const setUpSqlDB = async () => {
     await sqlHelper.initDB();
     await setUpTestGallery();
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
   };
 
 

test/backend/unit/model/sql/SearchManager.spec.ts

@@ -146,7 +146,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
   const setUpSqlDB = async () => {
     await sqlHelper.initDB();
     await setUpTestGallery();
-    await ObjectManagers.InitSQLManagers();
+    await ObjectManagers.InitManagers();
   };