Merge pull request #505 from bpatrik/feature/performance

Merging lossy gpx compression

src/backend/middlewares/MetaFileMWs.ts

@@ -0,0 +1,40 @@
+import {NextFunction, Request, Response} from 'express';
+import * as fs from 'fs';
+import { Config } from '../../common/config/private/Config';
+import { GPXProcessing } from '../model/GPXProcessing';
+
+export class MetaFileMWs {
+  public static async compressGPX(
+    req: Request,
+    res: Response,
+    next: NextFunction
+  ): Promise<void> {
+    if (!req.resultPipe) {
+      return next();
+    }
+    // if conversion is not enabled redirect, so browser can cache the full
+    if (Config.Client.MetaFile.GPXCompressing.enabled === false) {
+      return res.redirect(req.originalUrl.slice(0, -1 * '\\bestFit'.length));
+    }
+    const fullPath = req.resultPipe as string;
+
+    const compressedGPX = GPXProcessing.generateConvertedPath(
+      fullPath,
+    );
+
+    // check if converted photo exist
+    if (fs.existsSync(compressedGPX) === true) {
+      req.resultPipe = compressedGPX;
+      return next();
+    }
+
+    if (Config.Server.MetaFile.GPXCompressing.onTheFly === true) {
+      req.resultPipe = await GPXProcessing.compressGPX(fullPath);
+      return next();
+    }
+
+    // not converted and won't be now
+    return res.redirect(req.originalUrl.slice(0, -1 * '\\bestFit'.length));
+  }
+}
+

src/backend/middlewares/admin/SettingsMWs.ts

@@ -12,7 +12,7 @@ import {
   DatabaseType,
   ServerDataBaseConfig,
   ServerIndexingConfig,
-  ServerJobConfig,
+  ServerJobConfig, ServerMetaFileConfig,
   ServerPhotoConfig,
   ServerPreviewConfig,
   ServerThumbnailConfig,
@@ -21,7 +21,7 @@ import {
 import {
   ClientAlbumConfig,
   ClientFacesConfig,
-  ClientMapConfig,
+  ClientMapConfig, ClientMediaConfig,
   ClientMetaFileConfig,
   ClientPhotoConfig,
   ClientRandomPhotoConfig,
@@ -166,13 +166,21 @@ export class SettingsMWs {
     }
 
     try {
-      const original = await Config.original();
-      await ConfigDiagnostics.testMetaFileConfig(req.body.settings as ClientMetaFileConfig, original);
+      const settings: {
+        server: ServerMetaFileConfig,
+        client: ClientMetaFileConfig
+      } = req.body.settings;
 
-      Config.Client.MetaFile = (req.body.settings as ClientMetaFileConfig);
+      const original = await Config.original();
+      await ConfigDiagnostics.testClientMetaFileConfig(settings.client, original);
+      await ConfigDiagnostics.testServerMetaFileConfig(settings.server, original);
+
+      Config.Client.MetaFile = settings.client;
+      Config.Server.MetaFile = settings.server;
       // only updating explicitly set config (not saving config set by the diagnostics)
 
-      original.Client.MetaFile = (req.body.settings as ClientMetaFileConfig);
+      original.Client.MetaFile = settings.client;
+      original.Server.MetaFile = settings.server;
       original.save();
       await ConfigDiagnostics.runDiagnostics();
       Logger.info(LOG_TAG, 'new config:');

src/backend/model/GPXProcessing.ts

@@ -0,0 +1,127 @@
+import * as path from 'path';
+import {constants as fsConstants, promises as fsp} from 'fs';
+import * as xml2js from 'xml2js';
+import {ProjectPath} from '../ProjectPath';
+import {Config} from '../../common/config/private/Config';
+
+type gpxEntry = { '$': { lat: string, lon: string }, ele: string[], time: string[], extensions: unknown };
+
+export class GPXProcessing {
+
+
+  public static generateConvertedPath(filePath: string): string {
+    const file = path.basename(filePath);
+    return path.join(
+      ProjectPath.TranscodedFolder,
+      ProjectPath.getRelativePathToImages(path.dirname(filePath)),
+      file
+    );
+  }
+
+  public static async isValidConvertedPath(
+    convertedPath: string
+  ): Promise<boolean> {
+    const origFilePath = path.join(
+      ProjectPath.ImageFolder,
+      path.relative(
+        ProjectPath.TranscodedFolder,
+        convertedPath
+      )
+    );
+
+
+    try {
+      await fsp.access(origFilePath, fsConstants.R_OK);
+    } catch (e) {
+      return false;
+    }
+
+    return true;
+  }
+
+
+  static async compressedGPXExist(
+    filePath: string
+  ): Promise<boolean> {
+    // compressed gpx path
+    const outPath = GPXProcessing.generateConvertedPath(filePath);
+
+    // check if file already exist
+    try {
+      await fsp.access(outPath, fsConstants.R_OK);
+      return true;
+    } catch (e) {
+      // ignoring errors
+    }
+    return false;
+  }
+
+  public static async compressGPX(
+    filePath: string,
+  ): Promise<string> {
+    // generate compressed gpx path
+    const outPath = GPXProcessing.generateConvertedPath(filePath);
+
+    // check if file already exist
+    try {
+      await fsp.access(outPath, fsConstants.R_OK);
+      return outPath;
+    } catch (e) {
+      // ignoring errors
+    }
+
+
+    const outDir = path.dirname(outPath);
+
+    await fsp.mkdir(outDir, {recursive: true});
+    const gpxStr = await fsp.readFile(filePath);
+    const gpxObj = await (new xml2js.Parser()).parseStringPromise(gpxStr);
+    const items: gpxEntry[] = gpxObj.gpx.trk[0].trkseg[0].trkpt;
+
+    const distance = (entry1: gpxEntry, entry2: gpxEntry) => {
+      const lat1 = parseFloat(entry1.$.lat);
+      const lon1 = parseFloat(entry1.$.lon);
+      const lat2 = parseFloat(entry2.$.lat);
+      const lon2 = parseFloat(entry2.$.lon);
+
+      // credits to: https://www.movable-type.co.uk/scripts/latlong.html
+      const R = 6371e3; // metres
+      const φ1 = lat1 * Math.PI / 180; // φ, λ in radians
+      const φ2 = lat2 * Math.PI / 180;
+      const Δφ = (lat2 - lat1) * Math.PI / 180;
+      const Δλ = (lon2 - lon1) * Math.PI / 180;
+
+      const a = Math.sin(Δφ / 2) * Math.sin(Δφ / 2) +
+        Math.cos(φ1) * Math.cos(φ2) *
+        Math.sin(Δλ / 2) * Math.sin(Δλ / 2);
+      const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
+
+      const d = R * c; // in metres
+      return d;
+    };
+    const gpxEntryFilter = (value: gpxEntry, i: number, list: gpxEntry[]) => {
+      if (i === 0 || i >= list.length - 1) { // always keep the first and last items
+        return true;
+      }
+      const timeDelta = (Date.parse(list[i].time[0]) - Date.parse(list[i - 1].time[0])); // mill sec.
+      const dist = distance(list[i - 1], list[i]); // meters
+
+      return !(timeDelta < Config.Server.MetaFile.GPXCompressing.minTimeDistance &&
+        dist < Config.Server.MetaFile.GPXCompressing.minDistance);
+    };
+
+    gpxObj.gpx.trk[0].trkseg[0].trkpt = items.filter(gpxEntryFilter).map((v) => {
+      v.$.lon = parseFloat(v.$.lon).toFixed(6);
+      v.$.lat = parseFloat(v.$.lat).toFixed(6);
+      delete v.ele;
+      delete v.extensions;
+      return v;
+    });
+
+    await fsp.writeFile(outPath, (new xml2js.Builder({renderOpts: {pretty: false}})).buildObject(gpxObj));
+
+    return outPath;
+  }
+
+}
+

src/backend/model/diagnostics/ConfigDiagnostics.ts

@@ -23,6 +23,7 @@ import {
   IPrivateConfig,
   ServerDataBaseConfig,
   ServerJobConfig,
+  ServerMetaFileConfig,
   ServerPhotoConfig,
   ServerPreviewConfig,
   ServerThumbnailConfig,
@@ -82,7 +83,7 @@ export class ConfigDiagnostics {
     }
   }
 
-  static async testMetaFileConfig(
+  static async testClientMetaFileConfig(
     metaFileConfig: ClientMetaFileConfig,
     config: IPrivateConfig
   ): Promise<void> {
@@ -91,6 +92,13 @@ export class ConfigDiagnostics {
     }
   }
 
+  static async testServerMetaFileConfig(
+    metaFileConfig: ServerMetaFileConfig,
+    config: IPrivateConfig
+  ): Promise<void> {
+    // nothing to check at the moment
+  }
+
   static testClientVideoConfig(videoConfig: ClientVideoConfig): Promise<void> {
     return new Promise((resolve, reject) => {
       try {
@@ -150,7 +158,7 @@ export class ConfigDiagnostics {
   static testImageFolder(folder: string): Promise<void> {
     return new Promise((resolve, reject) => {
       if (!fs.existsSync(folder)) {
-        reject("Images folder not exists: '" + folder + "'");
+        reject('Images folder not exists: \'' + folder + '\'');
       }
       fs.access(folder, fs.constants.R_OK, (err) => {
         if (err) {
@@ -327,7 +335,7 @@ export class ConfigDiagnostics {
       Logger.warn(
         LOG_TAG,
         'Thumbnail hardware acceleration is not possible.' +
-          " 'sharp' node module is not found." +
+        ' \'sharp\' node module is not found.' +
         ' Falling back temporally to JS based thumbnail generation'
       );
       process.exit(1);
@@ -362,7 +370,7 @@ export class ConfigDiagnostics {
     }
 
     try {
-      await ConfigDiagnostics.testMetaFileConfig(
+      await ConfigDiagnostics.testClientMetaFileConfig(
         Config.Client.MetaFile,
         Config
       );
@@ -380,6 +388,25 @@ export class ConfigDiagnostics {
       Config.Client.MetaFile.gpx = false;
     }
 
+    try {
+      await ConfigDiagnostics.testServerMetaFileConfig(
+        Config.Server.MetaFile,
+        Config
+      );
+    } catch (ex) {
+      const err: Error = ex;
+      NotificationManager.warning(
+        'Meta file support error, switching off gpx..',
+        err.toString()
+      );
+      Logger.warn(
+        LOG_TAG,
+        'Meta file support error, switching off..',
+        err.toString()
+      );
+      Config.Client.MetaFile.gpx = false;
+    }
+
     try {
       await ConfigDiagnostics.testAlbumsConfig(Config.Client.Album, Config);
     } catch (ex) {

src/backend/routes/GalleryRouter.ts

@@ -9,6 +9,7 @@ import { VersionMWs } from '../middlewares/VersionMWs';
 import { SupportedFormats } from '../../common/SupportedFormats';
 import { PhotoConverterMWs } from '../middlewares/thumbnail/PhotoConverterMWs';
 import { ServerTimingMWs } from '../middlewares/ServerTimingMWs';
+import {MetaFileMWs} from '../middlewares/MetaFileMWs';
 
 export class GalleryRouter {
   public static route(app: Express): void {
@@ -21,6 +22,7 @@ export class GalleryRouter {
     this.addGetBestFitVideo(app);
     this.addGetVideo(app);
     this.addGetMetaFile(app);
+    this.addGetBestFitMetaFile(app);
     this.addRandom(app);
     this.addDirectoryList(app);
     this.addDirectoryZip(app);
@@ -158,6 +160,26 @@ export class GalleryRouter {
     );
   }
 
+  protected static addGetBestFitMetaFile(app: Express): void {
+    app.get(
+      [
+        '/api/gallery/content/:mediaPath(*.(' +
+        SupportedFormats.MetaFiles.join('|') +
+        '))/bestFit',
+      ],
+      // common part
+      AuthenticationMWs.authenticate,
+      AuthenticationMWs.normalizePathParam('mediaPath'),
+      AuthenticationMWs.authorisePath('mediaPath', false),
+
+      // specific part
+      GalleryMWs.loadFile,
+      MetaFileMWs.compressGPX,
+      ServerTimingMWs.addServerTiming,
+      RenderingMWs.renderFile
+    );
+  }
+
   protected static addRandom(app: Express): void {
     app.get(
       ['/api/gallery/random/:searchQueryDTO'],

src/common/config/private/PrivateConfig.ts

@@ -5,7 +5,7 @@ import {
   JobTrigger,
   JobTriggerType,
 } from '../../entities/job/JobScheduleDTO';
-import { ClientConfig } from '../public/ClientConfig';
+import {ClientConfig, ClientMetaFileConfig} from '../public/ClientConfig';
 import {SubConfigClass} from 'typeconfig/src/decorators/class/SubConfigClass';
 import {ConfigProperty} from 'typeconfig/src/decorators/property/ConfigPropoerty';
 import {DefaultsJobs} from '../../entities/job/JobDTO';
@@ -148,6 +148,25 @@ export class ServerThumbnailConfig {
   personFaceMargin: number = 0.6; // in ration [0-1]
 }
 
+@SubConfigClass()
+export class ServerGPXCompressingConfig {
+  @ConfigProperty({
+    description: 'Compresses gpx files on-the-fly, when they are requested.',
+  })
+  onTheFly: boolean = true;
+  @ConfigProperty({type: 'unsignedInt', description: 'Filters out entry that are closer than this in meters.'})
+  minDistance: number = 5;
+  @ConfigProperty({type: 'unsignedInt', description: 'Filters out entry that are closer than this in time in milliseconds.'})
+  minTimeDistance: number = 5000;
+}
+
+@SubConfigClass()
+export class ServerMetaFileConfig {
+  @ConfigProperty()
+  GPXCompressing: ServerGPXCompressingConfig = new ServerGPXCompressingConfig();
+}
+
+
 @SubConfigClass()
 export class ServerSharingConfig {
   @ConfigProperty()
@@ -163,9 +182,9 @@ export class ServerIndexingConfig {
   @ConfigProperty({
     arrayType: 'string',
     description:
-      "If an entry starts with '/' it is treated as an absolute path." +
-      " If it doesn't start with '/' but contains a '/', the path is relative to the image directory." +
-      " If it doesn't contain a '/', any folder with this name will be excluded.",
+      'If an entry starts with \'/\' it is treated as an absolute path.' +
+      ' If it doesn\'t start with \'/\' but contains a \'/\', the path is relative to the image directory.' +
+      ' If it doesn\'t contain a \'/\', any folder with this name will be excluded.',
   })
   excludeFolderList: string[] = ['.Trash-1000', '.dtrash', '$RECYCLE.BIN'];
   @ConfigProperty({
@@ -182,7 +201,7 @@ export class ServerThreadingConfig {
   enabled: boolean = true;
   @ConfigProperty({
     description:
-      "Number of threads that are used to generate thumbnails. If 0, number of 'CPU cores -1' threads will be used.",
+      'Number of threads that are used to generate thumbnails. If 0, number of \'CPU cores -1\' threads will be used.',
   })
   thumbnailThreads: number = 0; // if zero-> CPU count -1
 }
@@ -482,6 +501,8 @@ export class ServerConfig {
   Log: ServerLogConfig = new ServerLogConfig();
   @ConfigProperty()
   Jobs: ServerJobConfig = new ServerJobConfig();
+  @ConfigProperty()
+  MetaFile: ServerMetaFileConfig = new ServerMetaFileConfig();
 }
 
 export interface IPrivateConfig {

src/common/config/public/ClientConfig.ts

@@ -182,6 +182,12 @@ export class ClientPhotoConfig {
   loadFullImageOnZoom: boolean = true;
 }
 
+@SubConfigClass()
+export class ClientGPXCompressingConfig {
+  @ConfigProperty()
+  enabled: boolean = true;
+}
+
 @SubConfigClass()
 export class ClientMediaConfig {
   @ConfigProperty()
@@ -198,11 +204,16 @@ export class ClientMetaFileConfig {
     description: 'Reads *.gpx files and renders them on the map.',
   })
   gpx: boolean = true;
+
+  @ConfigProperty()
+  GPXCompressing: ClientGPXCompressingConfig = new ClientGPXCompressingConfig();
+
   @ConfigProperty({
     description:
       'Reads *.md files in a directory and shows the next to the map.',
   })
   markdown: boolean = true;
+
   @ConfigProperty({
     description:
       'Reads *.pg2conf files (You can use it for custom sorting and save search (albums)).',

src/frontend/app/ui/settings/metafiles/metafile.settings.component.html

@@ -12,21 +12,42 @@
         description="Reads *.gpx files and renders them on the map."
         i18n-description i18n-name
         [disabled]="!(settingsService.Settings | async).Client.Map.enabled"
-        [ngModel]="states.gpx">
+        [ngModel]="states.client.gpx">
       </app-settings-entry>
 
+      <app-settings-entry
+        name="*.gpx compression"
+        description="Enables *.gpx file compression."
+        link="https://github.com/bpatrik/pigallery2/issues/504"
+        linkText="See 504."
+        i18n-description i18n-name
+        [disabled]="!(settingsService.Settings | async).Client.Map.enabled || !states.client.gpx.value"
+        [ngModel]="states.client.GPXCompressing.enabled">
+      </app-settings-entry>
+
+      <app-settings-entry
+        name="OnTheFly *.gpx compression"
+        description="Enables on the fly *.gpx compression."
+        i18n-description i18n-name
+        [simplifiedMode]="simplifiedMode"
+        [disabled]="!(settingsService.Settings | async).Client.Map.enabled || !states.client.GPXCompressing.enabled.value || !states.client.gpx.value"
+        [ngModel]="states.server.GPXCompressing.onTheFly">
+      </app-settings-entry>
+
+      <hr/>
+
       <app-settings-entry
         name="Markdown files"
         description="Reads *.md files in a directory and shows the next to the map."
         i18n-description i18n-name
-        [ngModel]="states.markdown">
+        [ngModel]="states.client.markdown">
       </app-settings-entry>
 
       <app-settings-entry
         name="*.pg2conf files"
         description="Reads *.pg2conf files (You can use it for custom sorting and save search (albums))."
         i18n-description i18n-name
-        [ngModel]="states.pg2conf">
+        [ngModel]="states.client.pg2conf">
       </app-settings-entry>
 
 

src/frontend/app/ui/settings/metafiles/metafile.settings.component.ts

@@ -4,7 +4,8 @@ import { SettingsComponentDirective } from '../_abstract/abstract.settings.compo
 import { AuthenticationService } from '../../../model/network/authentication.service';
 import { NavigationService } from '../../../model/navigation.service';
 import { NotificationService } from '../../../model/notification.service';
-import { ClientMetaFileConfig } from '../../../../../common/config/public/ClientConfig';
+import {ClientMetaFileConfig, ClientPhotoConfig} from '../../../../../common/config/public/ClientConfig';
+import {ServerMetaFileConfig, ServerPhotoConfig} from '../../../../../common/config/private/PrivateConfig';
 
 @Component({
   selector: 'app-settings-meta-file',
@@ -15,7 +16,10 @@ import { ClientMetaFileConfig } from '../../../../../common/config/public/Client
   ],
   providers: [MetaFileSettingsService],
 })
-export class MetaFileSettingsComponent extends SettingsComponentDirective<ClientMetaFileConfig> {
+export class MetaFileSettingsComponent extends SettingsComponentDirective<{
+  server: ServerMetaFileConfig;
+  client: ClientMetaFileConfig;
+}> {
   constructor(
     authService: AuthenticationService,
     navigation: NavigationService,
@@ -29,7 +33,10 @@ export class MetaFileSettingsComponent extends SettingsComponentDirective<Client
       navigation,
       settingsService,
       notification,
-      (s) => s.Client.MetaFile
+      (s) => ({
+        client: s.Client.MetaFile,
+        server: s.Server.MetaFile,
+      })
     );
   }
 }

src/frontend/app/ui/settings/metafiles/metafile.settings.service.ts

@@ -3,9 +3,13 @@ import { NetworkService } from '../../../model/network/network.service';
 import {SettingsService} from '../settings.service';
 import {AbstractSettingsService} from '../_abstract/abstract.settings.service';
 import {ClientMetaFileConfig} from '../../../../../common/config/public/ClientConfig';
+import {ServerMetaFileConfig} from '../../../../../common/config/private/PrivateConfig';
 
 @Injectable()
-export class MetaFileSettingsService extends AbstractSettingsService<ClientMetaFileConfig> {
+export class MetaFileSettingsService extends AbstractSettingsService<{
+  server: ServerMetaFileConfig;
+  client: ClientMetaFileConfig;
+}> {
   constructor(
     private networkService: NetworkService,
     settingsService: SettingsService
@@ -21,7 +25,10 @@ export class MetaFileSettingsService extends AbstractSettingsService<ClientMetaF
     return false;
   }
 
-  public updateSettings(settings: ClientMetaFileConfig): Promise<void> {
+  public updateSettings(settings: {
+    server: ServerMetaFileConfig;
+    client: ClientMetaFileConfig;
+  }): Promise<void> {
     return this.networkService.putJson('/settings/metafile', {settings});
   }
 }