From ca0579cefd0e9d027975490945a8d69ab05578a6 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Sat, 25 Jun 2022 20:57:45 +0200 Subject: [PATCH 1/2] Implementing GPX compression job #504 --- src/backend/model/jobs/JobRepository.ts | 2 + src/backend/model/jobs/jobs/FileJob.ts | 107 +++++++++++------- .../model/jobs/jobs/GPXCompressionJob.ts | 37 ++++++ .../model/threading/DiskMangerWorker.ts | 4 +- src/common/entities/job/JobDTO.ts | 1 + src/frontend/app/model/backendtext.service.ts | 8 +- .../job-progress.settings.component.html | 4 +- .../metafile.settings.component.html | 46 +++++--- .../metafiles/metafile.settings.component.ts | 17 ++- .../photo/photo.settings.component.ts | 1 - 10 files changed, 166 insertions(+), 61 deletions(-) create mode 100644 src/backend/model/jobs/jobs/GPXCompressionJob.ts diff --git a/src/backend/model/jobs/JobRepository.ts b/src/backend/model/jobs/JobRepository.ts index 40c4298d..58a4d703 100644 --- a/src/backend/model/jobs/JobRepository.ts +++ b/src/backend/model/jobs/JobRepository.ts @@ -7,6 +7,7 @@ import { ThumbnailGenerationJob } from './jobs/ThumbnailGenerationJob'; import { TempFolderCleaningJob } from './jobs/TempFolderCleaningJob'; import { PreviewFillingJob } from './jobs/PreviewFillingJob'; import { PreviewRestJob } from './jobs/PreviewResetJob'; +import {GPXCompressionJob} from './jobs/GPXCompressionJob'; export class JobRepository { private static instance: JobRepository = null; @@ -38,4 +39,5 @@ JobRepository.Instance.register(new PreviewRestJob()); JobRepository.Instance.register(new VideoConvertingJob()); JobRepository.Instance.register(new PhotoConvertingJob()); JobRepository.Instance.register(new ThumbnailGenerationJob()); +JobRepository.Instance.register(new GPXCompressionJob()); JobRepository.Instance.register(new TempFolderCleaningJob()); diff --git a/src/backend/model/jobs/jobs/FileJob.ts b/src/backend/model/jobs/jobs/FileJob.ts index e4071e7c..0350a257 100644 --- a/src/backend/model/jobs/jobs/FileJob.ts +++ b/src/backend/model/jobs/jobs/FileJob.ts @@ -1,27 +1,28 @@ -import { ConfigTemplateEntry } from '../../../../common/entities/job/JobDTO'; -import { Job } from './Job'; +import {ConfigTemplateEntry} from '../../../../common/entities/job/JobDTO'; +import {Job} from './Job'; import * as path from 'path'; -import { DiskManager } from '../../DiskManger'; -import { DirectoryScanSettings } from '../../threading/DiskMangerWorker'; -import { Logger } from '../../../Logger'; -import { Config } from '../../../../common/config/private/Config'; -import { FileDTO } from '../../../../common/entities/FileDTO'; -import { SQLConnection } from '../../database/sql/SQLConnection'; -import { MediaEntity } from '../../database/sql/enitites/MediaEntity'; -import { PhotoEntity } from '../../database/sql/enitites/PhotoEntity'; -import { VideoEntity } from '../../database/sql/enitites/VideoEntity'; -import { backendTexts } from '../../../../common/BackendTexts'; -import { ProjectPath } from '../../../ProjectPath'; -import { DatabaseType } from '../../../../common/config/private/PrivateConfig'; +import {DiskManager} from '../../DiskManger'; +import {DirectoryScanSettings} from '../../threading/DiskMangerWorker'; +import {Logger} from '../../../Logger'; +import {Config} from '../../../../common/config/private/Config'; +import {FileDTO} from '../../../../common/entities/FileDTO'; +import {SQLConnection} from '../../database/sql/SQLConnection'; +import {MediaEntity} from '../../database/sql/enitites/MediaEntity'; +import {PhotoEntity} from '../../database/sql/enitites/PhotoEntity'; +import {VideoEntity} from '../../database/sql/enitites/VideoEntity'; +import {backendTexts} from '../../../../common/BackendTexts'; +import {ProjectPath} from '../../../ProjectPath'; +import {DatabaseType} from '../../../../common/config/private/PrivateConfig'; +import {FileEntity} from '../../database/sql/enitites/FileEntity'; +import {global} from '../../../../../node_modules/@angular/compiler/src/util'; +import {DirectoryBaseDTO, DirectoryDTOUtils} from '../../../../common/entities/DirectoryDTO'; const LOG_TAG = '[FileJob]'; /** * Abstract class for thumbnail creation, file deleting etc. */ -export abstract class FileJob< - S extends { indexedOnly: boolean } = { indexedOnly: boolean } -> extends Job { +export abstract class FileJob extends Job { public readonly ConfigTemplate: ConfigTemplateEntry[] = []; directoryQueue: string[] = []; fileQueue: string[] = []; @@ -109,6 +110,7 @@ export abstract class FileJob< for (const item of scanned.directories) { this.directoryQueue.push(path.join(item.path, item.name)); } + DirectoryDTOUtils.addReferences(scanned as DirectoryBaseDTO); if (this.scanFilter.noPhoto !== true || this.scanFilter.noVideo !== true) { const scannedAndFiltered = await this.filterMediaFiles(scanned.media); for (const item of scannedAndFiltered) { @@ -138,38 +140,67 @@ export abstract class FileJob< } private async loadAllMediaFilesFromDB(): Promise { - if (this.scanFilter.noVideo === true && this.scanFilter.noPhoto === true) { + if (this.scanFilter.noVideo === true && + this.scanFilter.noPhoto === true && + this.scanFilter.noMetaFile === true) { return; } + this.Progress.log('Loading files from db'); Logger.silly(LOG_TAG, 'Loading files from db'); const connection = await SQLConnection.getConnection(); + if (this.scanFilter.noVideo === false || + this.scanFilter.noPhoto === false) { - let usedEntity = MediaEntity; + let usedEntity = MediaEntity; - if (this.scanFilter.noVideo === true) { - usedEntity = PhotoEntity; - } else if (this.scanFilter.noPhoto === true) { - usedEntity = VideoEntity; + if (this.scanFilter.noVideo === true) { + usedEntity = PhotoEntity; + } else if (this.scanFilter.noPhoto === true) { + usedEntity = VideoEntity; + } + + const result = await connection + .getRepository(usedEntity) + .createQueryBuilder('media') + .select(['media.name', 'directory.name', 'directory.path']) + .leftJoin('media.directory', 'directory') + .getMany(); + + const scannedAndFiltered = await this.filterMediaFiles(result); + for (const item of scannedAndFiltered) { + this.fileQueue.push( + path.join( + ProjectPath.ImageFolder, + item.directory.path, + item.directory.name, + item.name + ) + ); + } } + if (this.scanFilter.noMetaFile === false) { - const result = await connection - .getRepository(usedEntity) - .createQueryBuilder('media') - .select(['media.name', 'media.id']) - .leftJoinAndSelect('media.directory', 'directory') - .getMany(); + const result = await connection + .getRepository(FileEntity) + .createQueryBuilder('file') + .select(['file.name', 'directory.name', 'directory.path']) + .leftJoin('file.directory', 'directory') + .getMany(); - for (const item of result) { - this.fileQueue.push( - path.join( - ProjectPath.ImageFolder, - item.directory.path, - item.directory.name, - item.name - ) - ); + + const scannedAndFiltered = await this.filterMetaFiles(result); + for (const item of scannedAndFiltered) { + this.fileQueue.push( + path.join( + ProjectPath.ImageFolder, + item.directory.path, + item.directory.name, + item.name + ) + ); + } } } } diff --git a/src/backend/model/jobs/jobs/GPXCompressionJob.ts b/src/backend/model/jobs/jobs/GPXCompressionJob.ts new file mode 100644 index 00000000..b2ec97ec --- /dev/null +++ b/src/backend/model/jobs/jobs/GPXCompressionJob.ts @@ -0,0 +1,37 @@ +import {Config} from '../../../../common/config/private/Config'; +import {DefaultsJobs} from '../../../../common/entities/job/JobDTO'; +import {FileJob} from './FileJob'; +import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing'; +import {GPXProcessing} from '../../GPXProcessing'; +import {FileDTO} from '../../../../common/entities/FileDTO'; +import {Logger} from '../../../Logger'; + +export class GPXCompressionJob extends FileJob { + public readonly Name = DefaultsJobs[DefaultsJobs['GPX Compression']]; + + constructor() { + super({noVideo: true, noPhoto: true, noMetaFile: false}); + } + + protected async filterMetaFiles(files: FileDTO[]): Promise { + return files.filter(file => file.name.toLowerCase().endsWith('.gpx')); + } + + public get Supported(): boolean { + return Config.Client.MetaFile.GPXCompressing.enabled === true; + } + + protected async shouldProcess(fPath: string): Promise { + return !(await GPXProcessing.compressedGPXExist( + fPath + )); + } + + protected async processFile(fPath: string): Promise { + try { + await GPXProcessing.compressGPX(fPath); + } catch (e) { + Logger.warn('GPXCompressionJob', ' Could not compress gpx at: ' + fPath); + } + } +} diff --git a/src/backend/model/threading/DiskMangerWorker.ts b/src/backend/model/threading/DiskMangerWorker.ts index 5280104f..fcdec1d9 100644 --- a/src/backend/model/threading/DiskMangerWorker.ts +++ b/src/backend/model/threading/DiskMangerWorker.ts @@ -133,7 +133,7 @@ export class DiskMangerWorker { // nothing to scan, we are here for the empty dir if ( settings.noPhoto === true && - settings.noMetadata === true && + settings.noMetaFile === true && settings.noVideo === true ) { return directory; @@ -270,6 +270,6 @@ export interface DirectoryScanSettings { noVideo?: boolean; noPhoto?: boolean; noDirectory?: boolean; - noMetadata?: boolean; + noMetadata?: boolean; // skip parsing images for metadata like exif, iptc noChildDirPhotos?: boolean; } diff --git a/src/common/entities/job/JobDTO.ts b/src/common/entities/job/JobDTO.ts index f0a23550..e2cc54c7 100644 --- a/src/common/entities/job/JobDTO.ts +++ b/src/common/entities/job/JobDTO.ts @@ -11,6 +11,7 @@ export enum DefaultsJobs { 'Temp Folder Cleaning' = 6, 'Preview Filling' = 7, 'Preview Reset' = 8, + 'GPX Compression' = 9, } export interface ConfigTemplateEntry { diff --git a/src/frontend/app/model/backendtext.service.ts b/src/frontend/app/model/backendtext.service.ts index a9db7fd7..803b2846 100644 --- a/src/frontend/app/model/backendtext.service.ts +++ b/src/frontend/app/model/backendtext.service.ts @@ -31,8 +31,6 @@ export class BackendtextService { switch (job as DefaultsJobs) { case DefaultsJobs.Indexing: return $localize`Indexing`; - case DefaultsJobs['Preview Filling']: - return $localize`Preview Filling`; case DefaultsJobs['Database Reset']: return $localize`Database Reset`; case DefaultsJobs['Thumbnail Generation']: @@ -43,6 +41,12 @@ export class BackendtextService { return $localize`Video Converting`; case DefaultsJobs['Temp Folder Cleaning']: return $localize`Temp Folder Cleaning`; + case DefaultsJobs['Preview Filling']: + return $localize`Preview Filling`; + case DefaultsJobs['Preview Reset']: + return $localize`Preview Reset`; + case DefaultsJobs['GPX Compression']: + return $localize`GPX Compression`; default: return DefaultsJobs[job as DefaultsJobs]; } diff --git a/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html b/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html index de5d56f3..75bac8b3 100644 --- a/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html +++ b/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html @@ -72,8 +72,8 @@
-

{{Name}}

-
diff --git a/src/frontend/app/ui/settings/metafiles/metafile.settings.component.html b/src/frontend/app/ui/settings/metafiles/metafile.settings.component.html index 423deb24..0db0488d 100644 --- a/src/frontend/app/ui/settings/metafiles/metafile.settings.component.html +++ b/src/frontend/app/ui/settings/metafiles/metafile.settings.component.html @@ -7,6 +7,22 @@ + + + + + + +
+ -
- - - - - - + +
+ + + +
+
+ + + +
diff --git a/src/frontend/app/ui/settings/metafiles/metafile.settings.component.ts b/src/frontend/app/ui/settings/metafiles/metafile.settings.component.ts index c931a22c..d2608d0b 100644 --- a/src/frontend/app/ui/settings/metafiles/metafile.settings.component.ts +++ b/src/frontend/app/ui/settings/metafiles/metafile.settings.component.ts @@ -6,6 +6,9 @@ import { NavigationService } from '../../../model/navigation.service'; import { NotificationService } from '../../../model/notification.service'; import {ClientMetaFileConfig, ClientPhotoConfig} from '../../../../../common/config/public/ClientConfig'; import {ServerMetaFileConfig, ServerPhotoConfig} from '../../../../../common/config/private/PrivateConfig'; +import {DefaultsJobs, JobDTOUtils} from '../../../../../common/entities/job/JobDTO'; +import {JobProgressDTO, JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO'; +import {ScheduledJobsService} from '../scheduled-jobs.service'; @Component({ selector: 'app-settings-meta-file', @@ -24,7 +27,8 @@ export class MetaFileSettingsComponent extends SettingsComponentDirective<{ authService: AuthenticationService, navigation: NavigationService, settingsService: MetaFileSettingsService, - notification: NotificationService + notification: NotificationService, + public jobsService: ScheduledJobsService, ) { super( $localize`Meta file`, @@ -39,6 +43,17 @@ export class MetaFileSettingsComponent extends SettingsComponentDirective<{ }) ); } + + + + readonly jobName = DefaultsJobs[DefaultsJobs['GPX Compression']]; + + + get Progress(): JobProgressDTO { + return this.jobsService.progress.value[ + JobDTOUtils.getHashName(DefaultsJobs[DefaultsJobs['GPX Compression']]) + ]; + } } diff --git a/src/frontend/app/ui/settings/photo/photo.settings.component.ts b/src/frontend/app/ui/settings/photo/photo.settings.component.ts index 8dcbddf8..b2c4650f 100644 --- a/src/frontend/app/ui/settings/photo/photo.settings.component.ts +++ b/src/frontend/app/ui/settings/photo/photo.settings.component.ts @@ -31,7 +31,6 @@ export class PhotoSettingsComponent extends SettingsComponentDirective<{ }> { readonly resolutionTypes = [720, 1080, 1440, 2160, 4320]; resolutions: { key: number; value: string }[] = []; - JobProgressStates = JobProgressStates; readonly jobName = DefaultsJobs[DefaultsJobs['Photo Converting']]; From d0b609b416e42c98e028fbae3787d9490ca396a9 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Sat, 25 Jun 2022 23:06:42 +0200 Subject: [PATCH 2/2] Adding missing bestFit url ending to gpx file request #504 --- src/backend/model/jobs/jobs/FileJob.ts | 1 - src/frontend/app/ui/gallery/map/map.service.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/model/jobs/jobs/FileJob.ts b/src/backend/model/jobs/jobs/FileJob.ts index 0350a257..7e380839 100644 --- a/src/backend/model/jobs/jobs/FileJob.ts +++ b/src/backend/model/jobs/jobs/FileJob.ts @@ -14,7 +14,6 @@ import {backendTexts} from '../../../../common/BackendTexts'; import {ProjectPath} from '../../../ProjectPath'; import {DatabaseType} from '../../../../common/config/private/PrivateConfig'; import {FileEntity} from '../../database/sql/enitites/FileEntity'; -import {global} from '../../../../../node_modules/@angular/compiler/src/util'; import {DirectoryBaseDTO, DirectoryDTOUtils} from '../../../../common/entities/DirectoryDTO'; const LOG_TAG = '[FileJob]'; diff --git a/src/frontend/app/ui/gallery/map/map.service.ts b/src/frontend/app/ui/gallery/map/map.service.ts index cd6edfe0..040db593 100644 --- a/src/frontend/app/ui/gallery/map/map.service.ts +++ b/src/frontend/app/ui/gallery/map/map.service.ts @@ -95,7 +95,7 @@ export class MapService { file.name ); const gpx = await this.networkService.getXML( - '/gallery/content/' + filePath + '/gallery/content/' + filePath + '/bestFit' ); const getCoordinates = (tagName: string): LatLngLiteral[] => { const elements = gpx.getElementsByTagName(tagName);