1
0
mirror of https://github.com/bpatrik/pigallery2.git synced 2024-12-25 02:04:15 +02:00

Merge pull request #508 from bpatrik/feature/performance

Merging GPX xompression job to master
This commit is contained in:
Patrik J. Braun 2022-06-25 23:07:26 +02:00 committed by GitHub
commit 486b0250b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 166 additions and 62 deletions

View File

@ -7,6 +7,7 @@ import { ThumbnailGenerationJob } from './jobs/ThumbnailGenerationJob';
import { TempFolderCleaningJob } from './jobs/TempFolderCleaningJob';
import { PreviewFillingJob } from './jobs/PreviewFillingJob';
import { PreviewRestJob } from './jobs/PreviewResetJob';
import {GPXCompressionJob} from './jobs/GPXCompressionJob';
export class JobRepository {
private static instance: JobRepository = null;
@ -38,4 +39,5 @@ JobRepository.Instance.register(new PreviewRestJob());
JobRepository.Instance.register(new VideoConvertingJob());
JobRepository.Instance.register(new PhotoConvertingJob());
JobRepository.Instance.register(new ThumbnailGenerationJob());
JobRepository.Instance.register(new GPXCompressionJob());
JobRepository.Instance.register(new TempFolderCleaningJob());

View File

@ -1,27 +1,27 @@
import { ConfigTemplateEntry } from '../../../../common/entities/job/JobDTO';
import { Job } from './Job';
import {ConfigTemplateEntry} from '../../../../common/entities/job/JobDTO';
import {Job} from './Job';
import * as path from 'path';
import { DiskManager } from '../../DiskManger';
import { DirectoryScanSettings } from '../../threading/DiskMangerWorker';
import { Logger } from '../../../Logger';
import { Config } from '../../../../common/config/private/Config';
import { FileDTO } from '../../../../common/entities/FileDTO';
import { SQLConnection } from '../../database/sql/SQLConnection';
import { MediaEntity } from '../../database/sql/enitites/MediaEntity';
import { PhotoEntity } from '../../database/sql/enitites/PhotoEntity';
import { VideoEntity } from '../../database/sql/enitites/VideoEntity';
import { backendTexts } from '../../../../common/BackendTexts';
import { ProjectPath } from '../../../ProjectPath';
import { DatabaseType } from '../../../../common/config/private/PrivateConfig';
import {DiskManager} from '../../DiskManger';
import {DirectoryScanSettings} from '../../threading/DiskMangerWorker';
import {Logger} from '../../../Logger';
import {Config} from '../../../../common/config/private/Config';
import {FileDTO} from '../../../../common/entities/FileDTO';
import {SQLConnection} from '../../database/sql/SQLConnection';
import {MediaEntity} from '../../database/sql/enitites/MediaEntity';
import {PhotoEntity} from '../../database/sql/enitites/PhotoEntity';
import {VideoEntity} from '../../database/sql/enitites/VideoEntity';
import {backendTexts} from '../../../../common/BackendTexts';
import {ProjectPath} from '../../../ProjectPath';
import {DatabaseType} from '../../../../common/config/private/PrivateConfig';
import {FileEntity} from '../../database/sql/enitites/FileEntity';
import {DirectoryBaseDTO, DirectoryDTOUtils} from '../../../../common/entities/DirectoryDTO';
const LOG_TAG = '[FileJob]';
/**
* Abstract class for thumbnail creation, file deleting etc.
*/
export abstract class FileJob<
S extends { indexedOnly: boolean } = { indexedOnly: boolean }
> extends Job<S> {
export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly: boolean }> extends Job<S> {
public readonly ConfigTemplate: ConfigTemplateEntry[] = [];
directoryQueue: string[] = [];
fileQueue: string[] = [];
@ -109,6 +109,7 @@ export abstract class FileJob<
for (const item of scanned.directories) {
this.directoryQueue.push(path.join(item.path, item.name));
}
DirectoryDTOUtils.addReferences(scanned as DirectoryBaseDTO);
if (this.scanFilter.noPhoto !== true || this.scanFilter.noVideo !== true) {
const scannedAndFiltered = await this.filterMediaFiles(scanned.media);
for (const item of scannedAndFiltered) {
@ -138,38 +139,67 @@ export abstract class FileJob<
}
private async loadAllMediaFilesFromDB(): Promise<void> {
if (this.scanFilter.noVideo === true && this.scanFilter.noPhoto === true) {
if (this.scanFilter.noVideo === true &&
this.scanFilter.noPhoto === true &&
this.scanFilter.noMetaFile === true) {
return;
}
this.Progress.log('Loading files from db');
Logger.silly(LOG_TAG, 'Loading files from db');
const connection = await SQLConnection.getConnection();
if (this.scanFilter.noVideo === false ||
this.scanFilter.noPhoto === false) {
let usedEntity = MediaEntity;
let usedEntity = MediaEntity;
if (this.scanFilter.noVideo === true) {
usedEntity = PhotoEntity;
} else if (this.scanFilter.noPhoto === true) {
usedEntity = VideoEntity;
if (this.scanFilter.noVideo === true) {
usedEntity = PhotoEntity;
} else if (this.scanFilter.noPhoto === true) {
usedEntity = VideoEntity;
}
const result = await connection
.getRepository(usedEntity)
.createQueryBuilder('media')
.select(['media.name', 'directory.name', 'directory.path'])
.leftJoin('media.directory', 'directory')
.getMany();
const scannedAndFiltered = await this.filterMediaFiles(result);
for (const item of scannedAndFiltered) {
this.fileQueue.push(
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
);
}
}
if (this.scanFilter.noMetaFile === false) {
const result = await connection
.getRepository(usedEntity)
.createQueryBuilder('media')
.select(['media.name', 'media.id'])
.leftJoinAndSelect('media.directory', 'directory')
.getMany();
const result = await connection
.getRepository(FileEntity)
.createQueryBuilder('file')
.select(['file.name', 'directory.name', 'directory.path'])
.leftJoin('file.directory', 'directory')
.getMany();
for (const item of result) {
this.fileQueue.push(
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
);
const scannedAndFiltered = await this.filterMetaFiles(result);
for (const item of scannedAndFiltered) {
this.fileQueue.push(
path.join(
ProjectPath.ImageFolder,
item.directory.path,
item.directory.name,
item.name
)
);
}
}
}
}

View File

@ -0,0 +1,37 @@
import {Config} from '../../../../common/config/private/Config';
import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
import {FileJob} from './FileJob';
import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
import {GPXProcessing} from '../../GPXProcessing';
import {FileDTO} from '../../../../common/entities/FileDTO';
import {Logger} from '../../../Logger';
export class GPXCompressionJob extends FileJob {
public readonly Name = DefaultsJobs[DefaultsJobs['GPX Compression']];
constructor() {
super({noVideo: true, noPhoto: true, noMetaFile: false});
}
protected async filterMetaFiles(files: FileDTO[]): Promise<FileDTO[]> {
return files.filter(file => file.name.toLowerCase().endsWith('.gpx'));
}
public get Supported(): boolean {
return Config.Client.MetaFile.GPXCompressing.enabled === true;
}
protected async shouldProcess(fPath: string): Promise<boolean> {
return !(await GPXProcessing.compressedGPXExist(
fPath
));
}
protected async processFile(fPath: string): Promise<void> {
try {
await GPXProcessing.compressGPX(fPath);
} catch (e) {
Logger.warn('GPXCompressionJob', ' Could not compress gpx at: ' + fPath);
}
}
}

View File

@ -133,7 +133,7 @@ export class DiskMangerWorker {
// nothing to scan, we are here for the empty dir
if (
settings.noPhoto === true &&
settings.noMetadata === true &&
settings.noMetaFile === true &&
settings.noVideo === true
) {
return directory;
@ -270,6 +270,6 @@ export interface DirectoryScanSettings {
noVideo?: boolean;
noPhoto?: boolean;
noDirectory?: boolean;
noMetadata?: boolean;
noMetadata?: boolean; // skip parsing images for metadata like exif, iptc
noChildDirPhotos?: boolean;
}

View File

@ -11,6 +11,7 @@ export enum DefaultsJobs {
'Temp Folder Cleaning' = 6,
'Preview Filling' = 7,
'Preview Reset' = 8,
'GPX Compression' = 9,
}
export interface ConfigTemplateEntry {

View File

@ -31,8 +31,6 @@ export class BackendtextService {
switch (job as DefaultsJobs) {
case DefaultsJobs.Indexing:
return $localize`Indexing`;
case DefaultsJobs['Preview Filling']:
return $localize`Preview Filling`;
case DefaultsJobs['Database Reset']:
return $localize`Database Reset`;
case DefaultsJobs['Thumbnail Generation']:
@ -43,6 +41,12 @@ export class BackendtextService {
return $localize`Video Converting`;
case DefaultsJobs['Temp Folder Cleaning']:
return $localize`Temp Folder Cleaning`;
case DefaultsJobs['Preview Filling']:
return $localize`Preview Filling`;
case DefaultsJobs['Preview Reset']:
return $localize`Preview Reset`;
case DefaultsJobs['GPX Compression']:
return $localize`GPX Compression`;
default:
return DefaultsJobs[job as DefaultsJobs];
}

View File

@ -95,7 +95,7 @@ export class MapService {
file.name
);
const gpx = await this.networkService.getXML(
'/gallery/content/' + filePath
'/gallery/content/' + filePath + '/bestFit'
);
const getCoordinates = (tagName: string): LatLngLiteral[] => {
const elements = gpx.getElementsByTagName(tagName);

View File

@ -72,8 +72,8 @@
<!-- Modal -->
<ng-template #template>
<div class="modal-header">
<h4 class="modal-title pull-left">{{Name}}</h4>
<button type="button" class="close pull-right" aria-label="Close" (click)="modalRef.hide()">
<h5 class="modal-title" >{{Name}}</h5>
<button type="button" class="btn-close" (click)="modalRef.hide()" data-dismiss="modal" aria-label="Close">
</button>
</div>
<div class="modal-body">

View File

@ -7,6 +7,22 @@
<div [hidden]="!error" class="alert alert-danger" role="alert"><strong>Error: </strong>{{error}}</div>
<app-settings-entry
name="Markdown files"
description="Reads *.md files in a directory and shows the next to the map."
i18n-description i18n-name
[ngModel]="states.client.markdown">
</app-settings-entry>
<app-settings-entry
name="*.pg2conf files"
description="Reads *.pg2conf files (You can use it for custom sorting and save search (albums))."
i18n-description i18n-name
[ngModel]="states.client.pg2conf">
</app-settings-entry>
<hr/>
<app-settings-entry
name="*.gpx files"
description="Reads *.gpx files and renders them on the map."
@ -34,21 +50,6 @@
[ngModel]="states.server.GPXCompressing.onTheFly">
</app-settings-entry>
<hr/>
<app-settings-entry
name="Markdown files"
description="Reads *.md files in a directory and shows the next to the map."
i18n-description i18n-name
[ngModel]="states.client.markdown">
</app-settings-entry>
<app-settings-entry
name="*.pg2conf files"
description="Reads *.pg2conf files (You can use it for custom sorting and save search (albums))."
i18n-description i18n-name
[ngModel]="states.client.pg2conf">
</app-settings-entry>
<button class="btn btn-success float-end"
@ -59,6 +60,21 @@
[disabled]=" !changed || inProgress"
(click)="reset()" i18n>Reset
</button>
<div [hidden]="!states.client.GPXCompressing.enabled.value || !states.client.gpx.value">
<app-settings-job-button class="mt-2 mt-md-0 float-left"
[soloRun]="true"
(jobError)="error=$event"
[allowParallelRun]="false"
[jobName]="jobName"></app-settings-job-button>
<ng-container *ngIf="Progress != null">
<br/>
<hr/>
<app-settings-job-progress [progress]="Progress"></app-settings-job-progress>
</ng-container>
</div>
</div>
</div>

View File

@ -6,6 +6,9 @@ import { NavigationService } from '../../../model/navigation.service';
import { NotificationService } from '../../../model/notification.service';
import {ClientMetaFileConfig, ClientPhotoConfig} from '../../../../../common/config/public/ClientConfig';
import {ServerMetaFileConfig, ServerPhotoConfig} from '../../../../../common/config/private/PrivateConfig';
import {DefaultsJobs, JobDTOUtils} from '../../../../../common/entities/job/JobDTO';
import {JobProgressDTO, JobProgressStates} from '../../../../../common/entities/job/JobProgressDTO';
import {ScheduledJobsService} from '../scheduled-jobs.service';
@Component({
selector: 'app-settings-meta-file',
@ -24,7 +27,8 @@ export class MetaFileSettingsComponent extends SettingsComponentDirective<{
authService: AuthenticationService,
navigation: NavigationService,
settingsService: MetaFileSettingsService,
notification: NotificationService
notification: NotificationService,
public jobsService: ScheduledJobsService,
) {
super(
$localize`Meta file`,
@ -39,6 +43,17 @@ export class MetaFileSettingsComponent extends SettingsComponentDirective<{
})
);
}
readonly jobName = DefaultsJobs[DefaultsJobs['GPX Compression']];
get Progress(): JobProgressDTO {
return this.jobsService.progress.value[
JobDTOUtils.getHashName(DefaultsJobs[DefaultsJobs['GPX Compression']])
];
}
}

View File

@ -31,7 +31,6 @@ export class PhotoSettingsComponent extends SettingsComponentDirective<{
}> {
readonly resolutionTypes = [720, 1080, 1440, 2160, 4320];
resolutions: { key: number; value: string }[] = [];
JobProgressStates = JobProgressStates;
readonly jobName = DefaultsJobs[DefaultsJobs['Photo Converting']];