self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2024-12-23 01:27:14 +02:00
Code Issues Releases Activity

Improving graceful degradation during indexing #704

Browse Source
This commit is contained in:
Patrik J. Braun 2023-09-02 14:17:35 +02:00
parent 930ea4f77a
commit 8a6b142229
3 changed files with 364 additions and 329 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
src/backend/model/jobs/jobs/IndexingJob.ts
View File

@ -11,6 +11,8 @@ import {ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO';
import {Logger} from '../../../Logger'; import {Logger} from '../../../Logger';
import {FileDTO} from '../../../../common/entities/FileDTO'; import {FileDTO} from '../../../../common/entities/FileDTO';
const LOG_TAG = '[IndexingJob]';
export class IndexingJob< export class IndexingJob<
S extends { indexChangesOnly: boolean } = { indexChangesOnly: boolean } S extends { indexChangesOnly: boolean } = { indexChangesOnly: boolean }
> extends Job<S> { > extends Job<S> {
@ -48,33 +50,50 @@ export class IndexingJob<
let scanned: ParentDirectoryDTO<FileDTO>; let scanned: ParentDirectoryDTO<FileDTO>;
let dirChanged = true; let dirChanged = true;
// check if the folder got modified if only changes need to be indexed try {
if (this.config.indexChangesOnly) {
const stat = fs.statSync(path.join(ProjectPath.ImageFolder, directory));
const lastModified = DiskMangerWorker.calcLastModified(stat);
scanned = await ObjectManagers.getInstance().GalleryManager.selectDirStructure(directory);
// If not modified and it was scanned before, dir is up-to-date
if (
scanned &&
scanned.lastModified === lastModified &&
scanned.lastScanned != null
) {
dirChanged = false;
}
}
// reindex const absDirPath = path.join(ProjectPath.ImageFolder, directory);
if (dirChanged || !this.config.indexChangesOnly) { if (!fs.existsSync(absDirPath)) {
this.Progress.log('Indexing: ' + directory); this.Progress.log('Skipping. Directory does not exist: ' + directory);
this.Progress.Processed++; this.Progress.Skipped++;
scanned = } else { // dir should exist now
await ObjectManagers.getInstance().IndexingManager.indexDirectory(
directory // check if the folder got modified if only changes need to be indexed
); if (this.config.indexChangesOnly) {
} else {
this.Progress.log('Skipped: ' + directory); const stat = fs.statSync(absDirPath);
const lastModified = DiskMangerWorker.calcLastModified(stat);
scanned = await ObjectManagers.getInstance().GalleryManager.selectDirStructure(directory);
// If not modified and it was scanned before, dir is up-to-date
if (
scanned &&
scanned.lastModified === lastModified &&
scanned.lastScanned != null
) {
dirChanged = false;
}
}
// reindex
if (dirChanged || !this.config.indexChangesOnly) {
this.Progress.log('Indexing: ' + directory);
this.Progress.Processed++;
scanned =
await ObjectManagers.getInstance().IndexingManager.indexDirectory(
directory
);
} else {
this.Progress.log('Skipping. No change for: ' + directory);
this.Progress.Skipped++;
Logger.silly(LOG_TAG, 'Skipping reindexing, no change for: ' + directory);
}
}
} catch (e) {
this.Progress.log('Skipping. Indexing failed for: ' + directory);
this.Progress.Skipped++; this.Progress.Skipped++;
Logger.silly('Skipping reindexing, no change for: ' + directory); Logger.warn(LOG_TAG, 'Skipping. Indexing failed for: ' + directory);
console.error(e);
} }
if (this.Progress.State !== JobProgressStates.running) { if (this.Progress.State !== JobProgressStates.running) {
return false; return false;

1
src/backend/model/jobs/jobs/Job.ts
View File

@ -153,6 +153,7 @@ export abstract class Job<T extends Record<string, any> = Record<string, any>> i
await new Promise(setImmediate); await new Promise(setImmediate);
this.run(); this.run();
} catch (e) { } catch (e) {
Logger.error(LOG_TAG, 'Job failed with:');
Logger.error(LOG_TAG, e); Logger.error(LOG_TAG, e);
this.Progress.log('Failed with: ' + (typeof e.toString === 'function') ? e.toString() : JSON.stringify(e)); this.Progress.log('Failed with: ' + (typeof e.toString === 'function') ? e.toString() : JSON.stringify(e));
this.Progress.State = JobProgressStates.failed; this.Progress.State = JobProgressStates.failed;

623
src/backend/model/threading/MetadataLoader.ts
View File

@ -92,334 +92,349 @@ export class MetadataLoader {
}); });
} }
private static readonly EMPTY_METADATA: PhotoMetadata = {
size: {width: 1, height: 1},
creationDate: 0,
fileSize: 0,
};
public static loadPhotoMetadata(fullPath: string): Promise<PhotoMetadata> { public static loadPhotoMetadata(fullPath: string): Promise<PhotoMetadata> {
return new Promise<PhotoMetadata>((resolve, reject) => { return new Promise<PhotoMetadata>((resolve, reject) => {
const fd = fs.openSync(fullPath, 'r'); try {
const fd = fs.openSync(fullPath, 'r');
const data = Buffer.allocUnsafe(Config.Media.photoMetadataSize); const data = Buffer.allocUnsafe(Config.Media.photoMetadataSize);
fs.read(fd, data, 0, Config.Media.photoMetadataSize, 0, (err) => { fs.read(fd, data, 0, Config.Media.photoMetadataSize, 0, (err) => {
fs.closeSync(fd); fs.closeSync(fd);
if (err) { const metadata: PhotoMetadata = {
return reject({file: fullPath, error: err}); size: {width: 1, height: 1},
} creationDate: 0,
const metadata: PhotoMetadata = { fileSize: 0,
size: {width: 1, height: 1}, };
creationDate: 0, if (err) {
fileSize: 0, Logger.error(LOG_TAG, 'Error during reading photo: ' + fullPath);
}; console.error(err);
try { return resolve(MetadataLoader.EMPTY_METADATA);
try {
const stat = fs.statSync(fullPath);
metadata.fileSize = stat.size;
metadata.creationDate = stat.mtime.getTime();
} catch (err) {
// ignoring errors
} }
try { try {
const exif = ExifParserFactory.create(data).parse();
if (
exif.tags.ISO ||
exif.tags.Model ||
exif.tags.Make ||
exif.tags.FNumber ||
exif.tags.ExposureTime ||
exif.tags.FocalLength ||
exif.tags.LensModel
) {
if (exif.tags.Model && exif.tags.Model !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.model = '' + exif.tags.Model;
}
if (exif.tags.Make && exif.tags.Make !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.make = '' + exif.tags.Make;
}
if (exif.tags.LensModel && exif.tags.LensModel !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.lens = '' + exif.tags.LensModel;
}
if (Utils.isUInt32(exif.tags.ISO)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.ISO = parseInt('' + exif.tags.ISO, 10);
}
if (Utils.isFloat32(exif.tags.FocalLength)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.focalLength = parseFloat(
'' + exif.tags.FocalLength
);
}
if (Utils.isFloat32(exif.tags.ExposureTime)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.exposure = parseFloat(
parseFloat('' + exif.tags.ExposureTime).toFixed(6)
);
}
if (Utils.isFloat32(exif.tags.FNumber)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.fStop = parseFloat(
parseFloat('' + exif.tags.FNumber).toFixed(2)
);
}
}
if (
!isNaN(exif.tags.GPSLatitude) ||
exif.tags.GPSLongitude ||
exif.tags.GPSAltitude
) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.GPSData = {};
if (Utils.isFloat32(exif.tags.GPSLongitude)) {
metadata.positionData.GPSData.longitude = parseFloat(
exif.tags.GPSLongitude.toFixed(6)
);
}
if (Utils.isFloat32(exif.tags.GPSLatitude)) {
metadata.positionData.GPSData.latitude = parseFloat(
exif.tags.GPSLatitude.toFixed(6)
);
}
}
if (
exif.tags.CreateDate ||
exif.tags.DateTimeOriginal ||
exif.tags.ModifyDate
) {
metadata.creationDate =
(exif.tags.DateTimeOriginal ||
exif.tags.CreateDate ||
exif.tags.ModifyDate) * 1000;
}
if (exif.imageSize) {
metadata.size = {
width: exif.imageSize.width,
height: exif.imageSize.height,
};
} else if (
exif.tags.RelatedImageWidth &&
exif.tags.RelatedImageHeight
) {
metadata.size = {
width: exif.tags.RelatedImageWidth,
height: exif.tags.RelatedImageHeight,
};
} else if (
exif.tags.ImageWidth &&
exif.tags.ImageHeight
) {
metadata.size = {
width: exif.tags.ImageWidth,
height: exif.tags.ImageHeight,
};
} else {
const info = imageSize(fullPath);
metadata.size = {width: info.width, height: info.height};
}
} catch (err) {
Logger.debug(LOG_TAG, 'Error parsing exif', fullPath, err);
try { try {
const info = imageSize(fullPath); const stat = fs.statSync(fullPath);
metadata.size = {width: info.width, height: info.height}; metadata.fileSize = stat.size;
} catch (e) { metadata.creationDate = stat.mtime.getTime();
metadata.size = {width: 1, height: 1}; } catch (err) {
// ignoring errors
} }
}
try { try {
const iptcData = IptcParser.parse(data); const exif = ExifParserFactory.create(data).parse();
if (iptcData.country_or_primary_location_name) { if (
metadata.positionData = metadata.positionData || {}; exif.tags.ISO ||
metadata.positionData.country = exif.tags.Model ||
iptcData.country_or_primary_location_name exif.tags.Make ||
exif.tags.FNumber ||
exif.tags.ExposureTime ||
exif.tags.FocalLength ||
exif.tags.LensModel
) {
if (exif.tags.Model && exif.tags.Model !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.model = '' + exif.tags.Model;
}
if (exif.tags.Make && exif.tags.Make !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.make = '' + exif.tags.Make;
}
if (exif.tags.LensModel && exif.tags.LensModel !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.lens = '' + exif.tags.LensModel;
}
if (Utils.isUInt32(exif.tags.ISO)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.ISO = parseInt('' + exif.tags.ISO, 10);
}
if (Utils.isFloat32(exif.tags.FocalLength)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.focalLength = parseFloat(
'' + exif.tags.FocalLength
);
}
if (Utils.isFloat32(exif.tags.ExposureTime)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.exposure = parseFloat(
parseFloat('' + exif.tags.ExposureTime).toFixed(6)
);
}
if (Utils.isFloat32(exif.tags.FNumber)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.fStop = parseFloat(
parseFloat('' + exif.tags.FNumber).toFixed(2)
);
}
}
if (
!isNaN(exif.tags.GPSLatitude) ||
exif.tags.GPSLongitude ||
exif.tags.GPSAltitude
) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.GPSData = {};
if (Utils.isFloat32(exif.tags.GPSLongitude)) {
metadata.positionData.GPSData.longitude = parseFloat(
exif.tags.GPSLongitude.toFixed(6)
);
}
if (Utils.isFloat32(exif.tags.GPSLatitude)) {
metadata.positionData.GPSData.latitude = parseFloat(
exif.tags.GPSLatitude.toFixed(6)
);
}
}
if (
exif.tags.CreateDate ||
exif.tags.DateTimeOriginal ||
exif.tags.ModifyDate
) {
metadata.creationDate =
(exif.tags.DateTimeOriginal ||
exif.tags.CreateDate ||
exif.tags.ModifyDate) * 1000;
}
if (exif.imageSize) {
metadata.size = {
width: exif.imageSize.width,
height: exif.imageSize.height,
};
} else if (
exif.tags.RelatedImageWidth &&
exif.tags.RelatedImageHeight
) {
metadata.size = {
width: exif.tags.RelatedImageWidth,
height: exif.tags.RelatedImageHeight,
};
} else if (
exif.tags.ImageWidth &&
exif.tags.ImageHeight
) {
metadata.size = {
width: exif.tags.ImageWidth,
height: exif.tags.ImageHeight,
};
} else {
const info = imageSize(fullPath);
metadata.size = {width: info.width, height: info.height};
}
} catch (err) {
Logger.debug(LOG_TAG, 'Error parsing exif', fullPath, err);
try {
const info = imageSize(fullPath);
metadata.size = {width: info.width, height: info.height};
} catch (e) {
metadata.size = {width: 1, height: 1};
}
}
try {
const iptcData = IptcParser.parse(data);
if (iptcData.country_or_primary_location_name) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.country =
iptcData.country_or_primary_location_name
.replace(/\0/g, '')
.trim();
}
if (iptcData.province_or_state) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.state = iptcData.province_or_state
.replace(/\0/g, '') .replace(/\0/g, '')
.trim(); .trim();
}
if (iptcData.province_or_state) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.state = iptcData.province_or_state
.replace(/\0/g, '')
.trim();
}
if (iptcData.city) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.city = iptcData.city
.replace(/\0/g, '')
.trim();
}
if (iptcData.caption) {
metadata.caption = iptcData.caption.replace(/\0/g, '').trim();
}
if (Array.isArray(iptcData.keywords)) {
metadata.keywords = iptcData.keywords;
}
if (iptcData.date_time) {
metadata.creationDate = iptcData.date_time.getTime();
}
} catch (err) {
// Logger.debug(LOG_TAG, 'Error parsing iptc data', fullPath, err);
}
if (!metadata.creationDate) {
// creationDate can be negative, when it was created before epoch (1970)
metadata.creationDate = 0;
}
try {
// TODO: clean up the three different exif readers,
// and keep the minimum amount only
const exif: ExifReader.Tags &ExifReader.XmpTags & ExifReader.IccTags = ExifReader.load(data);
if (exif.Rating) {
metadata.rating = parseInt(exif.Rating.value as string, 10) as 0 | 1 | 2 | 3 | 4 | 5;
if (metadata.rating < 0) {
metadata.rating = 0;
} }
} if (iptcData.city) {
if ( metadata.positionData = metadata.positionData || {};
exif.subject && metadata.positionData.city = iptcData.city
exif.subject.value && .replace(/\0/g, '')
exif.subject.value.length > 0 .trim();
) {
if (metadata.keywords === undefined) {
metadata.keywords = [];
} }
for (const kw of exif.subject.value as ExifReader.XmpTag[]) { if (iptcData.caption) {
if (metadata.keywords.indexOf(kw.description) === -1) { metadata.caption = iptcData.caption.replace(/\0/g, '').trim();
metadata.keywords.push(kw.description); }
if (Array.isArray(iptcData.keywords)) {
metadata.keywords = iptcData.keywords;
}
if (iptcData.date_time) {
metadata.creationDate = iptcData.date_time.getTime();
}
} catch (err) {
// Logger.debug(LOG_TAG, 'Error parsing iptc data', fullPath, err);
}
if (!metadata.creationDate) {
// creationDate can be negative, when it was created before epoch (1970)
metadata.creationDate = 0;
}
try {
// TODO: clean up the three different exif readers,
// and keep the minimum amount only
const exif: ExifReader.Tags & ExifReader.XmpTags & ExifReader.IccTags = ExifReader.load(data);
if (exif.Rating) {
metadata.rating = parseInt(exif.Rating.value as string, 10) as 0 | 1 | 2 | 3 | 4 | 5;
if (metadata.rating < 0) {
metadata.rating = 0;
} }
} }
}
let orientation = OrientationTypes.TOP_LEFT;
if (exif.Orientation) {
orientation = parseInt(
exif.Orientation.value as any,
10
) as number;
}
if (OrientationTypes.BOTTOM_LEFT < orientation) {
// noinspection JSSuspiciousNameCombination
const height = metadata.size.width;
// noinspection JSSuspiciousNameCombination
metadata.size.width = metadata.size.height;
metadata.size.height = height;
}
if (Config.Faces.enabled) {
const faces: FaceRegion[] = [];
if ( if (
((exif.Regions as any)?.value?.RegionList)?.value exif.subject &&
exif.subject.value &&
exif.subject.value.length > 0
) { ) {
for (const regionRoot of (exif.Regions as any).value.RegionList if (metadata.keywords === undefined) {
.value as any[]) { metadata.keywords = [];
let type; }
let name; for (const kw of exif.subject.value as ExifReader.XmpTag[]) {
let box; if (metadata.keywords.indexOf(kw.description) === -1) {
const createFaceBox = ( metadata.keywords.push(kw.description);
w: string, }
h: string, }
x: string, }
y: string let orientation = OrientationTypes.TOP_LEFT;
) => { if (exif.Orientation) {
if (OrientationTypes.BOTTOM_LEFT < orientation) { orientation = parseInt(
[x, y] = [y, x]; exif.Orientation.value as any,
[w, h] = [h, w]; 10
} ) as number;
let swapX = 0; }
let swapY = 0; if (OrientationTypes.BOTTOM_LEFT < orientation) {
switch (orientation) { // noinspection JSSuspiciousNameCombination
case OrientationTypes.TOP_RIGHT: const height = metadata.size.width;
case OrientationTypes.RIGHT_TOP: // noinspection JSSuspiciousNameCombination
swapX = 1; metadata.size.width = metadata.size.height;
break; metadata.size.height = height;
case OrientationTypes.BOTTOM_RIGHT: }
case OrientationTypes.RIGHT_BOTTOM:
swapX = 1; if (Config.Faces.enabled) {
swapY = 1; const faces: FaceRegion[] = [];
break; if (
case OrientationTypes.BOTTOM_LEFT: ((exif.Regions as any)?.value?.RegionList)?.value
case OrientationTypes.LEFT_BOTTOM: ) {
swapY = 1; for (const regionRoot of (exif.Regions as any).value.RegionList
break; .value as any[]) {
} let type;
// converting ratio to px let name;
return { let box;
width: Math.round(parseFloat(w) * metadata.size.width), const createFaceBox = (
height: Math.round(parseFloat(h) * metadata.size.height), w: string,
left: Math.round(Math.abs(parseFloat(x) - swapX) * metadata.size.width), h: string,
top: Math.round(Math.abs(parseFloat(y) - swapY) * metadata.size.height), x: string,
y: string
) => {
if (OrientationTypes.BOTTOM_LEFT < orientation) {
[x, y] = [y, x];
[w, h] = [h, w];
}
let swapX = 0;
let swapY = 0;
switch (orientation) {
case OrientationTypes.TOP_RIGHT:
case OrientationTypes.RIGHT_TOP:
swapX = 1;
break;
case OrientationTypes.BOTTOM_RIGHT:
case OrientationTypes.RIGHT_BOTTOM:
swapX = 1;
swapY = 1;
break;
case OrientationTypes.BOTTOM_LEFT:
case OrientationTypes.LEFT_BOTTOM:
swapY = 1;
break;
}
// converting ratio to px
return {
width: Math.round(parseFloat(w) * metadata.size.width),
height: Math.round(parseFloat(h) * metadata.size.height),
left: Math.round(Math.abs(parseFloat(x) - swapX) * metadata.size.width),
top: Math.round(Math.abs(parseFloat(y) - swapY) * metadata.size.height),
};
}; };
};
/* Adobe Lightroom based face region structure */ /* Adobe Lightroom based face region structure */
if ( if (
regionRoot.value && regionRoot.value &&
regionRoot.value['rdf:Description'] && regionRoot.value['rdf:Description'] &&
regionRoot.value['rdf:Description'].value && regionRoot.value['rdf:Description'].value &&
regionRoot.value['rdf:Description'].value['mwg-rs:Area'] regionRoot.value['rdf:Description'].value['mwg-rs:Area']
) { ) {
const region = regionRoot.value['rdf:Description']; const region = regionRoot.value['rdf:Description'];
const regionBox = region.value['mwg-rs:Area'].attributes; const regionBox = region.value['mwg-rs:Area'].attributes;
name = region.attributes['mwg-rs:Name']; name = region.attributes['mwg-rs:Name'];
type = region.attributes['mwg-rs:Type']; type = region.attributes['mwg-rs:Type'];
box = createFaceBox( box = createFaceBox(
regionBox['stArea:w'], regionBox['stArea:w'],
regionBox['stArea:h'], regionBox['stArea:h'],
regionBox['stArea:x'], regionBox['stArea:x'],
regionBox['stArea:y'] regionBox['stArea:y']
); );
/* Load exiftool edited face region structure, see github issue #191 */ /* Load exiftool edited face region structure, see github issue #191 */
} else if ( } else if (
regionRoot.Area && regionRoot.Area &&
regionRoot.Name && regionRoot.Name &&
regionRoot.Type regionRoot.Type
) { ) {
const regionBox = regionRoot.Area.value; const regionBox = regionRoot.Area.value;
name = regionRoot.Name.value; name = regionRoot.Name.value;
type = regionRoot.Type.value; type = regionRoot.Type.value;
box = createFaceBox( box = createFaceBox(
regionBox.w.value, regionBox.w.value,
regionBox.h.value, regionBox.h.value,
regionBox.x.value, regionBox.x.value,
regionBox.y.value regionBox.y.value
); );
}
if (type !== 'Face' || !name) {
continue;
}
// convert center base box to corner based box
box.left = Math.round(Math.max(0, box.left - box.width / 2));
box.top = Math.round(Math.max(0, box.top - box.height / 2));
faces.push({name, box});
}
}
if (faces.length > 0) {
metadata.faces = faces; // save faces
if (Config.Faces.keywordsToPersons) {
// remove faces from keywords
metadata.faces.forEach((f) => {
const index = metadata.keywords.indexOf(f.name);
if (index !== -1) {
metadata.keywords.splice(index, 1);
} }
});
if (type !== 'Face' || !name) {
continue;
}
// convert center base box to corner based box
box.left = Math.round(Math.max(0, box.left - box.width / 2));
box.top = Math.round(Math.max(0, box.top - box.height / 2));
faces.push({name, box});
}
}
if (faces.length > 0) {
metadata.faces = faces; // save faces
if (Config.Faces.keywordsToPersons) {
// remove faces from keywords
metadata.faces.forEach((f) => {
const index = metadata.keywords.indexOf(f.name);
if (index !== -1) {
metadata.keywords.splice(index, 1);
}
});
}
} }
} }
} catch (err) {
// ignoring errors
} }
} catch (err) {
// ignoring errors
}
return resolve(metadata); return resolve(metadata);
} catch (err) { } catch (err) {
return reject({file: fullPath, error: err}); return reject({file: fullPath, error: err});
} }
}); });
} catch (err) {
Logger.error(LOG_TAG, 'Error during reading photo: ' + fullPath);
console.error(err);
return resolve(MetadataLoader.EMPTY_METADATA);
}
}); });
} }
} }