1
0
mirror of https://github.com/bpatrik/pigallery2.git synced 2024-12-23 01:27:14 +02:00

Improving graceful degradation during indexing #704

This commit is contained in:
Patrik J. Braun 2023-09-02 14:17:35 +02:00
parent 930ea4f77a
commit 8a6b142229
3 changed files with 364 additions and 329 deletions

View File

@ -11,6 +11,8 @@ import {ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO';
import {Logger} from '../../../Logger';
import {FileDTO} from '../../../../common/entities/FileDTO';
const LOG_TAG = '[IndexingJob]';
export class IndexingJob<
S extends { indexChangesOnly: boolean } = { indexChangesOnly: boolean }
> extends Job<S> {
@ -48,33 +50,50 @@ export class IndexingJob<
let scanned: ParentDirectoryDTO<FileDTO>;
let dirChanged = true;
// check if the folder got modified if only changes need to be indexed
if (this.config.indexChangesOnly) {
const stat = fs.statSync(path.join(ProjectPath.ImageFolder, directory));
const lastModified = DiskMangerWorker.calcLastModified(stat);
scanned = await ObjectManagers.getInstance().GalleryManager.selectDirStructure(directory);
// If not modified and it was scanned before, dir is up-to-date
if (
scanned &&
scanned.lastModified === lastModified &&
scanned.lastScanned != null
) {
dirChanged = false;
}
}
try {
// reindex
if (dirChanged || !this.config.indexChangesOnly) {
this.Progress.log('Indexing: ' + directory);
this.Progress.Processed++;
scanned =
await ObjectManagers.getInstance().IndexingManager.indexDirectory(
directory
);
} else {
this.Progress.log('Skipped: ' + directory);
const absDirPath = path.join(ProjectPath.ImageFolder, directory);
if (!fs.existsSync(absDirPath)) {
this.Progress.log('Skipping. Directory does not exist: ' + directory);
this.Progress.Skipped++;
} else { // dir should exist now
// check if the folder got modified if only changes need to be indexed
if (this.config.indexChangesOnly) {
const stat = fs.statSync(absDirPath);
const lastModified = DiskMangerWorker.calcLastModified(stat);
scanned = await ObjectManagers.getInstance().GalleryManager.selectDirStructure(directory);
// If not modified and it was scanned before, dir is up-to-date
if (
scanned &&
scanned.lastModified === lastModified &&
scanned.lastScanned != null
) {
dirChanged = false;
}
}
// reindex
if (dirChanged || !this.config.indexChangesOnly) {
this.Progress.log('Indexing: ' + directory);
this.Progress.Processed++;
scanned =
await ObjectManagers.getInstance().IndexingManager.indexDirectory(
directory
);
} else {
this.Progress.log('Skipping. No change for: ' + directory);
this.Progress.Skipped++;
Logger.silly(LOG_TAG, 'Skipping reindexing, no change for: ' + directory);
}
}
} catch (e) {
this.Progress.log('Skipping. Indexing failed for: ' + directory);
this.Progress.Skipped++;
Logger.silly('Skipping reindexing, no change for: ' + directory);
Logger.warn(LOG_TAG, 'Skipping. Indexing failed for: ' + directory);
console.error(e);
}
if (this.Progress.State !== JobProgressStates.running) {
return false;

View File

@ -153,6 +153,7 @@ export abstract class Job<T extends Record<string, any> = Record<string, any>> i
await new Promise(setImmediate);
this.run();
} catch (e) {
Logger.error(LOG_TAG, 'Job failed with:');
Logger.error(LOG_TAG, e);
this.Progress.log('Failed with: ' + (typeof e.toString === 'function') ? e.toString() : JSON.stringify(e));
this.Progress.State = JobProgressStates.failed;

View File

@ -92,334 +92,349 @@ export class MetadataLoader {
});
}
private static readonly EMPTY_METADATA: PhotoMetadata = {
size: {width: 1, height: 1},
creationDate: 0,
fileSize: 0,
};
public static loadPhotoMetadata(fullPath: string): Promise<PhotoMetadata> {
return new Promise<PhotoMetadata>((resolve, reject) => {
const fd = fs.openSync(fullPath, 'r');
try {
const fd = fs.openSync(fullPath, 'r');
const data = Buffer.allocUnsafe(Config.Media.photoMetadataSize);
fs.read(fd, data, 0, Config.Media.photoMetadataSize, 0, (err) => {
fs.closeSync(fd);
if (err) {
return reject({file: fullPath, error: err});
}
const metadata: PhotoMetadata = {
size: {width: 1, height: 1},
creationDate: 0,
fileSize: 0,
};
try {
try {
const stat = fs.statSync(fullPath);
metadata.fileSize = stat.size;
metadata.creationDate = stat.mtime.getTime();
} catch (err) {
// ignoring errors
const data = Buffer.allocUnsafe(Config.Media.photoMetadataSize);
fs.read(fd, data, 0, Config.Media.photoMetadataSize, 0, (err) => {
fs.closeSync(fd);
const metadata: PhotoMetadata = {
size: {width: 1, height: 1},
creationDate: 0,
fileSize: 0,
};
if (err) {
Logger.error(LOG_TAG, 'Error during reading photo: ' + fullPath);
console.error(err);
return resolve(MetadataLoader.EMPTY_METADATA);
}
try {
const exif = ExifParserFactory.create(data).parse();
if (
exif.tags.ISO ||
exif.tags.Model ||
exif.tags.Make ||
exif.tags.FNumber ||
exif.tags.ExposureTime ||
exif.tags.FocalLength ||
exif.tags.LensModel
) {
if (exif.tags.Model && exif.tags.Model !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.model = '' + exif.tags.Model;
}
if (exif.tags.Make && exif.tags.Make !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.make = '' + exif.tags.Make;
}
if (exif.tags.LensModel && exif.tags.LensModel !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.lens = '' + exif.tags.LensModel;
}
if (Utils.isUInt32(exif.tags.ISO)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.ISO = parseInt('' + exif.tags.ISO, 10);
}
if (Utils.isFloat32(exif.tags.FocalLength)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.focalLength = parseFloat(
'' + exif.tags.FocalLength
);
}
if (Utils.isFloat32(exif.tags.ExposureTime)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.exposure = parseFloat(
parseFloat('' + exif.tags.ExposureTime).toFixed(6)
);
}
if (Utils.isFloat32(exif.tags.FNumber)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.fStop = parseFloat(
parseFloat('' + exif.tags.FNumber).toFixed(2)
);
}
}
if (
!isNaN(exif.tags.GPSLatitude) ||
exif.tags.GPSLongitude ||
exif.tags.GPSAltitude
) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.GPSData = {};
if (Utils.isFloat32(exif.tags.GPSLongitude)) {
metadata.positionData.GPSData.longitude = parseFloat(
exif.tags.GPSLongitude.toFixed(6)
);
}
if (Utils.isFloat32(exif.tags.GPSLatitude)) {
metadata.positionData.GPSData.latitude = parseFloat(
exif.tags.GPSLatitude.toFixed(6)
);
}
}
if (
exif.tags.CreateDate ||
exif.tags.DateTimeOriginal ||
exif.tags.ModifyDate
) {
metadata.creationDate =
(exif.tags.DateTimeOriginal ||
exif.tags.CreateDate ||
exif.tags.ModifyDate) * 1000;
}
if (exif.imageSize) {
metadata.size = {
width: exif.imageSize.width,
height: exif.imageSize.height,
};
} else if (
exif.tags.RelatedImageWidth &&
exif.tags.RelatedImageHeight
) {
metadata.size = {
width: exif.tags.RelatedImageWidth,
height: exif.tags.RelatedImageHeight,
};
} else if (
exif.tags.ImageWidth &&
exif.tags.ImageHeight
) {
metadata.size = {
width: exif.tags.ImageWidth,
height: exif.tags.ImageHeight,
};
} else {
const info = imageSize(fullPath);
metadata.size = {width: info.width, height: info.height};
}
} catch (err) {
Logger.debug(LOG_TAG, 'Error parsing exif', fullPath, err);
try {
const info = imageSize(fullPath);
metadata.size = {width: info.width, height: info.height};
} catch (e) {
metadata.size = {width: 1, height: 1};
const stat = fs.statSync(fullPath);
metadata.fileSize = stat.size;
metadata.creationDate = stat.mtime.getTime();
} catch (err) {
// ignoring errors
}
}
try {
const iptcData = IptcParser.parse(data);
if (iptcData.country_or_primary_location_name) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.country =
iptcData.country_or_primary_location_name
try {
const exif = ExifParserFactory.create(data).parse();
if (
exif.tags.ISO ||
exif.tags.Model ||
exif.tags.Make ||
exif.tags.FNumber ||
exif.tags.ExposureTime ||
exif.tags.FocalLength ||
exif.tags.LensModel
) {
if (exif.tags.Model && exif.tags.Model !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.model = '' + exif.tags.Model;
}
if (exif.tags.Make && exif.tags.Make !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.make = '' + exif.tags.Make;
}
if (exif.tags.LensModel && exif.tags.LensModel !== '') {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.lens = '' + exif.tags.LensModel;
}
if (Utils.isUInt32(exif.tags.ISO)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.ISO = parseInt('' + exif.tags.ISO, 10);
}
if (Utils.isFloat32(exif.tags.FocalLength)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.focalLength = parseFloat(
'' + exif.tags.FocalLength
);
}
if (Utils.isFloat32(exif.tags.ExposureTime)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.exposure = parseFloat(
parseFloat('' + exif.tags.ExposureTime).toFixed(6)
);
}
if (Utils.isFloat32(exif.tags.FNumber)) {
metadata.cameraData = metadata.cameraData || {};
metadata.cameraData.fStop = parseFloat(
parseFloat('' + exif.tags.FNumber).toFixed(2)
);
}
}
if (
!isNaN(exif.tags.GPSLatitude) ||
exif.tags.GPSLongitude ||
exif.tags.GPSAltitude
) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.GPSData = {};
if (Utils.isFloat32(exif.tags.GPSLongitude)) {
metadata.positionData.GPSData.longitude = parseFloat(
exif.tags.GPSLongitude.toFixed(6)
);
}
if (Utils.isFloat32(exif.tags.GPSLatitude)) {
metadata.positionData.GPSData.latitude = parseFloat(
exif.tags.GPSLatitude.toFixed(6)
);
}
}
if (
exif.tags.CreateDate ||
exif.tags.DateTimeOriginal ||
exif.tags.ModifyDate
) {
metadata.creationDate =
(exif.tags.DateTimeOriginal ||
exif.tags.CreateDate ||
exif.tags.ModifyDate) * 1000;
}
if (exif.imageSize) {
metadata.size = {
width: exif.imageSize.width,
height: exif.imageSize.height,
};
} else if (
exif.tags.RelatedImageWidth &&
exif.tags.RelatedImageHeight
) {
metadata.size = {
width: exif.tags.RelatedImageWidth,
height: exif.tags.RelatedImageHeight,
};
} else if (
exif.tags.ImageWidth &&
exif.tags.ImageHeight
) {
metadata.size = {
width: exif.tags.ImageWidth,
height: exif.tags.ImageHeight,
};
} else {
const info = imageSize(fullPath);
metadata.size = {width: info.width, height: info.height};
}
} catch (err) {
Logger.debug(LOG_TAG, 'Error parsing exif', fullPath, err);
try {
const info = imageSize(fullPath);
metadata.size = {width: info.width, height: info.height};
} catch (e) {
metadata.size = {width: 1, height: 1};
}
}
try {
const iptcData = IptcParser.parse(data);
if (iptcData.country_or_primary_location_name) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.country =
iptcData.country_or_primary_location_name
.replace(/\0/g, '')
.trim();
}
if (iptcData.province_or_state) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.state = iptcData.province_or_state
.replace(/\0/g, '')
.trim();
}
if (iptcData.province_or_state) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.state = iptcData.province_or_state
.replace(/\0/g, '')
.trim();
}
if (iptcData.city) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.city = iptcData.city
.replace(/\0/g, '')
.trim();
}
if (iptcData.caption) {
metadata.caption = iptcData.caption.replace(/\0/g, '').trim();
}
if (Array.isArray(iptcData.keywords)) {
metadata.keywords = iptcData.keywords;
}
if (iptcData.date_time) {
metadata.creationDate = iptcData.date_time.getTime();
}
} catch (err) {
// Logger.debug(LOG_TAG, 'Error parsing iptc data', fullPath, err);
}
if (!metadata.creationDate) {
// creationDate can be negative, when it was created before epoch (1970)
metadata.creationDate = 0;
}
try {
// TODO: clean up the three different exif readers,
// and keep the minimum amount only
const exif: ExifReader.Tags &ExifReader.XmpTags & ExifReader.IccTags = ExifReader.load(data);
if (exif.Rating) {
metadata.rating = parseInt(exif.Rating.value as string, 10) as 0 | 1 | 2 | 3 | 4 | 5;
if (metadata.rating < 0) {
metadata.rating = 0;
}
}
if (
exif.subject &&
exif.subject.value &&
exif.subject.value.length > 0
) {
if (metadata.keywords === undefined) {
metadata.keywords = [];
if (iptcData.city) {
metadata.positionData = metadata.positionData || {};
metadata.positionData.city = iptcData.city
.replace(/\0/g, '')
.trim();
}
for (const kw of exif.subject.value as ExifReader.XmpTag[]) {
if (metadata.keywords.indexOf(kw.description) === -1) {
metadata.keywords.push(kw.description);
if (iptcData.caption) {
metadata.caption = iptcData.caption.replace(/\0/g, '').trim();
}
if (Array.isArray(iptcData.keywords)) {
metadata.keywords = iptcData.keywords;
}
if (iptcData.date_time) {
metadata.creationDate = iptcData.date_time.getTime();
}
} catch (err) {
// Logger.debug(LOG_TAG, 'Error parsing iptc data', fullPath, err);
}
if (!metadata.creationDate) {
// creationDate can be negative, when it was created before epoch (1970)
metadata.creationDate = 0;
}
try {
// TODO: clean up the three different exif readers,
// and keep the minimum amount only
const exif: ExifReader.Tags & ExifReader.XmpTags & ExifReader.IccTags = ExifReader.load(data);
if (exif.Rating) {
metadata.rating = parseInt(exif.Rating.value as string, 10) as 0 | 1 | 2 | 3 | 4 | 5;
if (metadata.rating < 0) {
metadata.rating = 0;
}
}
}
let orientation = OrientationTypes.TOP_LEFT;
if (exif.Orientation) {
orientation = parseInt(
exif.Orientation.value as any,
10
) as number;
}
if (OrientationTypes.BOTTOM_LEFT < orientation) {
// noinspection JSSuspiciousNameCombination
const height = metadata.size.width;
// noinspection JSSuspiciousNameCombination
metadata.size.width = metadata.size.height;
metadata.size.height = height;
}
if (Config.Faces.enabled) {
const faces: FaceRegion[] = [];
if (
((exif.Regions as any)?.value?.RegionList)?.value
exif.subject &&
exif.subject.value &&
exif.subject.value.length > 0
) {
for (const regionRoot of (exif.Regions as any).value.RegionList
.value as any[]) {
let type;
let name;
let box;
const createFaceBox = (
w: string,
h: string,
x: string,
y: string
) => {
if (OrientationTypes.BOTTOM_LEFT < orientation) {
[x, y] = [y, x];
[w, h] = [h, w];
}
let swapX = 0;
let swapY = 0;
switch (orientation) {
case OrientationTypes.TOP_RIGHT:
case OrientationTypes.RIGHT_TOP:
swapX = 1;
break;
case OrientationTypes.BOTTOM_RIGHT:
case OrientationTypes.RIGHT_BOTTOM:
swapX = 1;
swapY = 1;
break;
case OrientationTypes.BOTTOM_LEFT:
case OrientationTypes.LEFT_BOTTOM:
swapY = 1;
break;
}
// converting ratio to px
return {
width: Math.round(parseFloat(w) * metadata.size.width),
height: Math.round(parseFloat(h) * metadata.size.height),
left: Math.round(Math.abs(parseFloat(x) - swapX) * metadata.size.width),
top: Math.round(Math.abs(parseFloat(y) - swapY) * metadata.size.height),
if (metadata.keywords === undefined) {
metadata.keywords = [];
}
for (const kw of exif.subject.value as ExifReader.XmpTag[]) {
if (metadata.keywords.indexOf(kw.description) === -1) {
metadata.keywords.push(kw.description);
}
}
}
let orientation = OrientationTypes.TOP_LEFT;
if (exif.Orientation) {
orientation = parseInt(
exif.Orientation.value as any,
10
) as number;
}
if (OrientationTypes.BOTTOM_LEFT < orientation) {
// noinspection JSSuspiciousNameCombination
const height = metadata.size.width;
// noinspection JSSuspiciousNameCombination
metadata.size.width = metadata.size.height;
metadata.size.height = height;
}
if (Config.Faces.enabled) {
const faces: FaceRegion[] = [];
if (
((exif.Regions as any)?.value?.RegionList)?.value
) {
for (const regionRoot of (exif.Regions as any).value.RegionList
.value as any[]) {
let type;
let name;
let box;
const createFaceBox = (
w: string,
h: string,
x: string,
y: string
) => {
if (OrientationTypes.BOTTOM_LEFT < orientation) {
[x, y] = [y, x];
[w, h] = [h, w];
}
let swapX = 0;
let swapY = 0;
switch (orientation) {
case OrientationTypes.TOP_RIGHT:
case OrientationTypes.RIGHT_TOP:
swapX = 1;
break;
case OrientationTypes.BOTTOM_RIGHT:
case OrientationTypes.RIGHT_BOTTOM:
swapX = 1;
swapY = 1;
break;
case OrientationTypes.BOTTOM_LEFT:
case OrientationTypes.LEFT_BOTTOM:
swapY = 1;
break;
}
// converting ratio to px
return {
width: Math.round(parseFloat(w) * metadata.size.width),
height: Math.round(parseFloat(h) * metadata.size.height),
left: Math.round(Math.abs(parseFloat(x) - swapX) * metadata.size.width),
top: Math.round(Math.abs(parseFloat(y) - swapY) * metadata.size.height),
};
};
};
/* Adobe Lightroom based face region structure */
if (
regionRoot.value &&
regionRoot.value['rdf:Description'] &&
regionRoot.value['rdf:Description'].value &&
regionRoot.value['rdf:Description'].value['mwg-rs:Area']
) {
const region = regionRoot.value['rdf:Description'];
const regionBox = region.value['mwg-rs:Area'].attributes;
/* Adobe Lightroom based face region structure */
if (
regionRoot.value &&
regionRoot.value['rdf:Description'] &&
regionRoot.value['rdf:Description'].value &&
regionRoot.value['rdf:Description'].value['mwg-rs:Area']
) {
const region = regionRoot.value['rdf:Description'];
const regionBox = region.value['mwg-rs:Area'].attributes;
name = region.attributes['mwg-rs:Name'];
type = region.attributes['mwg-rs:Type'];
box = createFaceBox(
regionBox['stArea:w'],
regionBox['stArea:h'],
regionBox['stArea:x'],
regionBox['stArea:y']
);
/* Load exiftool edited face region structure, see github issue #191 */
} else if (
regionRoot.Area &&
regionRoot.Name &&
regionRoot.Type
) {
const regionBox = regionRoot.Area.value;
name = regionRoot.Name.value;
type = regionRoot.Type.value;
box = createFaceBox(
regionBox.w.value,
regionBox.h.value,
regionBox.x.value,
regionBox.y.value
);
}
if (type !== 'Face' || !name) {
continue;
}
// convert center base box to corner based box
box.left = Math.round(Math.max(0, box.left - box.width / 2));
box.top = Math.round(Math.max(0, box.top - box.height / 2));
faces.push({name, box});
}
}
if (faces.length > 0) {
metadata.faces = faces; // save faces
if (Config.Faces.keywordsToPersons) {
// remove faces from keywords
metadata.faces.forEach((f) => {
const index = metadata.keywords.indexOf(f.name);
if (index !== -1) {
metadata.keywords.splice(index, 1);
name = region.attributes['mwg-rs:Name'];
type = region.attributes['mwg-rs:Type'];
box = createFaceBox(
regionBox['stArea:w'],
regionBox['stArea:h'],
regionBox['stArea:x'],
regionBox['stArea:y']
);
/* Load exiftool edited face region structure, see github issue #191 */
} else if (
regionRoot.Area &&
regionRoot.Name &&
regionRoot.Type
) {
const regionBox = regionRoot.Area.value;
name = regionRoot.Name.value;
type = regionRoot.Type.value;
box = createFaceBox(
regionBox.w.value,
regionBox.h.value,
regionBox.x.value,
regionBox.y.value
);
}
});
if (type !== 'Face' || !name) {
continue;
}
// convert center base box to corner based box
box.left = Math.round(Math.max(0, box.left - box.width / 2));
box.top = Math.round(Math.max(0, box.top - box.height / 2));
faces.push({name, box});
}
}
if (faces.length > 0) {
metadata.faces = faces; // save faces
if (Config.Faces.keywordsToPersons) {
// remove faces from keywords
metadata.faces.forEach((f) => {
const index = metadata.keywords.indexOf(f.name);
if (index !== -1) {
metadata.keywords.splice(index, 1);
}
});
}
}
}
} catch (err) {
// ignoring errors
}
} catch (err) {
// ignoring errors
}
return resolve(metadata);
} catch (err) {
return reject({file: fullPath, error: err});
}
});
return resolve(metadata);
} catch (err) {
return reject({file: fullPath, error: err});
}
});
} catch (err) {
Logger.error(LOG_TAG, 'Error during reading photo: ' + fullPath);
console.error(err);
return resolve(MetadataLoader.EMPTY_METADATA);
}
});
}
}