self-hosted/pigallery2
1
0
Fork 0
mirror of https://github.com/bpatrik/pigallery2.git synced 2024-12-23 01:27:14 +02:00
Code Issues Releases Activity

Merge branch 'master' into issue838

Browse Source 
# Conflicts:
#	src/backend/model/fileaccess/MetadataLoader.ts
This commit is contained in:
kagahd 2024-03-11 08:35:14 +01:00
commit f05ee7a867
No known key found for this signature in database
GPG Key ID: C7C8777FDE0053DF
17 changed files with 149 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/backend/model/database/SearchManager.ts
View File

@ -764,7 +764,7 @@ export class SearchManager {
tq.frequency === DatePatternFrequency.days_ago)) {
if (isNaN(tq.agoNumber)) {
throw new Error('ago number is missing on date patter search query with frequency: ' + DatePatternFrequency[tq.frequency] + ', ago number: ' + tq.agoNumber);
throw new Error('ago number is missing on date pattern search query with frequency: ' + DatePatternFrequency[tq.frequency] + ', ago number: ' + tq.agoNumber);
}
const to = new Date();
to.setHours(0, 0, 0, 0);
@ -853,15 +853,16 @@ export class SearchManager {
};
switch (tq.frequency) {
case DatePatternFrequency.every_year:
if (tq.daysLength >= 365) { // trivial result includes all photos
const d = new Date();
if (tq.daysLength >= (Utils.isDateFromLeapYear(d) ? 366: 365)) { // trivial result includes all photos
if (tq.negate) {
q.andWhere('FALSE');
}
return q;
}
const d = new Date();
const dayOfYear = Math.floor((d.getTime() - new Date(d.getFullYear(), 0, 0).getTime()) / 1000 / 60 / 60 / 24);
addWhere('%j', dayOfYear - tq.daysLength < 0);
const dayOfYear = Utils.getDayOfYear(d);
addWhere('%m%d', dayOfYear - tq.daysLength < 0);
break;
case DatePatternFrequency.every_month:
if (tq.daysLength >= 31) { // trivial result includes all photos

1
src/backend/model/extension/ExtensionManager.ts
View File

@ -55,6 +55,7 @@ export class ExtensionManager implements IObjectManager {
invalidateDirectoryCovers: new ExtensionEvent(),
},
DiskManager: {
excludeDir: new ExtensionEvent(),
scanDirectory: new ExtensionEvent()
},
ImageRenderer: {

5
src/backend/model/extension/IExtension.ts
View File

@ -73,6 +73,11 @@ export interface IExtensionEvents {
* photos, videos and metafiles
*/
DiskManager: {
excludeDir: IExtensionEvent<[{
name: string,
parentDirRelativeName: string,
parentDirAbsoluteName: string
}], boolean>,
scanDirectory: IExtensionEvent<[
string,
DirectoryScanSettings], ParentDirectoryDTO>

25
src/backend/model/fileaccess/DiskManager.ts
View File

@ -49,19 +49,20 @@ export class DiskManager {
return path.basename(dirPath);
}
public static async excludeDir(
@ExtensionDecorator(e => e.gallery.DiskManager.excludeDir)
public static async excludeDir(dir: {
name: string,
relativeDirectoryName: string,
absoluteDirectoryName: string
): Promise<boolean> {
parentDirRelativeName: string,
parentDirAbsoluteName: string
}): Promise<boolean> {
if (
Config.Indexing.excludeFolderList.length === 0 &&
Config.Indexing.excludeFileList.length === 0
) {
return false;
}
const absoluteName = path.normalize(path.join(absoluteDirectoryName, name));
const relativeName = path.normalize(path.join(relativeDirectoryName, name));
const absoluteName = path.normalize(path.join(dir.parentDirAbsoluteName, dir.name));
const relativeName = path.normalize(path.join(dir.parentDirRelativeName, dir.name));
for (const exclude of Config.Indexing.excludeFolderList) {
if (exclude.startsWith('/')) {
@ -73,7 +74,7 @@ export class DiskManager {
return true;
}
} else {
if (exclude === name) {
if (exclude === dir.name) {
return true;
}
}
@ -155,11 +156,11 @@ export class DiskManager {
if (
settings.noDirectory === true ||
settings.coverOnly === true ||
(await DiskManager.excludeDir(
file,
relativeDirectoryName,
absoluteDirectoryName
))
(await DiskManager.excludeDir({
name: file,
parentDirRelativeName: relativeDirectoryName,
parentDirAbsoluteName: absoluteDirectoryName
}))
) {
continue;
}

62
src/backend/model/fileaccess/MetadataLoader.ts
View File

@ -244,10 +244,21 @@ export class MetadataLoader {
}
try {
const data = Buffer.allocUnsafe(Config.Media.photoMetadataSize);
let bufferSize = Config.Media.photoMetadataSize;
try {
const stat = fs.statSync(fullPath);
metadata.fileSize = stat.size;
//No reason to make the buffer larger than the actual file
bufferSize = Math.min(Config.Media.photoMetadataSize, metadata.fileSize);
metadata.creationDate = stat.mtime.getTime();
} catch (err) {
// ignoring errors
}
const data = Buffer.allocUnsafe(bufferSize);
fileHandle = await fs.promises.open(fullPath, 'r');
try {
await fileHandle.read(data, 0, Config.Media.photoMetadataSize, 0);
await fileHandle.read(data, 0, bufferSize, 0);
} catch (err) {
Logger.error(LOG_TAG, 'Error during reading photo: ' + fullPath);
console.error(err);
@ -255,15 +266,7 @@ export class MetadataLoader {
} finally {
await fileHandle.close();
}
try {
try {
const stat = fs.statSync(fullPath);
metadata.fileSize = stat.size;
metadata.creationDate = stat.mtime.getTime();
} catch (err) {
// ignoring errors
}
try {
//read the actual image size, don't rely on tags for this
const info = imageSize(fullPath);
@ -363,34 +366,27 @@ export class MetadataLoader {
//Filesystem is the absolute last resort, and it's hard to write tests for, since file system dates are changed on e.g. git clone.
if (exif.exif.DateTimeOriginal) {
//DateTimeOriginal is when the camera shutter closed
if (exif.exif.OffsetTimeOriginal) { //OffsetTimeOriginal is the corresponding offset
metadata.creationDate = timestampToMS(exif.exif.DateTimeOriginal, exif.exif.OffsetTimeOriginal);
metadata.creationDateOffset = exif.exif.OffsetTimeOriginal;
} else {
const alt_offset = exif.exif.OffsetTimeDigitized || exif.exif.OffsetTime || getTimeOffsetByGPSStamp(exif.exif.DateTimeOriginal, exif.exif.GPSTimeStamp, exif.gps);
metadata.creationDate = timestampToMS(exif.exif.DateTimeOriginal, alt_offset);
metadata.creationDateOffset = alt_offset;
let offset = exif.exif.OffsetTimeOriginal; //OffsetTimeOriginal is the corresponding offset
if (!offset) { //Find offset among other options if possible
offset = exif.exif.OffsetTimeDigitized || exif.exif.OffsetTime || getTimeOffsetByGPSStamp(exif.exif.DateTimeOriginal, exif.exif.GPSTimeStamp, exif.gps);
}
metadata.creationDate = timestampToMS(exif.exif.DateTimeOriginal, offset);
metadata.creationDateOffset = offset;
} else if (exif.exif.CreateDate) { //using else if here, because DateTimeOriginal has preceedence
//Create is when the camera wrote the file (typically within the same ms as shutter close)
if (exif.exif.OffsetTimeDigitized) { //OffsetTimeDigitized is the corresponding offset
metadata.creationDate = timestampToMS(exif.exif.CreateDate, exif.exif.OffsetTimeDigitized);
metadata.creationDateOffset = exif.exif.OffsetTimeDigitized;
} else {
const alt_offset = exif.exif.OffsetTimeOriginal || exif.exif.OffsetTime || getTimeOffsetByGPSStamp(exif.exif.DateTimeOriginal, exif.exif.GPSTimeStamp, exif.gps);
metadata.creationDate = timestampToMS(exif.exif.DateTimeOriginal, alt_offset);
metadata.creationDateOffset = alt_offset;
let offset = exif.exif.OffsetTimeDigitized; //OffsetTimeDigitized is the corresponding offset
if (!offset) { //Find offset among other options if possible
offset = exif.exif.OffsetTimeOriginal || exif.exif.OffsetTime || getTimeOffsetByGPSStamp(exif.exif.DateTimeOriginal, exif.exif.GPSTimeStamp, exif.gps);
}
metadata.creationDate = timestampToMS(exif.exif.CreateDate, offset);
metadata.creationDateOffset = offset;
} else if (exif.ifd0?.ModifyDate) { //using else if here, because DateTimeOriginal and CreatDate have preceedence
if (exif.exif.OffsetTime) {
//exif.Offsettime is the offset corresponding to ifd0.ModifyDate
metadata.creationDate = timestampToMS(exif.ifd0.ModifyDate, exif.exif?.OffsetTime);
metadata.creationDateOffset = exif.exif?.OffsetTime
} else {
const alt_offset = exif.exif.DateTimeOriginal || exif.exif.OffsetTimeDigitized || getTimeOffsetByGPSStamp(exif.ifd0.ModifyDate, exif.exif.GPSTimeStamp, exif.gps);
metadata.creationDate = timestampToMS(exif.ifd0.ModifyDate, alt_offset);
metadata.creationDateOffset = alt_offset;
}
let offset = exif.exif.OffsetTime; //exif.Offsettime is the offset corresponding to ifd0.ModifyDate
if (!offset) { //Find offset among other options if possible
offset = exif.exif.DateTimeOriginal || exif.exif.OffsetTimeDigitized || getTimeOffsetByGPSStamp(exif.ifd0.ModifyDate, exif.exif.GPSTimeStamp, exif.gps);
}
metadata.creationDate = timestampToMS(exif.ifd0.ModifyDate, offset);
metadata.creationDateOffset = offset
} else if (exif.ihdr && exif.ihdr["Creation Time"]) {// again else if (another fallback date if the good ones aren't there) {
const any_offset = exif.exif.DateTimeOriginal || exif.exif.OffsetTimeDigitized || exif.exif.OffsetTime || getTimeOffsetByGPSStamp(exif.ifd0.ModifyDate, exif.exif.GPSTimeStamp, exif.gps);
metadata.creationDate = timestampToMS(exif.ihdr["Creation Time"], any_offset);

2
src/common/DataStructureVersion.ts
View File

@ -1,4 +1,4 @@
/**
* This version indicates that the sql/entities/*Entity.ts files got changed and the db needs to be recreated
*/
export const DataStructureVersion = 35;
export const DataStructureVersion = 36;

20
src/common/Utils.ts
View File

@ -147,6 +147,26 @@ export class Utils {
}
}
static isLeapYear(year: number) {
return (0 == year % 4) && (0 != year % 100) || (0 == year % 400)
}
static isDateFromLeapYear(date: Date) {
return Utils.isLeapYear(date.getFullYear());
}
// Get Day of Year
static getDayOfYear(date: Date) {
//Day-number at the start of Jan to Dec. A month baseline
const dayCount = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334];
const mn = date.getMonth();
let dayOfYear = dayCount[mn] + date.getDate(); //add the date to the month baseline
if (mn > 1 && Utils.isLeapYear((date.getFullYear()))) {
dayOfYear++; //Add an extra day for march to december (mn>1) on leap years
}
return dayOfYear;
}
static renderDataSize(size: number): string {
const postFixes = ['B', 'KB', 'MB', 'GB', 'TB'];
let index = 0;

7
src/common/config/private/subconfigs/ServerExtensionsConfig.ts
View File

@ -2,6 +2,13 @@
import {ConfigProperty, SubConfigClass} from 'typeconfig/common';
import {ClientExtensionsConfig, ConfigPriority, TAGS} from '../../public/ClientConfig';
import {GenericConfigType} from 'typeconfig/src/GenericConfigType';
declare let $localize: (s: TemplateStringsArray) => string;
if (typeof $localize === 'undefined') {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
global.$localize = (s) => s;
}
@SubConfigClass<TAGS>({softReadonly: true})
export class ServerExtensionsEntryConfig {

4
src/frontend/app/ui/gallery/grid/photo/photo.grid.gallery.component.ts
View File

@ -134,10 +134,10 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy {
}
getPositionText(): string {
if (!this.gridMedia || !this.gridMedia.isPhoto()) {
if (!this.gridMedia || !this.gridMedia.isPhoto() || !(this.gridMedia.media as PhotoDTO).metadata.positionData) {
return '';
}
return (
return ( //not much space in the gridview, so we only deliver city, or state or country
(this.gridMedia.media as PhotoDTO).metadata.positionData.city ||
(this.gridMedia.media as PhotoDTO).metadata.positionData.state ||
(this.gridMedia.media as PhotoDTO).metadata.positionData.country || ''

13
src/frontend/app/ui/gallery/lightbox/controls/controls.lightbox.gallery.component.ts
View File

@ -510,11 +510,14 @@ export class ControlsLightboxComponent implements OnDestroy, OnInit, OnChanges {
case LightBoxTitleTexts.date:
return this.datePipe.transform(m.metadata.creationDate, 'longDate', m.metadata.creationDateOffset);
case LightBoxTitleTexts.location:
return (
m.metadata.positionData?.city ||
m.metadata.positionData?.state ||
m.metadata.positionData?.country || ''
).trim();
if (!m.metadata.positionData) {
return '';
}
return [
m.metadata.positionData.city,
m.metadata.positionData.state,
m.metadata.positionData.country
].filter(elm => elm).join(', ').trim(); //Filter removes empty elements, join concats the values separated by ', '
case LightBoxTitleTexts.camera:
return m.metadata.cameraData?.model;
case LightBoxTitleTexts.lens:

16
src/frontend/app/ui/gallery/lightbox/infopanel/info-panel.lightbox.gallery.component.ts
View File

@ -183,17 +183,11 @@ export class InfoPanelLightboxComponent implements OnInit, OnChanges {
if (!(this.media as PhotoDTO).metadata.positionData) {
return '';
}
let str =
(this.media as PhotoDTO).metadata.positionData.city ||
(this.media as PhotoDTO).metadata.positionData.state ||
'';
if (str.length !== 0) {
str += ', ';
}
str += (this.media as PhotoDTO).metadata.positionData.country || '';
return str;
return [
(this.media as PhotoDTO).metadata.positionData.city,
(this.media as PhotoDTO).metadata.positionData.state,
(this.media as PhotoDTO).metadata.positionData.country
].filter(elm => elm).join(', ').trim(); //Filter removes empty elements, join concats the values separated by ', '
}
close(): void {

2
src/frontend/app/ui/gallery/navigator/sorting.service.ts
View File

@ -184,7 +184,7 @@ export class GallerySortingService {
private getGroupByNameFn(grouping: GroupingMethod) {
switch (grouping.method) {
case SortByTypes.Date:
return (m: MediaDTO) => this.datePipe.transform(m.metadata.creationDate, 'longDate', m.metadata.creationDateOffset);
return (m: MediaDTO) => this.datePipe.transform(m.metadata.creationDate, 'longDate', m.metadata.creationDateOffset ? m.metadata.creationDateOffset : 'UTC');
case SortByTypes.Name:
return (m: MediaDTO) => m.name.at(0).toUpperCase();

BIN
test/backend/assets/wild-1-small.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 58 KiB

20
test/backend/assets/wild-1-small.json Normal file
View File

@ -0,0 +1,20 @@
{
"size": {
"width": 306,
"height": 204
},
"creationDate": 1435910683000,
"fileSize": 59564,
"positionData": {
"country": "United States",
"state": "Arizona",
"city": "Williams"
},
"keywords": ["Akela the wolf", "Balu the bear", "Bearizona", "Hugin the raven", "USA", "USA Road trip"],
"cameraData": {
"make": "Canon",
"model": "Canon EOS 600D",
"lens": "EF-S15-85mm f/3.5-5.6 IS USM"
},
"creationDateOffset": "+02:00"
}

27
test/backend/unit/model/sql/SearchManager.spec.ts
View File

@ -115,16 +115,36 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
subDir2 = TestHelper.getDirectoryEntry(directory, 'Return of the Jedi');
p = TestHelper.getPhotoEntry1(directory);
p.metadata.creationDate = Date.now();
p.metadata.creationDateOffset = "+02:00";
p2 = TestHelper.getPhotoEntry2(directory);
p2.metadata.creationDate = Date.now() - 60 * 60 * 24 * 1000;
p2.metadata.creationDateOffset = "+02:00";
v = TestHelper.getVideoEntry1(directory);
v.metadata.creationDate = Date.now() - 60 * 60 * 24 * 7 * 1000;
v.metadata.creationDateOffset = "+02:00";
gpx = TestHelper.getRandomizedGPXEntry(directory);
p4 = TestHelper.getPhotoEntry4(subDir2);
p4.metadata.creationDate = Date.now() - 60 * 60 * 24 * 366 * 1000;
let d = new Date();
//set creation date to one year and one day earlier
p4.metadata.creationDate = d.getTime() - 60 * 60 * 24 * (Utils.isDateFromLeapYear(d) ? 367 : 366) * 1000;
p4.metadata.creationDateOffset = "+02:00";
const pFaceLessTmp = TestHelper.getPhotoEntry3(subDir);
delete pFaceLessTmp.metadata.faces;
pFaceLessTmp.metadata.creationDate = Date.now() - 60 * 60 * 24 * 32 * 1000;
d = new Date();
//we create a date 1 month and 1 day before now
if ([1, 3, 5, 7, 8, 10, 0].includes(d.getMonth())) {
//Now is a month after a long month: feb (1), april (3), june (5), august(7), september(8), november (10), january (0)
pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 32 * 1000;
} else if (d.getMonth() == 2 && Utils.isDateFromLeapYear(d)) {
//march on leap years
pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 30 * 1000;
} else if (d.getMonth() == 2) {
//march (and not leap years)
pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 29 * 1000;
} else { //all other months must come after a short month with 30 days, so we subtract 31
pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 31 * 1000;
}
pFaceLessTmp.metadata.creationDateOffset = "+02:00";
dir = await DBTestHelper.persistTestDir(directory);
subDir = dir.directories[0];
@ -937,13 +957,16 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
await setUpSqlDB();
p5 = TestHelper.getBasePhotoEntry(subDir2, 'p5-23h-ago.jpg');
p5.metadata.creationDate = Date.now() - 60 * 60 * 24 * 1000 - 1000;
//p5.metadata.creationDateOffset = "+02:00";
p6 = TestHelper.getBasePhotoEntry(subDir2, 'p6-300d-ago.jpg');
p6.metadata.creationDate = Date.now() - 60 * 60 * 24 * 300 * 1000;
//p6.metadata.creationDateOffset = "+02:00";
p7 = TestHelper.getBasePhotoEntry(subDir2, 'p7-1y-1min-ago.jpg');
const d = new Date();
d.setUTCFullYear(d.getUTCFullYear() - 1);
d.setUTCMinutes(d.getUTCMinutes() - 1);
p7.metadata.creationDate = d.getTime();
//p7.metadata.creationDateOffset = "+02:00";
subDir2 = await DBTestHelper.persistTestDir(subDir2) as any;
p4 = subDir2.media[0];

2
test/backend/unit/model/threading/DiskManagerWorker.spec.ts
View File

@ -24,7 +24,7 @@ describe('DiskMangerWorker', () => {
ProjectPath.ImageFolder = path.join(__dirname, '/../../../assets');
const dir = await DiskManager.scanDirectory('/');
// should match the number of media (photo/video) files in the assets folder
expect(dir.media.length).to.be.equals(14);
expect(dir.media.length).to.be.equals(15);
// eslint-disable-next-line @typescript-eslint/no-var-requires
const expected = require(path.join(__dirname, '/../../../assets/test image öüóőúéáű-.,.json'));
const i = dir.media.findIndex(m => m.name === 'test image öüóőúéáű-.,.jpg');

5
test/backend/unit/model/threading/MetaDataLoader.spec.ts
View File

@ -172,6 +172,11 @@ describe('MetadataLoader', () => {
expect(Utils.clone(data)).to.be.deep.equal(expected);
});
});
it('should load wild-1-small image with CreateDate from 2015, but no DateTimeOriginal', async () => {
const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../../assets/wild-1-small.jpg'));
const expected = require(path.join(__dirname, '/../../../assets/wild-1-small.json'));
expect(Utils.clone(data)).to.be.deep.equal(expected);
});
describe('should load jpg with edge case exif data', () => {
const root = path.join(__dirname, '/../../../assets/edge_case_exif_data');