From f11a133f43c51859ea8a2c2d2978027d2250857b Mon Sep 17 00:00:00 2001 From: grasdk Date: Wed, 6 Mar 2024 00:40:18 +0100 Subject: [PATCH] fix leap year searching and tests. small metadatabuffer size optimization. --- src/backend/model/database/SearchManager.ts | 11 ++++---- .../model/fileaccess/MetadataLoader.ts | 24 ++++++++++------- src/common/Utils.ts | 20 ++++++++++++++ .../ui/gallery/navigator/sorting.service.ts | 2 +- .../unit/model/sql/SearchManager.spec.ts | 27 +++++++++++++++++-- 5 files changed, 66 insertions(+), 18 deletions(-) diff --git a/src/backend/model/database/SearchManager.ts b/src/backend/model/database/SearchManager.ts index 3037ddfe..75010fdb 100644 --- a/src/backend/model/database/SearchManager.ts +++ b/src/backend/model/database/SearchManager.ts @@ -764,7 +764,7 @@ export class SearchManager { tq.frequency === DatePatternFrequency.days_ago)) { if (isNaN(tq.agoNumber)) { - throw new Error('ago number is missing on date patter search query with frequency: ' + DatePatternFrequency[tq.frequency] + ', ago number: ' + tq.agoNumber); + throw new Error('ago number is missing on date pattern search query with frequency: ' + DatePatternFrequency[tq.frequency] + ', ago number: ' + tq.agoNumber); } const to = new Date(); to.setHours(0, 0, 0, 0); @@ -853,15 +853,16 @@ export class SearchManager { }; switch (tq.frequency) { case DatePatternFrequency.every_year: - if (tq.daysLength >= 365) { // trivial result includes all photos + const d = new Date(); + if (tq.daysLength >= (Utils.isDateFromLeapYear(d) ? 366: 365)) { // trivial result includes all photos if (tq.negate) { q.andWhere('FALSE'); } return q; } - const d = new Date(); - const dayOfYear = Math.floor((d.getTime() - new Date(d.getFullYear(), 0, 0).getTime()) / 1000 / 60 / 60 / 24); - addWhere('%j', dayOfYear - tq.daysLength < 0); + + const dayOfYear = Utils.getDayOfYear(d); + addWhere('%m%d', dayOfYear - tq.daysLength < 0); break; case DatePatternFrequency.every_month: if (tq.daysLength >= 31) { // trivial result includes all photos diff --git a/src/backend/model/fileaccess/MetadataLoader.ts b/src/backend/model/fileaccess/MetadataLoader.ts index 908e0116..47a869de 100644 --- a/src/backend/model/fileaccess/MetadataLoader.ts +++ b/src/backend/model/fileaccess/MetadataLoader.ts @@ -194,7 +194,7 @@ export class MetadataLoader { translateValues: false, //don't translate orientation from numbers to strings etc. mergeOutput: false //don't merge output, because things like Microsoft Rating (percent) and xmp.rating will be merged }; - + //function to convert timestamp into milliseconds taking offset into account const timestampToMS = (timestamp: string, offset: string) => { if (!timestamp) { @@ -240,10 +240,21 @@ export class MetadataLoader { } try { - const data = Buffer.allocUnsafe(Config.Media.photoMetadataSize); + let bufferSize = Config.Media.photoMetadataSize; + try { + const stat = fs.statSync(fullPath); + metadata.fileSize = stat.size; + //No reason to make the buffer larger than the actual file + bufferSize = Math.min(Config.Media.photoMetadataSize, metadata.fileSize); + metadata.creationDate = stat.mtime.getTime(); + } catch (err) { + // ignoring errors + } + + const data = Buffer.allocUnsafe(bufferSize); fileHandle = await fs.promises.open(fullPath, 'r'); try { - await fileHandle.read(data, 0, Config.Media.photoMetadataSize, 0); + await fileHandle.read(data, 0, bufferSize, 0); } catch (err) { Logger.error(LOG_TAG, 'Error during reading photo: ' + fullPath); console.error(err); @@ -252,13 +263,6 @@ export class MetadataLoader { await fileHandle.close(); } try { - try { - const stat = fs.statSync(fullPath); - metadata.fileSize = stat.size; - metadata.creationDate = stat.mtime.getTime(); - } catch (err) { - // ignoring errors - } try { //read the actual image size, don't rely on tags for this const info = imageSize(fullPath); diff --git a/src/common/Utils.ts b/src/common/Utils.ts index e0a475d6..2e4f06b5 100644 --- a/src/common/Utils.ts +++ b/src/common/Utils.ts @@ -147,6 +147,26 @@ export class Utils { } } + static isLeapYear(year: number) { + return (0 == year % 4) && (0 != year % 100) || (0 == year % 400) + } + + static isDateFromLeapYear(date: Date) { + return Utils.isLeapYear(date.getFullYear()); + } + + // Get Day of Year + static getDayOfYear(date: Date) { + //Day-number at the start of Jan to Dec. A month baseline + const dayCount = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]; + const mn = date.getMonth(); + let dayOfYear = dayCount[mn] + date.getDate(); //add the date to the month baseline + if (mn > 1 && Utils.isLeapYear((date.getFullYear()))) { + dayOfYear++; //Add an extra day for march to december (mn>1) on leap years + } + return dayOfYear; + }; + static renderDataSize(size: number): string { const postFixes = ['B', 'KB', 'MB', 'GB', 'TB']; let index = 0; diff --git a/src/frontend/app/ui/gallery/navigator/sorting.service.ts b/src/frontend/app/ui/gallery/navigator/sorting.service.ts index 7e74d737..b5718c5a 100644 --- a/src/frontend/app/ui/gallery/navigator/sorting.service.ts +++ b/src/frontend/app/ui/gallery/navigator/sorting.service.ts @@ -184,7 +184,7 @@ export class GallerySortingService { private getGroupByNameFn(grouping: GroupingMethod) { switch (grouping.method) { case SortByTypes.Date: - return (m: MediaDTO) => this.datePipe.transform(m.metadata.creationDate, 'longDate', m.metadata.creationDateOffset); + return (m: MediaDTO) => this.datePipe.transform(m.metadata.creationDate, 'longDate', m.metadata.creationDateOffset ? m.metadata.creationDateOffset : 'UTC'); case SortByTypes.Name: return (m: MediaDTO) => m.name.at(0).toUpperCase(); diff --git a/test/backend/unit/model/sql/SearchManager.spec.ts b/test/backend/unit/model/sql/SearchManager.spec.ts index 216aa511..44f42ff3 100644 --- a/test/backend/unit/model/sql/SearchManager.spec.ts +++ b/test/backend/unit/model/sql/SearchManager.spec.ts @@ -115,16 +115,36 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => { subDir2 = TestHelper.getDirectoryEntry(directory, 'Return of the Jedi'); p = TestHelper.getPhotoEntry1(directory); p.metadata.creationDate = Date.now(); + p.metadata.creationDateOffset = "+02:00"; p2 = TestHelper.getPhotoEntry2(directory); p2.metadata.creationDate = Date.now() - 60 * 60 * 24 * 1000; + p2.metadata.creationDateOffset = "+02:00"; v = TestHelper.getVideoEntry1(directory); v.metadata.creationDate = Date.now() - 60 * 60 * 24 * 7 * 1000; + v.metadata.creationDateOffset = "+02:00"; gpx = TestHelper.getRandomizedGPXEntry(directory); p4 = TestHelper.getPhotoEntry4(subDir2); - p4.metadata.creationDate = Date.now() - 60 * 60 * 24 * 366 * 1000; + let d = new Date(); + //set creation date to one year and one day earlier + p4.metadata.creationDate = d.getTime() - 60 * 60 * 24 * (Utils.isDateFromLeapYear(d) ? 367 : 366) * 1000; + p4.metadata.creationDateOffset = "+02:00"; const pFaceLessTmp = TestHelper.getPhotoEntry3(subDir); delete pFaceLessTmp.metadata.faces; - pFaceLessTmp.metadata.creationDate = Date.now() - 60 * 60 * 24 * 32 * 1000; + d = new Date(); + //we create a date 1 month and 1 day before now + if ([1, 3, 5, 7, 8, 10, 0].includes(d.getMonth())) { + //Now is a month after a long month: feb (1), april (3), june (5), august(7), september(8), november (10), january (0) + pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 32 * 1000; + } else if (d.getMonth() == 2 && Utils.isDateFromLeapYear(d)) { + //march on leap years + pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 30 * 1000; + } else if (d.getMonth() == 2) { + //march (and not leap years) + pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 29 * 1000; + } else { //all other months must come after a short month with 30 days, so we subtract 31 + pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 31 * 1000; + } + pFaceLessTmp.metadata.creationDateOffset = "+02:00"; dir = await DBTestHelper.persistTestDir(directory); subDir = dir.directories[0]; @@ -937,13 +957,16 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => { await setUpSqlDB(); p5 = TestHelper.getBasePhotoEntry(subDir2, 'p5-23h-ago.jpg'); p5.metadata.creationDate = Date.now() - 60 * 60 * 24 * 1000 - 1000; + //p5.metadata.creationDateOffset = "+02:00"; p6 = TestHelper.getBasePhotoEntry(subDir2, 'p6-300d-ago.jpg'); p6.metadata.creationDate = Date.now() - 60 * 60 * 24 * 300 * 1000; + //p6.metadata.creationDateOffset = "+02:00"; p7 = TestHelper.getBasePhotoEntry(subDir2, 'p7-1y-1min-ago.jpg'); const d = new Date(); d.setUTCFullYear(d.getUTCFullYear() - 1); d.setUTCMinutes(d.getUTCMinutes() - 1); p7.metadata.creationDate = d.getTime(); + //p7.metadata.creationDateOffset = "+02:00"; subDir2 = await DBTestHelper.persistTestDir(subDir2) as any; p4 = subDir2.media[0];