diff --git a/src/backend/model/database/SearchManager.ts b/src/backend/model/database/SearchManager.ts index 3037ddfe..75010fdb 100644 --- a/src/backend/model/database/SearchManager.ts +++ b/src/backend/model/database/SearchManager.ts @@ -764,7 +764,7 @@ export class SearchManager { tq.frequency === DatePatternFrequency.days_ago)) { if (isNaN(tq.agoNumber)) { - throw new Error('ago number is missing on date patter search query with frequency: ' + DatePatternFrequency[tq.frequency] + ', ago number: ' + tq.agoNumber); + throw new Error('ago number is missing on date pattern search query with frequency: ' + DatePatternFrequency[tq.frequency] + ', ago number: ' + tq.agoNumber); } const to = new Date(); to.setHours(0, 0, 0, 0); @@ -853,15 +853,16 @@ export class SearchManager { }; switch (tq.frequency) { case DatePatternFrequency.every_year: - if (tq.daysLength >= 365) { // trivial result includes all photos + const d = new Date(); + if (tq.daysLength >= (Utils.isDateFromLeapYear(d) ? 366: 365)) { // trivial result includes all photos if (tq.negate) { q.andWhere('FALSE'); } return q; } - const d = new Date(); - const dayOfYear = Math.floor((d.getTime() - new Date(d.getFullYear(), 0, 0).getTime()) / 1000 / 60 / 60 / 24); - addWhere('%j', dayOfYear - tq.daysLength < 0); + + const dayOfYear = Utils.getDayOfYear(d); + addWhere('%m%d', dayOfYear - tq.daysLength < 0); break; case DatePatternFrequency.every_month: if (tq.daysLength >= 31) { // trivial result includes all photos diff --git a/src/backend/model/extension/ExtensionManager.ts b/src/backend/model/extension/ExtensionManager.ts index f5256861..8344b049 100644 --- a/src/backend/model/extension/ExtensionManager.ts +++ b/src/backend/model/extension/ExtensionManager.ts @@ -55,6 +55,7 @@ export class ExtensionManager implements IObjectManager { invalidateDirectoryCovers: new ExtensionEvent(), }, DiskManager: { + excludeDir: new ExtensionEvent(), scanDirectory: new ExtensionEvent() }, ImageRenderer: { diff --git a/src/backend/model/extension/IExtension.ts b/src/backend/model/extension/IExtension.ts index ea8da419..c6ed3de4 100644 --- a/src/backend/model/extension/IExtension.ts +++ b/src/backend/model/extension/IExtension.ts @@ -73,6 +73,11 @@ export interface IExtensionEvents { * photos, videos and metafiles */ DiskManager: { + excludeDir: IExtensionEvent<[{ + name: string, + parentDirRelativeName: string, + parentDirAbsoluteName: string + }], boolean>, scanDirectory: IExtensionEvent<[ string, DirectoryScanSettings], ParentDirectoryDTO> diff --git a/src/backend/model/fileaccess/DiskManager.ts b/src/backend/model/fileaccess/DiskManager.ts index 9226e25c..e07ff2e3 100644 --- a/src/backend/model/fileaccess/DiskManager.ts +++ b/src/backend/model/fileaccess/DiskManager.ts @@ -49,19 +49,20 @@ export class DiskManager { return path.basename(dirPath); } - public static async excludeDir( + @ExtensionDecorator(e => e.gallery.DiskManager.excludeDir) + public static async excludeDir(dir: { name: string, - relativeDirectoryName: string, - absoluteDirectoryName: string - ): Promise { + parentDirRelativeName: string, + parentDirAbsoluteName: string + }): Promise { if ( Config.Indexing.excludeFolderList.length === 0 && Config.Indexing.excludeFileList.length === 0 ) { return false; } - const absoluteName = path.normalize(path.join(absoluteDirectoryName, name)); - const relativeName = path.normalize(path.join(relativeDirectoryName, name)); + const absoluteName = path.normalize(path.join(dir.parentDirAbsoluteName, dir.name)); + const relativeName = path.normalize(path.join(dir.parentDirRelativeName, dir.name)); for (const exclude of Config.Indexing.excludeFolderList) { if (exclude.startsWith('/')) { @@ -73,7 +74,7 @@ export class DiskManager { return true; } } else { - if (exclude === name) { + if (exclude === dir.name) { return true; } } @@ -155,11 +156,11 @@ export class DiskManager { if ( settings.noDirectory === true || settings.coverOnly === true || - (await DiskManager.excludeDir( - file, - relativeDirectoryName, - absoluteDirectoryName - )) + (await DiskManager.excludeDir({ + name: file, + parentDirRelativeName: relativeDirectoryName, + parentDirAbsoluteName: absoluteDirectoryName + })) ) { continue; } diff --git a/src/backend/model/fileaccess/MetadataLoader.ts b/src/backend/model/fileaccess/MetadataLoader.ts index c4afbe8a..b45a4cc3 100644 --- a/src/backend/model/fileaccess/MetadataLoader.ts +++ b/src/backend/model/fileaccess/MetadataLoader.ts @@ -244,10 +244,21 @@ export class MetadataLoader { } try { - const data = Buffer.allocUnsafe(Config.Media.photoMetadataSize); + let bufferSize = Config.Media.photoMetadataSize; + try { + const stat = fs.statSync(fullPath); + metadata.fileSize = stat.size; + //No reason to make the buffer larger than the actual file + bufferSize = Math.min(Config.Media.photoMetadataSize, metadata.fileSize); + metadata.creationDate = stat.mtime.getTime(); + } catch (err) { + // ignoring errors + } + + const data = Buffer.allocUnsafe(bufferSize); fileHandle = await fs.promises.open(fullPath, 'r'); try { - await fileHandle.read(data, 0, Config.Media.photoMetadataSize, 0); + await fileHandle.read(data, 0, bufferSize, 0); } catch (err) { Logger.error(LOG_TAG, 'Error during reading photo: ' + fullPath); console.error(err); @@ -255,15 +266,7 @@ export class MetadataLoader { } finally { await fileHandle.close(); } - try { - try { - const stat = fs.statSync(fullPath); - metadata.fileSize = stat.size; - metadata.creationDate = stat.mtime.getTime(); - } catch (err) { - // ignoring errors - } try { //read the actual image size, don't rely on tags for this const info = imageSize(fullPath); @@ -363,34 +366,27 @@ export class MetadataLoader { //Filesystem is the absolute last resort, and it's hard to write tests for, since file system dates are changed on e.g. git clone. if (exif.exif.DateTimeOriginal) { //DateTimeOriginal is when the camera shutter closed - if (exif.exif.OffsetTimeOriginal) { //OffsetTimeOriginal is the corresponding offset - metadata.creationDate = timestampToMS(exif.exif.DateTimeOriginal, exif.exif.OffsetTimeOriginal); - metadata.creationDateOffset = exif.exif.OffsetTimeOriginal; - } else { - const alt_offset = exif.exif.OffsetTimeDigitized || exif.exif.OffsetTime || getTimeOffsetByGPSStamp(exif.exif.DateTimeOriginal, exif.exif.GPSTimeStamp, exif.gps); - metadata.creationDate = timestampToMS(exif.exif.DateTimeOriginal, alt_offset); - metadata.creationDateOffset = alt_offset; + let offset = exif.exif.OffsetTimeOriginal; //OffsetTimeOriginal is the corresponding offset + if (!offset) { //Find offset among other options if possible + offset = exif.exif.OffsetTimeDigitized || exif.exif.OffsetTime || getTimeOffsetByGPSStamp(exif.exif.DateTimeOriginal, exif.exif.GPSTimeStamp, exif.gps); } + metadata.creationDate = timestampToMS(exif.exif.DateTimeOriginal, offset); + metadata.creationDateOffset = offset; } else if (exif.exif.CreateDate) { //using else if here, because DateTimeOriginal has preceedence //Create is when the camera wrote the file (typically within the same ms as shutter close) - if (exif.exif.OffsetTimeDigitized) { //OffsetTimeDigitized is the corresponding offset - metadata.creationDate = timestampToMS(exif.exif.CreateDate, exif.exif.OffsetTimeDigitized); - metadata.creationDateOffset = exif.exif.OffsetTimeDigitized; - } else { - const alt_offset = exif.exif.OffsetTimeOriginal || exif.exif.OffsetTime || getTimeOffsetByGPSStamp(exif.exif.DateTimeOriginal, exif.exif.GPSTimeStamp, exif.gps); - metadata.creationDate = timestampToMS(exif.exif.DateTimeOriginal, alt_offset); - metadata.creationDateOffset = alt_offset; + let offset = exif.exif.OffsetTimeDigitized; //OffsetTimeDigitized is the corresponding offset + if (!offset) { //Find offset among other options if possible + offset = exif.exif.OffsetTimeOriginal || exif.exif.OffsetTime || getTimeOffsetByGPSStamp(exif.exif.DateTimeOriginal, exif.exif.GPSTimeStamp, exif.gps); } + metadata.creationDate = timestampToMS(exif.exif.CreateDate, offset); + metadata.creationDateOffset = offset; } else if (exif.ifd0?.ModifyDate) { //using else if here, because DateTimeOriginal and CreatDate have preceedence - if (exif.exif.OffsetTime) { - //exif.Offsettime is the offset corresponding to ifd0.ModifyDate - metadata.creationDate = timestampToMS(exif.ifd0.ModifyDate, exif.exif?.OffsetTime); - metadata.creationDateOffset = exif.exif?.OffsetTime - } else { - const alt_offset = exif.exif.DateTimeOriginal || exif.exif.OffsetTimeDigitized || getTimeOffsetByGPSStamp(exif.ifd0.ModifyDate, exif.exif.GPSTimeStamp, exif.gps); - metadata.creationDate = timestampToMS(exif.ifd0.ModifyDate, alt_offset); - metadata.creationDateOffset = alt_offset; - } + let offset = exif.exif.OffsetTime; //exif.Offsettime is the offset corresponding to ifd0.ModifyDate + if (!offset) { //Find offset among other options if possible + offset = exif.exif.DateTimeOriginal || exif.exif.OffsetTimeDigitized || getTimeOffsetByGPSStamp(exif.ifd0.ModifyDate, exif.exif.GPSTimeStamp, exif.gps); + } + metadata.creationDate = timestampToMS(exif.ifd0.ModifyDate, offset); + metadata.creationDateOffset = offset } else if (exif.ihdr && exif.ihdr["Creation Time"]) {// again else if (another fallback date if the good ones aren't there) { const any_offset = exif.exif.DateTimeOriginal || exif.exif.OffsetTimeDigitized || exif.exif.OffsetTime || getTimeOffsetByGPSStamp(exif.ifd0.ModifyDate, exif.exif.GPSTimeStamp, exif.gps); metadata.creationDate = timestampToMS(exif.ihdr["Creation Time"], any_offset); diff --git a/src/common/DataStructureVersion.ts b/src/common/DataStructureVersion.ts index 3f62a4e3..f4cf946b 100644 --- a/src/common/DataStructureVersion.ts +++ b/src/common/DataStructureVersion.ts @@ -1,4 +1,4 @@ /** * This version indicates that the sql/entities/*Entity.ts files got changed and the db needs to be recreated */ -export const DataStructureVersion = 35; +export const DataStructureVersion = 36; diff --git a/src/common/Utils.ts b/src/common/Utils.ts index e0a475d6..63e514cf 100644 --- a/src/common/Utils.ts +++ b/src/common/Utils.ts @@ -147,6 +147,26 @@ export class Utils { } } + static isLeapYear(year: number) { + return (0 == year % 4) && (0 != year % 100) || (0 == year % 400) + } + + static isDateFromLeapYear(date: Date) { + return Utils.isLeapYear(date.getFullYear()); + } + + // Get Day of Year + static getDayOfYear(date: Date) { + //Day-number at the start of Jan to Dec. A month baseline + const dayCount = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]; + const mn = date.getMonth(); + let dayOfYear = dayCount[mn] + date.getDate(); //add the date to the month baseline + if (mn > 1 && Utils.isLeapYear((date.getFullYear()))) { + dayOfYear++; //Add an extra day for march to december (mn>1) on leap years + } + return dayOfYear; + } + static renderDataSize(size: number): string { const postFixes = ['B', 'KB', 'MB', 'GB', 'TB']; let index = 0; diff --git a/src/common/config/private/subconfigs/ServerExtensionsConfig.ts b/src/common/config/private/subconfigs/ServerExtensionsConfig.ts index 0efcbf03..7aeaf2bd 100644 --- a/src/common/config/private/subconfigs/ServerExtensionsConfig.ts +++ b/src/common/config/private/subconfigs/ServerExtensionsConfig.ts @@ -2,6 +2,13 @@ import {ConfigProperty, SubConfigClass} from 'typeconfig/common'; import {ClientExtensionsConfig, ConfigPriority, TAGS} from '../../public/ClientConfig'; import {GenericConfigType} from 'typeconfig/src/GenericConfigType'; +declare let $localize: (s: TemplateStringsArray) => string; + +if (typeof $localize === 'undefined') { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + global.$localize = (s) => s; +} @SubConfigClass({softReadonly: true}) export class ServerExtensionsEntryConfig { diff --git a/src/frontend/app/ui/gallery/grid/photo/photo.grid.gallery.component.ts b/src/frontend/app/ui/gallery/grid/photo/photo.grid.gallery.component.ts index ce3716e0..ad69badb 100644 --- a/src/frontend/app/ui/gallery/grid/photo/photo.grid.gallery.component.ts +++ b/src/frontend/app/ui/gallery/grid/photo/photo.grid.gallery.component.ts @@ -134,10 +134,10 @@ export class GalleryPhotoComponent implements IRenderable, OnInit, OnDestroy { } getPositionText(): string { - if (!this.gridMedia || !this.gridMedia.isPhoto()) { + if (!this.gridMedia || !this.gridMedia.isPhoto() || !(this.gridMedia.media as PhotoDTO).metadata.positionData) { return ''; } - return ( + return ( //not much space in the gridview, so we only deliver city, or state or country (this.gridMedia.media as PhotoDTO).metadata.positionData.city || (this.gridMedia.media as PhotoDTO).metadata.positionData.state || (this.gridMedia.media as PhotoDTO).metadata.positionData.country || '' diff --git a/src/frontend/app/ui/gallery/lightbox/controls/controls.lightbox.gallery.component.ts b/src/frontend/app/ui/gallery/lightbox/controls/controls.lightbox.gallery.component.ts index 5638adc1..71c211ad 100644 --- a/src/frontend/app/ui/gallery/lightbox/controls/controls.lightbox.gallery.component.ts +++ b/src/frontend/app/ui/gallery/lightbox/controls/controls.lightbox.gallery.component.ts @@ -510,11 +510,14 @@ export class ControlsLightboxComponent implements OnDestroy, OnInit, OnChanges { case LightBoxTitleTexts.date: return this.datePipe.transform(m.metadata.creationDate, 'longDate', m.metadata.creationDateOffset); case LightBoxTitleTexts.location: - return ( - m.metadata.positionData?.city || - m.metadata.positionData?.state || - m.metadata.positionData?.country || '' - ).trim(); + if (!m.metadata.positionData) { + return ''; + } + return [ + m.metadata.positionData.city, + m.metadata.positionData.state, + m.metadata.positionData.country + ].filter(elm => elm).join(', ').trim(); //Filter removes empty elements, join concats the values separated by ', ' case LightBoxTitleTexts.camera: return m.metadata.cameraData?.model; case LightBoxTitleTexts.lens: diff --git a/src/frontend/app/ui/gallery/lightbox/infopanel/info-panel.lightbox.gallery.component.ts b/src/frontend/app/ui/gallery/lightbox/infopanel/info-panel.lightbox.gallery.component.ts index 542cc42f..fea2eb85 100644 --- a/src/frontend/app/ui/gallery/lightbox/infopanel/info-panel.lightbox.gallery.component.ts +++ b/src/frontend/app/ui/gallery/lightbox/infopanel/info-panel.lightbox.gallery.component.ts @@ -183,17 +183,11 @@ export class InfoPanelLightboxComponent implements OnInit, OnChanges { if (!(this.media as PhotoDTO).metadata.positionData) { return ''; } - let str = - (this.media as PhotoDTO).metadata.positionData.city || - (this.media as PhotoDTO).metadata.positionData.state || - ''; - - if (str.length !== 0) { - str += ', '; - } - str += (this.media as PhotoDTO).metadata.positionData.country || ''; - - return str; + return [ + (this.media as PhotoDTO).metadata.positionData.city, + (this.media as PhotoDTO).metadata.positionData.state, + (this.media as PhotoDTO).metadata.positionData.country + ].filter(elm => elm).join(', ').trim(); //Filter removes empty elements, join concats the values separated by ', ' } close(): void { diff --git a/src/frontend/app/ui/gallery/navigator/sorting.service.ts b/src/frontend/app/ui/gallery/navigator/sorting.service.ts index 7e74d737..b5718c5a 100644 --- a/src/frontend/app/ui/gallery/navigator/sorting.service.ts +++ b/src/frontend/app/ui/gallery/navigator/sorting.service.ts @@ -184,7 +184,7 @@ export class GallerySortingService { private getGroupByNameFn(grouping: GroupingMethod) { switch (grouping.method) { case SortByTypes.Date: - return (m: MediaDTO) => this.datePipe.transform(m.metadata.creationDate, 'longDate', m.metadata.creationDateOffset); + return (m: MediaDTO) => this.datePipe.transform(m.metadata.creationDate, 'longDate', m.metadata.creationDateOffset ? m.metadata.creationDateOffset : 'UTC'); case SortByTypes.Name: return (m: MediaDTO) => m.name.at(0).toUpperCase(); diff --git a/test/backend/assets/wild-1-small.jpg b/test/backend/assets/wild-1-small.jpg new file mode 100644 index 00000000..20102203 Binary files /dev/null and b/test/backend/assets/wild-1-small.jpg differ diff --git a/test/backend/assets/wild-1-small.json b/test/backend/assets/wild-1-small.json new file mode 100644 index 00000000..1c686195 --- /dev/null +++ b/test/backend/assets/wild-1-small.json @@ -0,0 +1,20 @@ +{ + "size": { + "width": 306, + "height": 204 + }, + "creationDate": 1435910683000, + "fileSize": 59564, + "positionData": { + "country": "United States", + "state": "Arizona", + "city": "Williams" + }, + "keywords": ["Akela the wolf", "Balu the bear", "Bearizona", "Hugin the raven", "USA", "USA Road trip"], + "cameraData": { + "make": "Canon", + "model": "Canon EOS 600D", + "lens": "EF-S15-85mm f/3.5-5.6 IS USM" + }, + "creationDateOffset": "+02:00" +} \ No newline at end of file diff --git a/test/backend/unit/model/sql/SearchManager.spec.ts b/test/backend/unit/model/sql/SearchManager.spec.ts index 216aa511..44f42ff3 100644 --- a/test/backend/unit/model/sql/SearchManager.spec.ts +++ b/test/backend/unit/model/sql/SearchManager.spec.ts @@ -115,16 +115,36 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => { subDir2 = TestHelper.getDirectoryEntry(directory, 'Return of the Jedi'); p = TestHelper.getPhotoEntry1(directory); p.metadata.creationDate = Date.now(); + p.metadata.creationDateOffset = "+02:00"; p2 = TestHelper.getPhotoEntry2(directory); p2.metadata.creationDate = Date.now() - 60 * 60 * 24 * 1000; + p2.metadata.creationDateOffset = "+02:00"; v = TestHelper.getVideoEntry1(directory); v.metadata.creationDate = Date.now() - 60 * 60 * 24 * 7 * 1000; + v.metadata.creationDateOffset = "+02:00"; gpx = TestHelper.getRandomizedGPXEntry(directory); p4 = TestHelper.getPhotoEntry4(subDir2); - p4.metadata.creationDate = Date.now() - 60 * 60 * 24 * 366 * 1000; + let d = new Date(); + //set creation date to one year and one day earlier + p4.metadata.creationDate = d.getTime() - 60 * 60 * 24 * (Utils.isDateFromLeapYear(d) ? 367 : 366) * 1000; + p4.metadata.creationDateOffset = "+02:00"; const pFaceLessTmp = TestHelper.getPhotoEntry3(subDir); delete pFaceLessTmp.metadata.faces; - pFaceLessTmp.metadata.creationDate = Date.now() - 60 * 60 * 24 * 32 * 1000; + d = new Date(); + //we create a date 1 month and 1 day before now + if ([1, 3, 5, 7, 8, 10, 0].includes(d.getMonth())) { + //Now is a month after a long month: feb (1), april (3), june (5), august(7), september(8), november (10), january (0) + pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 32 * 1000; + } else if (d.getMonth() == 2 && Utils.isDateFromLeapYear(d)) { + //march on leap years + pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 30 * 1000; + } else if (d.getMonth() == 2) { + //march (and not leap years) + pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 29 * 1000; + } else { //all other months must come after a short month with 30 days, so we subtract 31 + pFaceLessTmp.metadata.creationDate = d.getTime() - 60 * 60 * 24 * 31 * 1000; + } + pFaceLessTmp.metadata.creationDateOffset = "+02:00"; dir = await DBTestHelper.persistTestDir(directory); subDir = dir.directories[0]; @@ -937,13 +957,16 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => { await setUpSqlDB(); p5 = TestHelper.getBasePhotoEntry(subDir2, 'p5-23h-ago.jpg'); p5.metadata.creationDate = Date.now() - 60 * 60 * 24 * 1000 - 1000; + //p5.metadata.creationDateOffset = "+02:00"; p6 = TestHelper.getBasePhotoEntry(subDir2, 'p6-300d-ago.jpg'); p6.metadata.creationDate = Date.now() - 60 * 60 * 24 * 300 * 1000; + //p6.metadata.creationDateOffset = "+02:00"; p7 = TestHelper.getBasePhotoEntry(subDir2, 'p7-1y-1min-ago.jpg'); const d = new Date(); d.setUTCFullYear(d.getUTCFullYear() - 1); d.setUTCMinutes(d.getUTCMinutes() - 1); p7.metadata.creationDate = d.getTime(); + //p7.metadata.creationDateOffset = "+02:00"; subDir2 = await DBTestHelper.persistTestDir(subDir2) as any; p4 = subDir2.media[0]; diff --git a/test/backend/unit/model/threading/DiskManagerWorker.spec.ts b/test/backend/unit/model/threading/DiskManagerWorker.spec.ts index 8d23fdad..a04ee32f 100644 --- a/test/backend/unit/model/threading/DiskManagerWorker.spec.ts +++ b/test/backend/unit/model/threading/DiskManagerWorker.spec.ts @@ -24,7 +24,7 @@ describe('DiskMangerWorker', () => { ProjectPath.ImageFolder = path.join(__dirname, '/../../../assets'); const dir = await DiskManager.scanDirectory('/'); // should match the number of media (photo/video) files in the assets folder - expect(dir.media.length).to.be.equals(14); + expect(dir.media.length).to.be.equals(15); // eslint-disable-next-line @typescript-eslint/no-var-requires const expected = require(path.join(__dirname, '/../../../assets/test image öüóőúéáű-.,.json')); const i = dir.media.findIndex(m => m.name === 'test image öüóőúéáű-.,.jpg'); diff --git a/test/backend/unit/model/threading/MetaDataLoader.spec.ts b/test/backend/unit/model/threading/MetaDataLoader.spec.ts index 2286212b..298c6582 100644 --- a/test/backend/unit/model/threading/MetaDataLoader.spec.ts +++ b/test/backend/unit/model/threading/MetaDataLoader.spec.ts @@ -172,6 +172,11 @@ describe('MetadataLoader', () => { expect(Utils.clone(data)).to.be.deep.equal(expected); }); }); + it('should load wild-1-small image with CreateDate from 2015, but no DateTimeOriginal', async () => { + const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../../assets/wild-1-small.jpg')); + const expected = require(path.join(__dirname, '/../../../assets/wild-1-small.json')); + expect(Utils.clone(data)).to.be.deep.equal(expected); + }); describe('should load jpg with edge case exif data', () => { const root = path.join(__dirname, '/../../../assets/edge_case_exif_data');