diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7a3c97aa4..dcd3d344fd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -269,6 +269,9 @@ jobs: - name: Run existing migrations run: npm run typeorm:migrations:run + - name: Test npm run schema:reset command works + run: npm run typeorm:schema:reset + - name: Generate new migrations continue-on-error: true run: npm run typeorm:migrations:generate ./src/infra/migrations/TestMigration diff --git a/cli/src/api/open-api/api.ts b/cli/src/api/open-api/api.ts index 04cc134b65..05cac4bcff 100644 --- a/cli/src/api/open-api/api.ts +++ b/cli/src/api/open-api/api.ts @@ -3780,12 +3780,6 @@ export interface SystemConfigJobDto { * @memberof SystemConfigJobDto */ 'smartSearch': JobSettingsDto; - /** - * - * @type {JobSettingsDto} - * @memberof SystemConfigJobDto - */ - 'storageTemplateMigration': JobSettingsDto; /** * * @type {JobSettingsDto} @@ -4026,6 +4020,18 @@ export interface SystemConfigReverseGeocodingDto { * @interface SystemConfigStorageTemplateDto */ export interface SystemConfigStorageTemplateDto { + /** + * + * @type {boolean} + * @memberof SystemConfigStorageTemplateDto + */ + 'enabled': boolean; + /** + * + * @type {boolean} + * @memberof SystemConfigStorageTemplateDto + */ + 'hashVerificationEnabled': boolean; /** * * @type {string} diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index a45c7490af..ce1fe5447b 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -16,6 +16,7 @@ x-server-build: &server-common volumes: - ../server:/usr/src/app - ${UPLOAD_LOCATION}/photos:/usr/src/app/upload + - ${UPLOAD_LOCATION}/photos/upload:/usr/src/app/upload/upload - /usr/src/app/node_modules - /etc/localtime:/etc/localtime:ro env_file: diff --git a/mobile/openapi/doc/SystemConfigJobDto.md b/mobile/openapi/doc/SystemConfigJobDto.md index 15d6bc3c08..0e21413da1 100644 Binary files a/mobile/openapi/doc/SystemConfigJobDto.md and b/mobile/openapi/doc/SystemConfigJobDto.md differ diff --git a/mobile/openapi/doc/SystemConfigStorageTemplateDto.md b/mobile/openapi/doc/SystemConfigStorageTemplateDto.md index 88bfe4569b..c456480690 100644 Binary files a/mobile/openapi/doc/SystemConfigStorageTemplateDto.md and b/mobile/openapi/doc/SystemConfigStorageTemplateDto.md differ diff --git a/mobile/openapi/lib/model/system_config_job_dto.dart b/mobile/openapi/lib/model/system_config_job_dto.dart index 87e9f8db83..a3882f2c0b 100644 Binary files a/mobile/openapi/lib/model/system_config_job_dto.dart and b/mobile/openapi/lib/model/system_config_job_dto.dart differ diff --git a/mobile/openapi/lib/model/system_config_storage_template_dto.dart b/mobile/openapi/lib/model/system_config_storage_template_dto.dart index 604fe1256e..d2ed327733 100644 Binary files a/mobile/openapi/lib/model/system_config_storage_template_dto.dart and b/mobile/openapi/lib/model/system_config_storage_template_dto.dart differ diff --git a/mobile/openapi/test/system_config_job_dto_test.dart b/mobile/openapi/test/system_config_job_dto_test.dart index 9ba46e211a..e677834611 100644 Binary files a/mobile/openapi/test/system_config_job_dto_test.dart and b/mobile/openapi/test/system_config_job_dto_test.dart differ diff --git a/mobile/openapi/test/system_config_storage_template_dto_test.dart b/mobile/openapi/test/system_config_storage_template_dto_test.dart index 9f5a2e5e10..aa00b78fe4 100644 Binary files a/mobile/openapi/test/system_config_storage_template_dto_test.dart and b/mobile/openapi/test/system_config_storage_template_dto_test.dart differ diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index 99779609bb..9d23c8e555 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -9171,9 +9171,6 @@ "smartSearch": { "$ref": "#/components/schemas/JobSettingsDto" }, - "storageTemplateMigration": { - "$ref": "#/components/schemas/JobSettingsDto" - }, "thumbnailGeneration": { "$ref": "#/components/schemas/JobSettingsDto" }, @@ -9186,7 +9183,6 @@ "metadataExtraction", "videoConversion", "smartSearch", - "storageTemplateMigration", "migration", "backgroundTask", "search", @@ -9365,11 +9361,19 @@ }, "SystemConfigStorageTemplateDto": { "properties": { + "enabled": { + "type": "boolean" + }, + "hashVerificationEnabled": { + "type": "boolean" + }, "template": { "type": "string" } }, "required": [ + "enabled", + "hashVerificationEnabled", "template" ], "type": "object" diff --git a/server/package-lock.json b/server/package-lock.json index 9e928c45d0..d3e951ba21 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -41,7 +41,6 @@ "joi": "^17.10.0", "lodash": "^4.17.21", "luxon": "^3.4.2", - "mv": "^2.1.1", "nest-commander": "^3.11.1", "node-addon-api": "^7.0.0", "openid-client": "^5.4.3", @@ -72,7 +71,6 @@ "@types/lodash": "^4.14.197", "@types/mock-fs": "^4.13.1", "@types/multer": "^1.4.7", - "@types/mv": "^2.1.2", "@types/node": "^20.5.7", "@types/sharp": "^0.31.1", "@types/supertest": "^2.0.12", @@ -3548,12 +3546,6 @@ "@types/express": "*" } }, - "node_modules/@types/mv": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@types/mv/-/mv-2.1.4.tgz", - "integrity": "sha512-MgEHBpXnQo44Q43j8G0Bvp/Yi8LYbC8hxKrRFMgDEDZMmzDKZLgiyMWtW49B37ko+QupgZ3G5rtPUnOGe5ixLw==", - "dev": true - }, "node_modules/@types/node": { "version": "20.10.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.3.tgz", @@ -9293,45 +9285,6 @@ "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" }, - "node_modules/mv": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", - "integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==", - "dependencies": { - "mkdirp": "~0.5.1", - "ncp": "~2.0.0", - "rimraf": "~2.4.0" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/mv/node_modules/glob": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", - "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==", - "dependencies": { - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "2 || 3", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - } - }, - "node_modules/mv/node_modules/rimraf": { - "version": "2.4.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", - "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==", - "dependencies": { - "glob": "^6.0.1" - }, - "bin": { - "rimraf": "bin.js" - } - }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -9355,14 +9308,6 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, - "node_modules/ncp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", - "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==", - "bin": { - "ncp": "bin/ncp" - } - }, "node_modules/nearley": { "version": "2.20.1", "resolved": "https://registry.npmjs.org/nearley/-/nearley-2.20.1.tgz", @@ -15519,12 +15464,6 @@ "@types/express": "*" } }, - "@types/mv": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@types/mv/-/mv-2.1.4.tgz", - "integrity": "sha512-MgEHBpXnQo44Q43j8G0Bvp/Yi8LYbC8hxKrRFMgDEDZMmzDKZLgiyMWtW49B37ko+QupgZ3G5rtPUnOGe5ixLw==", - "dev": true - }, "@types/node": { "version": "20.10.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.3.tgz", @@ -19842,38 +19781,6 @@ "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" }, - "mv": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", - "integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==", - "requires": { - "mkdirp": "~0.5.1", - "ncp": "~2.0.0", - "rimraf": "~2.4.0" - }, - "dependencies": { - "glob": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", - "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==", - "requires": { - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "2 || 3", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "rimraf": { - "version": "2.4.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", - "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==", - "requires": { - "glob": "^6.0.1" - } - } - } - }, "mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -19897,11 +19804,6 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, - "ncp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", - "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==" - }, "nearley": { "version": "2.20.1", "resolved": "https://registry.npmjs.org/nearley/-/nearley-2.20.1.tgz", diff --git a/server/package.json b/server/package.json index cee957ebe0..9db511eeac 100644 --- a/server/package.json +++ b/server/package.json @@ -28,7 +28,7 @@ "typeorm:migrations:generate": "typeorm migration:generate -d ./dist/infra/database.config.js", "typeorm:migrations:run": "typeorm migration:run -d ./dist/infra/database.config.js", "typeorm:migrations:revert": "typeorm migration:revert -d ./dist/infra/database.config.js", - "typeorm:schema:drop": "typeorm schema:drop -d ./dist/infra/database.config.js", + "typeorm:schema:drop": "typeorm query -d ./dist/infra/database.config.js 'DROP schema public cascade; CREATE schema public;'", "typeorm:schema:reset": "npm run typeorm:schema:drop && npm run typeorm:migrations:run", "api:typescript": "bash ./bin/generate-open-api.sh web", "api:dart": "bash ./bin/generate-open-api.sh mobile", @@ -68,7 +68,6 @@ "joi": "^17.10.0", "lodash": "^4.17.21", "luxon": "^3.4.2", - "mv": "^2.1.1", "nest-commander": "^3.11.1", "node-addon-api": "^7.0.0", "openid-client": "^5.4.3", @@ -99,7 +98,6 @@ "@types/lodash": "^4.14.197", "@types/mock-fs": "^4.13.1", "@types/multer": "^1.4.7", - "@types/mv": "^2.1.2", "@types/node": "^20.5.7", "@types/sharp": "^0.31.1", "@types/supertest": "^2.0.12", diff --git a/server/src/domain/job/job.service.spec.ts b/server/src/domain/job/job.service.spec.ts index 27c73184d4..decf269987 100644 --- a/server/src/domain/job/job.service.spec.ts +++ b/server/src/domain/job/job.service.spec.ts @@ -226,7 +226,6 @@ describe(JobService.name, () => { [QueueName.SEARCH]: { concurrency: 10 }, [QueueName.SIDECAR]: { concurrency: 10 }, [QueueName.LIBRARY]: { concurrency: 10 }, - [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 }, [QueueName.MIGRATION]: { concurrency: 10 }, [QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 }, [QueueName.VIDEO_CONVERSION]: { concurrency: 10 }, @@ -239,7 +238,6 @@ describe(JobService.name, () => { expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.RECOGNIZE_FACES, 10); expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10); expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10); - expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10); expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.MIGRATION, 10); expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10); expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10); diff --git a/server/src/domain/job/job.service.ts b/server/src/domain/job/job.service.ts index 95fecd4275..312b862f69 100644 --- a/server/src/domain/job/job.service.ts +++ b/server/src/domain/job/job.service.ts @@ -123,7 +123,11 @@ export class JobService { async registerHandlers(jobHandlers: Record) { const config = await this.configCore.getConfig(); for (const queueName of Object.values(QueueName)) { - const concurrency = config.job[queueName].concurrency; + let concurrency = 1; + if (queueName !== QueueName.STORAGE_TEMPLATE_MIGRATION) { + concurrency = config.job[queueName].concurrency; + } + this.logger.debug(`Registering ${queueName} with a concurrency of ${concurrency}`); this.jobRepository.addHandler(queueName, concurrency, async (item: JobItem): Promise => { const { name, data } = item; @@ -143,7 +147,10 @@ export class JobService { this.configCore.config$.subscribe((config) => { this.logger.log(`Updating queue concurrency settings`); for (const queueName of Object.values(QueueName)) { - const concurrency = config.job[queueName].concurrency; + let concurrency = 1; + if (queueName !== QueueName.STORAGE_TEMPLATE_MIGRATION) { + concurrency = config.job[queueName].concurrency; + } this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`); this.jobRepository.setConcurrency(queueName, concurrency); } diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index 5cdcabdd85..2ab3def4f1 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -12,6 +12,7 @@ import { assetStub, faceStub, newAssetRepositoryMock, + newCryptoRepositoryMock, newJobRepositoryMock, newMediaRepositoryMock, newMoveRepositoryMock, @@ -24,6 +25,7 @@ import { import { JobName } from '../job'; import { IAssetRepository, + ICryptoRepository, IJobRepository, IMediaRepository, IMoveRepository, @@ -43,6 +45,7 @@ describe(MediaService.name, () => { let moveMock: jest.Mocked; let personMock: jest.Mocked; let storageMock: jest.Mocked; + let cryptoMock: jest.Mocked; beforeEach(async () => { assetMock = newAssetRepositoryMock(); @@ -52,8 +55,9 @@ describe(MediaService.name, () => { moveMock = newMoveRepositoryMock(); personMock = newPersonRepositoryMock(); storageMock = newStorageRepositoryMock(); + cryptoMock = newCryptoRepositoryMock(); - sut = new MediaService(assetMock, personMock, jobMock, mediaMock, storageMock, configMock, moveMock); + sut = new MediaService(assetMock, personMock, jobMock, mediaMock, storageMock, configMock, moveMock, cryptoMock); }); it('should be defined', () => { diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index e3e408913d..6f16a7b66e 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -14,6 +14,7 @@ import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } import { AudioStreamInfo, IAssetRepository, + ICryptoRepository, IJobRepository, IMediaRepository, IMoveRepository, @@ -52,9 +53,17 @@ export class MediaService { @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, @Inject(IMoveRepository) moveRepository: IMoveRepository, + @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, ) { this.configCore = SystemConfigCore.create(configRepository); - this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository); + this.storageCore = StorageCore.create( + assetRepository, + moveRepository, + personRepository, + cryptoRepository, + configRepository, + storageRepository, + ); } async handleQueueGenerateThumbnails({ force }: IBaseJob) { diff --git a/server/src/domain/metadata/metadata.service.spec.ts b/server/src/domain/metadata/metadata.service.spec.ts index 70589746f8..fd2a29d45c 100644 --- a/server/src/domain/metadata/metadata.service.spec.ts +++ b/server/src/domain/metadata/metadata.service.spec.ts @@ -239,15 +239,6 @@ describe(MetadataService.name, () => { expect(assetMock.save).not.toHaveBeenCalled(); }); - it('should handle an asset with isVisible set to false', async () => { - assetMock.getByIds.mockResolvedValue([{ ...assetStub.image, isVisible: false }]); - - await expect(sut.handleMetadataExtraction({ id: assetStub.image.id })).resolves.toBe(false); - expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]); - expect(assetMock.upsertExif).not.toHaveBeenCalled(); - expect(assetMock.save).not.toHaveBeenCalled(); - }); - it('should handle a date in a sidecar file', async () => { const originalDate = new Date('2023-11-21T16:13:17.517Z'); const sidecarDate = new Date('2022-01-01T00:00:00.000Z'); diff --git a/server/src/domain/metadata/metadata.service.ts b/server/src/domain/metadata/metadata.service.ts index cebb040231..544813d508 100644 --- a/server/src/domain/metadata/metadata.service.ts +++ b/server/src/domain/metadata/metadata.service.ts @@ -114,7 +114,14 @@ export class MetadataService { @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, ) { this.configCore = SystemConfigCore.create(configRepository); - this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository); + this.storageCore = StorageCore.create( + assetRepository, + moveRepository, + personRepository, + cryptoRepository, + configRepository, + storageRepository, + ); } async init() { @@ -199,7 +206,7 @@ export class MetadataService { async handleMetadataExtraction({ id }: IEntityJob) { const [asset] = await this.assetRepository.getByIds([id]); - if (!asset || !asset.isVisible) { + if (!asset) { return false; } diff --git a/server/src/domain/person/person.service.spec.ts b/server/src/domain/person/person.service.spec.ts index 2decdb2209..4ca718e417 100644 --- a/server/src/domain/person/person.service.spec.ts +++ b/server/src/domain/person/person.service.spec.ts @@ -7,6 +7,7 @@ import { faceStub, newAccessRepositoryMock, newAssetRepositoryMock, + newCryptoRepositoryMock, newJobRepositoryMock, newMachineLearningRepositoryMock, newMediaRepositoryMock, @@ -22,6 +23,7 @@ import { CacheControl, ImmichFileResponse } from '../domain.util'; import { JobName } from '../job'; import { IAssetRepository, + ICryptoRepository, IJobRepository, IMachineLearningRepository, IMediaRepository, @@ -73,6 +75,7 @@ describe(PersonService.name, () => { let personMock: jest.Mocked; let storageMock: jest.Mocked; let smartInfoMock: jest.Mocked; + let cryptoMock: jest.Mocked; let sut: PersonService; beforeEach(async () => { @@ -86,6 +89,7 @@ describe(PersonService.name, () => { personMock = newPersonRepositoryMock(); storageMock = newStorageRepositoryMock(); smartInfoMock = newSmartInfoRepositoryMock(); + cryptoMock = newCryptoRepositoryMock(); sut = new PersonService( accessMock, assetMock, @@ -97,6 +101,7 @@ describe(PersonService.name, () => { storageMock, jobMock, smartInfoMock, + cryptoMock, ); mediaMock.crop.mockResolvedValue(croppedFace); diff --git a/server/src/domain/person/person.service.ts b/server/src/domain/person/person.service.ts index fd80e444b2..d63eda3cea 100644 --- a/server/src/domain/person/person.service.ts +++ b/server/src/domain/person/person.service.ts @@ -13,6 +13,7 @@ import { CropOptions, IAccessRepository, IAssetRepository, + ICryptoRepository, IJobRepository, IMachineLearningRepository, IMediaRepository, @@ -59,10 +60,18 @@ export class PersonService { @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository, + @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, ) { this.access = AccessCore.create(accessRepository); this.configCore = SystemConfigCore.create(configRepository); - this.storageCore = StorageCore.create(assetRepository, moveRepository, repository, storageRepository); + this.storageCore = StorageCore.create( + assetRepository, + moveRepository, + repository, + cryptoRepository, + configRepository, + storageRepository, + ); } async getAll(auth: AuthDto, dto: PersonSearchDto): Promise { diff --git a/server/src/domain/repositories/storage.repository.ts b/server/src/domain/repositories/storage.repository.ts index 857783b701..902980a7b2 100644 --- a/server/src/domain/repositories/storage.repository.ts +++ b/server/src/domain/repositories/storage.repository.ts @@ -30,11 +30,12 @@ export interface IStorageRepository { unlink(filepath: string): Promise; unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise; removeEmptyDirs(folder: string, self?: boolean): Promise; - moveFile(source: string, target: string): Promise; checkFileExists(filepath: string, mode?: number): Promise; mkdirSync(filepath: string): void; checkDiskUsage(folder: string): Promise; readdir(folder: string): Promise; stat(filepath: string): Promise; crawl(crawlOptions: CrawlOptionsDto): Promise; + copyFile(source: string, target: string): Promise; + rename(source: string, target: string): Promise; } diff --git a/server/src/domain/storage-template/storage-template.service.spec.ts b/server/src/domain/storage-template/storage-template.service.spec.ts index 825a5cb25e..37c21827a2 100644 --- a/server/src/domain/storage-template/storage-template.service.spec.ts +++ b/server/src/domain/storage-template/storage-template.service.spec.ts @@ -1,8 +1,21 @@ -import { AssetPathType } from '@app/infra/entities'; +import { + IAlbumRepository, + IAssetRepository, + ICryptoRepository, + IMoveRepository, + IPersonRepository, + IStorageRepository, + ISystemConfigRepository, + IUserRepository, + StorageTemplateService, + defaults, +} from '@app/domain'; +import { AssetPathType, SystemConfigKey } from '@app/infra/entities'; import { assetStub, newAlbumRepositoryMock, newAssetRepositoryMock, + newCryptoRepositoryMock, newMoveRepositoryMock, newPersonRepositoryMock, newStorageRepositoryMock, @@ -11,17 +24,7 @@ import { userStub, } from '@test'; import { when } from 'jest-when'; -import { - IAlbumRepository, - IAssetRepository, - IMoveRepository, - IPersonRepository, - IStorageRepository, - ISystemConfigRepository, - IUserRepository, -} from '../repositories'; -import { defaults } from '../system-config/system-config.core'; -import { StorageTemplateService } from './storage-template.service'; +import { Stats } from 'node:fs'; describe(StorageTemplateService.name, () => { let sut: StorageTemplateService; @@ -32,19 +35,21 @@ describe(StorageTemplateService.name, () => { let personMock: jest.Mocked; let storageMock: jest.Mocked; let userMock: jest.Mocked; + let cryptoMock: jest.Mocked; it('should work', () => { expect(sut).toBeDefined(); }); beforeEach(async () => { + configMock = newSystemConfigRepositoryMock(); assetMock = newAssetRepositoryMock(); albumMock = newAlbumRepositoryMock(); - configMock = newSystemConfigRepositoryMock(); moveMock = newMoveRepositoryMock(); personMock = newPersonRepositoryMock(); storageMock = newStorageRepositoryMock(); userMock = newUserRepositoryMock(); + cryptoMock = newCryptoRepositoryMock(); sut = new StorageTemplateService( albumMock, @@ -55,27 +60,37 @@ describe(StorageTemplateService.name, () => { personMock, storageMock, userMock, + cryptoMock, ); + + configMock.load.mockResolvedValue([{ key: SystemConfigKey.STORAGE_TEMPLATE_ENABLED, value: true }]); }); describe('handleMigrationSingle', () => { + it('should skip when storage template is disabled', async () => { + configMock.load.mockResolvedValue([{ key: SystemConfigKey.STORAGE_TEMPLATE_ENABLED, value: false }]); + await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true); + expect(assetMock.getByIds).not.toHaveBeenCalled(); + expect(storageMock.checkFileExists).not.toHaveBeenCalled(); + expect(storageMock.rename).not.toHaveBeenCalled(); + expect(storageMock.copyFile).not.toHaveBeenCalled(); + expect(assetMock.save).not.toHaveBeenCalled(); + expect(moveMock.create).not.toHaveBeenCalled(); + expect(moveMock.update).not.toHaveBeenCalled(); + expect(storageMock.stat).not.toHaveBeenCalled(); + }); + it('should migrate single moving picture', async () => { userMock.get.mockResolvedValue(userStub.user1); - const path = (id: string) => `upload/library/${userStub.user1.id}/2023/2023-02-23/${id}.jpg`; - const newPath = (id: string) => `upload/library/${userStub.user1.id}/2023/2023-02-23/${id}+1.jpg`; - - when(storageMock.checkFileExists).calledWith(path(assetStub.livePhotoStillAsset.id)).mockResolvedValue(true); - when(storageMock.checkFileExists).calledWith(newPath(assetStub.livePhotoStillAsset.id)).mockResolvedValue(false); - - when(storageMock.checkFileExists).calledWith(path(assetStub.livePhotoMotionAsset.id)).mockResolvedValue(true); - when(storageMock.checkFileExists).calledWith(newPath(assetStub.livePhotoMotionAsset.id)).mockResolvedValue(false); + const newMotionPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.mp4`; + const newStillPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.jpeg`; when(assetMock.save) - .calledWith({ id: assetStub.livePhotoStillAsset.id, originalPath: newPath(assetStub.livePhotoStillAsset.id) }) + .calledWith({ id: assetStub.livePhotoStillAsset.id, originalPath: newStillPicturePath }) .mockResolvedValue(assetStub.livePhotoStillAsset); when(assetMock.save) - .calledWith({ id: assetStub.livePhotoMotionAsset.id, originalPath: newPath(assetStub.livePhotoMotionAsset.id) }) + .calledWith({ id: assetStub.livePhotoMotionAsset.id, originalPath: newMotionPicturePath }) .mockResolvedValue(assetStub.livePhotoMotionAsset); when(assetMock.getByIds) @@ -86,11 +101,265 @@ describe(StorageTemplateService.name, () => { .calledWith([assetStub.livePhotoMotionAsset.id]) .mockResolvedValue([assetStub.livePhotoMotionAsset]); + when(moveMock.create) + .calledWith({ + entityId: assetStub.livePhotoStillAsset.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.livePhotoStillAsset.originalPath, + newPath: newStillPicturePath, + }) + .mockResolvedValue({ + id: '123', + entityId: assetStub.livePhotoStillAsset.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.livePhotoStillAsset.originalPath, + newPath: newStillPicturePath, + }); + + when(moveMock.create) + .calledWith({ + entityId: assetStub.livePhotoMotionAsset.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.livePhotoMotionAsset.originalPath, + newPath: newMotionPicturePath, + }) + .mockResolvedValue({ + id: '124', + entityId: assetStub.livePhotoMotionAsset.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.livePhotoMotionAsset.originalPath, + newPath: newMotionPicturePath, + }); + await expect(sut.handleMigrationSingle({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(true); expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id]); expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]); + expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2); + expect(assetMock.save).toHaveBeenCalledWith({ + id: assetStub.livePhotoStillAsset.id, + originalPath: newStillPicturePath, + }); + expect(assetMock.save).toHaveBeenCalledWith({ + id: assetStub.livePhotoMotionAsset.id, + originalPath: newMotionPicturePath, + }); }); + it('should migrate previously failed move from original path when it still exists', async () => { + userMock.get.mockResolvedValue(userStub.user1); + const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`; + const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; + + when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(true); + when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(false); + + when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath: previousFailedNewPath, + }); + + when(assetMock.save) + .calledWith({ id: assetStub.image.id, originalPath: newPath }) + .mockResolvedValue(assetStub.image); + + when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]); + + when(moveMock.update) + .calledWith({ + id: '123', + oldPath: assetStub.image.originalPath, + newPath, + }) + .mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath, + }); + + await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true); + + expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]); + expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3); + expect(storageMock.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath); + expect(moveMock.update).toHaveBeenCalledWith({ + id: '123', + oldPath: assetStub.image.originalPath, + newPath, + }); + expect(assetMock.save).toHaveBeenCalledWith({ + id: assetStub.image.id, + originalPath: newPath, + }); + }); + it('should migrate previously failed move from previous new path when old path no longer exists, should validate file size still matches before moving', async () => { + userMock.get.mockResolvedValue(userStub.user1); + const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`; + const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; + + when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(false); + when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(true); + when(storageMock.stat) + .calledWith(previousFailedNewPath) + .mockResolvedValue({ size: 5000 } as Stats); + when(cryptoMock.hashFile).calledWith(previousFailedNewPath).mockResolvedValue(assetStub.image.checksum); + + when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath: previousFailedNewPath, + }); + + when(assetMock.save) + .calledWith({ id: assetStub.image.id, originalPath: newPath }) + .mockResolvedValue(assetStub.image); + + when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]); + + when(moveMock.update) + .calledWith({ + id: '123', + oldPath: previousFailedNewPath, + newPath, + }) + .mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: previousFailedNewPath, + newPath, + }); + + await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true); + + expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]); + expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3); + expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath); + expect(storageMock.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath); + expect(storageMock.copyFile).not.toHaveBeenCalled(); + expect(moveMock.update).toHaveBeenCalledWith({ + id: '123', + oldPath: previousFailedNewPath, + newPath, + }); + expect(assetMock.save).toHaveBeenCalledWith({ + id: assetStub.image.id, + originalPath: newPath, + }); + }); + + it('should fail move if copying and hash of asset and the new file do not match', async () => { + userMock.get.mockResolvedValue(userStub.user1); + const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; + + when(storageMock.rename).calledWith(assetStub.image.originalPath, newPath).mockRejectedValue({ code: 'EXDEV' }); + when(storageMock.stat) + .calledWith(newPath) + .mockResolvedValue({ size: 5000 } as Stats); + when(cryptoMock.hashFile).calledWith(newPath).mockResolvedValue(Buffer.from('different-hash', 'utf-8')); + + when(assetMock.save) + .calledWith({ id: assetStub.image.id, originalPath: newPath }) + .mockResolvedValue(assetStub.image); + + when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]); + + when(moveMock.create) + .calledWith({ + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath: newPath, + }) + .mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath, + }); + + await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true); + + expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]); + expect(storageMock.checkFileExists).toHaveBeenCalledTimes(1); + expect(storageMock.stat).toHaveBeenCalledWith(newPath); + expect(moveMock.create).toHaveBeenCalledWith({ + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath: newPath, + }); + expect(storageMock.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath); + expect(storageMock.copyFile).toHaveBeenCalledWith(assetStub.image.originalPath, newPath); + expect(storageMock.unlink).toHaveBeenCalledWith(newPath); + expect(storageMock.unlink).toHaveBeenCalledTimes(1); + expect(assetMock.save).not.toHaveBeenCalled(); + }); + + it.each` + failedPathChecksum | failedPathSize | reason + ${assetStub.image.checksum} | ${500} | ${'file size'} + ${Buffer.from('bad checksum', 'utf-8')} | ${assetStub.image.exifInfo?.fileSizeInByte} | ${'checksum'} + `( + 'should fail to migrate previously failed move from previous new path when old path no longer exists if $reason validation fails', + async ({ failedPathChecksum, failedPathSize }) => { + userMock.get.mockResolvedValue(userStub.user1); + const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`; + const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`; + + when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(false); + when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(true); + when(storageMock.stat) + .calledWith(previousFailedNewPath) + .mockResolvedValue({ size: failedPathSize } as Stats); + when(cryptoMock.hashFile).calledWith(previousFailedNewPath).mockResolvedValue(failedPathChecksum); + + when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath: previousFailedNewPath, + }); + + when(assetMock.save) + .calledWith({ id: assetStub.image.id, originalPath: newPath }) + .mockResolvedValue(assetStub.image); + + when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]); + + when(moveMock.update) + .calledWith({ + id: '123', + oldPath: previousFailedNewPath, + newPath, + }) + .mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: previousFailedNewPath, + newPath, + }); + + await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true); + + expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]); + expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3); + expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath); + expect(storageMock.rename).not.toHaveBeenCalled(); + expect(storageMock.copyFile).not.toHaveBeenCalled(); + expect(moveMock.update).not.toHaveBeenCalled(); + expect(assetMock.save).not.toHaveBeenCalled(); + }, + ); }); describe('handle template migration', () => { @@ -155,7 +424,8 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); expect(assetMock.getAll).toHaveBeenCalled(); - expect(storageMock.moveFile).not.toHaveBeenCalled(); + expect(storageMock.rename).not.toHaveBeenCalled(); + expect(storageMock.copyFile).not.toHaveBeenCalled(); expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2); expect(assetMock.save).not.toHaveBeenCalled(); }); @@ -175,7 +445,8 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); expect(assetMock.getAll).toHaveBeenCalled(); - expect(storageMock.moveFile).not.toHaveBeenCalled(); + expect(storageMock.rename).not.toHaveBeenCalled(); + expect(storageMock.copyFile).not.toHaveBeenCalled(); expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2); expect(assetMock.save).not.toHaveBeenCalled(); }); @@ -198,7 +469,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); expect(assetMock.getAll).toHaveBeenCalled(); - expect(storageMock.moveFile).toHaveBeenCalledWith( + expect(storageMock.rename).toHaveBeenCalledWith( '/original/path.jpg', 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', ); @@ -226,7 +497,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); expect(assetMock.getAll).toHaveBeenCalled(); - expect(storageMock.moveFile).toHaveBeenCalledWith( + expect(storageMock.rename).toHaveBeenCalledWith( '/original/path.jpg', 'upload/library/label-1/2023/2023-02-23/asset-id.jpg', ); @@ -236,12 +507,84 @@ describe(StorageTemplateService.name, () => { }); }); + it('should copy the file if rename fails due to EXDEV (rename across filesystems)', async () => { + const newPath = 'upload/library/user-id/2023/2023-02-23/asset-id.jpg'; + assetMock.getAll.mockResolvedValue({ + items: [assetStub.image], + hasNextPage: false, + }); + storageMock.rename.mockRejectedValue({ code: 'EXDEV' }); + userMock.getList.mockResolvedValue([userStub.user1]); + moveMock.create.mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath, + }); + when(storageMock.stat) + .calledWith(newPath) + .mockResolvedValue({ + size: 5000, + } as Stats); + when(cryptoMock.hashFile).calledWith(newPath).mockResolvedValue(assetStub.image.checksum); + + await sut.handleMigration(); + + expect(assetMock.getAll).toHaveBeenCalled(); + expect(storageMock.rename).toHaveBeenCalledWith('/original/path.jpg', newPath); + expect(storageMock.copyFile).toHaveBeenCalledWith('/original/path.jpg', newPath); + expect(storageMock.stat).toHaveBeenCalledWith(newPath); + expect(storageMock.unlink).toHaveBeenCalledWith(assetStub.image.originalPath); + expect(storageMock.unlink).toHaveBeenCalledTimes(1); + expect(assetMock.save).toHaveBeenCalledWith({ + id: assetStub.image.id, + originalPath: newPath, + }); + }); + + it('should not update the database if the move fails due to incorrect newPath filesize', async () => { + assetMock.getAll.mockResolvedValue({ + items: [assetStub.image], + hasNextPage: false, + }); + storageMock.rename.mockRejectedValue({ code: 'EXDEV' }); + userMock.getList.mockResolvedValue([userStub.user1]); + moveMock.create.mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + }); + when(storageMock.stat) + .calledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg') + .mockResolvedValue({ + size: 100, + } as Stats); + + await sut.handleMigration(); + + expect(assetMock.getAll).toHaveBeenCalled(); + expect(storageMock.rename).toHaveBeenCalledWith( + '/original/path.jpg', + 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + ); + expect(storageMock.copyFile).toHaveBeenCalledWith( + '/original/path.jpg', + 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + ); + expect(storageMock.stat).toHaveBeenCalledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg'); + expect(assetMock.save).not.toHaveBeenCalled(); + }); + it('should not update the database if the move fails', async () => { assetMock.getAll.mockResolvedValue({ items: [assetStub.image], hasNextPage: false, }); - storageMock.moveFile.mockRejectedValue(new Error('Read only system')); + storageMock.rename.mockRejectedValue(new Error('Read only system')); + storageMock.copyFile.mockRejectedValue(new Error('Read only system')); moveMock.create.mockResolvedValue({ id: 'move-123', entityId: '123', @@ -254,7 +597,7 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); expect(assetMock.getAll).toHaveBeenCalled(); - expect(storageMock.moveFile).toHaveBeenCalledWith( + expect(storageMock.rename).toHaveBeenCalledWith( '/original/path.jpg', 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', ); @@ -278,7 +621,8 @@ describe(StorageTemplateService.name, () => { await sut.handleMigration(); expect(assetMock.getAll).toHaveBeenCalled(); - expect(storageMock.moveFile).not.toHaveBeenCalled(); + expect(storageMock.rename).not.toHaveBeenCalled(); + expect(storageMock.copyFile).not.toHaveBeenCalled(); expect(assetMock.save).not.toHaveBeenCalled(); }); }); diff --git a/server/src/domain/storage-template/storage-template.service.ts b/server/src/domain/storage-template/storage-template.service.ts index cbaf554112..86ea0ae365 100644 --- a/server/src/domain/storage-template/storage-template.service.ts +++ b/server/src/domain/storage-template/storage-template.service.ts @@ -10,6 +10,7 @@ import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE } from '../job'; import { IAlbumRepository, IAssetRepository, + ICryptoRepository, IMoveRepository, IPersonRepository, IStorageRepository, @@ -61,6 +62,7 @@ export class StorageTemplateService { @Inject(IPersonRepository) personRepository: IPersonRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(IUserRepository) private userRepository: IUserRepository, + @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, ) { this.template = this.compile(config.storageTemplate.template); this.configCore = SystemConfigCore.create(configRepository); @@ -70,10 +72,22 @@ export class StorageTemplateService { this.logger.debug(`Received config, compiling storage template: ${template}`); this.template = this.compile(template); }); - this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository); + this.storageCore = StorageCore.create( + assetRepository, + moveRepository, + personRepository, + cryptoRepository, + configRepository, + storageRepository, + ); } async handleMigrationSingle({ id }: IEntityJob) { + const storageTemplateEnabled = (await this.configCore.getConfig()).storageTemplate.enabled; + if (!storageTemplateEnabled) { + return true; + } + const [asset] = await this.assetRepository.getByIds([id]); const user = await this.userRepository.get(asset.ownerId, {}); @@ -93,6 +107,11 @@ export class StorageTemplateService { async handleMigration() { this.logger.log('Starting storage template migration'); + const storageTemplateEnabled = (await this.configCore.getConfig()).storageTemplate.enabled; + if (!storageTemplateEnabled) { + this.logger.log('Storage template migration disabled, skipping'); + return true; + } const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => this.assetRepository.getAll(pagination), ); @@ -123,12 +142,23 @@ export class StorageTemplateService { return; } - const { id, sidecarPath, originalPath } = asset; + const { id, sidecarPath, originalPath, exifInfo } = asset; const oldPath = originalPath; const newPath = await this.getTemplatePath(asset, metadata); + if (!exifInfo || !exifInfo.fileSizeInByte) { + this.logger.error(`Asset ${id} missing exif info, skipping storage template migration`); + return; + } + try { - await this.storageCore.moveFile({ entityId: id, pathType: AssetPathType.ORIGINAL, oldPath, newPath }); + await this.storageCore.moveFile({ + entityId: id, + pathType: AssetPathType.ORIGINAL, + oldPath, + newPath, + assetInfo: { sizeInBytes: exifInfo.fileSizeInByte, checksum: asset.checksum }, + }); if (sidecarPath) { await this.storageCore.moveFile({ entityId: id, diff --git a/server/src/domain/storage/storage.core.ts b/server/src/domain/storage/storage.core.ts index 6a6e83087a..95a53c5c4d 100644 --- a/server/src/domain/storage/storage.core.ts +++ b/server/src/domain/storage/storage.core.ts @@ -1,8 +1,16 @@ +import { SystemConfigCore } from '@app/domain/system-config'; import { AssetEntity, AssetPathType, PathType, PersonEntity, PersonPathType } from '@app/infra/entities'; import { ImmichLogger } from '@app/infra/logger'; import { dirname, join, resolve } from 'node:path'; import { APP_MEDIA_LOCATION } from '../domain.constant'; -import { IAssetRepository, IMoveRepository, IPersonRepository, IStorageRepository } from '../repositories'; +import { + IAssetRepository, + ICryptoRepository, + IMoveRepository, + IPersonRepository, + IStorageRepository, + ISystemConfigRepository, +} from '../repositories'; export enum StorageFolder { ENCODED_VIDEO = 'encoded-video', @@ -17,6 +25,10 @@ export interface MoveRequest { pathType: PathType; oldPath: string | null; newPath: string; + assetInfo?: { + sizeInBytes: number; + checksum: Buffer; + }; } type GeneratedAssetPath = AssetPathType.JPEG_THUMBNAIL | AssetPathType.WEBP_THUMBNAIL | AssetPathType.ENCODED_VIDEO; @@ -25,22 +37,35 @@ let instance: StorageCore | null; export class StorageCore { private logger = new ImmichLogger(StorageCore.name); - + private configCore; private constructor( private assetRepository: IAssetRepository, private moveRepository: IMoveRepository, private personRepository: IPersonRepository, + private cryptoRepository: ICryptoRepository, + private systemConfigRepository: ISystemConfigRepository, private repository: IStorageRepository, - ) {} + ) { + this.configCore = SystemConfigCore.create(systemConfigRepository); + } static create( assetRepository: IAssetRepository, moveRepository: IMoveRepository, personRepository: IPersonRepository, + cryptoRepository: ICryptoRepository, + configRepository: ISystemConfigRepository, repository: IStorageRepository, ) { if (!instance) { - instance = new StorageCore(assetRepository, moveRepository, personRepository, repository); + instance = new StorageCore( + assetRepository, + moveRepository, + personRepository, + cryptoRepository, + configRepository, + repository, + ); } return instance; @@ -131,7 +156,7 @@ export class StorageCore { } async moveFile(request: MoveRequest) { - const { entityId, pathType, oldPath, newPath } = request; + const { entityId, pathType, oldPath, newPath, assetInfo } = request; if (!oldPath || oldPath === newPath) { return; } @@ -143,26 +168,94 @@ export class StorageCore { this.logger.log(`Attempting to finish incomplete move: ${move.oldPath} => ${move.newPath}`); const oldPathExists = await this.repository.checkFileExists(move.oldPath); const newPathExists = await this.repository.checkFileExists(move.newPath); - const actualPath = newPathExists ? move.newPath : oldPathExists ? move.oldPath : null; + const actualPath = oldPathExists ? move.oldPath : newPathExists ? move.newPath : null; if (!actualPath) { this.logger.warn('Unable to complete move. File does not exist at either location.'); return; } - this.logger.log(`Found file at ${actualPath === move.oldPath ? 'old' : 'new'} location`); + const fileAtNewLocation = actualPath === move.newPath; + this.logger.log(`Found file at ${fileAtNewLocation ? 'new' : 'old'} location`); + + if (fileAtNewLocation) { + if (!(await this.verifyNewPathContentsMatchesExpected(move.oldPath, move.newPath, assetInfo))) { + this.logger.fatal( + `Skipping move as file verification failed, old file is missing and new file is different to what was expected`, + ); + return; + } + } move = await this.moveRepository.update({ id: move.id, oldPath: actualPath, newPath }); } else { move = await this.moveRepository.create({ entityId, pathType, oldPath, newPath }); } - if (move.oldPath !== newPath) { - await this.repository.moveFile(move.oldPath, newPath); + if (pathType === AssetPathType.ORIGINAL && !assetInfo) { + this.logger.warn(`Unable to complete move. Missing asset info for ${entityId}`); + return; } + + if (move.oldPath !== newPath) { + try { + this.logger.debug(`Attempting to rename file: ${move.oldPath} => ${newPath}`); + await this.repository.rename(move.oldPath, newPath); + } catch (err: any) { + if (err.code !== 'EXDEV') { + this.logger.warn( + `Unable to complete move. Error renaming file with code ${err.code} and message: ${err.message}`, + ); + return; + } + this.logger.debug(`Unable to rename file. Falling back to copy, verify and delete`); + await this.repository.copyFile(move.oldPath, newPath); + + if (!(await this.verifyNewPathContentsMatchesExpected(move.oldPath, newPath, assetInfo))) { + this.logger.warn(`Skipping move due to file size mismatch`); + await this.repository.unlink(newPath); + return; + } + + try { + await this.repository.unlink(move.oldPath); + } catch (err: any) { + this.logger.warn(`Unable to delete old file, it will now no longer be tracked by Immich: ${err.message}`); + } + } + } + await this.savePath(pathType, entityId, newPath); await this.moveRepository.delete(move); } + private async verifyNewPathContentsMatchesExpected( + oldPath: string, + newPath: string, + assetInfo?: { sizeInBytes: number; checksum: Buffer }, + ) { + const oldPathSize = assetInfo ? assetInfo.sizeInBytes : (await this.repository.stat(oldPath)).size; + const newPathSize = (await this.repository.stat(newPath)).size; + this.logger.debug(`File size check: ${newPathSize} === ${oldPathSize}`); + if (newPathSize !== oldPathSize) { + this.logger.warn(`Unable to complete move. File size mismatch: ${newPathSize} !== ${oldPathSize}`); + return false; + } + if (assetInfo && (await this.configCore.getConfig()).storageTemplate.hashVerificationEnabled) { + const { checksum } = assetInfo; + const newChecksum = await this.cryptoRepository.hashFile(newPath); + if (!newChecksum.equals(checksum)) { + this.logger.warn( + `Unable to complete move. File checksum mismatch: ${newChecksum.toString('base64')} !== ${checksum.toString( + 'base64', + )}`, + ); + return false; + } + this.logger.debug(`File checksum check: ${newChecksum.toString('base64')} === ${checksum.toString('base64')}`); + } + return true; + } + ensureFolders(input: string) { this.repository.mkdirSync(dirname(input)); } diff --git a/server/src/domain/system-config/dto/system-config-job.dto.ts b/server/src/domain/system-config/dto/system-config-job.dto.ts index 1ab94378ce..0f081d1377 100644 --- a/server/src/domain/system-config/dto/system-config-job.dto.ts +++ b/server/src/domain/system-config/dto/system-config-job.dto.ts @@ -10,7 +10,9 @@ export class JobSettingsDto { concurrency!: number; } -export class SystemConfigJobDto implements Record { +export class SystemConfigJobDto + implements Record, JobSettingsDto> +{ @ApiProperty({ type: JobSettingsDto }) @ValidateNested() @IsObject() @@ -35,12 +37,6 @@ export class SystemConfigJobDto implements Record { @Type(() => JobSettingsDto) [QueueName.SMART_SEARCH]!: JobSettingsDto; - @ApiProperty({ type: JobSettingsDto }) - @ValidateNested() - @IsObject() - @Type(() => JobSettingsDto) - [QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto; - @ApiProperty({ type: JobSettingsDto }) @ValidateNested() @IsObject() diff --git a/server/src/domain/system-config/dto/system-config-storage-template.dto.ts b/server/src/domain/system-config/dto/system-config-storage-template.dto.ts index f00006583c..c09b5564aa 100644 --- a/server/src/domain/system-config/dto/system-config-storage-template.dto.ts +++ b/server/src/domain/system-config/dto/system-config-storage-template.dto.ts @@ -1,6 +1,10 @@ -import { IsNotEmpty, IsString } from 'class-validator'; +import { IsBoolean, IsNotEmpty, IsString } from 'class-validator'; export class SystemConfigStorageTemplateDto { + @IsBoolean() + enabled!: boolean; + @IsBoolean() + hashVerificationEnabled!: boolean; @IsNotEmpty() @IsString() template!: string; diff --git a/server/src/domain/system-config/system-config.core.ts b/server/src/domain/system-config/system-config.core.ts index e110c76d0d..5717e03085 100644 --- a/server/src/domain/system-config/system-config.core.ts +++ b/server/src/domain/system-config/system-config.core.ts @@ -53,7 +53,6 @@ export const defaults = Object.freeze({ [QueueName.SEARCH]: { concurrency: 5 }, [QueueName.SIDECAR]: { concurrency: 5 }, [QueueName.LIBRARY]: { concurrency: 5 }, - [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 }, [QueueName.MIGRATION]: { concurrency: 5 }, [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 }, [QueueName.VIDEO_CONVERSION]: { concurrency: 1 }, @@ -102,6 +101,8 @@ export const defaults = Object.freeze({ enabled: true, }, storageTemplate: { + enabled: false, + hashVerificationEnabled: true, template: '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}', }, thumbnail: { diff --git a/server/src/domain/system-config/system-config.service.spec.ts b/server/src/domain/system-config/system-config.service.spec.ts index c6c96706ce..9862477865 100644 --- a/server/src/domain/system-config/system-config.service.spec.ts +++ b/server/src/domain/system-config/system-config.service.spec.ts @@ -34,7 +34,6 @@ const updatedConfig = Object.freeze({ [QueueName.SEARCH]: { concurrency: 5 }, [QueueName.SIDECAR]: { concurrency: 5 }, [QueueName.LIBRARY]: { concurrency: 5 }, - [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 }, [QueueName.MIGRATION]: { concurrency: 5 }, [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 }, [QueueName.VIDEO_CONVERSION]: { concurrency: 1 }, @@ -102,6 +101,8 @@ const updatedConfig = Object.freeze({ enabled: true, }, storageTemplate: { + enabled: false, + hashVerificationEnabled: true, template: '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}', }, thumbnail: { diff --git a/server/src/infra/database.config.ts b/server/src/infra/database.config.ts index 88a46cd068..6e0d0babbb 100644 --- a/server/src/infra/database.config.ts +++ b/server/src/infra/database.config.ts @@ -20,6 +20,7 @@ export const databaseConfig: PostgresConnectionOptions = { subscribers: [__dirname + '/subscribers/*.{js,ts}'], migrationsRun: false, connectTimeoutMS: 10000, // 10 seconds + parseInt8: true, ...urlOrParts, }; diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index 16b00c9606..5c10c027f0 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -84,6 +84,8 @@ export enum SystemConfigKey { PASSWORD_LOGIN_ENABLED = 'passwordLogin.enabled', + STORAGE_TEMPLATE_ENABLED = 'storageTemplate.enabled', + STORAGE_TEMPLATE_HASH_VERIFICATION_ENABLED = 'storageTemplate.hashVerificationEnabled', STORAGE_TEMPLATE = 'storageTemplate.template', THUMBNAIL_WEBP_SIZE = 'thumbnail.webpSize', @@ -171,7 +173,7 @@ export interface SystemConfig { accel: TranscodeHWAccel; tonemap: ToneMapping; }; - job: Record; + job: Record, { concurrency: number }>; logging: { enabled: boolean; level: LogLevel; @@ -216,6 +218,8 @@ export interface SystemConfig { enabled: boolean; }; storageTemplate: { + enabled: boolean; + hashVerificationEnabled: boolean; template: string; }; thumbnail: { diff --git a/server/src/infra/migrations/1703288449127-DefaultStorageTemplateOnForExistingInstallations.ts b/server/src/infra/migrations/1703288449127-DefaultStorageTemplateOnForExistingInstallations.ts new file mode 100644 index 0000000000..4ea88db8eb --- /dev/null +++ b/server/src/infra/migrations/1703288449127-DefaultStorageTemplateOnForExistingInstallations.ts @@ -0,0 +1,16 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class DefaultStorageTemplateOnForExistingInstallations1703288449127 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + const adminCount = await queryRunner.query(`SELECT COUNT(*) FROM users WHERE "isAdmin" = true`) + if(adminCount[0].count > 0) { + await queryRunner.query(`INSERT INTO system_config (key, value) VALUES ('storageTemplate.enabled', 'true')`) + } + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DELETE FROM system_config WHERE key = 'storageTemplate.enabled'`) + } + +} diff --git a/server/src/infra/repositories/filesystem.provider.ts b/server/src/infra/repositories/filesystem.provider.ts index 417f80a109..c1b534edd8 100644 --- a/server/src/infra/repositories/filesystem.provider.ts +++ b/server/src/infra/repositories/filesystem.provider.ts @@ -9,14 +9,10 @@ import { import { ImmichLogger } from '@app/infra/logger'; import archiver from 'archiver'; import { constants, createReadStream, existsSync, mkdirSync } from 'fs'; -import fs, { readdir, writeFile } from 'fs/promises'; +import fs, { copyFile, readdir, rename, writeFile } from 'fs/promises'; import { glob } from 'glob'; -import mv from 'mv'; -import { promisify } from 'node:util'; import path from 'path'; -const moveFile = promisify(mv); - export class FilesystemProvider implements IStorageRepository { private logger = new ImmichLogger(FilesystemProvider.name); @@ -54,15 +50,9 @@ export class FilesystemProvider implements IStorageRepository { writeFile = writeFile; - async moveFile(source: string, destination: string): Promise { - this.logger.verbose(`Moving ${source} to ${destination}`); + rename = rename; - if (await this.checkFileExists(destination)) { - throw new Error(`Destination file already exists: ${destination}`); - } - - await moveFile(source, destination, { mkdirp: true, clobber: true }); - } + copyFile = copyFile; async checkFileExists(filepath: string, mode = constants.F_OK): Promise { try { diff --git a/server/test/repositories/storage.repository.mock.ts b/server/test/repositories/storage.repository.mock.ts index 2a485442d6..a6a088c543 100644 --- a/server/test/repositories/storage.repository.mock.ts +++ b/server/test/repositories/storage.repository.mock.ts @@ -13,12 +13,13 @@ export const newStorageRepositoryMock = (reset = true): jest.Mocked {#await getConfigs() then} -
+
+ + + + +
+

Variables

@@ -191,8 +208,8 @@