feat: storage template file move hardening (#5917)

* fix: pgvecto.rs extension breaks typeorm schema:drop command

* fix: parse postgres bigints to javascript number types when selecting data

* feat: verify file size is the same as original asset after copying file for storage template job

* feat: allow disabling of storage template job, defaults to disabled for new instances

* fix: don't allow setting concurrency for storage template migration, can cause race conditions above 1

* feat: add checksum verification when file is copied for storage template job

* fix: extract metadata for assets that aren't visible on timeline

.github/workflows/test.yml

@@ -269,6 +269,9 @@ jobs:
       - name: Run existing migrations
         run: npm run typeorm:migrations:run
 
+      - name: Test npm run schema:reset command works
+        run: npm run typeorm:schema:reset
+
       - name: Generate new migrations
         continue-on-error: true
         run: npm run typeorm:migrations:generate ./src/infra/migrations/TestMigration

cli/src/api/open-api/api.ts

@@ -3780,12 +3780,6 @@ export interface SystemConfigJobDto {
      * @memberof SystemConfigJobDto
      */
     'smartSearch': JobSettingsDto;
-    /**
-     * 
-     * @type {JobSettingsDto}
-     * @memberof SystemConfigJobDto
-     */
-    'storageTemplateMigration': JobSettingsDto;
     /**
      * 
      * @type {JobSettingsDto}
@@ -4026,6 +4020,18 @@ export interface SystemConfigReverseGeocodingDto {
  * @interface SystemConfigStorageTemplateDto
  */
 export interface SystemConfigStorageTemplateDto {
+    /**
+     * 
+     * @type {boolean}
+     * @memberof SystemConfigStorageTemplateDto
+     */
+    'enabled': boolean;
+    /**
+     * 
+     * @type {boolean}
+     * @memberof SystemConfigStorageTemplateDto
+     */
+    'hashVerificationEnabled': boolean;
     /**
      * 
      * @type {string}

docker/docker-compose.dev.yml

@@ -16,6 +16,7 @@ x-server-build: &server-common
   volumes:
     - ../server:/usr/src/app
     - ${UPLOAD_LOCATION}/photos:/usr/src/app/upload
+    - ${UPLOAD_LOCATION}/photos/upload:/usr/src/app/upload/upload
     - /usr/src/app/node_modules
     - /etc/localtime:/etc/localtime:ro
   env_file:

server/immich-openapi-specs.json

@@ -9171,9 +9171,6 @@
           "smartSearch": {
             "$ref": "#/components/schemas/JobSettingsDto"
           },
-          "storageTemplateMigration": {
-            "$ref": "#/components/schemas/JobSettingsDto"
-          },
           "thumbnailGeneration": {
             "$ref": "#/components/schemas/JobSettingsDto"
           },
@@ -9186,7 +9183,6 @@
           "metadataExtraction",
           "videoConversion",
           "smartSearch",
-          "storageTemplateMigration",
           "migration",
           "backgroundTask",
           "search",
@@ -9365,11 +9361,19 @@
       },
       "SystemConfigStorageTemplateDto": {
         "properties": {
+          "enabled": {
+            "type": "boolean"
+          },
+          "hashVerificationEnabled": {
+            "type": "boolean"
+          },
           "template": {
             "type": "string"
           }
         },
         "required": [
+          "enabled",
+          "hashVerificationEnabled",
           "template"
         ],
         "type": "object"

server/package-lock.json

@@ -41,7 +41,6 @@
         "joi": "^17.10.0",
         "lodash": "^4.17.21",
         "luxon": "^3.4.2",
-        "mv": "^2.1.1",
         "nest-commander": "^3.11.1",
         "node-addon-api": "^7.0.0",
         "openid-client": "^5.4.3",
@@ -72,7 +71,6 @@
         "@types/lodash": "^4.14.197",
         "@types/mock-fs": "^4.13.1",
         "@types/multer": "^1.4.7",
-        "@types/mv": "^2.1.2",
         "@types/node": "^20.5.7",
         "@types/sharp": "^0.31.1",
         "@types/supertest": "^2.0.12",
@@ -3548,12 +3546,6 @@
         "@types/express": "*"
       }
     },
-    "node_modules/@types/mv": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/@types/mv/-/mv-2.1.4.tgz",
-      "integrity": "sha512-MgEHBpXnQo44Q43j8G0Bvp/Yi8LYbC8hxKrRFMgDEDZMmzDKZLgiyMWtW49B37ko+QupgZ3G5rtPUnOGe5ixLw==",
-      "dev": true
-    },
     "node_modules/@types/node": {
       "version": "20.10.3",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.3.tgz",
@@ -9293,45 +9285,6 @@
       "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
       "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="
     },
-    "node_modules/mv": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
-      "integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==",
-      "dependencies": {
-        "mkdirp": "~0.5.1",
-        "ncp": "~2.0.0",
-        "rimraf": "~2.4.0"
-      },
-      "engines": {
-        "node": ">=0.8.0"
-      }
-    },
-    "node_modules/mv/node_modules/glob": {
-      "version": "6.0.4",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
-      "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==",
-      "dependencies": {
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "2 || 3",
-        "once": "^1.3.0",
-        "path-is-absolute": "^1.0.0"
-      },
-      "engines": {
-        "node": "*"
-      }
-    },
-    "node_modules/mv/node_modules/rimraf": {
-      "version": "2.4.5",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
-      "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==",
-      "dependencies": {
-        "glob": "^6.0.1"
-      },
-      "bin": {
-        "rimraf": "bin.js"
-      }
-    },
     "node_modules/mz": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -9355,14 +9308,6 @@
       "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
       "dev": true
     },
-    "node_modules/ncp": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
-      "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==",
-      "bin": {
-        "ncp": "bin/ncp"
-      }
-    },
     "node_modules/nearley": {
       "version": "2.20.1",
       "resolved": "https://registry.npmjs.org/nearley/-/nearley-2.20.1.tgz",
@@ -15519,12 +15464,6 @@
         "@types/express": "*"
       }
     },
-    "@types/mv": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/@types/mv/-/mv-2.1.4.tgz",
-      "integrity": "sha512-MgEHBpXnQo44Q43j8G0Bvp/Yi8LYbC8hxKrRFMgDEDZMmzDKZLgiyMWtW49B37ko+QupgZ3G5rtPUnOGe5ixLw==",
-      "dev": true
-    },
     "@types/node": {
       "version": "20.10.3",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.3.tgz",
@@ -19842,38 +19781,6 @@
       "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
       "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="
     },
-    "mv": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
-      "integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==",
-      "requires": {
-        "mkdirp": "~0.5.1",
-        "ncp": "~2.0.0",
-        "rimraf": "~2.4.0"
-      },
-      "dependencies": {
-        "glob": {
-          "version": "6.0.4",
-          "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
-          "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==",
-          "requires": {
-            "inflight": "^1.0.4",
-            "inherits": "2",
-            "minimatch": "2 || 3",
-            "once": "^1.3.0",
-            "path-is-absolute": "^1.0.0"
-          }
-        },
-        "rimraf": {
-          "version": "2.4.5",
-          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
-          "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==",
-          "requires": {
-            "glob": "^6.0.1"
-          }
-        }
-      }
-    },
     "mz": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -19897,11 +19804,6 @@
       "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
       "dev": true
     },
-    "ncp": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
-      "integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA=="
-    },
     "nearley": {
       "version": "2.20.1",
       "resolved": "https://registry.npmjs.org/nearley/-/nearley-2.20.1.tgz",

server/package.json

@@ -28,7 +28,7 @@
     "typeorm:migrations:generate": "typeorm migration:generate -d ./dist/infra/database.config.js",
     "typeorm:migrations:run": "typeorm migration:run -d ./dist/infra/database.config.js",
     "typeorm:migrations:revert": "typeorm migration:revert -d ./dist/infra/database.config.js",
-    "typeorm:schema:drop": "typeorm schema:drop -d ./dist/infra/database.config.js",
+    "typeorm:schema:drop": "typeorm query -d ./dist/infra/database.config.js 'DROP schema public cascade; CREATE schema public;'",
     "typeorm:schema:reset": "npm run typeorm:schema:drop && npm run typeorm:migrations:run",
     "api:typescript": "bash ./bin/generate-open-api.sh web",
     "api:dart": "bash ./bin/generate-open-api.sh mobile",
@@ -68,7 +68,6 @@
     "joi": "^17.10.0",
     "lodash": "^4.17.21",
     "luxon": "^3.4.2",
-    "mv": "^2.1.1",
     "nest-commander": "^3.11.1",
     "node-addon-api": "^7.0.0",
     "openid-client": "^5.4.3",
@@ -99,7 +98,6 @@
     "@types/lodash": "^4.14.197",
     "@types/mock-fs": "^4.13.1",
     "@types/multer": "^1.4.7",
-    "@types/mv": "^2.1.2",
     "@types/node": "^20.5.7",
     "@types/sharp": "^0.31.1",
     "@types/supertest": "^2.0.12",

server/src/domain/job/job.service.spec.ts

@@ -226,7 +226,6 @@ describe(JobService.name, () => {
           [QueueName.SEARCH]: { concurrency: 10 },
           [QueueName.SIDECAR]: { concurrency: 10 },
           [QueueName.LIBRARY]: { concurrency: 10 },
-          [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 },
           [QueueName.MIGRATION]: { concurrency: 10 },
           [QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 },
           [QueueName.VIDEO_CONVERSION]: { concurrency: 10 },
@@ -239,7 +238,6 @@ describe(JobService.name, () => {
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.RECOGNIZE_FACES, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10);
-      expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.MIGRATION, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10);

server/src/domain/job/job.service.ts

@@ -123,7 +123,11 @@ export class JobService {
   async registerHandlers(jobHandlers: Record<JobName, JobHandler>) {
     const config = await this.configCore.getConfig();
     for (const queueName of Object.values(QueueName)) {
-      const concurrency = config.job[queueName].concurrency;
+      let concurrency = 1;
+      if (queueName !== QueueName.STORAGE_TEMPLATE_MIGRATION) {
+        concurrency = config.job[queueName].concurrency;
+      }
+
       this.logger.debug(`Registering ${queueName} with a concurrency of ${concurrency}`);
       this.jobRepository.addHandler(queueName, concurrency, async (item: JobItem): Promise<void> => {
         const { name, data } = item;
@@ -143,7 +147,10 @@ export class JobService {
     this.configCore.config$.subscribe((config) => {
       this.logger.log(`Updating queue concurrency settings`);
       for (const queueName of Object.values(QueueName)) {
-        const concurrency = config.job[queueName].concurrency;
+        let concurrency = 1;
+        if (queueName !== QueueName.STORAGE_TEMPLATE_MIGRATION) {
+          concurrency = config.job[queueName].concurrency;
+        }
         this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
         this.jobRepository.setConcurrency(queueName, concurrency);
       }

server/src/domain/media/media.service.spec.ts

@@ -12,6 +12,7 @@ import {
   assetStub,
   faceStub,
   newAssetRepositoryMock,
+  newCryptoRepositoryMock,
   newJobRepositoryMock,
   newMediaRepositoryMock,
   newMoveRepositoryMock,
@@ -24,6 +25,7 @@ import {
 import { JobName } from '../job';
 import {
   IAssetRepository,
+  ICryptoRepository,
   IJobRepository,
   IMediaRepository,
   IMoveRepository,
@@ -43,6 +45,7 @@ describe(MediaService.name, () => {
   let moveMock: jest.Mocked<IMoveRepository>;
   let personMock: jest.Mocked<IPersonRepository>;
   let storageMock: jest.Mocked<IStorageRepository>;
+  let cryptoMock: jest.Mocked<ICryptoRepository>;
 
   beforeEach(async () => {
     assetMock = newAssetRepositoryMock();
@@ -52,8 +55,9 @@ describe(MediaService.name, () => {
     moveMock = newMoveRepositoryMock();
     personMock = newPersonRepositoryMock();
     storageMock = newStorageRepositoryMock();
+    cryptoMock = newCryptoRepositoryMock();
 
-    sut = new MediaService(assetMock, personMock, jobMock, mediaMock, storageMock, configMock, moveMock);
+    sut = new MediaService(assetMock, personMock, jobMock, mediaMock, storageMock, configMock, moveMock, cryptoMock);
   });
 
   it('should be defined', () => {

server/src/domain/media/media.service.ts

@@ -14,6 +14,7 @@ import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName }
 import {
   AudioStreamInfo,
   IAssetRepository,
+  ICryptoRepository,
   IJobRepository,
   IMediaRepository,
   IMoveRepository,
@@ -52,9 +53,17 @@ export class MediaService {
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
     @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
     @Inject(IMoveRepository) moveRepository: IMoveRepository,
+    @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
   ) {
     this.configCore = SystemConfigCore.create(configRepository);
-    this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
+    this.storageCore = StorageCore.create(
+      assetRepository,
+      moveRepository,
+      personRepository,
+      cryptoRepository,
+      configRepository,
+      storageRepository,
+    );
   }
 
   async handleQueueGenerateThumbnails({ force }: IBaseJob) {

server/src/domain/metadata/metadata.service.spec.ts

@@ -239,15 +239,6 @@ describe(MetadataService.name, () => {
       expect(assetMock.save).not.toHaveBeenCalled();
     });
 
-    it('should handle an asset with isVisible set to false', async () => {
-      assetMock.getByIds.mockResolvedValue([{ ...assetStub.image, isVisible: false }]);
-
-      await expect(sut.handleMetadataExtraction({ id: assetStub.image.id })).resolves.toBe(false);
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
-      expect(assetMock.upsertExif).not.toHaveBeenCalled();
-      expect(assetMock.save).not.toHaveBeenCalled();
-    });
-
     it('should handle a date in a sidecar file', async () => {
       const originalDate = new Date('2023-11-21T16:13:17.517Z');
       const sidecarDate = new Date('2022-01-01T00:00:00.000Z');

server/src/domain/metadata/metadata.service.ts

@@ -114,7 +114,14 @@ export class MetadataService {
     @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
   ) {
     this.configCore = SystemConfigCore.create(configRepository);
-    this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
+    this.storageCore = StorageCore.create(
+      assetRepository,
+      moveRepository,
+      personRepository,
+      cryptoRepository,
+      configRepository,
+      storageRepository,
+    );
   }
 
   async init() {
@@ -199,7 +206,7 @@ export class MetadataService {
 
   async handleMetadataExtraction({ id }: IEntityJob) {
     const [asset] = await this.assetRepository.getByIds([id]);
-    if (!asset || !asset.isVisible) {
+    if (!asset) {
       return false;
     }
 

server/src/domain/person/person.service.spec.ts

@@ -7,6 +7,7 @@ import {
   faceStub,
   newAccessRepositoryMock,
   newAssetRepositoryMock,
+  newCryptoRepositoryMock,
   newJobRepositoryMock,
   newMachineLearningRepositoryMock,
   newMediaRepositoryMock,
@@ -22,6 +23,7 @@ import { CacheControl, ImmichFileResponse } from '../domain.util';
 import { JobName } from '../job';
 import {
   IAssetRepository,
+  ICryptoRepository,
   IJobRepository,
   IMachineLearningRepository,
   IMediaRepository,
@@ -73,6 +75,7 @@ describe(PersonService.name, () => {
   let personMock: jest.Mocked<IPersonRepository>;
   let storageMock: jest.Mocked<IStorageRepository>;
   let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
+  let cryptoMock: jest.Mocked<ICryptoRepository>;
   let sut: PersonService;
 
   beforeEach(async () => {
@@ -86,6 +89,7 @@ describe(PersonService.name, () => {
     personMock = newPersonRepositoryMock();
     storageMock = newStorageRepositoryMock();
     smartInfoMock = newSmartInfoRepositoryMock();
+    cryptoMock = newCryptoRepositoryMock();
     sut = new PersonService(
       accessMock,
       assetMock,
@@ -97,6 +101,7 @@ describe(PersonService.name, () => {
       storageMock,
       jobMock,
       smartInfoMock,
+      cryptoMock,
     );
 
     mediaMock.crop.mockResolvedValue(croppedFace);

server/src/domain/person/person.service.ts

@@ -13,6 +13,7 @@ import {
   CropOptions,
   IAccessRepository,
   IAssetRepository,
+  ICryptoRepository,
   IJobRepository,
   IMachineLearningRepository,
   IMediaRepository,
@@ -59,10 +60,18 @@ export class PersonService {
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
+    @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
   ) {
     this.access = AccessCore.create(accessRepository);
     this.configCore = SystemConfigCore.create(configRepository);
-    this.storageCore = StorageCore.create(assetRepository, moveRepository, repository, storageRepository);
+    this.storageCore = StorageCore.create(
+      assetRepository,
+      moveRepository,
+      repository,
+      cryptoRepository,
+      configRepository,
+      storageRepository,
+    );
   }
 
   async getAll(auth: AuthDto, dto: PersonSearchDto): Promise<PeopleResponseDto> {

server/src/domain/repositories/storage.repository.ts

@@ -30,11 +30,12 @@ export interface IStorageRepository {
   unlink(filepath: string): Promise<void>;
   unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise<void>;
   removeEmptyDirs(folder: string, self?: boolean): Promise<void>;
-  moveFile(source: string, target: string): Promise<void>;
   checkFileExists(filepath: string, mode?: number): Promise<boolean>;
   mkdirSync(filepath: string): void;
   checkDiskUsage(folder: string): Promise<DiskUsage>;
   readdir(folder: string): Promise<string[]>;
   stat(filepath: string): Promise<Stats>;
   crawl(crawlOptions: CrawlOptionsDto): Promise<string[]>;
+  copyFile(source: string, target: string): Promise<void>;
+  rename(source: string, target: string): Promise<void>;
 }

server/src/domain/storage-template/storage-template.service.spec.ts

@@ -1,8 +1,21 @@
-import { AssetPathType } from '@app/infra/entities';
+import {
+  IAlbumRepository,
+  IAssetRepository,
+  ICryptoRepository,
+  IMoveRepository,
+  IPersonRepository,
+  IStorageRepository,
+  ISystemConfigRepository,
+  IUserRepository,
+  StorageTemplateService,
+  defaults,
+} from '@app/domain';
+import { AssetPathType, SystemConfigKey } from '@app/infra/entities';
 import {
   assetStub,
   newAlbumRepositoryMock,
   newAssetRepositoryMock,
+  newCryptoRepositoryMock,
   newMoveRepositoryMock,
   newPersonRepositoryMock,
   newStorageRepositoryMock,
@@ -11,17 +24,7 @@ import {
   userStub,
 } from '@test';
 import { when } from 'jest-when';
-import {
+import { Stats } from 'node:fs';
-  IAlbumRepository,
-  IAssetRepository,
-  IMoveRepository,
-  IPersonRepository,
-  IStorageRepository,
-  ISystemConfigRepository,
-  IUserRepository,
-} from '../repositories';
-import { defaults } from '../system-config/system-config.core';
-import { StorageTemplateService } from './storage-template.service';
 
 describe(StorageTemplateService.name, () => {
   let sut: StorageTemplateService;
@@ -32,19 +35,21 @@ describe(StorageTemplateService.name, () => {
   let personMock: jest.Mocked<IPersonRepository>;
   let storageMock: jest.Mocked<IStorageRepository>;
   let userMock: jest.Mocked<IUserRepository>;
+  let cryptoMock: jest.Mocked<ICryptoRepository>;
 
   it('should work', () => {
     expect(sut).toBeDefined();
   });
 
   beforeEach(async () => {
+    configMock = newSystemConfigRepositoryMock();
     assetMock = newAssetRepositoryMock();
     albumMock = newAlbumRepositoryMock();
-    configMock = newSystemConfigRepositoryMock();
     moveMock = newMoveRepositoryMock();
     personMock = newPersonRepositoryMock();
     storageMock = newStorageRepositoryMock();
     userMock = newUserRepositoryMock();
+    cryptoMock = newCryptoRepositoryMock();
 
     sut = new StorageTemplateService(
       albumMock,
@@ -55,27 +60,37 @@ describe(StorageTemplateService.name, () => {
       personMock,
       storageMock,
       userMock,
+      cryptoMock,
     );
+
+    configMock.load.mockResolvedValue([{ key: SystemConfigKey.STORAGE_TEMPLATE_ENABLED, value: true }]);
   });
 
   describe('handleMigrationSingle', () => {
+    it('should skip when storage template is disabled', async () => {
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.STORAGE_TEMPLATE_ENABLED, value: false }]);
+      await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
+      expect(assetMock.getByIds).not.toHaveBeenCalled();
+      expect(storageMock.checkFileExists).not.toHaveBeenCalled();
+      expect(storageMock.rename).not.toHaveBeenCalled();
+      expect(storageMock.copyFile).not.toHaveBeenCalled();
+      expect(assetMock.save).not.toHaveBeenCalled();
+      expect(moveMock.create).not.toHaveBeenCalled();
+      expect(moveMock.update).not.toHaveBeenCalled();
+      expect(storageMock.stat).not.toHaveBeenCalled();
+    });
+
     it('should migrate single moving picture', async () => {
       userMock.get.mockResolvedValue(userStub.user1);
-      const path = (id: string) => `upload/library/${userStub.user1.id}/2023/2023-02-23/${id}.jpg`;
+      const newMotionPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.mp4`;
-      const newPath = (id: string) => `upload/library/${userStub.user1.id}/2023/2023-02-23/${id}+1.jpg`;
+      const newStillPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.jpeg`;
-
-      when(storageMock.checkFileExists).calledWith(path(assetStub.livePhotoStillAsset.id)).mockResolvedValue(true);
-      when(storageMock.checkFileExists).calledWith(newPath(assetStub.livePhotoStillAsset.id)).mockResolvedValue(false);
-
-      when(storageMock.checkFileExists).calledWith(path(assetStub.livePhotoMotionAsset.id)).mockResolvedValue(true);
-      when(storageMock.checkFileExists).calledWith(newPath(assetStub.livePhotoMotionAsset.id)).mockResolvedValue(false);
 
       when(assetMock.save)
-        .calledWith({ id: assetStub.livePhotoStillAsset.id, originalPath: newPath(assetStub.livePhotoStillAsset.id) })
+        .calledWith({ id: assetStub.livePhotoStillAsset.id, originalPath: newStillPicturePath })
         .mockResolvedValue(assetStub.livePhotoStillAsset);
 
       when(assetMock.save)
-        .calledWith({ id: assetStub.livePhotoMotionAsset.id, originalPath: newPath(assetStub.livePhotoMotionAsset.id) })
+        .calledWith({ id: assetStub.livePhotoMotionAsset.id, originalPath: newMotionPicturePath })
         .mockResolvedValue(assetStub.livePhotoMotionAsset);
 
       when(assetMock.getByIds)
@@ -86,11 +101,265 @@ describe(StorageTemplateService.name, () => {
         .calledWith([assetStub.livePhotoMotionAsset.id])
         .mockResolvedValue([assetStub.livePhotoMotionAsset]);
 
+      when(moveMock.create)
+        .calledWith({
+          entityId: assetStub.livePhotoStillAsset.id,
+          pathType: AssetPathType.ORIGINAL,
+          oldPath: assetStub.livePhotoStillAsset.originalPath,
+          newPath: newStillPicturePath,
+        })
+        .mockResolvedValue({
+          id: '123',
+          entityId: assetStub.livePhotoStillAsset.id,
+          pathType: AssetPathType.ORIGINAL,
+          oldPath: assetStub.livePhotoStillAsset.originalPath,
+          newPath: newStillPicturePath,
+        });
+
+      when(moveMock.create)
+        .calledWith({
+          entityId: assetStub.livePhotoMotionAsset.id,
+          pathType: AssetPathType.ORIGINAL,
+          oldPath: assetStub.livePhotoMotionAsset.originalPath,
+          newPath: newMotionPicturePath,
+        })
+        .mockResolvedValue({
+          id: '124',
+          entityId: assetStub.livePhotoMotionAsset.id,
+          pathType: AssetPathType.ORIGINAL,
+          oldPath: assetStub.livePhotoMotionAsset.originalPath,
+          newPath: newMotionPicturePath,
+        });
+
       await expect(sut.handleMigrationSingle({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(true);
 
       expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id]);
       expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
+      expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2);
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetStub.livePhotoStillAsset.id,
+        originalPath: newStillPicturePath,
       });
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetStub.livePhotoMotionAsset.id,
+        originalPath: newMotionPicturePath,
+      });
+    });
+    it('should migrate previously failed move from original path when it still exists', async () => {
+      userMock.get.mockResolvedValue(userStub.user1);
+      const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
+      const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
+
+      when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(true);
+      when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(false);
+
+      when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({
+        id: '123',
+        entityId: assetStub.image.id,
+        pathType: AssetPathType.ORIGINAL,
+        oldPath: assetStub.image.originalPath,
+        newPath: previousFailedNewPath,
+      });
+
+      when(assetMock.save)
+        .calledWith({ id: assetStub.image.id, originalPath: newPath })
+        .mockResolvedValue(assetStub.image);
+
+      when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
+
+      when(moveMock.update)
+        .calledWith({
+          id: '123',
+          oldPath: assetStub.image.originalPath,
+          newPath,
+        })
+        .mockResolvedValue({
+          id: '123',
+          entityId: assetStub.image.id,
+          pathType: AssetPathType.ORIGINAL,
+          oldPath: assetStub.image.originalPath,
+          newPath,
+        });
+
+      await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
+
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+      expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
+      expect(storageMock.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
+      expect(moveMock.update).toHaveBeenCalledWith({
+        id: '123',
+        oldPath: assetStub.image.originalPath,
+        newPath,
+      });
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetStub.image.id,
+        originalPath: newPath,
+      });
+    });
+    it('should migrate previously failed move from previous new path when old path no longer exists, should validate file size still matches before moving', async () => {
+      userMock.get.mockResolvedValue(userStub.user1);
+      const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
+      const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
+
+      when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(false);
+      when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(true);
+      when(storageMock.stat)
+        .calledWith(previousFailedNewPath)
+        .mockResolvedValue({ size: 5000 } as Stats);
+      when(cryptoMock.hashFile).calledWith(previousFailedNewPath).mockResolvedValue(assetStub.image.checksum);
+
+      when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({
+        id: '123',
+        entityId: assetStub.image.id,
+        pathType: AssetPathType.ORIGINAL,
+        oldPath: assetStub.image.originalPath,
+        newPath: previousFailedNewPath,
+      });
+
+      when(assetMock.save)
+        .calledWith({ id: assetStub.image.id, originalPath: newPath })
+        .mockResolvedValue(assetStub.image);
+
+      when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
+
+      when(moveMock.update)
+        .calledWith({
+          id: '123',
+          oldPath: previousFailedNewPath,
+          newPath,
+        })
+        .mockResolvedValue({
+          id: '123',
+          entityId: assetStub.image.id,
+          pathType: AssetPathType.ORIGINAL,
+          oldPath: previousFailedNewPath,
+          newPath,
+        });
+
+      await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
+
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+      expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
+      expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath);
+      expect(storageMock.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);
+      expect(storageMock.copyFile).not.toHaveBeenCalled();
+      expect(moveMock.update).toHaveBeenCalledWith({
+        id: '123',
+        oldPath: previousFailedNewPath,
+        newPath,
+      });
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetStub.image.id,
+        originalPath: newPath,
+      });
+    });
+
+    it('should fail move if copying and hash of asset and the new file do not match', async () => {
+      userMock.get.mockResolvedValue(userStub.user1);
+      const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
+
+      when(storageMock.rename).calledWith(assetStub.image.originalPath, newPath).mockRejectedValue({ code: 'EXDEV' });
+      when(storageMock.stat)
+        .calledWith(newPath)
+        .mockResolvedValue({ size: 5000 } as Stats);
+      when(cryptoMock.hashFile).calledWith(newPath).mockResolvedValue(Buffer.from('different-hash', 'utf-8'));
+
+      when(assetMock.save)
+        .calledWith({ id: assetStub.image.id, originalPath: newPath })
+        .mockResolvedValue(assetStub.image);
+
+      when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
+
+      when(moveMock.create)
+        .calledWith({
+          entityId: assetStub.image.id,
+          pathType: AssetPathType.ORIGINAL,
+          oldPath: assetStub.image.originalPath,
+          newPath: newPath,
+        })
+        .mockResolvedValue({
+          id: '123',
+          entityId: assetStub.image.id,
+          pathType: AssetPathType.ORIGINAL,
+          oldPath: assetStub.image.originalPath,
+          newPath,
+        });
+
+      await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
+
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+      expect(storageMock.checkFileExists).toHaveBeenCalledTimes(1);
+      expect(storageMock.stat).toHaveBeenCalledWith(newPath);
+      expect(moveMock.create).toHaveBeenCalledWith({
+        entityId: assetStub.image.id,
+        pathType: AssetPathType.ORIGINAL,
+        oldPath: assetStub.image.originalPath,
+        newPath: newPath,
+      });
+      expect(storageMock.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
+      expect(storageMock.copyFile).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
+      expect(storageMock.unlink).toHaveBeenCalledWith(newPath);
+      expect(storageMock.unlink).toHaveBeenCalledTimes(1);
+      expect(assetMock.save).not.toHaveBeenCalled();
+    });
+
+    it.each`
+      failedPathChecksum                      | failedPathSize                              | reason
+      ${assetStub.image.checksum}             | ${500}                                      | ${'file size'}
+      ${Buffer.from('bad checksum', 'utf-8')} | ${assetStub.image.exifInfo?.fileSizeInByte} | ${'checksum'}
+    `(
+      'should fail to migrate previously failed move from previous new path when old path no longer exists if $reason validation fails',
+      async ({ failedPathChecksum, failedPathSize }) => {
+        userMock.get.mockResolvedValue(userStub.user1);
+        const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
+        const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
+
+        when(storageMock.checkFileExists).calledWith(assetStub.image.originalPath).mockResolvedValue(false);
+        when(storageMock.checkFileExists).calledWith(previousFailedNewPath).mockResolvedValue(true);
+        when(storageMock.stat)
+          .calledWith(previousFailedNewPath)
+          .mockResolvedValue({ size: failedPathSize } as Stats);
+        when(cryptoMock.hashFile).calledWith(previousFailedNewPath).mockResolvedValue(failedPathChecksum);
+
+        when(moveMock.getByEntity).calledWith(assetStub.image.id, AssetPathType.ORIGINAL).mockResolvedValue({
+          id: '123',
+          entityId: assetStub.image.id,
+          pathType: AssetPathType.ORIGINAL,
+          oldPath: assetStub.image.originalPath,
+          newPath: previousFailedNewPath,
+        });
+
+        when(assetMock.save)
+          .calledWith({ id: assetStub.image.id, originalPath: newPath })
+          .mockResolvedValue(assetStub.image);
+
+        when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
+
+        when(moveMock.update)
+          .calledWith({
+            id: '123',
+            oldPath: previousFailedNewPath,
+            newPath,
+          })
+          .mockResolvedValue({
+            id: '123',
+            entityId: assetStub.image.id,
+            pathType: AssetPathType.ORIGINAL,
+            oldPath: previousFailedNewPath,
+            newPath,
+          });
+
+        await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
+
+        expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+        expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
+        expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath);
+        expect(storageMock.rename).not.toHaveBeenCalled();
+        expect(storageMock.copyFile).not.toHaveBeenCalled();
+        expect(moveMock.update).not.toHaveBeenCalled();
+        expect(assetMock.save).not.toHaveBeenCalled();
+      },
+    );
   });
 
   describe('handle template migration', () => {
@@ -155,7 +424,8 @@ describe(StorageTemplateService.name, () => {
       await sut.handleMigration();
 
       expect(assetMock.getAll).toHaveBeenCalled();
-      expect(storageMock.moveFile).not.toHaveBeenCalled();
+      expect(storageMock.rename).not.toHaveBeenCalled();
+      expect(storageMock.copyFile).not.toHaveBeenCalled();
       expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
       expect(assetMock.save).not.toHaveBeenCalled();
     });
@@ -175,7 +445,8 @@ describe(StorageTemplateService.name, () => {
       await sut.handleMigration();
 
       expect(assetMock.getAll).toHaveBeenCalled();
-      expect(storageMock.moveFile).not.toHaveBeenCalled();
+      expect(storageMock.rename).not.toHaveBeenCalled();
+      expect(storageMock.copyFile).not.toHaveBeenCalled();
       expect(storageMock.checkFileExists).not.toHaveBeenCalledTimes(2);
       expect(assetMock.save).not.toHaveBeenCalled();
     });
@@ -198,7 +469,7 @@ describe(StorageTemplateService.name, () => {
       await sut.handleMigration();
 
       expect(assetMock.getAll).toHaveBeenCalled();
-      expect(storageMock.moveFile).toHaveBeenCalledWith(
+      expect(storageMock.rename).toHaveBeenCalledWith(
         '/original/path.jpg',
         'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
       );
@@ -226,7 +497,7 @@ describe(StorageTemplateService.name, () => {
       await sut.handleMigration();
 
       expect(assetMock.getAll).toHaveBeenCalled();
-      expect(storageMock.moveFile).toHaveBeenCalledWith(
+      expect(storageMock.rename).toHaveBeenCalledWith(
         '/original/path.jpg',
         'upload/library/label-1/2023/2023-02-23/asset-id.jpg',
       );
@@ -236,12 +507,84 @@ describe(StorageTemplateService.name, () => {
       });
     });
 
+    it('should copy the file if rename fails due to EXDEV (rename across filesystems)', async () => {
+      const newPath = 'upload/library/user-id/2023/2023-02-23/asset-id.jpg';
+      assetMock.getAll.mockResolvedValue({
+        items: [assetStub.image],
+        hasNextPage: false,
+      });
+      storageMock.rename.mockRejectedValue({ code: 'EXDEV' });
+      userMock.getList.mockResolvedValue([userStub.user1]);
+      moveMock.create.mockResolvedValue({
+        id: '123',
+        entityId: assetStub.image.id,
+        pathType: AssetPathType.ORIGINAL,
+        oldPath: assetStub.image.originalPath,
+        newPath,
+      });
+      when(storageMock.stat)
+        .calledWith(newPath)
+        .mockResolvedValue({
+          size: 5000,
+        } as Stats);
+      when(cryptoMock.hashFile).calledWith(newPath).mockResolvedValue(assetStub.image.checksum);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.rename).toHaveBeenCalledWith('/original/path.jpg', newPath);
+      expect(storageMock.copyFile).toHaveBeenCalledWith('/original/path.jpg', newPath);
+      expect(storageMock.stat).toHaveBeenCalledWith(newPath);
+      expect(storageMock.unlink).toHaveBeenCalledWith(assetStub.image.originalPath);
+      expect(storageMock.unlink).toHaveBeenCalledTimes(1);
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: assetStub.image.id,
+        originalPath: newPath,
+      });
+    });
+
+    it('should not update the database if the move fails due to incorrect newPath filesize', async () => {
+      assetMock.getAll.mockResolvedValue({
+        items: [assetStub.image],
+        hasNextPage: false,
+      });
+      storageMock.rename.mockRejectedValue({ code: 'EXDEV' });
+      userMock.getList.mockResolvedValue([userStub.user1]);
+      moveMock.create.mockResolvedValue({
+        id: '123',
+        entityId: assetStub.image.id,
+        pathType: AssetPathType.ORIGINAL,
+        oldPath: assetStub.image.originalPath,
+        newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
+      });
+      when(storageMock.stat)
+        .calledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg')
+        .mockResolvedValue({
+          size: 100,
+        } as Stats);
+
+      await sut.handleMigration();
+
+      expect(assetMock.getAll).toHaveBeenCalled();
+      expect(storageMock.rename).toHaveBeenCalledWith(
+        '/original/path.jpg',
+        'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
+      );
+      expect(storageMock.copyFile).toHaveBeenCalledWith(
+        '/original/path.jpg',
+        'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
+      );
+      expect(storageMock.stat).toHaveBeenCalledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg');
+      expect(assetMock.save).not.toHaveBeenCalled();
+    });
+
     it('should not update the database if the move fails', async () => {
       assetMock.getAll.mockResolvedValue({
         items: [assetStub.image],
         hasNextPage: false,
       });
-      storageMock.moveFile.mockRejectedValue(new Error('Read only system'));
+      storageMock.rename.mockRejectedValue(new Error('Read only system'));
+      storageMock.copyFile.mockRejectedValue(new Error('Read only system'));
       moveMock.create.mockResolvedValue({
         id: 'move-123',
         entityId: '123',
@@ -254,7 +597,7 @@ describe(StorageTemplateService.name, () => {
       await sut.handleMigration();
 
       expect(assetMock.getAll).toHaveBeenCalled();
-      expect(storageMock.moveFile).toHaveBeenCalledWith(
+      expect(storageMock.rename).toHaveBeenCalledWith(
         '/original/path.jpg',
         'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
       );
@@ -278,7 +621,8 @@ describe(StorageTemplateService.name, () => {
       await sut.handleMigration();
 
       expect(assetMock.getAll).toHaveBeenCalled();
-      expect(storageMock.moveFile).not.toHaveBeenCalled();
+      expect(storageMock.rename).not.toHaveBeenCalled();
+      expect(storageMock.copyFile).not.toHaveBeenCalled();
       expect(assetMock.save).not.toHaveBeenCalled();
     });
   });

server/src/domain/storage-template/storage-template.service.ts

@@ -10,6 +10,7 @@ import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE } from '../job';
 import {
   IAlbumRepository,
   IAssetRepository,
+  ICryptoRepository,
   IMoveRepository,
   IPersonRepository,
   IStorageRepository,
@@ -61,6 +62,7 @@ export class StorageTemplateService {
     @Inject(IPersonRepository) personRepository: IPersonRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
     @Inject(IUserRepository) private userRepository: IUserRepository,
+    @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
   ) {
     this.template = this.compile(config.storageTemplate.template);
     this.configCore = SystemConfigCore.create(configRepository);
@@ -70,10 +72,22 @@ export class StorageTemplateService {
       this.logger.debug(`Received config, compiling storage template: ${template}`);
       this.template = this.compile(template);
     });
-    this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
+    this.storageCore = StorageCore.create(
+      assetRepository,
+      moveRepository,
+      personRepository,
+      cryptoRepository,
+      configRepository,
+      storageRepository,
+    );
   }
 
   async handleMigrationSingle({ id }: IEntityJob) {
+    const storageTemplateEnabled = (await this.configCore.getConfig()).storageTemplate.enabled;
+    if (!storageTemplateEnabled) {
+      return true;
+    }
+
     const [asset] = await this.assetRepository.getByIds([id]);
 
     const user = await this.userRepository.get(asset.ownerId, {});
@@ -93,6 +107,11 @@ export class StorageTemplateService {
 
   async handleMigration() {
     this.logger.log('Starting storage template migration');
+    const storageTemplateEnabled = (await this.configCore.getConfig()).storageTemplate.enabled;
+    if (!storageTemplateEnabled) {
+      this.logger.log('Storage template migration disabled, skipping');
+      return true;
+    }
     const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
       this.assetRepository.getAll(pagination),
     );
@@ -123,12 +142,23 @@ export class StorageTemplateService {
       return;
     }
 
-    const { id, sidecarPath, originalPath } = asset;
+    const { id, sidecarPath, originalPath, exifInfo } = asset;
     const oldPath = originalPath;
     const newPath = await this.getTemplatePath(asset, metadata);
 
+    if (!exifInfo || !exifInfo.fileSizeInByte) {
+      this.logger.error(`Asset ${id} missing exif info, skipping storage template migration`);
+      return;
+    }
+
     try {
-      await this.storageCore.moveFile({ entityId: id, pathType: AssetPathType.ORIGINAL, oldPath, newPath });
+      await this.storageCore.moveFile({
+        entityId: id,
+        pathType: AssetPathType.ORIGINAL,
+        oldPath,
+        newPath,
+        assetInfo: { sizeInBytes: exifInfo.fileSizeInByte, checksum: asset.checksum },
+      });
       if (sidecarPath) {
         await this.storageCore.moveFile({
           entityId: id,

server/src/domain/storage/storage.core.ts

@@ -1,8 +1,16 @@
+import { SystemConfigCore } from '@app/domain/system-config';
 import { AssetEntity, AssetPathType, PathType, PersonEntity, PersonPathType } from '@app/infra/entities';
 import { ImmichLogger } from '@app/infra/logger';
 import { dirname, join, resolve } from 'node:path';
 import { APP_MEDIA_LOCATION } from '../domain.constant';
-import { IAssetRepository, IMoveRepository, IPersonRepository, IStorageRepository } from '../repositories';
+import {
+  IAssetRepository,
+  ICryptoRepository,
+  IMoveRepository,
+  IPersonRepository,
+  IStorageRepository,
+  ISystemConfigRepository,
+} from '../repositories';
 
 export enum StorageFolder {
   ENCODED_VIDEO = 'encoded-video',
@@ -17,6 +25,10 @@ export interface MoveRequest {
   pathType: PathType;
   oldPath: string | null;
   newPath: string;
+  assetInfo?: {
+    sizeInBytes: number;
+    checksum: Buffer;
+  };
 }
 
 type GeneratedAssetPath = AssetPathType.JPEG_THUMBNAIL | AssetPathType.WEBP_THUMBNAIL | AssetPathType.ENCODED_VIDEO;
@@ -25,22 +37,35 @@ let instance: StorageCore | null;
 
 export class StorageCore {
   private logger = new ImmichLogger(StorageCore.name);
-
+  private configCore;
   private constructor(
     private assetRepository: IAssetRepository,
     private moveRepository: IMoveRepository,
     private personRepository: IPersonRepository,
+    private cryptoRepository: ICryptoRepository,
+    private systemConfigRepository: ISystemConfigRepository,
     private repository: IStorageRepository,
-  ) {}
+  ) {
+    this.configCore = SystemConfigCore.create(systemConfigRepository);
+  }
 
   static create(
     assetRepository: IAssetRepository,
     moveRepository: IMoveRepository,
     personRepository: IPersonRepository,
+    cryptoRepository: ICryptoRepository,
+    configRepository: ISystemConfigRepository,
     repository: IStorageRepository,
   ) {
     if (!instance) {
-      instance = new StorageCore(assetRepository, moveRepository, personRepository, repository);
+      instance = new StorageCore(
+        assetRepository,
+        moveRepository,
+        personRepository,
+        cryptoRepository,
+        configRepository,
+        repository,
+      );
     }
 
     return instance;
@@ -131,7 +156,7 @@ export class StorageCore {
   }
 
   async moveFile(request: MoveRequest) {
-    const { entityId, pathType, oldPath, newPath } = request;
+    const { entityId, pathType, oldPath, newPath, assetInfo } = request;
     if (!oldPath || oldPath === newPath) {
       return;
     }
@@ -143,26 +168,94 @@ export class StorageCore {
       this.logger.log(`Attempting to finish incomplete move: ${move.oldPath} => ${move.newPath}`);
       const oldPathExists = await this.repository.checkFileExists(move.oldPath);
       const newPathExists = await this.repository.checkFileExists(move.newPath);
-      const actualPath = newPathExists ? move.newPath : oldPathExists ? move.oldPath : null;
+      const actualPath = oldPathExists ? move.oldPath : newPathExists ? move.newPath : null;
       if (!actualPath) {
         this.logger.warn('Unable to complete move. File does not exist at either location.');
         return;
       }
 
-      this.logger.log(`Found file at ${actualPath === move.oldPath ? 'old' : 'new'} location`);
+      const fileAtNewLocation = actualPath === move.newPath;
+      this.logger.log(`Found file at ${fileAtNewLocation ? 'new' : 'old'} location`);
+
+      if (fileAtNewLocation) {
+        if (!(await this.verifyNewPathContentsMatchesExpected(move.oldPath, move.newPath, assetInfo))) {
+          this.logger.fatal(
+            `Skipping move as file verification failed, old file is missing and new file is different to what was expected`,
+          );
+          return;
+        }
+      }
 
       move = await this.moveRepository.update({ id: move.id, oldPath: actualPath, newPath });
     } else {
       move = await this.moveRepository.create({ entityId, pathType, oldPath, newPath });
     }
 
-    if (move.oldPath !== newPath) {
+    if (pathType === AssetPathType.ORIGINAL && !assetInfo) {
-      await this.repository.moveFile(move.oldPath, newPath);
+      this.logger.warn(`Unable to complete move. Missing asset info for ${entityId}`);
+      return;
     }
+
+    if (move.oldPath !== newPath) {
+      try {
+        this.logger.debug(`Attempting to rename file: ${move.oldPath} => ${newPath}`);
+        await this.repository.rename(move.oldPath, newPath);
+      } catch (err: any) {
+        if (err.code !== 'EXDEV') {
+          this.logger.warn(
+            `Unable to complete move. Error renaming file with code ${err.code} and message: ${err.message}`,
+          );
+          return;
+        }
+        this.logger.debug(`Unable to rename file. Falling back to copy, verify and delete`);
+        await this.repository.copyFile(move.oldPath, newPath);
+
+        if (!(await this.verifyNewPathContentsMatchesExpected(move.oldPath, newPath, assetInfo))) {
+          this.logger.warn(`Skipping move due to file size mismatch`);
+          await this.repository.unlink(newPath);
+          return;
+        }
+
+        try {
+          await this.repository.unlink(move.oldPath);
+        } catch (err: any) {
+          this.logger.warn(`Unable to delete old file, it will now no longer be tracked by Immich: ${err.message}`);
+        }
+      }
+    }
+
     await this.savePath(pathType, entityId, newPath);
     await this.moveRepository.delete(move);
   }
 
+  private async verifyNewPathContentsMatchesExpected(
+    oldPath: string,
+    newPath: string,
+    assetInfo?: { sizeInBytes: number; checksum: Buffer },
+  ) {
+    const oldPathSize = assetInfo ? assetInfo.sizeInBytes : (await this.repository.stat(oldPath)).size;
+    const newPathSize = (await this.repository.stat(newPath)).size;
+    this.logger.debug(`File size check: ${newPathSize} === ${oldPathSize}`);
+    if (newPathSize !== oldPathSize) {
+      this.logger.warn(`Unable to complete move. File size mismatch: ${newPathSize} !== ${oldPathSize}`);
+      return false;
+    }
+    if (assetInfo && (await this.configCore.getConfig()).storageTemplate.hashVerificationEnabled) {
+      const { checksum } = assetInfo;
+      const newChecksum = await this.cryptoRepository.hashFile(newPath);
+      if (!newChecksum.equals(checksum)) {
+        this.logger.warn(
+          `Unable to complete move. File checksum mismatch: ${newChecksum.toString('base64')} !== ${checksum.toString(
+            'base64',
+          )}`,
+        );
+        return false;
+      }
+      this.logger.debug(`File checksum check: ${newChecksum.toString('base64')} === ${checksum.toString('base64')}`);
+    }
+    return true;
+  }
+
   ensureFolders(input: string) {
     this.repository.mkdirSync(dirname(input));
   }

server/src/domain/system-config/dto/system-config-job.dto.ts

@@ -10,7 +10,9 @@ export class JobSettingsDto {
   concurrency!: number;
 }
 
-export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
+export class SystemConfigJobDto
+  implements Record<Exclude<QueueName, QueueName.STORAGE_TEMPLATE_MIGRATION>, JobSettingsDto>
+{
   @ApiProperty({ type: JobSettingsDto })
   @ValidateNested()
   @IsObject()
@@ -35,12 +37,6 @@ export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
   @Type(() => JobSettingsDto)
   [QueueName.SMART_SEARCH]!: JobSettingsDto;
 
-  @ApiProperty({ type: JobSettingsDto })
-  @ValidateNested()
-  @IsObject()
-  @Type(() => JobSettingsDto)
-  [QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto;
-
   @ApiProperty({ type: JobSettingsDto })
   @ValidateNested()
   @IsObject()

server/src/domain/system-config/dto/system-config-storage-template.dto.ts

@@ -1,6 +1,10 @@
-import { IsNotEmpty, IsString } from 'class-validator';
+import { IsBoolean, IsNotEmpty, IsString } from 'class-validator';
 
 export class SystemConfigStorageTemplateDto {
+  @IsBoolean()
+  enabled!: boolean;
+  @IsBoolean()
+  hashVerificationEnabled!: boolean;
   @IsNotEmpty()
   @IsString()
   template!: string;

server/src/domain/system-config/system-config.core.ts

@@ -53,7 +53,6 @@ export const defaults = Object.freeze<SystemConfig>({
     [QueueName.SEARCH]: { concurrency: 5 },
     [QueueName.SIDECAR]: { concurrency: 5 },
     [QueueName.LIBRARY]: { concurrency: 5 },
-    [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
     [QueueName.MIGRATION]: { concurrency: 5 },
     [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
     [QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
@@ -102,6 +101,8 @@ export const defaults = Object.freeze<SystemConfig>({
     enabled: true,
   },
   storageTemplate: {
+    enabled: false,
+    hashVerificationEnabled: true,
     template: '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}',
   },
   thumbnail: {

server/src/domain/system-config/system-config.service.spec.ts

@@ -34,7 +34,6 @@ const updatedConfig = Object.freeze<SystemConfig>({
     [QueueName.SEARCH]: { concurrency: 5 },
     [QueueName.SIDECAR]: { concurrency: 5 },
     [QueueName.LIBRARY]: { concurrency: 5 },
-    [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
     [QueueName.MIGRATION]: { concurrency: 5 },
     [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
     [QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
@@ -102,6 +101,8 @@ const updatedConfig = Object.freeze<SystemConfig>({
     enabled: true,
   },
   storageTemplate: {
+    enabled: false,
+    hashVerificationEnabled: true,
     template: '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}',
   },
   thumbnail: {

server/src/infra/database.config.ts

@@ -20,6 +20,7 @@ export const databaseConfig: PostgresConnectionOptions = {
   subscribers: [__dirname + '/subscribers/*.{js,ts}'],
   migrationsRun: false,
   connectTimeoutMS: 10000, // 10 seconds
+  parseInt8: true,
   ...urlOrParts,
 };
 

server/src/infra/entities/system-config.entity.ts

@@ -84,6 +84,8 @@ export enum SystemConfigKey {
 
   PASSWORD_LOGIN_ENABLED = 'passwordLogin.enabled',
 
+  STORAGE_TEMPLATE_ENABLED = 'storageTemplate.enabled',
+  STORAGE_TEMPLATE_HASH_VERIFICATION_ENABLED = 'storageTemplate.hashVerificationEnabled',
   STORAGE_TEMPLATE = 'storageTemplate.template',
 
   THUMBNAIL_WEBP_SIZE = 'thumbnail.webpSize',
@@ -171,7 +173,7 @@ export interface SystemConfig {
     accel: TranscodeHWAccel;
     tonemap: ToneMapping;
   };
-  job: Record<QueueName, { concurrency: number }>;
+  job: Record<Exclude<QueueName, QueueName.STORAGE_TEMPLATE_MIGRATION>, { concurrency: number }>;
   logging: {
     enabled: boolean;
     level: LogLevel;
@@ -216,6 +218,8 @@ export interface SystemConfig {
     enabled: boolean;
   };
   storageTemplate: {
+    enabled: boolean;
+    hashVerificationEnabled: boolean;
     template: string;
   };
   thumbnail: {

server/src/infra/migrations/1703288449127-DefaultStorageTemplateOnForExistingInstallations.ts

@@ -0,0 +1,16 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class DefaultStorageTemplateOnForExistingInstallations1703288449127 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const adminCount = await queryRunner.query(`SELECT COUNT(*) FROM users WHERE "isAdmin" = true`)
+        if(adminCount[0].count > 0) {
+          await queryRunner.query(`INSERT INTO system_config (key, value) VALUES ('storageTemplate.enabled', 'true')`)
+        }
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+      await queryRunner.query(`DELETE FROM system_config WHERE key = 'storageTemplate.enabled'`)
+    }
+
+}

server/src/infra/repositories/filesystem.provider.ts

@@ -9,14 +9,10 @@ import {
 import { ImmichLogger } from '@app/infra/logger';
 import archiver from 'archiver';
 import { constants, createReadStream, existsSync, mkdirSync } from 'fs';
-import fs, { readdir, writeFile } from 'fs/promises';
+import fs, { copyFile, readdir, rename, writeFile } from 'fs/promises';
 import { glob } from 'glob';
-import mv from 'mv';
-import { promisify } from 'node:util';
 import path from 'path';
 
-const moveFile = promisify<string, string, mv.Options>(mv);
-
 export class FilesystemProvider implements IStorageRepository {
   private logger = new ImmichLogger(FilesystemProvider.name);
 
@@ -54,15 +50,9 @@ export class FilesystemProvider implements IStorageRepository {
 
   writeFile = writeFile;
 
-  async moveFile(source: string, destination: string): Promise<void> {
+  rename = rename;
-    this.logger.verbose(`Moving ${source} to ${destination}`);
 
-    if (await this.checkFileExists(destination)) {
+  copyFile = copyFile;
-      throw new Error(`Destination file already exists: ${destination}`);
-    }
-
-    await moveFile(source, destination, { mkdirp: true, clobber: true });
-  }
 
   async checkFileExists(filepath: string, mode = constants.F_OK): Promise<boolean> {
     try {

server/test/repositories/storage.repository.mock.ts

@@ -13,12 +13,13 @@ export const newStorageRepositoryMock = (reset = true): jest.Mocked<IStorageRepo
     unlink: jest.fn(),
     unlinkDir: jest.fn().mockResolvedValue(true),
     removeEmptyDirs: jest.fn(),
-    moveFile: jest.fn(),
     checkFileExists: jest.fn(),
     mkdirSync: jest.fn(),
     checkDiskUsage: jest.fn(),
     readdir: jest.fn(),
     stat: jest.fn(),
     crawl: jest.fn(),
+    rename: jest.fn(),
+    copyFile: jest.fn(),
   };
 };

web/src/api/open-api/api.ts

@@ -3780,12 +3780,6 @@ export interface SystemConfigJobDto {
      * @memberof SystemConfigJobDto
      */
     'smartSearch': JobSettingsDto;
-    /**
-     * 
-     * @type {JobSettingsDto}
-     * @memberof SystemConfigJobDto
-     */
-    'storageTemplateMigration': JobSettingsDto;
     /**
      * 
      * @type {JobSettingsDto}
@@ -4026,6 +4020,18 @@ export interface SystemConfigReverseGeocodingDto {
  * @interface SystemConfigStorageTemplateDto
  */
 export interface SystemConfigStorageTemplateDto {
+    /**
+     * 
+     * @type {boolean}
+     * @memberof SystemConfigStorageTemplateDto
+     */
+    'enabled': boolean;
+    /**
+     * 
+     * @type {boolean}
+     * @memberof SystemConfigStorageTemplateDto
+     */
+    'hashVerificationEnabled': boolean;
     /**
      * 
      * @type {string}

web/src/lib/components/admin-page/settings/job-settings/job-settings.svelte

@@ -25,7 +25,6 @@
     JobName.SmartSearch,
     JobName.RecognizeFaces,
     JobName.VideoConversion,
-    JobName.StorageTemplateMigration,
     JobName.Migration,
   ];
 

web/src/lib/components/admin-page/settings/storage-template/storage-template-settings.svelte

@@ -15,6 +15,7 @@
   import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
   import { user } from '$lib/stores/user.store';
   import type { ResetOptions } from '$lib/utils/dipatch';
+  import SettingSwitch from '$lib/components/admin-page/settings/setting-switch.svelte';
 
   export let storageConfig: SystemConfigStorageTemplateDto;
   export let disabled = false;
@@ -145,7 +146,23 @@
 
 <section class="dark:text-immich-dark-fg">
   {#await getConfigs() then}
-    <div id="directory-path-builder" class="m-4">
+    <div id="directory-path-builder" class="flex flex-col gap-4 m-4">
+      <SettingSwitch
+        title="ENABLED"
+        {disabled}
+        subtitle="Enable storage template engine"
+        bind:checked={storageConfig.enabled}
+      />
+
+      <SettingSwitch
+        title="HASH VERIFICATION ENABLED"
+        {disabled}
+        subtitle="Enables hash verification, don't disable this unless you're certain of the implications"
+        bind:checked={storageConfig.hashVerificationEnabled}
+      />
+
+      <hr />
+
       <h3 class="text-base font-medium text-immich-primary dark:text-immich-dark-primary">Variables</h3>
 
       <section class="support-date">
@@ -191,8 +208,8 @@
           <div class="flex flex-col my-2">
             <label class="text-sm" for="preset-select">PRESET</label>
             <select
-              class="p-2 mt-2 text-sm rounded-lg bg-slate-200 hover:cursor-pointer dark:bg-gray-600"
+              class="immich-form-input p-2 mt-2 text-sm rounded-lg bg-slate-200 hover:cursor-pointer dark:bg-gray-600"
-              {disabled}
+              disabled={disabled || !storageConfig.enabled}
               name="presets"
               id="preset-select"
               bind:value={selectedPreset}
@@ -206,7 +223,7 @@
           <div class="flex gap-2 align-bottom">
             <SettingInputField
               label="TEMPLATE"
-              {disabled}
+              disabled={disabled || !storageConfig.enabled}
               required
               inputType={SettingInputFieldType.TEXT}
               bind:value={storageConfig.template}
@@ -239,7 +256,6 @@
               </p>
             </section>
           </div>
-
           <SettingButtonsRow
             on:reset={({ detail }) => handleReset(detail)}
             on:save={saveSetting}