feat(server): in upload folder, split the files into folders based on the first four of the files uuid (#6175)

server/src/domain/asset/asset.service.spec.ts

@@ -63,6 +63,7 @@ const uploadFile = {
     auth: null,
     fieldName: UploadFieldName.ASSET_DATA,
     file: {
+      uuid: 'random-uuid',
       checksum: Buffer.from('checksum', 'utf8'),
       originalPath: 'upload/admin/image.jpeg',
       originalName: 'image.jpeg',
@@ -73,6 +74,7 @@ const uploadFile = {
       auth: authStub.admin,
       fieldName,
       file: {
+        uuid: 'random-uuid',
         mimeType: 'image/jpeg',
         checksum: Buffer.from('checksum', 'utf8'),
         originalPath: `upload/admin/${filename}`,
@@ -280,9 +282,9 @@ describe(AssetService.name, () => {
 
     it('should return upload for everything else', () => {
       expect(sut.getUploadFolder(uploadFile.filename(UploadFieldName.ASSET_DATA, 'image.jpg'))).toEqual(
-        'upload/upload/admin_id',
+        'upload/upload/admin_id/ra/nd',
       );
-      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/upload/admin_id');
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/upload/admin_id/ra/nd');
     });
   });
 

server/src/domain/asset/asset.service.ts

@@ -71,6 +71,7 @@ export interface UploadRequest {
 }
 
 export interface UploadFile {
+  uuid: string;
   checksum: Buffer;
   originalPath: string;
   originalName: string;
@@ -170,13 +171,13 @@ export class AssetService {
       [UploadFieldName.PROFILE_DATA]: originalExt,
     };
 
-    return sanitize(`${this.cryptoRepository.randomUUID()}${lookup[fieldName]}`);
+    return sanitize(`${file.uuid}${lookup[fieldName]}`);
   }
 
-  getUploadFolder({ auth, fieldName }: UploadRequest): string {
+  getUploadFolder({ auth, fieldName, file }: UploadRequest): string {
     auth = this.access.requireUploadAccess(auth);
 
-    let folder = StorageCore.getFolderLocation(StorageFolder.UPLOAD, auth.user.id);
+    let folder = StorageCore.getNestedFolder(StorageFolder.UPLOAD, auth.user.id, file.uuid);
     if (fieldName === UploadFieldName.PROFILE_DATA) {
       folder = StorageCore.getFolderLocation(StorageFolder.PROFILE, auth.user.id);
     }

server/src/domain/storage/storage.core.ts

@@ -281,12 +281,11 @@ export class StorageCore {
     }
   }
 
-  private static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
-    return join(
-      StorageCore.getFolderLocation(folder, ownerId),
-      filename.substring(0, 2),
-      filename.substring(2, 4),
-      filename,
-    );
+  static getNestedFolder(folder: StorageFolder, ownerId: string, filename: string): string {
+    return join(StorageCore.getFolderLocation(folder, ownerId), filename.substring(0, 2), filename.substring(2, 4));
+  }
+
+  static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
+    return join(this.getNestedFolder(folder, ownerId, filename), filename);
   }
 }

server/src/immich/api-v1/asset/asset.service.spec.ts

@@ -122,6 +122,7 @@ describe('AssetService', () => {
     it('should handle a file upload', async () => {
       const assetEntity = _getAsset_1();
       const file = {
+        uuid: 'random-uuid',
         originalPath: 'fake_path/asset_1.jpeg',
         mimeType: 'image/jpeg',
         checksum: Buffer.from('file hash', 'utf8'),
@@ -139,6 +140,7 @@ describe('AssetService', () => {
 
     it('should handle a duplicate', async () => {
       const file = {
+        uuid: 'random-uuid',
         originalPath: 'fake_path/asset_1.jpeg',
         mimeType: 'image/jpeg',
         checksum: Buffer.from('file hash', 'utf8'),

server/src/immich/interceptors/file-upload.interceptor.ts

@@ -4,7 +4,7 @@ import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nes
 import { PATH_METADATA } from '@nestjs/common/constants';
 import { Reflector } from '@nestjs/core';
 import { transformException } from '@nestjs/platform-express/multer/multer/multer.utils';
-import { createHash } from 'crypto';
+import { createHash, randomUUID } from 'crypto';
 import { NextFunction, RequestHandler } from 'express';
 import multer, { StorageEngine, diskStorage } from 'multer';
 import { Observable } from 'rxjs';
@@ -17,11 +17,13 @@ export enum Route {
 
 export interface ImmichFile extends Express.Multer.File {
   /** sha1 hash of file */
+  uuid: string;
   checksum: Buffer;
 }
 
 export function mapToUploadFile(file: ImmichFile): UploadFile {
   return {
+    uuid: file.uuid,
     checksum: file.checksum,
     originalPath: file.path,
     originalName: Buffer.from(file.originalname, 'latin1').toString('utf8'),
@@ -30,6 +32,8 @@ export function mapToUploadFile(file: ImmichFile): UploadFile {
 
 type DiskStorageCallback = (error: Error | null, result: string) => void;
 
+type ImmichMulterFile = Express.Multer.File & { uuid: string };
+
 interface Callback<T> {
   (error: Error): void;
   (error: null, result: T): void;
@@ -118,6 +122,7 @@ export class FileUploadInterceptor implements NestInterceptor {
   }
 
   private handleFile(req: AuthRequest, file: Express.Multer.File, callback: Callback<Partial<ImmichFile>>) {
+    (file as ImmichMulterFile).uuid = randomUUID();
     if (!this.isAssetUploadFile(file)) {
       this.defaultStorage._handleFile(req, file, callback);
       return;

server/test/fixtures/file.stub.ts

@@ -1,10 +1,12 @@
 export const fileStub = {
   livePhotoStill: Object.freeze({
+    uuid: 'random-uuid',
     originalPath: 'fake_path/asset_1.jpeg',
     checksum: Buffer.from('file hash', 'utf8'),
     originalName: 'asset_1.jpeg',
   }),
   livePhotoMotion: Object.freeze({
+    uuid: 'random-uuid',
     originalPath: 'fake_path/asset_1.mp4',
     checksum: Buffer.from('live photo file hash', 'utf8'),
     originalName: 'asset_1.mp4',