refactor(server): asset service - upload asset (#1438)

* refactor: asset upload

* refactor: background service

* chore: tests

* Regenerate api

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/apps/immich/src/api-v1/asset/asset-repository.ts

@@ -1,10 +1,9 @@
 import { SearchPropertiesDto } from './dto/search-properties.dto';
 import { CuratedLocationsResponseDto } from './response-dto/curated-locations-response.dto';
 import { AssetEntity, AssetType } from '@app/infra';
-import { BadRequestException, Inject, Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm/repository/Repository';
-import { CreateAssetDto } from './dto/create-asset.dto';
 import { CuratedObjectsResponseDto } from './response-dto/curated-objects-response.dto';
 import { AssetCountByTimeBucket } from './response-dto/asset-count-by-time-group-response.dto';
 import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
@@ -19,15 +18,10 @@ import { IsNull, Not } from 'typeorm';
 import { AssetSearchDto } from './dto/asset-search.dto';
 
 export interface IAssetRepository {
-  create(
-    createAssetDto: CreateAssetDto,
-    ownerId: string,
-    originalPath: string,
-    mimeType: string,
-    isVisible: boolean,
-    checksum?: Buffer,
-    livePhotoAssetEntity?: AssetEntity,
-  ): Promise<AssetEntity>;
+  get(id: string): Promise<AssetEntity | null>;
+  create(asset: Omit<AssetEntity, 'id'>): Promise<AssetEntity>;
+  remove(asset: AssetEntity): Promise<void>;
+
   update(userId: string, asset: AssetEntity, dto: UpdateAssetDto): Promise<AssetEntity>;
   getAll(): Promise<AssetEntity[]>;
   getAllVideos(): Promise<AssetEntity[]>;
@@ -282,44 +276,16 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
-  /**
-   * Create new asset information in database
-   * @param createAssetDto
-   * @param ownerId
-   * @param originalPath
-   * @param mimeType
-   * @returns Promise<AssetEntity>
-   */
-  async create(
-    createAssetDto: CreateAssetDto,
-    ownerId: string,
-    originalPath: string,
-    mimeType: string,
-    isVisible: boolean,
-    checksum?: Buffer,
-    livePhotoAssetEntity?: AssetEntity,
-  ): Promise<AssetEntity> {
-    const asset = new AssetEntity();
-    asset.deviceAssetId = createAssetDto.deviceAssetId;
-    asset.userId = ownerId;
-    asset.deviceId = createAssetDto.deviceId;
-    asset.type = !isVisible ? AssetType.VIDEO : createAssetDto.assetType || AssetType.OTHER; // If an asset is not visible, it is a LivePhotos video portion, therefore we can confidently assign the type as VIDEO here
-    asset.originalPath = originalPath;
-    asset.createdAt = createAssetDto.createdAt;
-    asset.modifiedAt = createAssetDto.modifiedAt;
-    asset.isFavorite = createAssetDto.isFavorite;
-    asset.mimeType = mimeType;
-    asset.duration = createAssetDto.duration || null;
-    asset.checksum = checksum || null;
-    asset.isVisible = isVisible;
-    asset.livePhotoVideoId = livePhotoAssetEntity ? livePhotoAssetEntity.id : null;
-
-    const createdAsset = await this.assetRepository.save(asset);
-
-    if (!createdAsset) {
-      throw new BadRequestException('Asset not created');
+  get(id: string): Promise<AssetEntity | null> {
+    return this.assetRepository.findOne({ where: { id } });
   }
-    return createdAsset;
+
+  async create(asset: Omit<AssetEntity, 'id'>): Promise<AssetEntity> {
+    return this.assetRepository.save(asset);
+  }
+
+  async remove(asset: AssetEntity): Promise<void> {
+    await this.assetRepository.remove(asset);
   }
 
   /**

server/apps/immich/src/api-v1/asset/asset.controller.ts

@@ -19,11 +19,9 @@ import {
 import { Authenticated } from '../../decorators/authenticated.decorator';
 import { AssetService } from './asset.service';
 import { FileFieldsInterceptor } from '@nestjs/platform-express';
-import { assetUploadOption, ImmichFile } from '../../config/asset-upload.config';
 import { AuthUserDto, GetAuthUser } from '../../decorators/auth-user.decorator';
 import { ServeFileDto } from './dto/serve-file.dto';
 import { Response as Res } from 'express';
-import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
 import { DeleteAssetDto } from './dto/delete-asset.dto';
 import { SearchAssetDto } from './dto/search-asset.dto';
 import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto';
@@ -33,9 +31,9 @@ import { CuratedLocationsResponseDto } from './response-dto/curated-locations-re
 import { AssetResponseDto } from '@app/domain';
 import { CheckDuplicateAssetResponseDto } from './response-dto/check-duplicate-asset-response.dto';
 import { AssetFileUploadDto } from './dto/asset-file-upload.dto';
-import { CreateAssetDto } from './dto/create-asset.dto';
+import { CreateAssetDto, mapToUploadFile } from './dto/create-asset.dto';
 import { AssetFileUploadResponseDto } from './response-dto/asset-file-upload-response.dto';
-import { DeleteAssetResponseDto, DeleteAssetStatusEnum } from './response-dto/delete-asset-response.dto';
+import { DeleteAssetResponseDto } from './response-dto/delete-asset-response.dto';
 import { GetAssetThumbnailDto } from './dto/get-asset-thumbnail.dto';
 import { AssetCountByTimeBucketResponseDto } from './response-dto/asset-count-by-time-group-response.dto';
 import { GetAssetCountByTimeBucketDto } from './dto/get-asset-count-by-time-bucket.dto';
@@ -55,12 +53,13 @@ import { CreateAssetsShareLinkDto } from './dto/create-asset-shared-link.dto';
 import { SharedLinkResponseDto } from '@app/domain';
 import { UpdateAssetsToSharedLinkDto } from './dto/add-assets-to-shared-link.dto';
 import { AssetSearchDto } from './dto/asset-search.dto';
+import { assetUploadOption, ImmichFile } from '../../config/asset-upload.config';
 
 @ApiBearerAuth()
 @ApiTags('Asset')
 @Controller('asset')
 export class AssetController {
-  constructor(private assetService: AssetService, private backgroundTaskService: BackgroundTaskService) {}
+  constructor(private assetService: AssetService) {}
 
   @Authenticated({ isShared: true })
   @Post('upload')
@@ -81,13 +80,22 @@ export class AssetController {
   async uploadFile(
     @GetAuthUser() authUser: AuthUserDto,
     @UploadedFiles() files: { assetData: ImmichFile[]; livePhotoData?: ImmichFile[] },
-    @Body(ValidationPipe) createAssetDto: CreateAssetDto,
+    @Body(ValidationPipe) dto: CreateAssetDto,
     @Response({ passthrough: true }) res: Res,
   ): Promise<AssetFileUploadResponseDto> {
-    const originalAssetData = files.assetData[0];
-    const livePhotoAssetData = files.livePhotoData?.[0];
+    const file = mapToUploadFile(files.assetData[0]);
+    const _livePhotoFile = files.livePhotoData?.[0];
+    let livePhotoFile;
+    if (_livePhotoFile) {
+      livePhotoFile = mapToUploadFile(_livePhotoFile);
+    }
 
-    return this.assetService.handleUploadedAsset(authUser, createAssetDto, res, originalAssetData, livePhotoAssetData);
+    const responseDto = await this.assetService.uploadFile(authUser, dto, file, livePhotoFile);
+    if (responseDto.duplicate) {
+      res.send(200);
+    }
+
+    return responseDto;
   }
 
   @Authenticated({ isShared: true })
@@ -276,37 +284,10 @@ export class AssetController {
   @Delete('/')
   async deleteAsset(
     @GetAuthUser() authUser: AuthUserDto,
-    @Body(ValidationPipe) assetIds: DeleteAssetDto,
+    @Body(ValidationPipe) dto: DeleteAssetDto,
   ): Promise<DeleteAssetResponseDto[]> {
-    await this.assetService.checkAssetsAccess(authUser, assetIds.ids, true);
-
-    const deleteAssetList: AssetResponseDto[] = [];
-
-    for (const id of assetIds.ids) {
-      const assets = await this.assetService.getAssetById(authUser, id);
-      if (!assets) {
-        continue;
-      }
-      deleteAssetList.push(assets);
-
-      if (assets.livePhotoVideoId) {
-        const livePhotoVideo = await this.assetService.getAssetById(authUser, assets.livePhotoVideoId);
-        if (livePhotoVideo) {
-          deleteAssetList.push(livePhotoVideo);
-          assetIds.ids = [...assetIds.ids, livePhotoVideo.id];
-        }
-      }
-    }
-
-    const result = await this.assetService.deleteAssetById(assetIds);
-
-    result.forEach((res) => {
-      deleteAssetList.filter((a) => a.id == res.id && res.status == DeleteAssetStatusEnum.SUCCESS);
-    });
-
-    await this.backgroundTaskService.deleteFileOnDisk(deleteAssetList as any[]);
-
-    return result;
+    await this.assetService.checkAssetsAccess(authUser, dto.ids, true);
+    return this.assetService.deleteAll(authUser, dto);
   }
 
   /**

server/apps/immich/src/api-v1/asset/asset.core.ts

@@ -0,0 +1,52 @@
+import { timeUtils } from '@app/common';
+import { AuthUserDto, IJobRepository, JobName } from '@app/domain';
+import { AssetEntity } from '@app/infra/db/entities';
+import { StorageService } from '@app/storage';
+import { IAssetRepository } from './asset-repository';
+import { CreateAssetDto, UploadFile } from './dto/create-asset.dto';
+
+export class AssetCore {
+  constructor(
+    private repository: IAssetRepository,
+    private jobRepository: IJobRepository,
+    private storageService: StorageService,
+  ) {}
+
+  async create(
+    authUser: AuthUserDto,
+    dto: CreateAssetDto,
+    file: UploadFile,
+    livePhotoAssetId?: string,
+  ): Promise<AssetEntity> {
+    let asset = await this.repository.create({
+      userId: authUser.id,
+
+      mimeType: file.mimeType,
+      checksum: file.checksum || null,
+      originalPath: file.originalPath,
+
+      createdAt: timeUtils.checkValidTimestamp(dto.createdAt) ? dto.createdAt : new Date().toISOString(),
+      modifiedAt: timeUtils.checkValidTimestamp(dto.modifiedAt) ? dto.modifiedAt : new Date().toISOString(),
+
+      deviceAssetId: dto.deviceAssetId,
+      deviceId: dto.deviceId,
+
+      type: dto.assetType,
+      isFavorite: dto.isFavorite,
+      duration: dto.duration || null,
+      isVisible: dto.isVisible ?? true,
+      livePhotoVideoId: livePhotoAssetId || null,
+      resizePath: null,
+      webpPath: null,
+      encodedVideoPath: null,
+      tags: [],
+      sharedLinks: [],
+    });
+
+    asset = await this.storageService.moveAsset(asset, file.originalName);
+
+    await this.jobRepository.add({ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } });
+
+    return asset;
+  }
+}

server/apps/immich/src/api-v1/asset/asset.module.ts

@@ -3,8 +3,6 @@ import { AssetService } from './asset.service';
 import { AssetController } from './asset.controller';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { AssetEntity } from '@app/infra';
-import { BackgroundTaskModule } from '../../modules/background-task/background-task.module';
-import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
 import { CommunicationModule } from '../communication/communication.module';
 import { AssetRepository, IAssetRepository } from './asset-repository';
 import { DownloadModule } from '../../modules/download/download.module';
@@ -21,14 +19,13 @@ const ASSET_REPOSITORY_PROVIDER = {
   imports: [
     TypeOrmModule.forFeature([AssetEntity]),
     CommunicationModule,
-    BackgroundTaskModule,
     DownloadModule,
     TagModule,
     StorageModule,
     forwardRef(() => AlbumModule),
   ],
   controllers: [AssetController],
-  providers: [AssetService, BackgroundTaskService, ASSET_REPOSITORY_PROVIDER],
+  providers: [AssetService, ASSET_REPOSITORY_PROVIDER],
   exports: [ASSET_REPOSITORY_PROVIDER],
 })
 export class AssetModule {}

server/apps/immich/src/api-v1/asset/asset.service.spec.ts

@@ -1,17 +1,15 @@
 import { IAssetRepository } from './asset-repository';
-import { AuthUserDto } from '../../decorators/auth-user.decorator';
 import { AssetService } from './asset.service';
-import { Repository } from 'typeorm';
+import { QueryFailedError, Repository } from 'typeorm';
 import { AssetEntity, AssetType } from '@app/infra';
 import { CreateAssetDto } from './dto/create-asset.dto';
 import { AssetCountByTimeBucket } from './response-dto/asset-count-by-time-group-response.dto';
 import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto';
 import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
 import { DownloadService } from '../../modules/download/download.service';
-import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
 import { AlbumRepository, IAlbumRepository } from '../album/album-repository';
 import { StorageService } from '@app/storage';
-import { ICryptoRepository, IJobRepository, ISharedLinkRepository } from '@app/domain';
+import { ICryptoRepository, IJobRepository, ISharedLinkRepository, JobName } from '@app/domain';
 import {
   authStub,
   newCryptoRepositoryMock,
@@ -23,24 +21,7 @@ import {
 import { CreateAssetsShareLinkDto } from './dto/create-asset-shared-link.dto';
 import { BadRequestException, ForbiddenException } from '@nestjs/common';
 
-describe('AssetService', () => {
-  let sui: AssetService;
-  let a: Repository<AssetEntity>; // TO BE DELETED AFTER FINISHED REFACTORING
-  let assetRepositoryMock: jest.Mocked<IAssetRepository>;
-  let albumRepositoryMock: jest.Mocked<IAlbumRepository>;
-  let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
-  let backgroundTaskServiceMock: jest.Mocked<BackgroundTaskService>;
-  let storageSeriveMock: jest.Mocked<StorageService>;
-  let sharedLinkRepositoryMock: jest.Mocked<ISharedLinkRepository>;
-  let cryptoMock: jest.Mocked<ICryptoRepository>;
-  let jobMock: jest.Mocked<IJobRepository>;
-  const authUser: AuthUserDto = Object.freeze({
-    id: 'user_id_1',
-    email: 'auth@test.com',
-    isAdmin: false,
-  });
-
-  const _getCreateAssetDto = (): CreateAssetDto => {
+const _getCreateAssetDto = (): CreateAssetDto => {
   const createAssetDto = new CreateAssetDto();
   createAssetDto.deviceAssetId = 'deviceAssetId';
   createAssetDto.deviceId = 'deviceId';
@@ -51,9 +32,9 @@ describe('AssetService', () => {
   createAssetDto.duration = '0:00:00.000000';
 
   return createAssetDto;
-  };
+};
 
-  const _getAsset_1 = () => {
+const _getAsset_1 = () => {
   const asset_1 = new AssetEntity();
 
   asset_1.id = 'id_1';
@@ -71,9 +52,9 @@ describe('AssetService', () => {
   asset_1.encodedVideoPath = '';
   asset_1.duration = '0:00:00.000000';
   return asset_1;
-  };
+};
 
-  const _getAsset_2 = () => {
+const _getAsset_2 = () => {
   const asset_2 = new AssetEntity();
 
   asset_2.id = 'id_2';
@@ -92,13 +73,13 @@ describe('AssetService', () => {
   asset_2.duration = '0:00:00.000000';
 
   return asset_2;
-  };
+};
 
-  const _getAssets = () => {
+const _getAssets = () => {
   return [_getAsset_1(), _getAsset_2()];
-  };
+};
 
-  const _getAssetCountByTimeBucket = (): AssetCountByTimeBucket[] => {
+const _getAssetCountByTimeBucket = (): AssetCountByTimeBucket[] => {
   const result1 = new AssetCountByTimeBucket();
   result1.count = 2;
   result1.timeBucket = '2022-06-01T00:00:00.000Z';
@@ -108,20 +89,34 @@ describe('AssetService', () => {
   result1.timeBucket = '2022-07-01T00:00:00.000Z';
 
   return [result1, result2];
-  };
+};
 
-  const _getAssetCountByUserId = (): AssetCountByUserIdResponseDto => {
+const _getAssetCountByUserId = (): AssetCountByUserIdResponseDto => {
   const result = new AssetCountByUserIdResponseDto();
 
   result.videos = 2;
   result.photos = 2;
 
   return result;
-  };
+};
 
-  beforeAll(() => {
+describe('AssetService', () => {
+  let sut: AssetService;
+  let a: Repository<AssetEntity>; // TO BE DELETED AFTER FINISHED REFACTORING
+  let assetRepositoryMock: jest.Mocked<IAssetRepository>;
+  let albumRepositoryMock: jest.Mocked<IAlbumRepository>;
+  let downloadServiceMock: jest.Mocked<Partial<DownloadService>>;
+  let storageServiceMock: jest.Mocked<StorageService>;
+  let sharedLinkRepositoryMock: jest.Mocked<ISharedLinkRepository>;
+  let cryptoMock: jest.Mocked<ICryptoRepository>;
+  let jobMock: jest.Mocked<IJobRepository>;
+
+  beforeEach(() => {
     assetRepositoryMock = {
+      get: jest.fn(),
       create: jest.fn(),
+      remove: jest.fn(),
+
       update: jest.fn(),
       getAll: jest.fn(),
       getAllVideos: jest.fn(),
@@ -151,18 +146,21 @@ describe('AssetService', () => {
       downloadArchive: jest.fn(),
     };
 
-    sharedLinkRepositoryMock = newSharedLinkRepositoryMock();
+    storageServiceMock = {
+      moveAsset: jest.fn(),
+      removeEmptyDirectories: jest.fn(),
+    } as unknown as jest.Mocked<StorageService>;
 
+    sharedLinkRepositoryMock = newSharedLinkRepositoryMock();
     jobMock = newJobRepositoryMock();
     cryptoMock = newCryptoRepositoryMock();
 
-    sui = new AssetService(
+    sut = new AssetService(
       assetRepositoryMock,
       albumRepositoryMock,
       a,
-      backgroundTaskServiceMock,
       downloadServiceMock as DownloadService,
-      storageSeriveMock,
+      storageServiceMock,
       sharedLinkRepositoryMock,
       jobMock,
       cryptoMock,
@@ -178,7 +176,7 @@ describe('AssetService', () => {
       assetRepositoryMock.countByIdAndUser.mockResolvedValue(1);
       sharedLinkRepositoryMock.create.mockResolvedValue(sharedLinkStub.valid);
 
-      await expect(sui.createAssetsSharedLink(authStub.user1, dto)).resolves.toEqual(sharedLinkResponseStub.valid);
+      await expect(sut.createAssetsSharedLink(authStub.user1, dto)).resolves.toEqual(sharedLinkResponseStub.valid);
 
       expect(assetRepositoryMock.getById).toHaveBeenCalledWith(asset1.id);
       expect(assetRepositoryMock.countByIdAndUser).toHaveBeenCalledWith(asset1.id, authStub.user1.id);
@@ -196,7 +194,7 @@ describe('AssetService', () => {
       sharedLinkRepositoryMock.get.mockResolvedValue(null);
       sharedLinkRepositoryMock.hasAssetAccess.mockResolvedValue(true);
 
-      await expect(sui.updateAssetsInSharedLink(authDto, dto)).rejects.toBeInstanceOf(BadRequestException);
+      await expect(sut.updateAssetsInSharedLink(authDto, dto)).rejects.toBeInstanceOf(BadRequestException);
 
       expect(assetRepositoryMock.getById).toHaveBeenCalledWith(asset1.id);
       expect(sharedLinkRepositoryMock.get).toHaveBeenCalledWith(authDto.id, authDto.sharedLinkId);
@@ -215,7 +213,7 @@ describe('AssetService', () => {
       sharedLinkRepositoryMock.hasAssetAccess.mockResolvedValue(true);
       sharedLinkRepositoryMock.save.mockResolvedValue(sharedLinkStub.valid);
 
-      await expect(sui.updateAssetsInSharedLink(authDto, dto)).resolves.toEqual(sharedLinkResponseStub.valid);
+      await expect(sut.updateAssetsInSharedLink(authDto, dto)).resolves.toEqual(sharedLinkResponseStub.valid);
 
       expect(assetRepositoryMock.getById).toHaveBeenCalledWith(asset1.id);
       expect(sharedLinkRepositoryMock.get).toHaveBeenCalledWith(authDto.id, authDto.sharedLinkId);
@@ -223,27 +221,94 @@ describe('AssetService', () => {
     });
   });
 
-  // Currently failing due to calculate checksum from a file
-  it('create an asset', async () => {
+  describe('uploadFile', () => {
+    it('should handle a file upload', async () => {
       const assetEntity = _getAsset_1();
+      const file = {
+        originalPath: 'fake_path/asset_1.jpeg',
+        mimeType: 'image/jpeg',
+        checksum: Buffer.from('file hash', 'utf8'),
+        originalName: 'asset_1.jpeg',
+      };
+      const dto = _getCreateAssetDto();
 
-    assetRepositoryMock.create.mockImplementation(() => Promise.resolve<AssetEntity>(assetEntity));
+      assetRepositoryMock.create.mockImplementation(() => Promise.resolve(assetEntity));
+      storageServiceMock.moveAsset.mockResolvedValue({ ...assetEntity, originalPath: 'fake_new_path/asset_123.jpeg' });
 
-    const originalPath = 'fake_path/asset_1.jpeg';
-    const mimeType = 'image/jpeg';
-    const createAssetDto = _getCreateAssetDto();
-    const result = await sui.createUserAsset(
-      authUser,
-      createAssetDto,
-      originalPath,
-      mimeType,
-      Buffer.from('0x5041E6328F7DF8AFF650BEDAED9251897D9A6241', 'hex'),
-      true,
-    );
+      await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: false, id: 'id_1' });
+    });
 
-    expect(result.userId).toEqual(authUser.id);
-    expect(result.resizePath).toEqual('');
-    expect(result.webpPath).toEqual('');
+    it('should handle a duplicate', async () => {
+      const file = {
+        originalPath: 'fake_path/asset_1.jpeg',
+        mimeType: 'image/jpeg',
+        checksum: Buffer.from('file hash', 'utf8'),
+        originalName: 'asset_1.jpeg',
+      };
+      const dto = _getCreateAssetDto();
+      const error = new QueryFailedError('', [], '');
+      (error as any).constraint = 'UQ_userid_checksum';
+
+      assetRepositoryMock.create.mockRejectedValue(error);
+      assetRepositoryMock.getAssetByChecksum.mockResolvedValue(_getAsset_1());
+
+      await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: true, id: 'id_1' });
+
+      expect(jobMock.add).toHaveBeenCalledWith({
+        name: JobName.DELETE_FILE_ON_DISK,
+        data: { assets: [{ originalPath: 'fake_path/asset_1.jpeg', resizePath: null }] },
+      });
+      expect(storageServiceMock.moveAsset).not.toHaveBeenCalled();
+    });
+
+    it('should handle a live photo', async () => {
+      const file = {
+        originalPath: 'fake_path/asset_1.jpeg',
+        mimeType: 'image/jpeg',
+        checksum: Buffer.from('file hash', 'utf8'),
+        originalName: 'asset_1.jpeg',
+      };
+      const asset = {
+        id: 'live-photo-asset',
+        originalPath: file.originalPath,
+        userId: authStub.user1.id,
+        type: AssetType.IMAGE,
+        isVisible: true,
+      } as AssetEntity;
+
+      const livePhotoFile = {
+        originalPath: 'fake_path/asset_1.mp4',
+        mimeType: 'image/jpeg',
+        checksum: Buffer.from('live photo file hash', 'utf8'),
+        originalName: 'asset_1.jpeg',
+      };
+
+      const livePhotoAsset = {
+        id: 'live-photo-motion',
+        originalPath: livePhotoFile.originalPath,
+        userId: authStub.user1.id,
+        type: AssetType.VIDEO,
+        isVisible: false,
+      } as AssetEntity;
+
+      const dto = _getCreateAssetDto();
+      const error = new QueryFailedError('', [], '');
+      (error as any).constraint = 'UQ_userid_checksum';
+
+      assetRepositoryMock.create.mockResolvedValueOnce(livePhotoAsset);
+      assetRepositoryMock.create.mockResolvedValueOnce(asset);
+      storageServiceMock.moveAsset.mockImplementation((asset) => Promise.resolve(asset));
+
+      await expect(sut.uploadFile(authStub.user1, dto, file, livePhotoFile)).resolves.toEqual({
+        duplicate: false,
+        id: 'live-photo-asset',
+      });
+
+      expect(jobMock.add.mock.calls).toEqual([
+        [{ name: JobName.ASSET_UPLOADED, data: { asset: livePhotoAsset, fileName: file.originalName } }],
+        [{ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } }],
+      ]);
+    });
   });
 
   it('get assets by device id', async () => {
@@ -254,7 +319,7 @@ describe('AssetService', () => {
     );
 
     const deviceId = 'device_id_1';
-    const result = await sui.getUserAssetsByDeviceId(authUser, deviceId);
+    const result = await sut.getUserAssetsByDeviceId(authStub.user1, deviceId);
 
     expect(result.length).toEqual(2);
     expect(result).toEqual(assets.map((asset) => asset.deviceAssetId));
@@ -267,7 +332,7 @@ describe('AssetService', () => {
       Promise.resolve<AssetCountByTimeBucket[]>(assetCountByTimeBucket),
     );
 
-    const result = await sui.getAssetCountByTimeBucket(authUser, {
+    const result = await sut.getAssetCountByTimeBucket(authStub.user1, {
       timeGroup: TimeGroupEnum.Month,
     });
 
@@ -282,18 +347,70 @@ describe('AssetService', () => {
       Promise.resolve<AssetCountByUserIdResponseDto>(assetCount),
     );
 
-    const result = await sui.getAssetCountByUserId(authUser);
+    const result = await sut.getAssetCountByUserId(authStub.user1);
 
     expect(result).toEqual(assetCount);
   });
 
+  describe('deleteAll', () => {
+    it('should return failed status when an asset is missing', async () => {
+      assetRepositoryMock.get.mockResolvedValue(null);
+
+      await expect(sut.deleteAll(authStub.user1, { ids: ['asset1'] })).resolves.toEqual([
+        { id: 'asset1', status: 'FAILED' },
+      ]);
+
+      expect(jobMock.add).not.toHaveBeenCalled();
+    });
+
+    it('should return failed status a delete fails', async () => {
+      assetRepositoryMock.get.mockResolvedValue({ id: 'asset1' } as AssetEntity);
+      assetRepositoryMock.remove.mockRejectedValue('delete failed');
+
+      await expect(sut.deleteAll(authStub.user1, { ids: ['asset1'] })).resolves.toEqual([
+        { id: 'asset1', status: 'FAILED' },
+      ]);
+
+      expect(jobMock.add).not.toHaveBeenCalled();
+    });
+
+    it('should delete a live photo', async () => {
+      assetRepositoryMock.get.mockResolvedValueOnce({ id: 'asset1', livePhotoVideoId: 'live-photo' } as AssetEntity);
+      assetRepositoryMock.get.mockResolvedValueOnce({ id: 'live-photo' } as AssetEntity);
+
+      await expect(sut.deleteAll(authStub.user1, { ids: ['asset1'] })).resolves.toEqual([
+        { id: 'asset1', status: 'SUCCESS' },
+        { id: 'live-photo', status: 'SUCCESS' },
+      ]);
+
+      expect(jobMock.add).toHaveBeenCalledWith({
+        name: JobName.DELETE_FILE_ON_DISK,
+        data: { assets: [{ id: 'asset1', livePhotoVideoId: 'live-photo' }, { id: 'live-photo' }] },
+      });
+    });
+
+    it('should delete a batch of assets', async () => {
+      assetRepositoryMock.get.mockImplementation((id) => Promise.resolve({ id } as AssetEntity));
+      assetRepositoryMock.remove.mockImplementation(() => Promise.resolve());
+
+      await expect(sut.deleteAll(authStub.user1, { ids: ['asset1', 'asset2'] })).resolves.toEqual([
+        { id: 'asset1', status: 'SUCCESS' },
+        { id: 'asset2', status: 'SUCCESS' },
+      ]);
+
+      expect(jobMock.add.mock.calls).toEqual([
+        [{ name: JobName.DELETE_FILE_ON_DISK, data: { assets: [{ id: 'asset1' }, { id: 'asset2' }] } }],
+      ]);
+    });
+  });
+
   describe('checkDownloadAccess', () => {
     it('should validate download access', async () => {
-      await sui.checkDownloadAccess(authStub.adminSharedLink);
+      await sut.checkDownloadAccess(authStub.adminSharedLink);
     });
 
     it('should not allow when user is not allowed to download', async () => {
-      expect(() => sui.checkDownloadAccess(authStub.readonlySharedLink)).toThrow(ForbiddenException);
+      expect(() => sut.checkDownloadAccess(authStub.readonlySharedLink)).toThrow(ForbiddenException);
     });
   });
 });

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -23,8 +23,8 @@ import { SearchAssetDto } from './dto/search-asset.dto';
 import fs from 'fs/promises';
 import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto';
 import { CuratedObjectsResponseDto } from './response-dto/curated-objects-response.dto';
-import { AssetResponseDto, mapAsset, mapAssetWithoutExif } from '@app/domain';
-import { CreateAssetDto } from './dto/create-asset.dto';
+import { AssetResponseDto, JobName, mapAsset, mapAssetWithoutExif } from '@app/domain';
+import { CreateAssetDto, UploadFile } from './dto/create-asset.dto';
 import { DeleteAssetResponseDto, DeleteAssetStatusEnum } from './response-dto/delete-asset-response.dto';
 import { GetAssetThumbnailDto, GetAssetThumbnailFormatEnum } from './dto/get-asset-thumbnail.dto';
 import { CheckDuplicateAssetResponseDto } from './response-dto/check-duplicate-asset-response.dto';
@@ -37,13 +37,12 @@ import {
 import { GetAssetCountByTimeBucketDto } from './dto/get-asset-count-by-time-bucket.dto';
 import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
 import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto';
-import { timeUtils } from '@app/common/utils';
+import { AssetCore } from './asset.core';
 import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
 import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto';
 import { UpdateAssetDto } from './dto/update-asset.dto';
 import { AssetFileUploadResponseDto } from './response-dto/asset-file-upload-response.dto';
-import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
-import { ICryptoRepository, IJobRepository, JobName } from '@app/domain';
+import { ICryptoRepository, IJobRepository } from '@app/domain';
 import { DownloadService } from '../../modules/download/download.service';
 import { DownloadDto } from './dto/download-library.dto';
 import { IAlbumRepository } from '../album/album-repository';
@@ -55,7 +54,6 @@ import { CreateAssetsShareLinkDto } from './dto/create-asset-shared-link.dto';
 import { mapSharedLink, SharedLinkResponseDto } from '@app/domain';
 import { UpdateAssetsToSharedLinkDto } from './dto/add-assets-to-shared-link.dto';
 import { AssetSearchDto } from './dto/asset-search.dto';
-import { ImmichFile } from '../../config/asset-upload.config';
 
 const fileInfo = promisify(stat);
 
@@ -63,140 +61,67 @@ const fileInfo = promisify(stat);
 export class AssetService {
   readonly logger = new Logger(AssetService.name);
   private shareCore: ShareCore;
+  private assetCore: AssetCore;
 
   constructor(
     @Inject(IAssetRepository) private _assetRepository: IAssetRepository,
     @Inject(IAlbumRepository) private _albumRepository: IAlbumRepository,
     @InjectRepository(AssetEntity)
     private assetRepository: Repository<AssetEntity>,
-    private backgroundTaskService: BackgroundTaskService,
     private downloadService: DownloadService,
-    private storageService: StorageService,
+    storageService: StorageService,
     @Inject(ISharedLinkRepository) sharedLinkRepository: ISharedLinkRepository,
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
   ) {
+    this.assetCore = new AssetCore(_assetRepository, jobRepository, storageService);
     this.shareCore = new ShareCore(sharedLinkRepository, cryptoRepository);
   }
 
-  public async handleUploadedAsset(
+  public async uploadFile(
     authUser: AuthUserDto,
-    createAssetDto: CreateAssetDto,
-    res: Res,
-    originalAssetData: ImmichFile,
-    livePhotoAssetData?: ImmichFile,
-  ) {
-    const checksum = originalAssetData.checksum;
-    const isLivePhoto = livePhotoAssetData !== undefined;
-    let livePhotoAssetEntity: AssetEntity | undefined;
+    dto: CreateAssetDto,
+    file: UploadFile,
+    livePhotoFile?: UploadFile,
+  ): Promise<AssetFileUploadResponseDto> {
+    if (livePhotoFile) {
+      livePhotoFile.originalName = file.originalName;
+    }
+
+    let livePhotoAsset: AssetEntity | null = null;
 
     try {
-      if (isLivePhoto) {
-        const livePhotoChecksum = livePhotoAssetData.checksum;
-        livePhotoAssetEntity = await this.createUserAsset(
-          authUser,
-          createAssetDto,
-          livePhotoAssetData.path,
-          livePhotoAssetData.mimetype,
-          livePhotoChecksum,
-          false,
-        );
-
-        if (!livePhotoAssetEntity) {
-          await this.backgroundTaskService.deleteFileOnDisk([
-            {
-              originalPath: livePhotoAssetData.path,
-            } as any,
-          ]);
-          throw new BadRequestException('Asset not created');
+      if (livePhotoFile) {
+        const livePhotoDto = { ...dto, assetType: AssetType.VIDEO, isVisible: false };
+        livePhotoAsset = await this.assetCore.create(authUser, livePhotoDto, livePhotoFile);
       }
 
-        await this.storageService.moveAsset(livePhotoAssetEntity, originalAssetData.originalname);
-
-        await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset: livePhotoAssetEntity } });
-      }
-
-      const assetEntity = await this.createUserAsset(
-        authUser,
-        createAssetDto,
-        originalAssetData.path,
-        originalAssetData.mimetype,
-        checksum,
-        true,
-        livePhotoAssetEntity,
-      );
-
-      if (!assetEntity) {
-        await this.backgroundTaskService.deleteFileOnDisk([
-          {
-            originalPath: originalAssetData.path,
-          } as any,
-        ]);
-        throw new BadRequestException('Asset not created');
-      }
-
-      const movedAsset = await this.storageService.moveAsset(assetEntity, originalAssetData.originalname);
+      const asset = await this.assetCore.create(authUser, dto, file, livePhotoAsset?.id);
 
+      return { id: asset.id, duplicate: false };
+    } catch (error: any) {
+      // clean up files
       await this.jobRepository.add({
-        name: JobName.ASSET_UPLOADED,
-        data: { asset: movedAsset, fileName: originalAssetData.originalname },
+        name: JobName.DELETE_FILE_ON_DISK,
+        data: {
+          assets: [
+            {
+              originalPath: file.originalPath,
+              resizePath: livePhotoFile?.originalPath || null,
+            } as AssetEntity,
+          ],
+        },
       });
 
-      return new AssetFileUploadResponseDto(movedAsset.id);
-    } catch (err) {
-      await this.backgroundTaskService.deleteFileOnDisk([
-        {
-          originalPath: originalAssetData.path,
-        } as any,
-      ]); // simulate asset to make use of delete queue (or use fs.unlink instead)
-
-      if (isLivePhoto) {
-        await this.backgroundTaskService.deleteFileOnDisk([
-          {
-            originalPath: livePhotoAssetData.path,
-          } as any,
-        ]);
+      // handle duplicates with a success response
+      if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_userid_checksum') {
+        const duplicate = await this.getAssetByChecksum(authUser.id, file.checksum);
+        return { id: duplicate.id, duplicate: true };
       }
 
-      if (err instanceof QueryFailedError && (err as any).constraint === 'UQ_userid_checksum') {
-        const existedAsset = await this.getAssetByChecksum(authUser.id, checksum);
-        res.status(200); // normal POST is 201. we use 200 to indicate the asset already exists
-        return new AssetFileUploadResponseDto(existedAsset.id);
+      this.logger.error(`Error uploading file ${error}`, error?.stack);
+      throw new BadRequestException(`Error uploading file`, `${error}`);
     }
-
-      Logger.error(`Error uploading file ${err}`);
-      throw new BadRequestException(`Error uploading file`, `${err}`);
-    }
-  }
-
-  public async createUserAsset(
-    authUser: AuthUserDto,
-    createAssetDto: CreateAssetDto,
-    originalPath: string,
-    mimeType: string,
-    checksum: Buffer,
-    isVisible: boolean,
-    livePhotoAssetEntity?: AssetEntity,
-  ): Promise<AssetEntity> {
-    if (!timeUtils.checkValidTimestamp(createAssetDto.createdAt)) {
-      createAssetDto.createdAt = new Date().toISOString();
-    }
-
-    if (!timeUtils.checkValidTimestamp(createAssetDto.modifiedAt)) {
-      createAssetDto.modifiedAt = new Date().toISOString();
-    }
-
-    const assetEntity = await this._assetRepository.create(
-      createAssetDto,
-      authUser.id,
-      originalPath,
-      mimeType,
-      isVisible,
-      checksum,
-      livePhotoAssetEntity,
-    );
-
-    return assetEntity;
   }
 
   public async getUserAssetsByDeviceId(authUser: AuthUserDto, deviceId: string) {
@@ -520,26 +445,35 @@ export class AssetService {
     }
   }
 
-  public async deleteAssetById(assetIds: DeleteAssetDto): Promise<DeleteAssetResponseDto[]> {
+  public async deleteAll(authUser: AuthUserDto, dto: DeleteAssetDto): Promise<DeleteAssetResponseDto[]> {
+    const deleteQueue: AssetEntity[] = [];
     const result: DeleteAssetResponseDto[] = [];
 
-    const target = assetIds.ids;
-    for (const assetId of target) {
-      const res = await this.assetRepository.delete({
-        id: assetId,
-      });
-
-      if (res.affected) {
-        result.push({
-          id: assetId,
-          status: DeleteAssetStatusEnum.SUCCESS,
-        });
-      } else {
-        result.push({
-          id: assetId,
-          status: DeleteAssetStatusEnum.FAILED,
-        });
+    const ids = dto.ids.slice();
+    for (const id of ids) {
+      const asset = await this._assetRepository.get(id);
+      if (!asset) {
+        result.push({ id, status: DeleteAssetStatusEnum.FAILED });
+        continue;
       }
+
+      try {
+        await this._assetRepository.remove(asset);
+
+        result.push({ id: asset.id, status: DeleteAssetStatusEnum.SUCCESS });
+        deleteQueue.push(asset as any);
+
+        // TODO refactor this to use cascades
+        if (asset.livePhotoVideoId && !ids.includes(asset.livePhotoVideoId)) {
+          ids.push(asset.livePhotoVideoId);
+        }
+      } catch {
+        result.push({ id, status: DeleteAssetStatusEnum.FAILED });
+      }
+    }
+
+    if (deleteQueue.length > 0) {
+      await this.jobRepository.add({ name: JobName.DELETE_FILE_ON_DISK, data: { assets: deleteQueue } });
     }
 
     return result;

server/apps/immich/src/api-v1/asset/dto/create-asset.dto.ts

@@ -1,6 +1,7 @@
-import { IsNotEmpty, IsOptional } from 'class-validator';
 import { AssetType } from '@app/infra';
 import { ApiProperty } from '@nestjs/swagger';
+import { IsBoolean, IsNotEmpty, IsOptional } from 'class-validator';
+import { ImmichFile } from '../../../config/asset-upload.config';
 
 export class CreateAssetDto {
   @IsNotEmpty()
@@ -22,9 +23,29 @@ export class CreateAssetDto {
   @IsNotEmpty()
   isFavorite!: boolean;
 
+  @IsOptional()
+  @IsBoolean()
+  isVisible?: boolean;
+
   @IsNotEmpty()
   fileExtension!: string;
 
   @IsOptional()
   duration?: string;
 }
+
+export interface UploadFile {
+  mimeType: string;
+  checksum: Buffer;
+  originalPath: string;
+  originalName: string;
+}
+
+export function mapToUploadFile(file: ImmichFile): UploadFile {
+  return {
+    checksum: file.checksum,
+    mimeType: file.mimetype,
+    originalPath: file.path,
+    originalName: file.originalname,
+  };
+}

server/apps/immich/src/api-v1/asset/response-dto/asset-file-upload-response.dto.ts

@@ -1,7 +1,4 @@
 export class AssetFileUploadResponseDto {
-  constructor(id: string) {
-    this.id = id;
-  }
-
-  id: string;
+  id!: string;
+  duplicate!: boolean;
 }

server/apps/immich/src/app.module.ts

@@ -4,7 +4,6 @@ import { AssetModule } from './api-v1/asset/asset.module';
 import { DeviceInfoModule } from './api-v1/device-info/device-info.module';
 import { ConfigModule } from '@nestjs/config';
 import { ServerInfoModule } from './api-v1/server-info/server-info.module';
-import { BackgroundTaskModule } from './modules/background-task/background-task.module';
 import { CommunicationModule } from './api-v1/communication/communication.module';
 import { AlbumModule } from './api-v1/album/album.module';
 import { AppController } from './app.controller';
@@ -40,8 +39,6 @@ import { UserAuthStrategy } from './modules/immich-auth/strategies/user-auth.str
 
     ServerInfoModule,
 
-    BackgroundTaskModule,
-
     CommunicationModule,
 
     AlbumModule,

server/apps/immich/src/modules/background-task/background-task.module.ts

@@ -1,9 +0,0 @@
-import { Module } from '@nestjs/common';
-import { BackgroundTaskProcessor } from './background-task.processor';
-import { BackgroundTaskService } from './background-task.service';
-
-@Module({
-  providers: [BackgroundTaskService, BackgroundTaskProcessor],
-  exports: [BackgroundTaskService],
-})
-export class BackgroundTaskModule {}

server/apps/immich/src/modules/background-task/background-task.service.ts

@@ -1,12 +0,0 @@
-import { IJobRepository, JobName } from '@app/domain';
-import { AssetEntity } from '@app/infra';
-import { Inject, Injectable } from '@nestjs/common';
-
-@Injectable()
-export class BackgroundTaskService {
-  constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {}
-
-  async deleteFileOnDisk(assets: AssetEntity[]) {
-    await this.jobRepository.add({ name: JobName.DELETE_FILE_ON_DISK, data: { assets } });
-  }
-}

server/apps/microservices/src/microservices.module.ts

@@ -14,6 +14,7 @@ import { StorageMigrationProcessor } from './processors/storage-migration.proces
 import { ThumbnailGeneratorProcessor } from './processors/thumbnail.processor';
 import { UserDeletionProcessor } from './processors/user-deletion.processor';
 import { VideoTranscodeProcessor } from './processors/video-transcode.processor';
+import { BackgroundTaskProcessor } from './processors/background-task.processor';
 import { DomainModule } from '@app/domain';
 
 @Module({
@@ -37,6 +38,7 @@ import { DomainModule } from '@app/domain';
     MachineLearningProcessor,
     UserDeletionProcessor,
     StorageMigrationProcessor,
+    BackgroundTaskProcessor,
   ],
 })
 export class MicroservicesModule {}

server/apps/microservices/src/processors/background-task.processor.ts

@@ -2,7 +2,7 @@ import { assetUtils } from '@app/common/utils';
 import { Process, Processor } from '@nestjs/bull';
 import { Job } from 'bull';
 import { JobName, QueueName } from '@app/domain';
-import { AssetEntity } from '@app/infra';
+import { AssetEntity } from '@app/infra/db/entities';
 
 @Processor(QueueName.BACKGROUND_TASK)
 export class BackgroundTaskProcessor {

server/apps/microservices/src/processors/metadata-extraction.processor.ts

@@ -235,6 +235,10 @@ export class MetadataExtractionProcessor {
   async extractVideoMetadata(job: Job<IVideoLengthExtractionProcessor>) {
     const { asset, fileName } = job.data;
 
+    if (!asset.isVisible) {
+      return;
+    }
+
     try {
       const data = await new Promise<ffmpeg.FfprobeData>((resolve, reject) =>
         ffmpeg.ffprobe(asset.originalPath, (err, data) => {

server/immich-openapi-specs.json

@@ -3725,10 +3725,14 @@
         "properties": {
           "id": {
             "type": "string"
+          },
+          "duplicate": {
+            "type": "boolean"
           }
         },
         "required": [
-          "id"
+          "id",
+          "duplicate"
         ]
       },
       "DownloadFilesDto": {

server/libs/infra/src/db/entities/asset.entity.ts

@@ -32,7 +32,7 @@ export class AssetEntity {
   webpPath!: string | null;
 
   @Column({ type: 'varchar', nullable: true, default: '' })
-  encodedVideoPath!: string;
+  encodedVideoPath!: string | null;
 
   @Column({ type: 'timestamptz' })
   createdAt!: string;

server/libs/storage/src/storage.service.ts

@@ -25,7 +25,7 @@ const moveFile = promisify<string, string, mv.Options>(mv);
 
 @Injectable()
 export class StorageService {
-  readonly logger = new Logger(StorageService.name);
+  private readonly logger = new Logger(StorageService.name);
 
   private storageTemplate: HandlebarsTemplateDelegate<any>;
 

web/src/api/open-api/api.ts

@@ -4,7 +4,7 @@
  * Immich
  * Immich API
  *
- * The version of the OpenAPI document: 1.42.0
+ * The version of the OpenAPI document: 1.43.0
  * 
  *
  * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
@@ -395,6 +395,12 @@ export interface AssetFileUploadResponseDto {
      * @memberof AssetFileUploadResponseDto
      */
     'id': string;
+    /**
+     * 
+     * @type {boolean}
+     * @memberof AssetFileUploadResponseDto
+     */
+    'duplicate': boolean;
 }
 /**
  * 

web/src/api/open-api/base.ts

@@ -4,7 +4,7 @@
  * Immich
  * Immich API
  *
- * The version of the OpenAPI document: 1.42.0
+ * The version of the OpenAPI document: 1.43.0
  * 
  *
  * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

web/src/api/open-api/common.ts

@@ -4,7 +4,7 @@
  * Immich
  * Immich API
  *
- * The version of the OpenAPI document: 1.42.0
+ * The version of the OpenAPI document: 1.43.0
  * 
  *
  * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

web/src/api/open-api/configuration.ts

@@ -4,7 +4,7 @@
  * Immich
  * Immich API
  *
- * The version of the OpenAPI document: 1.42.0
+ * The version of the OpenAPI document: 1.43.0
  * 
  *
  * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

web/src/api/open-api/index.ts

@@ -4,7 +4,7 @@
  * Immich
  * Immich API
  *
- * The version of the OpenAPI document: 1.42.0
+ * The version of the OpenAPI document: 1.43.0
  * 
  *
  * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).