feat(server): calculate sha1 checksum (#525)

* feat(server): override multer storage

* feat(server): calc sha1 of uploaded file

* feat(server): add checksum into asset

* chore(server): add package-lock for mkdirp package

* fix(server): free hash stream

* chore(server): rollback this changes, not refactor here

* refactor(server): re-arrange import statement

* fix(server): make sure hash done before callback

* refactor(server): replace varchar to char for checksum, reserve pixelChecksum for future

* refactor(server): remove pixelChecksum

* refactor(server): convert checksum from string to bytea

* feat(server): add index to checksum

* refactor(): rollback package.json changes

* feat(server): remove uploaded file when progress fail

* feat(server): calculate hash in sequence

server/apps/immich/src/api-v1/asset/asset-repository.ts

@@ -10,7 +10,7 @@ import { AssetCountByTimeGroupDto } from './response-dto/asset-count-by-time-gro
 import { TimeGroupEnum } from './dto/get-asset-count-by-time-group.dto';
 
 export interface IAssetRepository {
-  create(createAssetDto: CreateAssetDto, ownerId: string, originalPath: string, mimeType: string): Promise<AssetEntity>;
+  create(createAssetDto: CreateAssetDto, ownerId: string, originalPath: string, mimeType: string, checksum?: Buffer): Promise<AssetEntity>;
   getAllByUserId(userId: string): Promise<AssetEntity[]>;
   getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
   getById(assetId: string): Promise<AssetEntity>;
@@ -143,6 +143,7 @@ export class AssetRepository implements IAssetRepository {
     ownerId: string,
     originalPath: string,
     mimeType: string,
+    checksum?: Buffer,
   ): Promise<AssetEntity> {
     const asset = new AssetEntity();
     asset.deviceAssetId = createAssetDto.deviceAssetId;
@@ -155,6 +156,7 @@ export class AssetRepository implements IAssetRepository {
     asset.isFavorite = createAssetDto.isFavorite;
     asset.mimeType = mimeType;
     asset.duration = createAssetDto.duration || null;
+    asset.checksum = checksum || null;
 
     const createdAsset = await this.assetRepository.save(asset);
 

server/apps/immich/src/api-v1/asset/asset.controller.ts

@@ -75,6 +75,9 @@ export class AssetController {
     try {
       const savedAsset = await this.assetService.createUserAsset(authUser, assetInfo, file.path, file.mimetype);
       if (!savedAsset) {
+        await this.backgroundTaskService.deleteFileOnDisk([{
+          originalPath: file.path
+        } as any]); // simulate asset to make use of delete queue (or use fs.unlink instead)
         throw new BadRequestException('Asset not created');
       }
 
@@ -87,6 +90,9 @@ export class AssetController {
       return new AssetFileUploadResponseDto(savedAsset.id);
     } catch (e) {
       Logger.error(`Error uploading file ${e}`);
+      await this.backgroundTaskService.deleteFileOnDisk([{
+        originalPath: file.path
+      } as any]); // simulate asset to make use of delete queue (or use fs.unlink instead)
       throw new BadRequestException(`Error uploading file`, `${e}`);
     }
   }

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -9,6 +9,7 @@ import {
   StreamableFile,
 } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
+import { createHash } from 'node:crypto';
 import { Repository } from 'typeorm';
 import { AuthUserDto } from '../../decorators/auth-user.decorator';
 import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
@@ -53,7 +54,8 @@ export class AssetService {
     originalPath: string,
     mimeType: string,
   ): Promise<AssetEntity> {
-    const assetEntity = await this._assetRepository.create(createAssetDto, authUser.id, originalPath, mimeType);
+    const checksum = await this.calculateChecksum(originalPath);
+    const assetEntity = await this._assetRepository.create(createAssetDto, authUser.id, originalPath, mimeType, checksum);
 
     return assetEntity;
   }
@@ -444,4 +446,16 @@ export class AssetService {
 
     return mapAssetCountByTimeGroupResponse(result);
   }
+
+  private calculateChecksum(filePath: string): Promise<Buffer> {
+    const fileReadStream = createReadStream(filePath);
+    const sha1Hash = createHash('sha1');
+    const deferred = new Promise<Buffer>((resolve, reject) => {
+      sha1Hash.once('error', (err) => reject(err));
+      sha1Hash.once('finish', () => resolve(sha1Hash.read()));
+    });
+
+    fileReadStream.pipe(sha1Hash);
+    return deferred;
+  }
 }

server/libs/database/src/entities/asset.entity.ts

@@ -1,4 +1,4 @@
-import { Column, Entity, OneToOne, PrimaryGeneratedColumn, Unique } from 'typeorm';
+import { Column, Entity, Index, OneToOne, PrimaryGeneratedColumn, Unique } from 'typeorm';
 import { ExifEntity } from './exif.entity';
 import { SmartInfoEntity } from './smart-info.entity';
 
@@ -44,6 +44,10 @@ export class AssetEntity {
   @Column({ type: 'varchar', nullable: true })
   mimeType!: string | null;
 
+  @Column({ type: 'bytea', nullable: true, select: false })
+  @Index({ where: `'checksum' IS NOT NULL` }) // avoid null index
+  checksum?: Buffer | null; // sha1 checksum
+
   @Column({ type: 'varchar', nullable: true })
   duration!: string | null;
 

server/libs/database/src/migrations/1661881837496-AddAssetChecksum.ts

@@ -0,0 +1,16 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddAssetChecksum1661881837496 implements MigrationInterface {
+  name = 'AddAssetChecksum1661881837496'
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "assets" ADD "checksum" bytea`);
+    await queryRunner.query(`CREATE INDEX "IDX_64c507300988dd1764f9a6530c" ON "assets" ("checksum") WHERE 'checksum' IS NOT NULL`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "public"."IDX_64c507300988dd1764f9a6530c"`);
+    await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "checksum"`);
+  }
+
+}