From b80dca74ef098fdf422a58a9b30128dd264849ff Mon Sep 17 00:00:00 2001 From: Thanh Pham Date: Wed, 31 Aug 2022 21:27:17 +0700 Subject: [PATCH] feat(server): calculate sha1 checksum (#525) * feat(server): override multer storage * feat(server): calc sha1 of uploaded file * feat(server): add checksum into asset * chore(server): add package-lock for mkdirp package * fix(server): free hash stream * chore(server): rollback this changes, not refactor here * refactor(server): re-arrange import statement * fix(server): make sure hash done before callback * refactor(server): replace varchar to char for checksum, reserve pixelChecksum for future * refactor(server): remove pixelChecksum * refactor(server): convert checksum from string to bytea * feat(server): add index to checksum * refactor(): rollback package.json changes * feat(server): remove uploaded file when progress fail * feat(server): calculate hash in sequence --- .../immich/src/api-v1/asset/asset-repository.ts | 4 +++- .../immich/src/api-v1/asset/asset.controller.ts | 6 ++++++ .../immich/src/api-v1/asset/asset.service.ts | 16 +++++++++++++++- .../libs/database/src/entities/asset.entity.ts | 6 +++++- .../migrations/1661881837496-AddAssetChecksum.ts | 16 ++++++++++++++++ 5 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 server/libs/database/src/migrations/1661881837496-AddAssetChecksum.ts diff --git a/server/apps/immich/src/api-v1/asset/asset-repository.ts b/server/apps/immich/src/api-v1/asset/asset-repository.ts index 9424750776..89f6bbb933 100644 --- a/server/apps/immich/src/api-v1/asset/asset-repository.ts +++ b/server/apps/immich/src/api-v1/asset/asset-repository.ts @@ -10,7 +10,7 @@ import { AssetCountByTimeGroupDto } from './response-dto/asset-count-by-time-gro import { TimeGroupEnum } from './dto/get-asset-count-by-time-group.dto'; export interface IAssetRepository { - create(createAssetDto: CreateAssetDto, ownerId: string, originalPath: string, mimeType: string): Promise; + create(createAssetDto: CreateAssetDto, ownerId: string, originalPath: string, mimeType: string, checksum?: Buffer): Promise; getAllByUserId(userId: string): Promise; getAllByDeviceId(userId: string, deviceId: string): Promise; getById(assetId: string): Promise; @@ -143,6 +143,7 @@ export class AssetRepository implements IAssetRepository { ownerId: string, originalPath: string, mimeType: string, + checksum?: Buffer, ): Promise { const asset = new AssetEntity(); asset.deviceAssetId = createAssetDto.deviceAssetId; @@ -155,6 +156,7 @@ export class AssetRepository implements IAssetRepository { asset.isFavorite = createAssetDto.isFavorite; asset.mimeType = mimeType; asset.duration = createAssetDto.duration || null; + asset.checksum = checksum || null; const createdAsset = await this.assetRepository.save(asset); diff --git a/server/apps/immich/src/api-v1/asset/asset.controller.ts b/server/apps/immich/src/api-v1/asset/asset.controller.ts index 7009f1e11a..dde8077a09 100644 --- a/server/apps/immich/src/api-v1/asset/asset.controller.ts +++ b/server/apps/immich/src/api-v1/asset/asset.controller.ts @@ -75,6 +75,9 @@ export class AssetController { try { const savedAsset = await this.assetService.createUserAsset(authUser, assetInfo, file.path, file.mimetype); if (!savedAsset) { + await this.backgroundTaskService.deleteFileOnDisk([{ + originalPath: file.path + } as any]); // simulate asset to make use of delete queue (or use fs.unlink instead) throw new BadRequestException('Asset not created'); } @@ -87,6 +90,9 @@ export class AssetController { return new AssetFileUploadResponseDto(savedAsset.id); } catch (e) { Logger.error(`Error uploading file ${e}`); + await this.backgroundTaskService.deleteFileOnDisk([{ + originalPath: file.path + } as any]); // simulate asset to make use of delete queue (or use fs.unlink instead) throw new BadRequestException(`Error uploading file`, `${e}`); } } diff --git a/server/apps/immich/src/api-v1/asset/asset.service.ts b/server/apps/immich/src/api-v1/asset/asset.service.ts index 4d32212671..ef7282b838 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.ts @@ -9,6 +9,7 @@ import { StreamableFile, } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; +import { createHash } from 'node:crypto'; import { Repository } from 'typeorm'; import { AuthUserDto } from '../../decorators/auth-user.decorator'; import { AssetEntity, AssetType } from '@app/database/entities/asset.entity'; @@ -53,7 +54,8 @@ export class AssetService { originalPath: string, mimeType: string, ): Promise { - const assetEntity = await this._assetRepository.create(createAssetDto, authUser.id, originalPath, mimeType); + const checksum = await this.calculateChecksum(originalPath); + const assetEntity = await this._assetRepository.create(createAssetDto, authUser.id, originalPath, mimeType, checksum); return assetEntity; } @@ -444,4 +446,16 @@ export class AssetService { return mapAssetCountByTimeGroupResponse(result); } + + private calculateChecksum(filePath: string): Promise { + const fileReadStream = createReadStream(filePath); + const sha1Hash = createHash('sha1'); + const deferred = new Promise((resolve, reject) => { + sha1Hash.once('error', (err) => reject(err)); + sha1Hash.once('finish', () => resolve(sha1Hash.read())); + }); + + fileReadStream.pipe(sha1Hash); + return deferred; + } } diff --git a/server/libs/database/src/entities/asset.entity.ts b/server/libs/database/src/entities/asset.entity.ts index ed5e4ee216..d963d56a69 100644 --- a/server/libs/database/src/entities/asset.entity.ts +++ b/server/libs/database/src/entities/asset.entity.ts @@ -1,4 +1,4 @@ -import { Column, Entity, OneToOne, PrimaryGeneratedColumn, Unique } from 'typeorm'; +import { Column, Entity, Index, OneToOne, PrimaryGeneratedColumn, Unique } from 'typeorm'; import { ExifEntity } from './exif.entity'; import { SmartInfoEntity } from './smart-info.entity'; @@ -44,6 +44,10 @@ export class AssetEntity { @Column({ type: 'varchar', nullable: true }) mimeType!: string | null; + @Column({ type: 'bytea', nullable: true, select: false }) + @Index({ where: `'checksum' IS NOT NULL` }) // avoid null index + checksum?: Buffer | null; // sha1 checksum + @Column({ type: 'varchar', nullable: true }) duration!: string | null; diff --git a/server/libs/database/src/migrations/1661881837496-AddAssetChecksum.ts b/server/libs/database/src/migrations/1661881837496-AddAssetChecksum.ts new file mode 100644 index 0000000000..a5f1140d9a --- /dev/null +++ b/server/libs/database/src/migrations/1661881837496-AddAssetChecksum.ts @@ -0,0 +1,16 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddAssetChecksum1661881837496 implements MigrationInterface { + name = 'AddAssetChecksum1661881837496' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "assets" ADD "checksum" bytea`); + await queryRunner.query(`CREATE INDEX "IDX_64c507300988dd1764f9a6530c" ON "assets" ("checksum") WHERE 'checksum' IS NOT NULL`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_64c507300988dd1764f9a6530c"`); + await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "checksum"`); + } + +}