1
0
mirror of https://github.com/immich-app/immich.git synced 2024-12-25 10:43:13 +02:00

feat(server): calculate sha1 checksum (#525)

* feat(server): override multer storage

* feat(server): calc sha1 of uploaded file

* feat(server): add checksum into asset

* chore(server): add package-lock for mkdirp package

* fix(server): free hash stream

* chore(server): rollback this changes, not refactor here

* refactor(server): re-arrange import statement

* fix(server): make sure hash done before callback

* refactor(server): replace varchar to char for checksum, reserve pixelChecksum for future

* refactor(server): remove pixelChecksum

* refactor(server): convert checksum from string to bytea

* feat(server): add index to checksum

* refactor(): rollback package.json changes

* feat(server): remove uploaded file when progress fail

* feat(server): calculate hash in sequence
This commit is contained in:
Thanh Pham 2022-08-31 21:27:17 +07:00 committed by GitHub
parent f5f00e0f6c
commit b80dca74ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 45 additions and 3 deletions

View File

@ -10,7 +10,7 @@ import { AssetCountByTimeGroupDto } from './response-dto/asset-count-by-time-gro
import { TimeGroupEnum } from './dto/get-asset-count-by-time-group.dto';
export interface IAssetRepository {
create(createAssetDto: CreateAssetDto, ownerId: string, originalPath: string, mimeType: string): Promise<AssetEntity>;
create(createAssetDto: CreateAssetDto, ownerId: string, originalPath: string, mimeType: string, checksum?: Buffer): Promise<AssetEntity>;
getAllByUserId(userId: string): Promise<AssetEntity[]>;
getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
getById(assetId: string): Promise<AssetEntity>;
@ -143,6 +143,7 @@ export class AssetRepository implements IAssetRepository {
ownerId: string,
originalPath: string,
mimeType: string,
checksum?: Buffer,
): Promise<AssetEntity> {
const asset = new AssetEntity();
asset.deviceAssetId = createAssetDto.deviceAssetId;
@ -155,6 +156,7 @@ export class AssetRepository implements IAssetRepository {
asset.isFavorite = createAssetDto.isFavorite;
asset.mimeType = mimeType;
asset.duration = createAssetDto.duration || null;
asset.checksum = checksum || null;
const createdAsset = await this.assetRepository.save(asset);

View File

@ -75,6 +75,9 @@ export class AssetController {
try {
const savedAsset = await this.assetService.createUserAsset(authUser, assetInfo, file.path, file.mimetype);
if (!savedAsset) {
await this.backgroundTaskService.deleteFileOnDisk([{
originalPath: file.path
} as any]); // simulate asset to make use of delete queue (or use fs.unlink instead)
throw new BadRequestException('Asset not created');
}
@ -87,6 +90,9 @@ export class AssetController {
return new AssetFileUploadResponseDto(savedAsset.id);
} catch (e) {
Logger.error(`Error uploading file ${e}`);
await this.backgroundTaskService.deleteFileOnDisk([{
originalPath: file.path
} as any]); // simulate asset to make use of delete queue (or use fs.unlink instead)
throw new BadRequestException(`Error uploading file`, `${e}`);
}
}

View File

@ -9,6 +9,7 @@ import {
StreamableFile,
} from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { createHash } from 'node:crypto';
import { Repository } from 'typeorm';
import { AuthUserDto } from '../../decorators/auth-user.decorator';
import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
@ -53,7 +54,8 @@ export class AssetService {
originalPath: string,
mimeType: string,
): Promise<AssetEntity> {
const assetEntity = await this._assetRepository.create(createAssetDto, authUser.id, originalPath, mimeType);
const checksum = await this.calculateChecksum(originalPath);
const assetEntity = await this._assetRepository.create(createAssetDto, authUser.id, originalPath, mimeType, checksum);
return assetEntity;
}
@ -444,4 +446,16 @@ export class AssetService {
return mapAssetCountByTimeGroupResponse(result);
}
private calculateChecksum(filePath: string): Promise<Buffer> {
const fileReadStream = createReadStream(filePath);
const sha1Hash = createHash('sha1');
const deferred = new Promise<Buffer>((resolve, reject) => {
sha1Hash.once('error', (err) => reject(err));
sha1Hash.once('finish', () => resolve(sha1Hash.read()));
});
fileReadStream.pipe(sha1Hash);
return deferred;
}
}

View File

@ -1,4 +1,4 @@
import { Column, Entity, OneToOne, PrimaryGeneratedColumn, Unique } from 'typeorm';
import { Column, Entity, Index, OneToOne, PrimaryGeneratedColumn, Unique } from 'typeorm';
import { ExifEntity } from './exif.entity';
import { SmartInfoEntity } from './smart-info.entity';
@ -44,6 +44,10 @@ export class AssetEntity {
@Column({ type: 'varchar', nullable: true })
mimeType!: string | null;
@Column({ type: 'bytea', nullable: true, select: false })
@Index({ where: `'checksum' IS NOT NULL` }) // avoid null index
checksum?: Buffer | null; // sha1 checksum
@Column({ type: 'varchar', nullable: true })
duration!: string | null;

View File

@ -0,0 +1,16 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class AddAssetChecksum1661881837496 implements MigrationInterface {
name = 'AddAssetChecksum1661881837496'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "assets" ADD "checksum" bytea`);
await queryRunner.query(`CREATE INDEX "IDX_64c507300988dd1764f9a6530c" ON "assets" ("checksum") WHERE 'checksum' IS NOT NULL`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX "public"."IDX_64c507300988dd1764f9a6530c"`);
await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "checksum"`);
}
}