diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 094bcbe40a..eafeb6851f 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -17,6 +17,10 @@ doc/AlbumCountResponseDto.md doc/AlbumResponseDto.md doc/AllJobStatusResponseDto.md doc/AssetApi.md +doc/AssetBulkUploadCheckDto.md +doc/AssetBulkUploadCheckItem.md +doc/AssetBulkUploadCheckResponseDto.md +doc/AssetBulkUploadCheckResult.md doc/AssetCountByTimeBucket.md doc/AssetCountByTimeBucketResponseDto.md doc/AssetCountByUserIdResponseDto.md @@ -142,6 +146,10 @@ lib/model/api_key_create_dto.dart lib/model/api_key_create_response_dto.dart lib/model/api_key_response_dto.dart lib/model/api_key_update_dto.dart +lib/model/asset_bulk_upload_check_dto.dart +lib/model/asset_bulk_upload_check_item.dart +lib/model/asset_bulk_upload_check_response_dto.dart +lib/model/asset_bulk_upload_check_result.dart lib/model/asset_count_by_time_bucket.dart lib/model/asset_count_by_time_bucket_response_dto.dart lib/model/asset_count_by_user_id_response_dto.dart @@ -236,6 +244,10 @@ test/api_key_create_response_dto_test.dart test/api_key_response_dto_test.dart test/api_key_update_dto_test.dart test/asset_api_test.dart +test/asset_bulk_upload_check_dto_test.dart +test/asset_bulk_upload_check_item_test.dart +test/asset_bulk_upload_check_response_dto_test.dart +test/asset_bulk_upload_check_result_test.dart test/asset_count_by_time_bucket_response_dto_test.dart test/asset_count_by_time_bucket_test.dart test/asset_count_by_user_id_response_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 2c31764325..b6b3ea18df 100644 Binary files a/mobile/openapi/README.md and b/mobile/openapi/README.md differ diff --git a/mobile/openapi/doc/AssetApi.md b/mobile/openapi/doc/AssetApi.md index 2e5a4641fd..fbe6d02a70 100644 Binary files a/mobile/openapi/doc/AssetApi.md and b/mobile/openapi/doc/AssetApi.md differ diff --git a/mobile/openapi/doc/AssetBulkUploadCheckDto.md b/mobile/openapi/doc/AssetBulkUploadCheckDto.md new file mode 100644 index 0000000000..e3d8419ec7 Binary files /dev/null and b/mobile/openapi/doc/AssetBulkUploadCheckDto.md differ diff --git a/mobile/openapi/doc/AssetBulkUploadCheckItem.md b/mobile/openapi/doc/AssetBulkUploadCheckItem.md new file mode 100644 index 0000000000..d0cb998320 Binary files /dev/null and b/mobile/openapi/doc/AssetBulkUploadCheckItem.md differ diff --git a/mobile/openapi/doc/AssetBulkUploadCheckResponseDto.md b/mobile/openapi/doc/AssetBulkUploadCheckResponseDto.md new file mode 100644 index 0000000000..5cdea7d3b8 Binary files /dev/null and b/mobile/openapi/doc/AssetBulkUploadCheckResponseDto.md differ diff --git a/mobile/openapi/doc/AssetBulkUploadCheckResult.md b/mobile/openapi/doc/AssetBulkUploadCheckResult.md new file mode 100644 index 0000000000..670d1d9fa4 Binary files /dev/null and b/mobile/openapi/doc/AssetBulkUploadCheckResult.md differ diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index e22e9ac694..833628d62e 100644 Binary files a/mobile/openapi/lib/api.dart and b/mobile/openapi/lib/api.dart differ diff --git a/mobile/openapi/lib/api/asset_api.dart b/mobile/openapi/lib/api/asset_api.dart index e6cde800db..44acdb2919 100644 Binary files a/mobile/openapi/lib/api/asset_api.dart and b/mobile/openapi/lib/api/asset_api.dart differ diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index e159f60578..7b46ea15f1 100644 Binary files a/mobile/openapi/lib/api_client.dart and b/mobile/openapi/lib/api_client.dart differ diff --git a/mobile/openapi/lib/model/asset_bulk_upload_check_dto.dart b/mobile/openapi/lib/model/asset_bulk_upload_check_dto.dart new file mode 100644 index 0000000000..6d5fdf5142 Binary files /dev/null and b/mobile/openapi/lib/model/asset_bulk_upload_check_dto.dart differ diff --git a/mobile/openapi/lib/model/asset_bulk_upload_check_item.dart b/mobile/openapi/lib/model/asset_bulk_upload_check_item.dart new file mode 100644 index 0000000000..89ee3cf341 Binary files /dev/null and b/mobile/openapi/lib/model/asset_bulk_upload_check_item.dart differ diff --git a/mobile/openapi/lib/model/asset_bulk_upload_check_response_dto.dart b/mobile/openapi/lib/model/asset_bulk_upload_check_response_dto.dart new file mode 100644 index 0000000000..9a0ed965f4 Binary files /dev/null and b/mobile/openapi/lib/model/asset_bulk_upload_check_response_dto.dart differ diff --git a/mobile/openapi/lib/model/asset_bulk_upload_check_result.dart b/mobile/openapi/lib/model/asset_bulk_upload_check_result.dart new file mode 100644 index 0000000000..016342de0a Binary files /dev/null and b/mobile/openapi/lib/model/asset_bulk_upload_check_result.dart differ diff --git a/mobile/openapi/test/asset_api_test.dart b/mobile/openapi/test/asset_api_test.dart index 085f1560f8..af0bc44c25 100644 Binary files a/mobile/openapi/test/asset_api_test.dart and b/mobile/openapi/test/asset_api_test.dart differ diff --git a/mobile/openapi/test/asset_bulk_upload_check_dto_test.dart b/mobile/openapi/test/asset_bulk_upload_check_dto_test.dart new file mode 100644 index 0000000000..830cf2e29e Binary files /dev/null and b/mobile/openapi/test/asset_bulk_upload_check_dto_test.dart differ diff --git a/mobile/openapi/test/asset_bulk_upload_check_item_test.dart b/mobile/openapi/test/asset_bulk_upload_check_item_test.dart new file mode 100644 index 0000000000..688e5b1fcd Binary files /dev/null and b/mobile/openapi/test/asset_bulk_upload_check_item_test.dart differ diff --git a/mobile/openapi/test/asset_bulk_upload_check_response_dto_test.dart b/mobile/openapi/test/asset_bulk_upload_check_response_dto_test.dart new file mode 100644 index 0000000000..1af1fede08 Binary files /dev/null and b/mobile/openapi/test/asset_bulk_upload_check_response_dto_test.dart differ diff --git a/mobile/openapi/test/asset_bulk_upload_check_result_test.dart b/mobile/openapi/test/asset_bulk_upload_check_result_test.dart new file mode 100644 index 0000000000..dc1bf68a52 Binary files /dev/null and b/mobile/openapi/test/asset_bulk_upload_check_result_test.dart differ diff --git a/server/apps/immich/src/api-v1/asset/asset-repository.ts b/server/apps/immich/src/api-v1/asset/asset-repository.ts index a5500a020c..cf033c783a 100644 --- a/server/apps/immich/src/api-v1/asset/asset-repository.ts +++ b/server/apps/immich/src/api-v1/asset/asset-repository.ts @@ -10,13 +10,17 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto'; import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto'; import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto'; import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto'; -import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto'; import { In } from 'typeorm/find-options/operator/In'; import { UpdateAssetDto } from './dto/update-asset.dto'; import { ITagRepository } from '../tag/tag.repository'; import { IsNull, Not } from 'typeorm'; import { AssetSearchDto } from './dto/asset-search.dto'; +export interface AssetCheck { + id: string; + checksum: Buffer; +} + export interface IAssetRepository { get(id: string): Promise; create( @@ -38,11 +42,8 @@ export interface IAssetRepository { getAssetCountByUserId(userId: string): Promise; getArchivedAssetCountByUserId(userId: string): Promise; getAssetByTimeBucket(userId: string, getAssetByTimeBucketDto: GetAssetByTimeBucketDto): Promise; - getAssetByChecksum(userId: string, checksum: Buffer): Promise; - getExistingAssets( - userId: string, - checkDuplicateAssetDto: CheckExistingAssetsDto, - ): Promise; + getAssetsByChecksums(userId: string, checksums: Buffer[]): Promise; + getExistingAssets(userId: string, checkDuplicateAssetDto: CheckExistingAssetsDto): Promise; countByIdAndUser(assetId: string, userId: string): Promise; } @@ -310,41 +311,39 @@ export class AssetRepository implements IAssetRepository { * @returns Promise - Array of assetIds belong to the device */ async getAllByDeviceId(ownerId: string, deviceId: string): Promise { - const rows = await this.assetRepository.find({ + const items = await this.assetRepository.find({ + select: { deviceAssetId: true }, where: { ownerId, deviceId, isVisible: true, }, - select: ['deviceAssetId'], }); - const res: string[] = []; - rows.forEach((v) => res.push(v.deviceAssetId)); - return res; + return items.map((asset) => asset.deviceAssetId); } /** - * Get asset by checksum on the database + * Get assets by checksums on the database * @param ownerId - * @param checksum + * @param checksums * */ - getAssetByChecksum(ownerId: string, checksum: Buffer): Promise { - return this.assetRepository.findOneOrFail({ + async getAssetsByChecksums(ownerId: string, checksums: Buffer[]): Promise { + return this.assetRepository.find({ + select: { + id: true, + checksum: true, + }, where: { ownerId, - checksum, + checksum: In(checksums), }, - relations: ['exifInfo'], }); } - async getExistingAssets( - ownerId: string, - checkDuplicateAssetDto: CheckExistingAssetsDto, - ): Promise { - const existingAssets = await this.assetRepository.find({ + async getExistingAssets(ownerId: string, checkDuplicateAssetDto: CheckExistingAssetsDto): Promise { + const assets = await this.assetRepository.find({ select: { deviceAssetId: true }, where: { deviceAssetId: In(checkDuplicateAssetDto.deviceAssetIds), @@ -352,7 +351,7 @@ export class AssetRepository implements IAssetRepository { ownerId, }, }); - return new CheckExistingAssetsResponseDto(existingAssets.map((a) => a.deviceAssetId)); + return assets.map((asset) => asset.deviceAssetId); } async countByIdAndUser(assetId: string, ownerId: string): Promise { diff --git a/server/apps/immich/src/api-v1/asset/asset.controller.ts b/server/apps/immich/src/api-v1/asset/asset.controller.ts index 6088b81222..774a72ea9b 100644 --- a/server/apps/immich/src/api-v1/asset/asset.controller.ts +++ b/server/apps/immich/src/api-v1/asset/asset.controller.ts @@ -57,6 +57,8 @@ import { AssetSearchDto } from './dto/asset-search.dto'; import { assetUploadOption, ImmichFile } from '../../config/asset-upload.config'; import FileNotEmptyValidator from '../validation/file-not-empty-validator'; import { RemoveAssetsDto } from '../album/dto/remove-assets.dto'; +import { AssetBulkUploadCheckDto } from './dto/asset-check.dto'; +import { AssetBulkUploadCheckResponseDto } from './response-dto/asset-check-response.dto'; import { AssetIdDto } from './dto/asset-id.dto'; import { DeviceIdDto } from './dto/device-id.dto'; @@ -332,6 +334,19 @@ export class AssetController { return await this.assetService.checkExistingAssets(authUser, checkExistingAssetsDto); } + /** + * Checks if assets exist by checksums + */ + @Authenticated() + @Post('/bulk-upload-check') + @HttpCode(200) + bulkUploadCheck( + @GetAuthUser() authUser: AuthUserDto, + @Body(ValidationPipe) dto: AssetBulkUploadCheckDto, + ): Promise { + return this.assetService.bulkUploadCheck(authUser, dto); + } + @Authenticated() @Post('/shared-link') async createAssetsSharedLink( diff --git a/server/apps/immich/src/api-v1/asset/asset.core.ts b/server/apps/immich/src/api-v1/asset/asset.core.ts index 8d3992d389..34e014fc72 100644 --- a/server/apps/immich/src/api-v1/asset/asset.core.ts +++ b/server/apps/immich/src/api-v1/asset/asset.core.ts @@ -17,7 +17,7 @@ export class AssetCore { owner: { id: authUser.id } as UserEntity, mimeType: file.mimeType, - checksum: file.checksum || null, + checksum: file.checksum, originalPath: file.originalPath, deviceAssetId: dto.deviceAssetId, diff --git a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts index 80d5cc91da..cacacc606a 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts @@ -157,7 +157,7 @@ describe('AssetService', () => { getLocationsByUserId: jest.fn(), getSearchPropertiesByUserId: jest.fn(), getAssetByTimeBucket: jest.fn(), - getAssetByChecksum: jest.fn(), + getAssetsByChecksums: jest.fn(), getAssetCountByUserId: jest.fn(), getArchivedAssetCountByUserId: jest.fn(), getExistingAssets: jest.fn(), @@ -299,7 +299,7 @@ describe('AssetService', () => { (error as any).constraint = 'UQ_userid_checksum'; assetRepositoryMock.create.mockRejectedValue(error); - assetRepositoryMock.getAssetByChecksum.mockResolvedValue(_getAsset_1()); + assetRepositoryMock.getAssetsByChecksums.mockResolvedValue([_getAsset_1()]); await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: true, id: 'id_1' }); diff --git a/server/apps/immich/src/api-v1/asset/asset.service.ts b/server/apps/immich/src/api-v1/asset/asset.service.ts index e130b6b07b..145cd717de 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.ts @@ -63,6 +63,12 @@ import { mapSharedLink, SharedLinkResponseDto } from '@app/domain'; import { AssetSearchDto } from './dto/asset-search.dto'; import { AddAssetsDto } from '../album/dto/add-assets.dto'; import { RemoveAssetsDto } from '../album/dto/remove-assets.dto'; +import { AssetBulkUploadCheckDto } from './dto/asset-check.dto'; +import { + AssetUploadAction, + AssetRejectReason, + AssetBulkUploadCheckResponseDto, +} from './response-dto/asset-check-response.dto'; const fileInfo = promisify(stat); @@ -128,7 +134,8 @@ export class AssetService { // handle duplicates with a success response if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_userid_checksum') { - const duplicate = await this.getAssetByChecksum(authUser.id, file.checksum); + const checksums = [file.checksum, livePhotoFile?.checksum].filter((checksum): checksum is Buffer => !!checksum); + const [duplicate] = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums); return { id: duplicate.id, duplicate: true }; } @@ -463,7 +470,40 @@ export class AssetService { authUser: AuthUserDto, checkExistingAssetsDto: CheckExistingAssetsDto, ): Promise { - return this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto); + return { + existingIds: await this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto), + }; + } + + async bulkUploadCheck(authUser: AuthUserDto, dto: AssetBulkUploadCheckDto): Promise { + const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex')); + const results = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums); + const resultsMap: Record = {}; + + for (const { id, checksum } of results) { + resultsMap[checksum.toString('hex')] = id; + } + + return { + results: dto.assets.map(({ id, checksum }) => { + const duplicate = resultsMap[checksum]; + if (duplicate) { + return { + id, + assetId: duplicate, + action: AssetUploadAction.REJECT, + reason: AssetRejectReason.DUPLICATE, + }; + } + + // TODO mime-check + + return { + id, + action: AssetUploadAction.ACCEPT, + }; + }), + }; } async getAssetCountByTimeBucket( @@ -482,10 +522,6 @@ export class AssetService { return mapAssetCountByTimeBucket(result); } - getAssetByChecksum(userId: string, checksum: Buffer) { - return this._assetRepository.getAssetByChecksum(userId, checksum); - } - getAssetCountByUserId(authUser: AuthUserDto): Promise { return this._assetRepository.getAssetCountByUserId(authUser.id); } diff --git a/server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts b/server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts new file mode 100644 index 0000000000..6fab46d631 --- /dev/null +++ b/server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts @@ -0,0 +1,19 @@ +import { Type } from 'class-transformer'; +import { IsArray, IsNotEmpty, IsString, ValidateNested } from 'class-validator'; + +export class AssetBulkUploadCheckItem { + @IsString() + @IsNotEmpty() + id!: string; + + @IsString() + @IsNotEmpty() + checksum!: string; +} + +export class AssetBulkUploadCheckDto { + @IsArray() + @ValidateNested({ each: true }) + @Type(() => AssetBulkUploadCheckItem) + assets!: AssetBulkUploadCheckItem[]; +} diff --git a/server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts b/server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts new file mode 100644 index 0000000000..1a51dc53f2 --- /dev/null +++ b/server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts @@ -0,0 +1,20 @@ +export class AssetBulkUploadCheckResult { + id!: string; + action!: AssetUploadAction; + reason?: AssetRejectReason; + assetId?: string; +} + +export class AssetBulkUploadCheckResponseDto { + results!: AssetBulkUploadCheckResult[]; +} + +export enum AssetUploadAction { + ACCEPT = 'accept', + REJECT = 'reject', +} + +export enum AssetRejectReason { + DUPLICATE = 'duplicate', + UNSUPPORTED_FORMAT = 'unsupported-format', +} diff --git a/server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts b/server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts index 9a159c308a..c39a79606b 100644 --- a/server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts +++ b/server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts @@ -1,6 +1,3 @@ export class CheckExistingAssetsResponseDto { - constructor(existingIds: string[]) { - this.existingIds = existingIds; - } - existingIds: string[]; + existingIds!: string[]; } diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index e0c021f4b8..10704b9bd3 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -3251,6 +3251,49 @@ ] } }, + "/asset/bulk-upload-check": { + "post": { + "operationId": "bulkUploadCheck", + "description": "Checks if assets exist by checksums", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssetBulkUploadCheckDto" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssetBulkUploadCheckResponseDto" + } + } + } + } + }, + "tags": [ + "Asset" + ], + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ] + } + }, "/asset/shared-link": { "post": { "operationId": "createAssetsSharedLink", @@ -6046,6 +6089,78 @@ "existingIds" ] }, + "AssetBulkUploadCheckItem": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "checksum": { + "type": "string" + } + }, + "required": [ + "id", + "checksum" + ] + }, + "AssetBulkUploadCheckDto": { + "type": "object", + "properties": { + "assets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AssetBulkUploadCheckItem" + } + } + }, + "required": [ + "assets" + ] + }, + "AssetBulkUploadCheckResult": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "action": { + "type": "string", + "enum": [ + "accept", + "reject" + ] + }, + "reason": { + "type": "string", + "enum": [ + "duplicate", + "unsupported-format" + ] + }, + "assetId": { + "type": "string" + } + }, + "required": [ + "id", + "action" + ] + }, + "AssetBulkUploadCheckResponseDto": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AssetBulkUploadCheckResult" + } + } + }, + "required": [ + "results" + ] + }, "CreateAssetsShareLinkDto": { "type": "object", "properties": { diff --git a/server/libs/domain/test/fixtures.ts b/server/libs/domain/test/fixtures.ts index c963709e7f..664347da0c 100644 --- a/server/libs/domain/test/fixtures.ts +++ b/server/libs/domain/test/fixtures.ts @@ -147,6 +147,7 @@ export const assetEntityStub = { deviceId: 'device-id', originalPath: 'upload/upload/path.ext', resizePath: null, + checksum: Buffer.from('file hash', 'utf8'), type: AssetType.IMAGE, webpPath: null, encodedVideoPath: null, @@ -173,6 +174,7 @@ export const assetEntityStub = { deviceId: 'device-id', originalPath: '/original/path.ext', resizePath: '/uploads/user-id/thumbs/path.ext', + checksum: Buffer.from('file hash', 'utf8'), type: AssetType.IMAGE, webpPath: null, encodedVideoPath: null, @@ -201,6 +203,7 @@ export const assetEntityStub = { deviceId: 'device-id', originalPath: '/original/path.ext', resizePath: '/uploads/user-id/thumbs/path.ext', + checksum: Buffer.from('file hash', 'utf8'), type: AssetType.VIDEO, webpPath: null, encodedVideoPath: null, @@ -246,6 +249,7 @@ export const assetEntityStub = { owner: userEntityStub.user1, ownerId: 'user-id', deviceId: 'device-id', + checksum: Buffer.from('file hash', 'utf8'), originalPath: '/original/path.ext', resizePath: '/uploads/user-id/thumbs/path.ext', type: AssetType.IMAGE, @@ -663,6 +667,7 @@ export const sharedLinkStub = { type: AssetType.VIDEO, originalPath: 'fake_path/jpeg', resizePath: '', + checksum: Buffer.from('file hash', 'utf8'), fileModifiedAt: today.toISOString(), fileCreatedAt: today.toISOString(), createdAt: today.toISOString(), diff --git a/server/libs/infra/src/entities/asset.entity.ts b/server/libs/infra/src/entities/asset.entity.ts index cba5518ea9..3e6356e2c6 100644 --- a/server/libs/infra/src/entities/asset.entity.ts +++ b/server/libs/infra/src/entities/asset.entity.ts @@ -75,9 +75,9 @@ export class AssetEntity { @Column({ type: 'varchar', nullable: true }) mimeType!: string | null; - @Column({ type: 'bytea', nullable: true, select: false }) - @Index({ where: `'checksum' IS NOT NULL` }) // avoid null index - checksum?: Buffer | null; // sha1 checksum + @Column({ type: 'bytea' }) + @Index() + checksum!: Buffer; // sha1 checksum @Column({ type: 'varchar', nullable: true }) duration!: string | null; diff --git a/server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts b/server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts new file mode 100644 index 0000000000..6da8f32622 --- /dev/null +++ b/server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts @@ -0,0 +1,19 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class RequireChecksumNotNull1684328185099 implements MigrationInterface { + name = 'removeNotNullFromChecksumIndex1684328185099'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_64c507300988dd1764f9a6530c"`); + await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "checksum" SET NOT NULL`); + await queryRunner.query(`CREATE INDEX "IDX_8d3efe36c0755849395e6ea866" ON "assets" ("checksum") `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_8d3efe36c0755849395e6ea866"`); + await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "checksum" DROP NOT NULL`); + await queryRunner.query( + `CREATE INDEX "IDX_64c507300988dd1764f9a6530c" ON "assets" ("checksum") WHERE ('checksum' IS NOT NULL)`, + ); + } +} diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index 685964b291..37d1ce4491 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -346,6 +346,96 @@ export interface AllJobStatusResponseDto { */ 'recognize-faces-queue': JobStatusDto; } +/** + * + * @export + * @interface AssetBulkUploadCheckDto + */ +export interface AssetBulkUploadCheckDto { + /** + * + * @type {Array} + * @memberof AssetBulkUploadCheckDto + */ + 'assets': Array; +} +/** + * + * @export + * @interface AssetBulkUploadCheckItem + */ +export interface AssetBulkUploadCheckItem { + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckItem + */ + 'id': string; + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckItem + */ + 'checksum': string; +} +/** + * + * @export + * @interface AssetBulkUploadCheckResponseDto + */ +export interface AssetBulkUploadCheckResponseDto { + /** + * + * @type {Array} + * @memberof AssetBulkUploadCheckResponseDto + */ + 'results': Array; +} +/** + * + * @export + * @interface AssetBulkUploadCheckResult + */ +export interface AssetBulkUploadCheckResult { + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckResult + */ + 'id': string; + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckResult + */ + 'action': AssetBulkUploadCheckResultActionEnum; + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckResult + */ + 'reason'?: AssetBulkUploadCheckResultReasonEnum; + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckResult + */ + 'assetId'?: string; +} + +export const AssetBulkUploadCheckResultActionEnum = { + Accept: 'accept', + Reject: 'reject' +} as const; + +export type AssetBulkUploadCheckResultActionEnum = typeof AssetBulkUploadCheckResultActionEnum[keyof typeof AssetBulkUploadCheckResultActionEnum]; +export const AssetBulkUploadCheckResultReasonEnum = { + Duplicate: 'duplicate', + UnsupportedFormat: 'unsupported-format' +} as const; + +export type AssetBulkUploadCheckResultReasonEnum = typeof AssetBulkUploadCheckResultReasonEnum[keyof typeof AssetBulkUploadCheckResultReasonEnum]; + /** * * @export @@ -4120,6 +4210,50 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration options: localVarRequestOptions, }; }, + /** + * Checks if assets exist by checksums + * @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + bulkUploadCheck: async (assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options: AxiosRequestConfig = {}): Promise => { + // verify required parameter 'assetBulkUploadCheckDto' is not null or undefined + assertParamExists('bulkUploadCheck', 'assetBulkUploadCheckDto', assetBulkUploadCheckDto) + const localVarPath = `/asset/bulk-upload-check`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + localVarHeaderParameter['Content-Type'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(assetBulkUploadCheckDto, localVarRequestOptions, configuration) + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, /** * Check duplicated asset before uploading - for Web upload used * @param {CheckDuplicateAssetDto} checkDuplicateAssetDto @@ -5312,6 +5446,16 @@ export const AssetApiFp = function(configuration?: Configuration) { const localVarAxiosArgs = await localVarAxiosParamCreator.addAssetsToSharedLink(addAssetsDto, key, options); return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); }, + /** + * Checks if assets exist by checksums + * @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async bulkUploadCheck(assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.bulkUploadCheck(assetBulkUploadCheckDto, options); + return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); + }, /** * Check duplicated asset before uploading - for Web upload used * @param {CheckDuplicateAssetDto} checkDuplicateAssetDto @@ -5595,6 +5739,15 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath addAssetsToSharedLink(addAssetsDto: AddAssetsDto, key?: string, options?: any): AxiosPromise { return localVarFp.addAssetsToSharedLink(addAssetsDto, key, options).then((request) => request(axios, basePath)); }, + /** + * Checks if assets exist by checksums + * @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + bulkUploadCheck(assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options?: any): AxiosPromise { + return localVarFp.bulkUploadCheck(assetBulkUploadCheckDto, options).then((request) => request(axios, basePath)); + }, /** * Check duplicated asset before uploading - for Web upload used * @param {CheckDuplicateAssetDto} checkDuplicateAssetDto @@ -5856,6 +6009,17 @@ export class AssetApi extends BaseAPI { return AssetApiFp(this.configuration).addAssetsToSharedLink(addAssetsDto, key, options).then((request) => request(this.axios, this.basePath)); } + /** + * Checks if assets exist by checksums + * @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof AssetApi + */ + public bulkUploadCheck(assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options?: AxiosRequestConfig) { + return AssetApiFp(this.configuration).bulkUploadCheck(assetBulkUploadCheckDto, options).then((request) => request(this.axios, this.basePath)); + } + /** * Check duplicated asset before uploading - for Web upload used * @param {CheckDuplicateAssetDto} checkDuplicateAssetDto diff --git a/web/src/lib/utils/file-uploader.ts b/web/src/lib/utils/file-uploader.ts index bad793e5da..e825d0eb74 100644 --- a/web/src/lib/utils/file-uploader.ts +++ b/web/src/lib/utils/file-uploader.ts @@ -4,7 +4,7 @@ import { } from './../components/shared-components/notification/notification'; import { uploadAssetsStore } from '$lib/stores/upload'; import type { UploadAsset } from '../models/upload-asset'; -import { api, AssetFileUploadResponseDto } from '@api'; +import { AssetFileUploadResponseDto } from '@api'; import { addAssetsToAlbum, getFileMimeType, getFilenameExtension } from '$lib/utils/asset-utils'; import { mergeMap, filter, firstValueFrom, from, of, combineLatestAll } from 'rxjs'; import axios from 'axios'; @@ -73,7 +73,7 @@ async function fileUploader( const deviceAssetId = 'web' + '-' + asset.name + '-' + asset.lastModified; try { - // Create and add Unique ID of asset on the device + // Create and add pseudo-unique ID of asset on the device formData.append('deviceAssetId', deviceAssetId); // Get device id - for web -> use WEB @@ -102,23 +102,6 @@ async function fileUploader( // failed uploads. formData.append('assetData', new File([asset], asset.name, { type: mimeType })); - // Check if asset upload on server before performing upload - const { data, status } = await api.assetApi.checkDuplicateAsset( - { - deviceAssetId: String(deviceAssetId), - deviceId: 'WEB' - }, - sharedKey - ); - - if (status === 200 && data.isExist && data.id) { - if (albumId) { - await addAssetsToAlbum(albumId, [data.id], sharedKey); - } - - return data.id; - } - const newUploadAsset: UploadAsset = { id: deviceAssetId, file: asset,