feat(server): return asset checksum (#2582)

* feat: return asset checksum

* chore: generate open api

* chore: coverage

* feat(server): support base64 hashes in bulk upload check:

* chore: generate open api

server/apps/immich/src/api-v1/asset/asset.service.spec.ts

@@ -30,6 +30,7 @@ import {
 import { CreateAssetsShareLinkDto } from './dto/create-asset-shared-link.dto';
 import { BadRequestException, ForbiddenException } from '@nestjs/common';
 import { when } from 'jest-when';
+import { AssetRejectReason, AssetUploadAction } from './response-dto/asset-check-response.dto';
 
 const _getCreateAssetDto = (): CreateAssetDto => {
   const createAssetDto = new CreateAssetDto();
@@ -504,4 +505,32 @@ describe('AssetService', () => {
       expect(storageMock.createReadStream).toHaveBeenCalledWith('fake_path/asset_1.jpeg', 'image/jpeg');
     });
   });
+
+  describe('bulkUploadCheck', () => {
+    it('should accept hex and base64 checksums', async () => {
+      const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex');
+      const file2 = Buffer.from('53be335e99f18a66ff12e9a901c7a6171dd76573', 'hex');
+
+      assetRepositoryMock.getAssetsByChecksums.mockResolvedValue([
+        { id: 'asset-1', checksum: file1 },
+        { id: 'asset-2', checksum: file2 },
+      ]);
+
+      await expect(
+        sut.bulkUploadCheck(authStub.admin, {
+          assets: [
+            { id: '1', checksum: file1.toString('hex') },
+            { id: '2', checksum: file2.toString('base64') },
+          ],
+        }),
+      ).resolves.toEqual({
+        results: [
+          { id: '1', assetId: 'asset-1', action: AssetUploadAction.REJECT, reason: AssetRejectReason.DUPLICATE },
+          { id: '2', assetId: 'asset-2', action: AssetUploadAction.REJECT, reason: AssetRejectReason.DUPLICATE },
+        ],
+      });
+
+      expect(assetRepositoryMock.getAssetsByChecksums).toHaveBeenCalledWith(authStub.admin.id, [file1, file2]);
+    });
+  });
 });

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -486,17 +486,24 @@ export class AssetService {
   }
 
   async bulkUploadCheck(authUser: AuthUserDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
+    // support base64 and hex checksums
+    for (const asset of dto.assets) {
+      if (asset.checksum.length === 28) {
+        asset.checksum = Buffer.from(asset.checksum, 'base64').toString('hex');
+      }
+    }
+
     const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
     const results = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
-    const resultsMap: Record<string, string> = {};
+    const checksumMap: Record<string, string> = {};
 
     for (const { id, checksum } of results) {
-      resultsMap[checksum.toString('hex')] = id;
+      checksumMap[checksum.toString('hex')] = id;
     }
 
     return {
       results: dto.assets.map(({ id, checksum }) => {
-        const duplicate = resultsMap[checksum];
+        const duplicate = checksumMap[checksum];
         if (duplicate) {
           return {
             id,

server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts

@@ -6,6 +6,7 @@ export class AssetBulkUploadCheckItem {
   @IsNotEmpty()
   id!: string;
 
+  /** base64 or hex encoded sha1 hash */
   @IsString()
   @IsNotEmpty()
   checksum!: string;

server/immich-openapi-specs.json

@@ -4446,9 +4446,8 @@
           "originalFileName": {
             "type": "string"
           },
-          "resizePath": {
+          "resized": {
-            "type": "string",
+            "type": "boolean"
-            "nullable": true
           },
           "fileCreatedAt": {
             "type": "string"
@@ -4472,14 +4471,6 @@
           "duration": {
             "type": "string"
           },
-          "webpPath": {
-            "type": "string",
-            "nullable": true
-          },
-          "encodedVideoPath": {
-            "type": "string",
-            "nullable": true
-          },
           "exifInfo": {
             "$ref": "#/components/schemas/ExifResponseDto"
           },
@@ -4501,6 +4492,10 @@
             "items": {
               "$ref": "#/components/schemas/PersonResponseDto"
             }
+          },
+          "checksum": {
+            "type": "string",
+            "description": "base64 encoded sha1 hash"
           }
         },
         "required": [
@@ -4511,7 +4506,7 @@
           "deviceId",
           "originalPath",
           "originalFileName",
-          "resizePath",
+          "resized",
           "fileCreatedAt",
           "fileModifiedAt",
           "updatedAt",
@@ -4519,7 +4514,7 @@
           "isArchived",
           "mimeType",
           "duration",
-          "webpPath"
+          "checksum"
         ]
       },
       "AlbumResponseDto": {
@@ -6173,7 +6168,8 @@
             "type": "string"
           },
           "checksum": {
-            "type": "string"
+            "type": "string",
+            "description": "base64 or hex encoded sha1 hash"
           }
         },
         "required": [

server/libs/domain/src/album/album.service.ts

@@ -1,6 +1,6 @@
 import { AlbumEntity, AssetEntity, UserEntity } from '@app/infra/entities';
 import { BadRequestException, ForbiddenException, Inject, Injectable } from '@nestjs/common';
-import { IAssetRepository } from '../asset';
+import { IAssetRepository, mapAsset } from '../asset';
 import { AuthUserDto } from '../auth';
 import { IJobRepository, JobName } from '../job';
 import { IAlbumRepository } from './album.repository';
@@ -40,6 +40,7 @@ export class AlbumService {
     return albums.map((album) => {
       return {
         ...album,
+        assets: album?.assets?.map(mapAsset),
         sharedLinks: undefined, // Don't return shared links
         shared: album.sharedLinks?.length > 0 || album.sharedUsers?.length > 0,
         assetCount: albumsAssetCountObj[album.id],

server/libs/domain/src/asset/response-dto/asset-response.dto.ts

@@ -15,7 +15,7 @@ export class AssetResponseDto {
   type!: AssetType;
   originalPath!: string;
   originalFileName!: string;
-  resizePath!: string | null;
+  resized!: boolean;
   fileCreatedAt!: string;
   fileModifiedAt!: string;
   updatedAt!: string;
@@ -23,13 +23,13 @@ export class AssetResponseDto {
   isArchived!: boolean;
   mimeType!: string | null;
   duration!: string;
-  webpPath!: string | null;
-  encodedVideoPath?: string | null;
   exifInfo?: ExifResponseDto;
   smartInfo?: SmartInfoResponseDto;
   livePhotoVideoId?: string | null;
   tags?: TagResponseDto[];
   people?: PersonResponseDto[];
+  /**base64 encoded sha1 hash */
+  checksum!: string;
 }
 
 export function mapAsset(entity: AssetEntity): AssetResponseDto {
@@ -41,21 +41,20 @@ export function mapAsset(entity: AssetEntity): AssetResponseDto {
     type: entity.type,
     originalPath: entity.originalPath,
     originalFileName: entity.originalFileName,
-    resizePath: entity.resizePath,
+    resized: !!entity.resizePath,
     fileCreatedAt: entity.fileCreatedAt,
     fileModifiedAt: entity.fileModifiedAt,
     updatedAt: entity.updatedAt,
     isFavorite: entity.isFavorite,
     isArchived: entity.isArchived,
     mimeType: entity.mimeType,
-    webpPath: entity.webpPath,
-    encodedVideoPath: entity.encodedVideoPath,
     duration: entity.duration ?? '0:00:00.00000',
     exifInfo: entity.exifInfo ? mapExif(entity.exifInfo) : undefined,
     smartInfo: entity.smartInfo ? mapSmartInfo(entity.smartInfo) : undefined,
     livePhotoVideoId: entity.livePhotoVideoId,
     tags: entity.tags?.map(mapTag),
     people: entity.faces?.map(mapFace),
+    checksum: entity.checksum.toString('base64'),
   };
 }
 
@@ -68,20 +67,19 @@ export function mapAssetWithoutExif(entity: AssetEntity): AssetResponseDto {
     type: entity.type,
     originalPath: entity.originalPath,
     originalFileName: entity.originalFileName,
-    resizePath: entity.resizePath,
+    resized: !!entity.resizePath,
     fileCreatedAt: entity.fileCreatedAt,
     fileModifiedAt: entity.fileModifiedAt,
     updatedAt: entity.updatedAt,
     isFavorite: entity.isFavorite,
     isArchived: entity.isArchived,
     mimeType: entity.mimeType,
-    webpPath: entity.webpPath,
-    encodedVideoPath: entity.encodedVideoPath,
     duration: entity.duration ?? '0:00:00.00000',
     exifInfo: undefined,
     smartInfo: entity.smartInfo ? mapSmartInfo(entity.smartInfo) : undefined,
     livePhotoVideoId: entity.livePhotoVideoId,
     tags: entity.tags?.map(mapTag),
     people: entity.faces?.map(mapFace),
+    checksum: entity.checksum.toString('base64'),
   };
 }

server/libs/domain/src/search/search.service.ts

@@ -3,7 +3,7 @@ import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common'
 import { ConfigService } from '@nestjs/config';
 import { mapAlbum } from '../album';
 import { IAlbumRepository } from '../album/album.repository';
-import { mapAsset } from '../asset';
+import { AssetResponseDto, mapAsset } from '../asset';
 import { IAssetRepository } from '../asset/asset.repository';
 import { AuthUserDto } from '../auth';
 import { MACHINE_LEARNING_ENABLED } from '../domain.constant';
@@ -103,9 +103,13 @@ export class SearchService {
     }
   }
 
-  async getExploreData(authUser: AuthUserDto): Promise<SearchExploreItem<AssetEntity>[]> {
+  async getExploreData(authUser: AuthUserDto): Promise<SearchExploreItem<AssetResponseDto>[]> {
     this.assertEnabled();
-    return this.searchRepository.explore(authUser.id);
+    const results = await this.searchRepository.explore(authUser.id);
+    return results.map(({ fieldName, items }) => ({
+      fieldName,
+      items: items.map(({ value, data }) => ({ value, data: mapAsset(data) })),
+    }));
   }
 
   async search(authUser: AuthUserDto, dto: SearchDto): Promise<SearchResponseDto> {

server/libs/domain/test/fixtures.ts

@@ -446,7 +446,7 @@ const assetResponse: AssetResponseDto = {
   type: AssetType.VIDEO,
   originalPath: 'fake_path/jpeg',
   originalFileName: 'asset_1.jpeg',
-  resizePath: '',
+  resized: false,
   fileModifiedAt: today.toISOString(),
   fileCreatedAt: today.toISOString(),
   updatedAt: today.toISOString(),
@@ -457,13 +457,12 @@ const assetResponse: AssetResponseDto = {
     tags: [],
     objects: ['a', 'b', 'c'],
   },
-  webpPath: '',
-  encodedVideoPath: '',
   duration: '0:00:00.00000',
   exifInfo: assetInfo,
   livePhotoVideoId: null,
   tags: [],
   people: [],
+  checksum: 'ZmlsZSBoYXNo',
 };
 
 const albumResponse: AlbumResponseDto = {

web/src/api/open-api/api.ts

@@ -378,7 +378,7 @@ export interface AssetBulkUploadCheckItem {
      */
     'id': string;
     /**
-     * 
+     * base64 or hex encoded sha1 hash
      * @type {string}
      * @memberof AssetBulkUploadCheckItem
      */
@@ -586,10 +586,10 @@ export interface AssetResponseDto {
     'originalFileName': string;
     /**
      * 
-     * @type {string}
+     * @type {boolean}
      * @memberof AssetResponseDto
      */
-    'resizePath': string | null;
+    'resized': boolean;
     /**
      * 
      * @type {string}
@@ -632,18 +632,6 @@ export interface AssetResponseDto {
      * @memberof AssetResponseDto
      */
     'duration': string;
-    /**
-     * 
-     * @type {string}
-     * @memberof AssetResponseDto
-     */
-    'webpPath': string | null;
-    /**
-     * 
-     * @type {string}
-     * @memberof AssetResponseDto
-     */
-    'encodedVideoPath'?: string | null;
     /**
      * 
      * @type {ExifResponseDto}
@@ -674,6 +662,12 @@ export interface AssetResponseDto {
      * @memberof AssetResponseDto
      */
     'people'?: Array<PersonResponseDto>;
+    /**
+     * base64 encoded sha1 hash
+     * @type {string}
+     * @memberof AssetResponseDto
+     */
+    'checksum': string;
 }
 
 

web/src/lib/components/asset-viewer/asset-viewer.svelte

@@ -350,7 +350,7 @@
 
 	<div class="row-start-1 row-span-full col-start-1 col-span-4">
 		{#key asset.id}
-			{#if !asset.resizePath}
+			{#if !asset.resized}
 				<div class="h-full w-full flex justify-center">
 					<div
 						class="h-full bg-gray-100 dark:bg-immich-dark-gray flex items-center justify-center aspect-square px-auto"

web/src/lib/components/assets/thumbnail/thumbnail.svelte

@@ -123,7 +123,7 @@
 					</div>
 				{/if}
 
-				{#if asset.resizePath}
+				{#if asset.resized}
 					<ImageThumbnail
 						url={api.getAssetThumbnailUrl(asset.id, format, publicSharedKey)}
 						altText={asset.originalFileName}