feat(web/server): Face thumbnail selection (#3081)

* add migration

* verify running migration populate new value

* implemented service

* generate api

* FE works

* FR Works

* fix test

* fix test fixture

* fix test

* fix test

* consolidate api

* fix test

* added test

* pr feedback

* refactor

* click ont humbnail to show feature selection as well

server/immich-openapi-specs.json

@@ -5816,12 +5816,14 @@
         "type": "object",
         "properties": {
           "name": {
-            "type": "string"
-          }
+            "type": "string",
+            "description": "Person name."
           },
-        "required": [
-          "name"
-        ]
+          "featureFaceAssetId": {
+            "type": "string",
+            "description": "Asset is used to get the feature face thumbnail."
+          }
+        }
       },
       "QueueStatusDto": {
         "type": "object",

server/src/domain/facial-recognition/facial-recognition.service.spec.ts

@@ -191,6 +191,12 @@ describe(FacialRecognitionService.name, () => {
         personId: 'person-1',
         assetId: 'asset-id',
         embedding: [1, 2, 3, 4],
+        boundingBoxX1: 100,
+        boundingBoxY1: 100,
+        boundingBoxX2: 200,
+        boundingBoxY2: 200,
+        imageHeight: 500,
+        imageWidth: 400,
       });
     });
 
@@ -207,6 +213,12 @@ describe(FacialRecognitionService.name, () => {
         personId: 'person-1',
         assetId: 'asset-id',
         embedding: [1, 2, 3, 4],
+        boundingBoxX1: 100,
+        boundingBoxY1: 100,
+        boundingBoxX2: 200,
+        boundingBoxY2: 200,
+        imageHeight: 500,
+        imageWidth: 400,
       });
       expect(jobMock.queue.mock.calls).toEqual([
         [

server/src/domain/facial-recognition/facial-recognition.services.ts

@@ -83,7 +83,16 @@ export class FacialRecognitionService {
 
       const faceId: AssetFaceId = { assetId: asset.id, personId };
 
-      await this.faceRepository.create({ ...faceId, embedding });
+      await this.faceRepository.create({
+        ...faceId,
+        embedding,
+        imageHeight: rest.imageHeight,
+        imageWidth: rest.imageWidth,
+        boundingBoxX1: rest.boundingBox.x1,
+        boundingBoxX2: rest.boundingBox.x2,
+        boundingBoxY1: rest.boundingBox.y1,
+        boundingBoxY2: rest.boundingBox.y2,
+      });
       await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACE, data: faceId });
     }
 

server/src/domain/person/person.dto.ts

@@ -1,10 +1,20 @@
 import { AssetFaceEntity, PersonEntity } from '@app/infra/entities';
-import { IsNotEmpty, IsString } from 'class-validator';
+import { IsOptional, IsString } from 'class-validator';
 
 export class PersonUpdateDto {
-  @IsNotEmpty()
+  /**
+   * Person name.
+   */
+  @IsOptional()
   @IsString()
-  name!: string;
+  name?: string;
+
+  /**
+   * Asset is used to get the feature face thumbnail.
+   */
+  @IsOptional()
+  @IsString()
+  featureFaceAssetId?: string;
 }
 
 export class PersonResponseDto {

server/src/domain/person/person.repository.ts

@@ -1,5 +1,5 @@
-import { AssetEntity, PersonEntity } from '@app/infra/entities';
-
+import { AssetFaceId } from '@app/domain';
+import { AssetEntity, AssetFaceEntity, PersonEntity } from '@app/infra/entities';
 export const IPersonRepository = 'IPersonRepository';
 
 export interface PersonSearchOptions {
@@ -16,4 +16,6 @@ export interface IPersonRepository {
   update(entity: Partial<PersonEntity>): Promise<PersonEntity>;
   delete(entity: PersonEntity): Promise<PersonEntity | null>;
   deleteAll(): Promise<number>;
+
+  getFaceById(payload: AssetFaceId): Promise<AssetFaceEntity | null>;
 }

server/src/domain/person/person.service.spec.ts

@@ -2,6 +2,7 @@ import { BadRequestException, NotFoundException } from '@nestjs/common';
 import {
   assetEntityStub,
   authStub,
+  faceStub,
   newJobRepositoryMock,
   newPersonRepositoryMock,
   newStorageRepositoryMock,
@@ -108,6 +109,36 @@ describe(PersonService.name, () => {
         data: { ids: [assetEntityStub.image.id] },
       });
     });
+
+    it("should update a person's thumbnailPath", async () => {
+      personMock.getById.mockResolvedValue(personStub.withName);
+      personMock.getFaceById.mockResolvedValue(faceStub.face1);
+
+      await expect(
+        sut.update(authStub.admin, 'person-1', { featureFaceAssetId: faceStub.face1.assetId }),
+      ).resolves.toEqual(responseDto);
+
+      expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1');
+      expect(personMock.getFaceById).toHaveBeenCalledWith({
+        assetId: faceStub.face1.assetId,
+        personId: 'person-1',
+      });
+      expect(jobMock.queue).toHaveBeenCalledWith({
+        name: JobName.GENERATE_FACE_THUMBNAIL,
+        data: {
+          assetId: faceStub.face1.assetId,
+          personId: 'person-1',
+          boundingBox: {
+            x1: faceStub.face1.boundingBoxX1,
+            x2: faceStub.face1.boundingBoxX2,
+            y1: faceStub.face1.boundingBoxY1,
+            y2: faceStub.face1.boundingBoxY2,
+          },
+          imageHeight: faceStub.face1.imageHeight,
+          imageWidth: faceStub.face1.imageWidth,
+        },
+      });
+    });
   });
 
   describe('handlePersonCleanup', () => {

server/src/domain/person/person.service.ts

@@ -1,3 +1,4 @@
+import { PersonEntity } from '@app/infra/entities';
 import { BadRequestException, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common';
 import { AssetResponseDto, mapAsset } from '../asset';
 import { AuthUserDto } from '../auth';
@@ -52,18 +53,54 @@ export class PersonService {
   }
 
   async update(authUser: AuthUserDto, personId: string, dto: PersonUpdateDto): Promise<PersonResponseDto> {
-    const exists = await this.repository.getById(authUser.id, personId);
-    if (!exists) {
+    let person = await this.repository.getById(authUser.id, personId);
+    if (!person) {
       throw new BadRequestException();
     }
 
-    const person = await this.repository.update({ id: personId, name: dto.name });
+    if (dto.name) {
+      person = await this.updateName(authUser, personId, dto.name);
+    }
+
+    if (dto.featureFaceAssetId) {
+      await this.updateFaceThumbnail(personId, dto.featureFaceAssetId);
+    }
+
+    return mapPerson(person);
+  }
+
+  private async updateName(authUser: AuthUserDto, personId: string, name: string): Promise<PersonEntity> {
+    const person = await this.repository.update({ id: personId, name });
 
     const relatedAsset = await this.getAssets(authUser, personId);
     const assetIds = relatedAsset.map((asset) => asset.id);
     await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: assetIds } });
 
-    return mapPerson(person);
+    return person;
+  }
+
+  private async updateFaceThumbnail(personId: string, assetId: string): Promise<void> {
+    const face = await this.repository.getFaceById({ assetId, personId });
+
+    if (!face) {
+      throw new BadRequestException();
+    }
+
+    return await this.jobRepository.queue({
+      name: JobName.GENERATE_FACE_THUMBNAIL,
+      data: {
+        assetId: assetId,
+        personId,
+        boundingBox: {
+          x1: face.boundingBoxX1,
+          x2: face.boundingBoxX2,
+          y1: face.boundingBoxY1,
+          y2: face.boundingBoxY2,
+        },
+        imageHeight: face.imageHeight,
+        imageWidth: face.imageWidth,
+      },
+    });
   }
 
   async handlePersonCleanup() {

server/src/infra/entities/asset-face.entity.ts

@@ -17,6 +17,24 @@ export class AssetFaceEntity {
   })
   embedding!: number[] | null;
 
+  @Column({ default: 0, type: 'int' })
+  imageWidth!: number;
+
+  @Column({ default: 0, type: 'int' })
+  imageHeight!: number;
+
+  @Column({ default: 0, type: 'int' })
+  boundingBoxX1!: number;
+
+  @Column({ default: 0, type: 'int' })
+  boundingBoxY1!: number;
+
+  @Column({ default: 0, type: 'int' })
+  boundingBoxX2!: number;
+
+  @Column({ default: 0, type: 'int' })
+  boundingBoxY2!: number;
+
   @ManyToOne(() => AssetEntity, (asset) => asset.faces, { onDelete: 'CASCADE', onUpdate: 'CASCADE' })
   asset!: AssetEntity;
 

server/src/infra/migrations/1688241394489-AddDetectFaceResultInfo.ts

@@ -0,0 +1,23 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddDetectFaceResultInfo1688241394489 implements MigrationInterface {
+  name = 'AddDetectFaceResultInfo1688241394489';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "imageWidth" integer NOT NULL DEFAULT '0'`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "imageHeight" integer NOT NULL DEFAULT '0'`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "boundingBoxX1" integer NOT NULL DEFAULT '0'`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "boundingBoxY1" integer NOT NULL DEFAULT '0'`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "boundingBoxX2" integer NOT NULL DEFAULT '0'`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" ADD "boundingBoxY2" integer NOT NULL DEFAULT '0'`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "boundingBoxY2"`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "boundingBoxX2"`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "boundingBoxY1"`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "boundingBoxX1"`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "imageHeight"`);
+    await queryRunner.query(`ALTER TABLE "asset_faces" DROP COLUMN "imageWidth"`);
+  }
+}

server/src/infra/repositories/person.repository.ts

@@ -1,4 +1,4 @@
-import { IPersonRepository, PersonSearchOptions } from '@app/domain';
+import { AssetFaceId, IPersonRepository, PersonSearchOptions } from '@app/domain';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities';
@@ -77,4 +77,8 @@ export class PersonRepository implements IPersonRepository {
     const { id } = await this.personRepository.save(entity);
     return this.personRepository.findOneByOrFail({ id });
   }
+
+  async getFaceById({ personId, assetId }: AssetFaceId): Promise<AssetFaceEntity | null> {
+    return this.assetFaceRepository.findOneBy({ assetId, personId });
+  }
 }

server/test/fixtures.ts

@@ -1157,6 +1157,16 @@ export const personStub = {
     thumbnailPath: '',
     faces: [],
   }),
+  newThumbnail: Object.freeze<PersonEntity>({
+    id: 'person-1',
+    createdAt: new Date('2021-01-01'),
+    updatedAt: new Date('2021-01-01'),
+    ownerId: userEntityStub.admin.id,
+    owner: userEntityStub.admin,
+    name: '',
+    thumbnailPath: '/new/path/to/thumbnail',
+    faces: [],
+  }),
 };
 
 export const partnerStub = {
@@ -1185,6 +1195,12 @@ export const faceStub = {
     personId: personStub.withName.id,
     person: personStub.withName,
     embedding: [1, 2, 3, 4],
+    boundingBoxX1: 0,
+    boundingBoxY1: 0,
+    boundingBoxX2: 1,
+    boundingBoxY2: 1,
+    imageHeight: 1024,
+    imageWidth: 1024,
   }),
 };
 

server/test/repositories/person.repository.mock.ts

@@ -11,5 +11,7 @@ export const newPersonRepositoryMock = (): jest.Mocked<IPersonRepository> => {
     update: jest.fn(),
     deleteAll: jest.fn(),
     delete: jest.fn(),
+
+    getFaceById: jest.fn(),
   };
 };

web/src/api/open-api/api.ts

@@ -1772,11 +1772,17 @@ export interface PersonResponseDto {
  */
 export interface PersonUpdateDto {
     /**
-     * 
+     * Person name.
      * @type {string}
      * @memberof PersonUpdateDto
      */
-    'name': string;
+    'name'?: string;
+    /**
+     * Asset is used to get the feature face thumbnail.
+     * @type {string}
+     * @memberof PersonUpdateDto
+     */
+    'featureFaceAssetId'?: string;
 }
 /**
  * 

web/src/lib/components/faces-page/face-thumbnail-selector.svelte

@@ -0,0 +1,36 @@
+<script lang="ts">
+  import type { AssetResponseDto } from '@api';
+  import { createEventDispatcher } from 'svelte';
+  import { quintOut } from 'svelte/easing';
+  import { fly } from 'svelte/transition';
+  import ControlAppBar from '../shared-components/control-app-bar.svelte';
+  import AssetSelectionViewer from '../shared-components/gallery-viewer/asset-selection-viewer.svelte';
+
+  const dispatch = createEventDispatcher();
+
+  export let assets: AssetResponseDto[];
+
+  let selectedAsset: AssetResponseDto | undefined = undefined;
+
+  const handleSelectedAsset = async (event: CustomEvent) => {
+    const { asset }: { asset: AssetResponseDto } = event.detail;
+    selectedAsset = asset;
+    onClose();
+  };
+
+  const onClose = () => {
+    dispatch('go-back', { selectedAsset });
+  };
+</script>
+
+<section
+  transition:fly={{ y: 500, duration: 100, easing: quintOut }}
+  class="absolute top-0 left-0 w-full h-full bg-immich-bg dark:bg-immich-dark-bg z-[9999]"
+>
+  <ControlAppBar on:close-button-click={onClose}>
+    <svelte:fragment slot="leading">Select feature photo</svelte:fragment>
+  </ControlAppBar>
+  <section class="pt-[100px] pl-[70px] bg-immich-bg dark:bg-immich-dark-bg">
+    <AssetSelectionViewer {assets} on:select={handleSelectedAsset} />
+  </section>
+</section>

web/src/lib/components/shared-components/gallery-viewer/asset-selection-viewer.svelte

@@ -0,0 +1,44 @@
+<script lang="ts">
+  import Thumbnail from '$lib/components/assets/thumbnail/thumbnail.svelte';
+  import { getThumbnailSize } from '$lib/utils/thumbnail-util';
+  import { AssetResponseDto, ThumbnailFormat } from '@api';
+  import { createEventDispatcher } from 'svelte';
+  import { flip } from 'svelte/animate';
+
+  export let assets: AssetResponseDto[];
+  export let selectedAssets: Set<AssetResponseDto> = new Set();
+
+  let viewWidth: number;
+  $: thumbnailSize = getThumbnailSize(assets.length, viewWidth);
+
+  let dispatch = createEventDispatcher();
+
+  const selectAssetHandler = (event: CustomEvent) => {
+    const { asset }: { asset: AssetResponseDto } = event.detail;
+    let temp = new Set(selectedAssets);
+    if (selectedAssets.has(asset)) {
+      temp.delete(asset);
+    } else {
+      temp.add(asset);
+    }
+
+    selectedAssets = temp;
+    dispatch('select', { asset, selectedAssets });
+  };
+</script>
+
+{#if assets.length > 0}
+  <div class="flex flex-wrap gap-1 w-full pb-20" bind:clientWidth={viewWidth}>
+    {#each assets as asset (asset.id)}
+      <div animate:flip={{ duration: 500 }}>
+        <Thumbnail
+          {asset}
+          {thumbnailSize}
+          format={assets.length < 7 ? ThumbnailFormat.Jpeg : ThumbnailFormat.Webp}
+          on:click={selectAssetHandler}
+          selected={selectedAssets.has(asset)}
+        />
+      </div>
+    {/each}
+  </div>
+{/if}

web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte

@@ -10,6 +10,7 @@
   import AssetViewer from '../../asset-viewer/asset-viewer.svelte';
   import { flip } from 'svelte/animate';
   import { archivedAsset } from '$lib/stores/archived-asset.store';
+  import { getThumbnailSize } from '$lib/utils/thumbnail-util';
 
   export let assets: AssetResponseDto[];
   export let sharedLink: SharedLinkResponseDto | undefined = undefined;
@@ -24,22 +25,10 @@
   let currentViewAssetIndex = 0;
 
   let viewWidth: number;
-  let thumbnailSize = 300;
+  $: thumbnailSize = getThumbnailSize(assets.length, viewWidth);
 
   $: isMultiSelectionMode = selectedAssets.size > 0;
 
-  $: {
-    if (assets.length < 6) {
-      thumbnailSize = Math.min(320, Math.floor(viewWidth / assets.length - assets.length));
-    } else {
-      if (viewWidth > 600) thumbnailSize = Math.floor(viewWidth / 7 - 7);
-      else if (viewWidth > 400) thumbnailSize = Math.floor(viewWidth / 4 - 6);
-      else if (viewWidth > 300) thumbnailSize = Math.floor(viewWidth / 2 - 6);
-      else if (viewWidth > 200) thumbnailSize = Math.floor(viewWidth / 2 - 6);
-      else if (viewWidth > 100) thumbnailSize = Math.floor(viewWidth / 1 - 6);
-    }
-  }
-
   const viewAssetHandler = (event: CustomEvent) => {
     const { asset }: { asset: AssetResponseDto } = event.detail;
 

web/src/lib/utils/thumbnail-util.ts

@@ -0,0 +1,19 @@
+/**
+ * Calculate thumbnail size based on number of assets and viewport width
+ * @param assetCount Number of assets in the view
+ * @param viewWidth viewport width
+ * @returns thumbnail size
+ */
+export function getThumbnailSize(assetCount: number, viewWidth: number): number {
+  if (assetCount < 6) {
+    return Math.min(320, Math.floor(viewWidth / assetCount - assetCount));
+  } else {
+    if (viewWidth > 600) return viewWidth / 7 - 7;
+    else if (viewWidth > 400) return viewWidth / 4 - 6;
+    else if (viewWidth > 300) return viewWidth / 2 - 6;
+    else if (viewWidth > 200) return viewWidth / 2 - 6;
+    else if (viewWidth > 100) return viewWidth / 1 - 6;
+  }
+
+  return 300;
+}

web/src/routes/(user)/people/[personId]/+page.svelte

@@ -22,10 +22,17 @@
   import type { PageData } from './$types';
   import CircleIconButton from '$lib/components/elements/buttons/circle-icon-button.svelte';
   import SelectAll from 'svelte-material-icons/SelectAll.svelte';
+  import MenuOption from '$lib/components/shared-components/context-menu/menu-option.svelte';
+  import FaceThumbnailSelector from '$lib/components/faces-page/face-thumbnail-selector.svelte';
+  import {
+    NotificationType,
+    notificationController,
+  } from '$lib/components/shared-components/notification/notification';
 
   export let data: PageData;
 
-  let isEditName = false;
+  let isEditingName = false;
+  let isSelectingFace = false;
   let previousRoute: string = AppRoute.EXPLORE;
   let selectedAssets: Set<AssetResponseDto> = new Set();
   $: isMultiSelectionMode = selectedAssets.size > 0;
@@ -41,7 +48,7 @@
 
   const handleNameChange = async (name: string) => {
     try {
-      isEditName = false;
+      isEditingName = false;
       data.person.name = name;
       await api.personApi.updatePerson({ id: data.person.id, personUpdateDto: { name } });
     } catch (error) {
@@ -55,6 +62,25 @@
   const handleSelectAll = () => {
     selectedAssets = new Set(data.assets);
   };
+
+  const handleSelectFeaturePhoto = async (event: CustomEvent) => {
+    isSelectingFace = false;
+
+    const { selectedAsset }: { selectedAsset: AssetResponseDto | undefined } = event.detail;
+
+    if (selectedAsset) {
+      await api.personApi.updatePerson({
+        id: data.person.id,
+        personUpdateDto: { featureFaceAssetId: selectedAsset.id },
+      });
+
+      // TODO: Replace by Websocket in the future
+      notificationController.show({
+        message: 'Feature photo updated, refresh page to see changes',
+        type: NotificationType.Info,
+      });
+    }
+  };
 </script>
 
 {#if isMultiSelectionMode}
@@ -73,18 +99,25 @@
     </AssetSelectContextMenu>
   </AssetSelectControlBar>
 {:else}
-  <ControlAppBar showBackButton backIcon={ArrowLeft} on:close-button-click={() => goto(previousRoute)} />
+  <ControlAppBar showBackButton backIcon={ArrowLeft} on:close-button-click={() => goto(previousRoute)}>
+    <svelte:fragment slot="trailing">
+      <AssetSelectContextMenu icon={DotsVertical} title="Menu">
+        <MenuOption text="Change feature photo" on:click={() => (isSelectingFace = true)} />
+      </AssetSelectContextMenu>
+    </svelte:fragment>
+  </ControlAppBar>
 {/if}
 
 <!-- Face information block -->
 <section class="pt-24 px-4 sm:px-6 flex place-items-center">
-  {#if isEditName}
+  {#if isEditingName}
     <EditNameInput
       person={data.person}
       on:change={(event) => handleNameChange(event.detail)}
-      on:cancel={() => (isEditName = false)}
+      on:cancel={() => (isEditingName = false)}
     />
   {:else}
+    <button on:click={() => (isSelectingFace = true)}>
       <ImageThumbnail
         circle
         shadow
@@ -93,10 +126,12 @@
         widthStyle="3.375rem"
         heightStyle="3.375rem"
       />
+    </button>
+
     <button
       title="Edit name"
       class="px-4 text-immich-primary dark:text-immich-dark-primary"
-      on:click={() => (isEditName = true)}
+      on:click={() => (isEditingName = true)}
     >
       {#if data.person.name}
         <p class="font-medium py-2">{data.person.name}</p>
@@ -109,6 +144,7 @@
 </section>
 
 <!-- Gallery Block -->
+{#if !isSelectingFace}
   <section class="relative pt-8 sm:px-4 mb-12 bg-immich-bg dark:bg-immich-dark-bg">
     <section class="overflow-y-auto relative immich-scrollbar">
       <section id="search-content" class="relative bg-immich-bg dark:bg-immich-dark-bg">
@@ -116,3 +152,8 @@
       </section>
     </section>
   </section>
+{/if}
+
+{#if isSelectingFace}
+  <FaceThumbnailSelector assets={data.assets} on:go-back={handleSelectFeaturePhoto} />
+{/if}