feat(web): Option to assign people to unassigned faces (#9773)

* added unassigned faces to people edit

* svelte fix

* fix format

* Captialized unassigned person name, removed person id from alttext, fixed problem with multiple faces per person

* Added faces to the getAssetInfo API endpoint

* Updated openApi clients

* Readded the photoeditor dependency

* fixed lint/format

* fixed photoViewer type

* changes getAssetInfo.faces to only include unassigned faces

* fix: bad merge

* title

* logic

---------

Co-authored-by: Jan108 <dasJan108@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

open-api/immich-openapi-specs.json

@@ -7725,6 +7725,12 @@
           "type": {
             "$ref": "#/components/schemas/AssetTypeEnum"
           },
+          "unassignedFaces": {
+            "items": {
+              "$ref": "#/components/schemas/AssetFaceWithoutPersonResponseDto"
+            },
+            "type": "array"
+          },
           "updatedAt": {
             "format": "date-time",
             "type": "string"

open-api/typescript-sdk/src/fetch-client.ts

@@ -194,6 +194,7 @@ export type AssetResponseDto = {
     tags?: TagResponseDto[];
     thumbhash: string | null;
     "type": AssetTypeEnum;
+    unassignedFaces?: AssetFaceWithoutPersonResponseDto[];
     updatedAt: string;
 };
 export type AlbumResponseDto = {

server/src/dtos/asset-response.dto.ts

@@ -2,7 +2,12 @@ import { ApiProperty } from '@nestjs/swagger';
 import { PropertyLifecycle } from 'src/decorators';
 import { AuthDto } from 'src/dtos/auth.dto';
 import { ExifResponseDto, mapExif } from 'src/dtos/exif.dto';
-import { PersonWithFacesResponseDto, mapFacesWithoutPerson, mapPerson } from 'src/dtos/person.dto';
+import {
+  AssetFaceWithoutPersonResponseDto,
+  PersonWithFacesResponseDto,
+  mapFacesWithoutPerson,
+  mapPerson,
+} from 'src/dtos/person.dto';
 import { TagResponseDto, mapTag } from 'src/dtos/tag.dto';
 import { UserResponseDto, mapUser } from 'src/dtos/user.dto';
 import { AssetFaceEntity } from 'src/entities/asset-face.entity';
@@ -41,6 +46,7 @@ export class AssetResponseDto extends SanitizedAssetResponseDto {
   smartInfo?: SmartInfoResponseDto;
   tags?: TagResponseDto[];
   people?: PersonWithFacesResponseDto[];
+  unassignedFaces?: AssetFaceWithoutPersonResponseDto[];
   /**base64 encoded sha1 hash */
   checksum!: string;
   stackParentId?: string | null;
@@ -116,6 +122,7 @@ export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): As
     livePhotoVideoId: entity.livePhotoVideoId,
     tags: entity.tags?.map(mapTag),
     people: peopleWithFaces(entity.faces),
+    unassignedFaces: entity.faces?.filter((face) => !face.person).map((a) => mapFacesWithoutPerson(a)),
     checksum: entity.checksum.toString('base64'),
     stackParentId: withStack ? entity.stack?.primaryAssetId : undefined,
     stack: withStack

web/src/lib/components/asset-viewer/detail-panel.svelte

@@ -27,6 +27,7 @@
     mdiImageOutline,
     mdiInformationOutline,
     mdiPencil,
+    mdiAccountOff,
   } from '@mdi/js';
   import { DateTime } from 'luxon';
   import { createEventDispatcher, onMount } from 'svelte';
@@ -76,6 +77,7 @@
     if (newAsset.id && !isSharedLink()) {
       const data = await getAssetInfo({ id: asset.id });
       people = data?.people || [];
+      unassignedFaces = data?.unassignedFaces || [];
     }
   };
 
@@ -93,6 +95,8 @@
   $: people = asset.people || [];
   $: showingHiddenPeople = false;
 
+  $: unassignedFaces = asset.unassignedFaces || [];
+
   onMount(() => {
     return websocketEvents.on('on_asset_update', (assetUpdate) => {
       if (assetUpdate.id === asset.id) {
@@ -118,6 +122,7 @@
   const handleRefreshPeople = async () => {
     await getAssetInfo({ id: asset.id }).then((data) => {
       people = data?.people || [];
+      unassignedFaces = data?.unassignedFaces || [];
     });
     showEditFaces = false;
   };
@@ -158,11 +163,20 @@
 
   <DetailPanelDescription {asset} {isOwner} />
 
-  {#if !isSharedLink() && people.length > 0}
+  {#if (!isSharedLink() && unassignedFaces.length > 0) || people.length > 0}
     <section class="px-4 py-4 text-sm">
       <div class="flex h-10 w-full items-center justify-between">
         <h2>{$t('people').toUpperCase()}</h2>
         <div class="flex gap-2 items-center">
+          {#if unassignedFaces.length > 0}
+            <Icon
+              ariaLabel="Asset has unassigned faces"
+              title="Asset has unassigned faces"
+              color="currentColor"
+              path={mdiAccountOff}
+              size="24"
+            />
+          {/if}
           {#if people.some((person) => person.isHidden)}
             <CircleIconButton
               title={$t('show_hidden_people')}

web/src/lib/components/faces-page/assign-face-side-panel.svelte

@@ -1,24 +1,24 @@
 <script lang="ts">
   import { timeBeforeShowLoadingSpinner } from '$lib/constants';
-  import { photoViewer } from '$lib/stores/assets.store';
-  import { getAssetThumbnailUrl, getPeopleThumbnailUrl } from '$lib/utils';
   import { getPersonNameWithHiddenValue } from '$lib/utils/person';
+  import { getPeopleThumbnailUrl } from '$lib/utils';
   import { AssetTypeEnum, type AssetFaceResponseDto, type PersonResponseDto } from '@immich/sdk';
   import { mdiArrowLeftThin, mdiClose, mdiMagnify, mdiPlus } from '@mdi/js';
   import { createEventDispatcher } from 'svelte';
   import { linear } from 'svelte/easing';
   import { fly } from 'svelte/transition';
+  import { photoViewer } from '$lib/stores/assets.store';
   import ImageThumbnail from '../assets/thumbnail/image-thumbnail.svelte';
   import LoadingSpinner from '../shared-components/loading-spinner.svelte';
   import SearchPeople from '$lib/components/faces-page/people-search.svelte';
   import CircleIconButton from '$lib/components/elements/buttons/circle-icon-button.svelte';
+  import { zoomImageToBase64 } from '$lib/utils/people-utils';
   import { t } from 'svelte-i18n';
 
-  export let peopleWithFaces: AssetFaceResponseDto[];
   export let allPeople: PersonResponseDto[];
-  export let editedPerson: PersonResponseDto;
-  export let assetType: AssetTypeEnum;
+  export let editedFace: AssetFaceResponseDto;
   export let assetId: string;
+  export let assetType: AssetTypeEnum;
 
   // loading spinners
   let isShowLoadingNewPerson = false;
@@ -39,71 +39,11 @@
   const handleBackButton = () => {
     dispatch('close');
   };
-  const zoomImageToBase64 = async (face: AssetFaceResponseDto): Promise<string | null> => {
-    let image: HTMLImageElement | null = null;
-    if (assetType === AssetTypeEnum.Image) {
-      image = $photoViewer;
-    } else if (assetType === AssetTypeEnum.Video) {
-      const data = getAssetThumbnailUrl(assetId);
-      const img: HTMLImageElement = new Image();
-      img.src = data;
-
-      await new Promise<void>((resolve) => {
-        img.addEventListener('load', () => resolve());
-        img.addEventListener('error', () => resolve());
-      });
-
-      image = img;
-    }
-    if (image === null) {
-      return null;
-    }
-    const {
-      boundingBoxX1: x1,
-      boundingBoxX2: x2,
-      boundingBoxY1: y1,
-      boundingBoxY2: y2,
-      imageWidth,
-      imageHeight,
-    } = face;
-
-    const coordinates = {
-      x1: (image.naturalWidth / imageWidth) * x1,
-      x2: (image.naturalWidth / imageWidth) * x2,
-      y1: (image.naturalHeight / imageHeight) * y1,
-      y2: (image.naturalHeight / imageHeight) * y2,
-    };
-
-    const faceWidth = coordinates.x2 - coordinates.x1;
-    const faceHeight = coordinates.y2 - coordinates.y1;
-
-    const faceImage = new Image();
-    faceImage.src = image.src;
-
-    await new Promise((resolve) => {
-      faceImage.addEventListener('load', resolve);
-      faceImage.addEventListener('error', () => resolve(null));
-    });
-
-    const canvas = document.createElement('canvas');
-    canvas.width = faceWidth;
-    canvas.height = faceHeight;
-
-    const context = canvas.getContext('2d');
-    if (context) {
-      context.drawImage(faceImage, coordinates.x1, coordinates.y1, faceWidth, faceHeight, 0, 0, faceWidth, faceHeight);
-
-      return canvas.toDataURL();
-    } else {
-      return null;
-    }
-  };
 
   const handleCreatePerson = async () => {
     const timeout = setTimeout(() => (isShowLoadingNewPerson = true), timeBeforeShowLoadingSpinner);
-    const personToUpdate = peopleWithFaces.find((face) => face.person?.id === editedPerson.id);
 
-    const newFeaturePhoto = personToUpdate ? await zoomImageToBase64(personToUpdate) : null;
+    const newFeaturePhoto = await zoomImageToBase64(editedFace, assetId, assetType, $photoViewer);
 
     dispatch('createPerson', newFeaturePhoto);
 
@@ -161,7 +101,7 @@
     <h2 class="mb-8 mt-4 uppercase">{$t('all_people')}</h2>
     <div class="immich-scrollbar mt-4 flex flex-wrap gap-2 overflow-y-auto">
       {#each showPeople as person (person.id)}
-        {#if person.id !== editedPerson.id}
+        {#if !editedFace.person || person.id !== editedFace.person.id}
           <div class="w-fit">
             <button type="button" class="w-[90px]" on:click={() => dispatch('reassign', person)}>
               <div class="relative">

web/src/lib/components/faces-page/person-side-panel.svelte

@@ -7,14 +7,16 @@
   import { handleError } from '$lib/utils/handle-error';
   import { getPersonNameWithHiddenValue } from '$lib/utils/person';
   import {
-    AssetTypeEnum,
     createPerson,
     getAllPeople,
     getFaces,
     reassignFacesById,
+    AssetTypeEnum,
     type AssetFaceResponseDto,
     type PersonResponseDto,
   } from '@immich/sdk';
+  import { mdiAccountOff } from '@mdi/js';
+  import Icon from '$lib/components/elements/icon.svelte';
   import { mdiArrowLeftThin, mdiMinus, mdiRestart } from '@mdi/js';
   import { createEventDispatcher, onMount } from 'svelte';
   import { linear } from 'svelte/easing';
@@ -23,6 +25,8 @@
   import { NotificationType, notificationController } from '../shared-components/notification/notification';
   import AssignFaceSidePanel from './assign-face-side-panel.svelte';
   import CircleIconButton from '$lib/components/elements/buttons/circle-icon-button.svelte';
+  import { zoomImageToBase64 } from '$lib/utils/people-utils';
+  import { photoViewer } from '$lib/stores/assets.store';
   import { t } from 'svelte-i18n';
 
   export let assetId: string;
@@ -36,7 +40,6 @@
   let peopleWithFaces: AssetFaceResponseDto[] = [];
   let selectedPersonToReassign: Record<string, PersonResponseDto> = {};
   let selectedPersonToCreate: Record<string, string> = {};
-  let editedPerson: PersonResponseDto;
   let editedFace: AssetFaceResponseDto;
 
   // loading spinners
@@ -171,11 +174,8 @@
   };
 
   const handleFacePicker = (face: AssetFaceResponseDto) => {
-    if (face.person) {
-      editedFace = face;
-      editedPerson = face.person;
-      showSelectedFaces = true;
-    }
+    editedFace = face;
+    showSelectedFaces = true;
   };
 </script>
 
@@ -209,91 +209,125 @@
         </div>
       {:else}
         {#each peopleWithFaces as face, index}
-          {#if face.person}
-            <div class="relative z-[20001] h-[115px] w-[95px]">
-              <div
-                role="button"
-                tabindex={index}
-                class="absolute left-0 top-0 h-[90px] w-[90px] cursor-default"
-                on:focus={() => ($boundingBoxesArray = [peopleWithFaces[index]])}
-                on:mouseover={() => ($boundingBoxesArray = [peopleWithFaces[index]])}
-                on:mouseleave={() => ($boundingBoxesArray = [])}
-              >
-                <div class="relative">
-                  {#if selectedPersonToCreate[face.id]}
+          {@const personName = face.person ? face.person?.name : 'Unassigned'}
+          <div class="relative z-[20001] h-[115px] w-[95px]">
+            <div
+              role="button"
+              tabindex={index}
+              class="absolute left-0 top-0 h-[90px] w-[90px] cursor-default"
+              on:focus={() => ($boundingBoxesArray = [peopleWithFaces[index]])}
+              on:mouseover={() => ($boundingBoxesArray = [peopleWithFaces[index]])}
+              on:mouseleave={() => ($boundingBoxesArray = [])}
+            >
+              <div class="relative">
+                {#if selectedPersonToCreate[face.id]}
+                  <ImageThumbnail
+                    curve
+                    shadow
+                    url={selectedPersonToCreate[face.id]}
+                    altText={'New person'}
+                    title={'New person'}
+                    widthStyle={thumbnailWidth}
+                    heightStyle={thumbnailWidth}
+                  />
+                {:else if selectedPersonToReassign[face.id]}
+                  <ImageThumbnail
+                    curve
+                    shadow
+                    url={getPeopleThumbnailUrl(selectedPersonToReassign[face.id].id)}
+                    altText={selectedPersonToReassign[face.id].name}
+                    title={getPersonNameWithHiddenValue(
+                      selectedPersonToReassign[face.id].name,
+                      selectedPersonToReassign[face.id]?.isHidden,
+                    )}
+                    widthStyle={thumbnailWidth}
+                    heightStyle={thumbnailWidth}
+                    hidden={selectedPersonToReassign[face.id].isHidden}
+                  />
+                {:else if face.person}
+                  <ImageThumbnail
+                    curve
+                    shadow
+                    url={getPeopleThumbnailUrl(face.person.id)}
+                    altText={face.person.name}
+                    title={getPersonNameWithHiddenValue(face.person.name, face.person.isHidden)}
+                    widthStyle={thumbnailWidth}
+                    heightStyle={thumbnailWidth}
+                    hidden={face.person.isHidden}
+                  />
+                {:else}
+                  {#await zoomImageToBase64(face, assetId, assetType, $photoViewer)}
                     <ImageThumbnail
                       curve
                       shadow
-                      url={selectedPersonToCreate[face.id]}
-                      altText={selectedPersonToCreate[face.id]}
-                      title={$t('new_person')}
-                      widthStyle={thumbnailWidth}
-                      heightStyle={thumbnailWidth}
+                      url="/src/lib/assets/no-thumbnail.png"
+                      altText="Unassigned"
+                      title="Unassigned"
+                      widthStyle="90px"
+                      heightStyle="90px"
+                      thumbhash={null}
+                      hidden={false}
                     />
-                  {:else if selectedPersonToReassign[face.id]}
+                  {:then data}
                     <ImageThumbnail
                       curve
                       shadow
-                      url={getPeopleThumbnailUrl(selectedPersonToReassign[face.id].id)}
-                      altText={selectedPersonToReassign[face.id]?.name || selectedPersonToReassign[face.id].id}
-                      title={getPersonNameWithHiddenValue(
-                        selectedPersonToReassign[face.id].name,
-                        face.person?.isHidden,
-                      )}
-                      widthStyle={thumbnailWidth}
-                      heightStyle={thumbnailWidth}
-                      hidden={selectedPersonToReassign[face.id].isHidden}
+                      url={data === null ? '/src/lib/assets/no-thumbnail.png' : data}
+                      altText="Unassigned"
+                      title="Unassigned"
+                      widthStyle="90px"
+                      heightStyle="90px"
+                      thumbhash={null}
+                      hidden={false}
                     />
-                  {:else}
-                    <ImageThumbnail
-                      curve
-                      shadow
-                      url={getPeopleThumbnailUrl(face.person.id)}
-                      altText={face.person.name || face.person.id}
-                      title={getPersonNameWithHiddenValue(face.person.name, face.person.isHidden)}
-                      widthStyle={thumbnailWidth}
-                      heightStyle={thumbnailWidth}
-                      hidden={face.person.isHidden}
-                    />
-                  {/if}
-                </div>
-
-                {#if !selectedPersonToCreate[face.id]}
-                  <p class="relative mt-1 truncate font-medium" title={face.person?.name}>
-                    {#if selectedPersonToReassign[face.id]?.id}
-                      {selectedPersonToReassign[face.id]?.name}
-                    {:else}
-                      {face.person?.name}
-                    {/if}
-                  </p>
+                  {/await}
                 {/if}
+              </div>
 
-                <div class="absolute -right-[5px] -top-[5px] h-[20px] w-[20px] rounded-full">
-                  {#if selectedPersonToCreate[face.id] || selectedPersonToReassign[face.id]}
-                    <CircleIconButton
-                      color="primary"
-                      icon={mdiRestart}
-                      title={$t('reset')}
-                      size="18"
-                      padding="1"
-                      class="absolute left-1/2 top-1/2 translate-x-[-50%] translate-y-[-50%] transform"
-                      on:click={() => handleReset(face.id)}
-                    />
+              {#if !selectedPersonToCreate[face.id]}
+                <p class="relative mt-1 truncate font-medium" title={personName}>
+                  {#if selectedPersonToReassign[face.id]?.id}
+                    {selectedPersonToReassign[face.id]?.name}
                   {:else}
-                    <CircleIconButton
-                      color="primary"
-                      icon={mdiMinus}
-                      title={$t('select_new_face')}
-                      size="18"
-                      padding="1"
-                      class="absolute left-1/2 top-1/2 translate-x-[-50%] translate-y-[-50%] transform"
-                      on:click={() => handleFacePicker(face)}
-                    />
+                    <span class={personName == 'Unassigned' ? 'dark:text-gray-500' : ''}>{personName}</span>
                   {/if}
-                </div>
+                </p>
+              {/if}
+
+              <div class="absolute -right-[5px] -top-[5px] h-[20px] w-[20px] rounded-full">
+                {#if selectedPersonToCreate[face.id] || selectedPersonToReassign[face.id]}
+                  <CircleIconButton
+                    color="primary"
+                    icon={mdiRestart}
+                    title="Reset"
+                    size="18"
+                    padding="1"
+                    class="absolute left-1/2 top-1/2 translate-x-[-50%] translate-y-[-50%] transform"
+                    on:click={() => handleReset(face.id)}
+                  />
+                {:else}
+                  <CircleIconButton
+                    color="primary"
+                    icon={mdiMinus}
+                    title="Select new face"
+                    size="18"
+                    padding="1"
+                    class="absolute left-1/2 top-1/2 translate-x-[-50%] translate-y-[-50%] transform"
+                    on:click={() => handleFacePicker(face)}
+                  />
+                {/if}
+              </div>
+              <div class="absolute right-[25px] -top-[5px] h-[20px] w-[20px] rounded-full">
+                {#if !selectedPersonToCreate[face.id] && !selectedPersonToReassign[face.id] && !face.person}
+                  <div
+                    class="flex place-content-center place-items-center rounded-full bg-[#d3d3d3] p-1 transition-all absolute left-1/2 top-1/2 translate-x-[-50%] translate-y-[-50%] transform"
+                  >
+                    <Icon color="primary" path={mdiAccountOff} ariaLabel="Just a face" size="18" />
+                  </div>
+                {/if}
               </div>
             </div>
-          {/if}
+          </div>
         {/each}
       {/if}
     </div>
@@ -302,11 +336,10 @@
 
 {#if showSelectedFaces}
   <AssignFaceSidePanel
-    {peopleWithFaces}
     {allPeople}
-    {editedPerson}
-    {assetType}
+    {editedFace}
     {assetId}
+    {assetType}
     on:close={() => (showSelectedFaces = false)}
     on:createPerson={(event) => handleCreatePerson(event.detail)}
     on:reassign={(event) => handleReassignFace(event.detail)}

web/src/lib/utils/people-utils.ts

@@ -1,4 +1,6 @@
 import type { Faces } from '$lib/stores/people.store';
+import { getAssetThumbnailUrl } from '$lib/utils';
+import { AssetTypeEnum, type AssetFaceResponseDto } from '@immich/sdk';
 import type { ZoomImageWheelState } from '@zoom-image/core';
 
 const getContainedSize = (img: HTMLImageElement): { width: number; height: number } => {
@@ -69,3 +71,61 @@ export const getBoundingBox = (
   }
   return boxes;
 };
+
+export const zoomImageToBase64 = async (
+  face: AssetFaceResponseDto,
+  assetId: string,
+  assetType: AssetTypeEnum,
+  photoViewer: HTMLImageElement | null,
+): Promise<string | null> => {
+  let image: HTMLImageElement | null = null;
+  if (assetType === AssetTypeEnum.Image) {
+    image = photoViewer;
+  } else if (assetType === AssetTypeEnum.Video) {
+    const data = getAssetThumbnailUrl(assetId);
+    const img: HTMLImageElement = new Image();
+    img.src = data;
+
+    await new Promise<void>((resolve) => {
+      img.addEventListener('load', () => resolve());
+      img.addEventListener('error', () => resolve());
+    });
+
+    image = img;
+  }
+  if (image === null) {
+    return null;
+  }
+  const { boundingBoxX1: x1, boundingBoxX2: x2, boundingBoxY1: y1, boundingBoxY2: y2, imageWidth, imageHeight } = face;
+
+  const coordinates = {
+    x1: (image.naturalWidth / imageWidth) * x1,
+    x2: (image.naturalWidth / imageWidth) * x2,
+    y1: (image.naturalHeight / imageHeight) * y1,
+    y2: (image.naturalHeight / imageHeight) * y2,
+  };
+
+  const faceWidth = coordinates.x2 - coordinates.x1;
+  const faceHeight = coordinates.y2 - coordinates.y1;
+
+  const faceImage = new Image();
+  faceImage.src = image.src;
+
+  await new Promise((resolve) => {
+    faceImage.addEventListener('load', resolve);
+    faceImage.addEventListener('error', () => resolve(null));
+  });
+
+  const canvas = document.createElement('canvas');
+  canvas.width = faceWidth;
+  canvas.height = faceHeight;
+
+  const context = canvas.getContext('2d');
+  if (context) {
+    context.drawImage(faceImage, coordinates.x1, coordinates.y1, faceWidth, faceHeight, 0, 0, faceWidth, faceHeight);
+
+    return canvas.toDataURL();
+  } else {
+    return null;
+  }
+};