feat: photo-viewer; use <img> instead of blob urls, simplify/refactor, avoid window.events (#9883)

* Photoviewer

* make copyImage/zoomToggle optional

* Add e2e test

* lint

* Accept bo0tzz suggestion

Co-authored-by: bo0tzz <git@bo0tzz.me>

* Bad merge and review comments

* unused import

---------

Co-authored-by: bo0tzz <git@bo0tzz.me>
Co-authored-by: Alex <alex.tran1502@gmail.com>

e2e/src/utils.ts

@@ -323,6 +323,40 @@ export const utils = {
     return body as AssetMediaResponseDto;
   },
 
+  replaceAsset: async (
+    accessToken: string,
+    assetId: string,
+    dto?: Partial<Omit<AssetMediaCreateDto, 'assetData'>> & { assetData?: AssetData },
+  ) => {
+    const _dto = {
+      deviceAssetId: 'test-1',
+      deviceId: 'test',
+      fileCreatedAt: new Date().toISOString(),
+      fileModifiedAt: new Date().toISOString(),
+      ...dto,
+    };
+
+    const assetData = dto?.assetData?.bytes || makeRandomImage();
+    const filename = dto?.assetData?.filename || 'example.png';
+
+    if (dto?.assetData?.bytes) {
+      console.log(`Uploading ${filename}`);
+    }
+
+    const builder = request(app)
+      .put(`/assets/${assetId}/original`)
+      .attach('assetData', assetData, filename)
+      .set('Authorization', `Bearer ${accessToken}`);
+
+    for (const [key, value] of Object.entries(_dto)) {
+      void builder.field(key, String(value));
+    }
+
+    const { body } = await builder;
+
+    return body as AssetMediaResponseDto;
+  },
+
   createImageFile: (path: string) => {
     if (!existsSync(dirname(path))) {
       mkdirSync(dirname(path), { recursive: true });

e2e/src/web/specs/photo-viewer.e2e-spec.ts

@@ -0,0 +1,57 @@
+import { AssetMediaResponseDto, LoginResponseDto } from '@immich/sdk';
+import { Page, expect, test } from '@playwright/test';
+import { utils } from 'src/utils';
+
+function imageLocator(page: Page) {
+  return page.getByAltText('Image taken on').locator('visible=true');
+}
+test.describe('Photo Viewer', () => {
+  let admin: LoginResponseDto;
+  let asset: AssetMediaResponseDto;
+
+  test.beforeAll(async () => {
+    utils.initSdk();
+    await utils.resetDatabase();
+    admin = await utils.adminSetup();
+    asset = await utils.createAsset(admin.accessToken);
+  });
+
+  test.beforeEach(async ({ context, page }) => {
+    // before each test, login as user
+    await utils.setAuthCookies(context, admin.accessToken);
+    await page.goto('/photos');
+    await page.waitForLoadState('networkidle');
+  });
+
+  test('initially shows a loading spinner', async ({ page }) => {
+    await page.route(`/api/assets/${asset.id}/thumbnail**`, async (route) => {
+      // slow down the request for thumbnail, so spiner has chance to show up
+      await new Promise((f) => setTimeout(f, 2000));
+      await route.continue();
+    });
+    await page.goto(`/photos/${asset.id}`);
+    await page.waitForLoadState('load');
+    // this is the spinner
+    await page.waitForSelector('svg[role=status]');
+    await expect(page.getByRole('status')).toBeVisible();
+  });
+
+  test('loads high resolution photo when zoomed', async ({ page }) => {
+    await page.goto(`/photos/${asset.id}`);
+    await expect.poll(async () => await imageLocator(page).getAttribute('src')).toContain('thumbnail');
+    const box = await imageLocator(page).boundingBox();
+    expect(box).toBeTruthy;
+    const { x, y, width, height } = box!;
+    await page.mouse.move(x + width / 2, y + height / 2);
+    await page.mouse.wheel(0, -1);
+    await expect.poll(async () => await imageLocator(page).getAttribute('src')).toContain('original');
+  });
+
+  test('reloads photo when checksum changes', async ({ page }) => {
+    await page.goto(`/photos/${asset.id}`);
+    await expect.poll(async () => await imageLocator(page).getAttribute('src')).toContain('thumbnail');
+    const initialSrc = await imageLocator(page).getAttribute('src');
+    await utils.replaceAsset(admin.accessToken, asset.id);
+    await expect.poll(async () => await imageLocator(page).getAttribute('src')).not.toBe(initialSrc);
+  });
+});

web/src/lib/actions/zoom-image.ts

@@ -0,0 +1,31 @@
+import { photoZoomState, zoomed } from '$lib/stores/zoom-image.store';
+import { useZoomImageWheel } from '@zoom-image/svelte';
+import { get } from 'svelte/store';
+
+export { zoomed } from '$lib/stores/zoom-image.store';
+
+export const zoomImageAction = (node: HTMLElement) => {
+  const { createZoomImage, zoomImageState, setZoomImageState } = useZoomImageWheel();
+
+  createZoomImage(node, {
+    maxZoom: 10,
+    wheelZoomRatio: 0.2,
+  });
+
+  const state = get(photoZoomState);
+  if (state) {
+    setZoomImageState(state);
+  }
+
+  const unsubscribes = [
+    zoomed.subscribe((state) => setZoomImageState({ currentZoom: state ? 2 : 1 })),
+    zoomImageState.subscribe((state) => photoZoomState.set(state)),
+  ];
+  return {
+    destroy() {
+      for (const unsubscribe of unsubscribes) {
+        unsubscribe();
+      }
+    },
+  };
+};

web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte

@@ -51,6 +51,8 @@
   export let showShareButton: boolean;
   export let showSlideshow = false;
   export let hasStackChildren = false;
+  export let onZoomImage: () => void;
+  export let onCopyImage: () => void;
 
   $: isOwner = $user && asset.ownerId === $user?.id;
 
@@ -144,22 +146,11 @@
         hideMobile={true}
         icon={$photoZoomState && $photoZoomState.currentZoom > 1 ? mdiMagnifyMinusOutline : mdiMagnifyPlusOutline}
         title={$t('zoom_image')}
-        on:click={() => {
+        on:click={onZoomImage}
-          const zoomImage = new CustomEvent('zoomImage');
-          window.dispatchEvent(zoomImage);
-        }}
       />
     {/if}
     {#if showCopyButton}
-      <CircleIconButton
+      <CircleIconButton color="opaque" icon={mdiContentCopy} title={$t('copy_image')} on:click={onCopyImage} />
-        color="opaque"
-        icon={mdiContentCopy}
-        title={$t('copy_image')}
-        on:click={() => {
-          const copyEvent = new CustomEvent('copyImage');
-          window.dispatchEvent(copyEvent);
-        }}
-      />
     {/if}
 
     {#if !isOwner && showDownloadButton}

web/src/lib/components/asset-viewer/asset-viewer.svelte

@@ -59,6 +59,7 @@
   import VideoViewer from './video-wrapper-viewer.svelte';
   import { navigate } from '$lib/utils/navigation';
   import { websocketEvents } from '$lib/stores/websocket';
+  import { canCopyImagesToClipboard } from 'copy-image-clipboard';
   import { t } from 'svelte-i18n';
 
   export let assetStore: AssetStore | null = null;
@@ -98,7 +99,6 @@
   let shouldShowDownloadButton = sharedLink ? sharedLink.allowDownload : !asset.isOffline;
   let enableDetailPanel = asset.hasMetadata;
   let shouldShowShareModal = !asset.isTrashed;
-  let canCopyImagesToClipboard: boolean;
   let slideshowStateUnsubscribe: () => void;
   let shuffleSlideshowUnsubscribe: () => void;
   let previewStackedAsset: AssetResponseDto | undefined;
@@ -107,6 +107,8 @@
   let numberOfComments: number;
   let fullscreenElement: Element;
   let unsubscribe: () => void;
+  let zoomToggle = () => void 0;
+  let copyImage: () => Promise<void>;
   $: isFullScreen = fullscreenElement !== null;
 
   $: {
@@ -227,11 +229,6 @@
       await handleGetAllAlbums();
     }
 
-    // Import hack :( see https://github.com/vadimkorr/svelte-carousel/issues/27#issuecomment-851022295
-    // TODO: Move to regular import once the package correctly supports ESM.
-    const module = await import('copy-image-clipboard');
-    canCopyImagesToClipboard = module.canCopyImagesToClipboard();
-
     if (asset.stackCount && asset.stack) {
       $stackAssetsStore = asset.stack;
       $stackAssetsStore = [...$stackAssetsStore, asset].sort(
@@ -568,7 +565,7 @@
           {asset}
           {album}
           isMotionPhotoPlaying={shouldPlayMotionPhoto}
-          showCopyButton={canCopyImagesToClipboard && asset.type === AssetTypeEnum.Image}
+          showCopyButton={canCopyImagesToClipboard() && asset.type === AssetTypeEnum.Image}
           showZoomButton={asset.type === AssetTypeEnum.Image}
           showMotionPlayButton={!!asset.livePhotoVideoId}
           showDownloadButton={shouldShowDownloadButton}
@@ -576,6 +573,8 @@
           showSlideshow={!!assetStore}
           hasStackChildren={$stackAssetsStore.length > 0}
           showShareButton={shouldShowShareModal}
+          onZoomImage={zoomToggle}
+          onCopyImage={copyImage}
           on:back={closeViewer}
           on:showDetail={showDetailInfoHandler}
           on:download={() => downloadFile(asset)}
@@ -623,6 +622,8 @@
         {#key previewStackedAsset.id}
           {#if previewStackedAsset.type === AssetTypeEnum.Image}
             <PhotoViewer
+              bind:zoomToggle
+              bind:copyImage
               asset={previewStackedAsset}
               {preloadAssets}
               on:close={closeViewer}
@@ -665,7 +666,7 @@
                   .endsWith('.insp'))}
               <PanoramaViewer {asset} />
             {:else}
-              <PhotoViewer {asset} {preloadAssets} on:close={closeViewer} />
+              <PhotoViewer bind:zoomToggle bind:copyImage {asset} {preloadAssets} on:close={closeViewer} />
             {/if}
           {:else}
             <VideoViewer

web/src/lib/components/asset-viewer/photo-viewer.spec.ts

@@ -1,83 +0,0 @@
-import * as utils from '$lib/utils';
-import type { AssetResponseDto } from '@immich/sdk';
-import { assetFactory } from '@test-data/factories/asset-factory';
-import '@testing-library/jest-dom';
-import { fireEvent, render, screen, waitFor } from '@testing-library/svelte';
-import type { Mock, MockInstance } from 'vitest';
-import PhotoViewer from './photo-viewer.svelte';
-
-vi.mock('$lib/utils', async (originalImport) => {
-  const meta = await originalImport<typeof import('$lib/utils')>();
-  return {
-    ...meta,
-    downloadRequest: vi.fn(),
-  };
-});
-
-describe('PhotoViewer component', () => {
-  let downloadRequestMock: MockInstance;
-  let createObjectURLMock: Mock<[obj: Blob], string>;
-  let asset: AssetResponseDto;
-
-  beforeAll(() => {
-    downloadRequestMock = vi.spyOn(utils, 'downloadRequest').mockResolvedValue({
-      data: new Blob(),
-      status: 200,
-    });
-    createObjectURLMock = vi.fn();
-    window.URL.createObjectURL = createObjectURLMock;
-    asset = assetFactory.build({ originalPath: 'image.png' });
-  });
-
-  afterAll(() => {
-    vi.resetAllMocks();
-  });
-
-  it('initially shows a loading spinner', () => {
-    render(PhotoViewer, { asset });
-    expect(screen.getByRole('status')).toBeInTheDocument();
-  });
-
-  it('loads and shows a photo', async () => {
-    createObjectURLMock.mockReturnValueOnce('url-one');
-    render(PhotoViewer, { asset });
-
-    expect(downloadRequestMock).toBeCalledWith(
-      expect.objectContaining({
-        url: `/api/assets/${asset.id}/thumbnail?size=preview&c=${asset.checksum}`,
-      }),
-    );
-    await waitFor(() => expect(screen.getByRole('img')).toBeInTheDocument());
-    expect(screen.getByRole('img')).toHaveAttribute('src', 'url-one');
-  });
-
-  it('loads high resolution photo when zoomed', async () => {
-    createObjectURLMock.mockReturnValueOnce('url-one');
-    render(PhotoViewer, { asset });
-    createObjectURLMock.mockReturnValueOnce('url-two');
-
-    await waitFor(() => expect(screen.getByRole('img')).toBeInTheDocument());
-    await fireEvent(window, new CustomEvent('zoomImage'));
-    await waitFor(() => expect(screen.getByRole('img')).toHaveAttribute('src', 'url-two'));
-    expect(downloadRequestMock).toBeCalledWith(
-      expect.objectContaining({
-        url: `/api/assets/${asset.id}/original?c=${asset.checksum}`,
-      }),
-    );
-  });
-
-  it('reloads photo when checksum changes', async () => {
-    const { component } = render(PhotoViewer, { asset });
-    createObjectURLMock.mockReturnValueOnce('url-two');
-
-    await waitFor(() => expect(screen.getByRole('img')).toBeInTheDocument());
-    component.$set({ asset: { ...asset, checksum: 'new-checksum' } });
-
-    await waitFor(() => expect(screen.getByRole('img')).toHaveAttribute('src', 'url-two'));
-    expect(downloadRequestMock).toBeCalledWith(
-      expect.objectContaining({
-        url: `/api/assets/${asset.id}/thumbnail?size=preview&c=new-checksum`,
-      }),
-    );
-  });
-});

web/src/lib/components/asset-viewer/photo-viewer.svelte

@@ -1,103 +1,83 @@
 <script lang="ts">
+  import { shortcuts } from '$lib/actions/shortcut';
   import { photoViewer } from '$lib/stores/assets.store';
   import { boundingBoxesArray } from '$lib/stores/people.store';
   import { alwaysLoadOriginalFile } from '$lib/stores/preferences.store';
+  import { SlideshowLook, SlideshowState, slideshowLookCssMapping, slideshowStore } from '$lib/stores/slideshow.store';
   import { photoZoomState } from '$lib/stores/zoom-image.store';
-  import { downloadRequest, getAssetOriginalUrl, getAssetThumbnailUrl, handlePromiseError } from '$lib/utils';
+  import { getAssetOriginalUrl, getAssetThumbnailUrl, handlePromiseError } from '$lib/utils';
   import { isWebCompatibleImage } from '$lib/utils/asset-utils';
   import { getBoundingBox } from '$lib/utils/people-utils';
-  import { shortcuts } from '$lib/actions/shortcut';
+  import { getAltText } from '$lib/utils/thumbnail-util';
-  import { type AssetResponseDto, AssetTypeEnum, AssetMediaSize } from '@immich/sdk';
+  import { AssetTypeEnum, type AssetResponseDto, AssetMediaSize } from '@immich/sdk';
-  import { useZoomImageWheel } from '@zoom-image/svelte';
+  import { zoomImageAction, zoomed } from '$lib/actions/zoom-image';
-  import { onDestroy, onMount } from 'svelte';
+  import { canCopyImagesToClipboard, copyImageToClipboard } from 'copy-image-clipboard';
+  import { onDestroy } from 'svelte';
+
   import { fade } from 'svelte/transition';
   import LoadingSpinner from '../shared-components/loading-spinner.svelte';
   import { NotificationType, notificationController } from '../shared-components/notification/notification';
-  import { getAltText } from '$lib/utils/thumbnail-util';
-  import { SlideshowLook, slideshowLookCssMapping, SlideshowState, slideshowStore } from '$lib/stores/slideshow.store';
   import { t } from 'svelte-i18n';
 
-  const { slideshowState, slideshowLook } = slideshowStore;
-
   export let asset: AssetResponseDto;
-  export let preloadAssets: AssetResponseDto[] | null = null;
+  export let preloadAssets: AssetResponseDto[] | undefined = undefined;
   export let element: HTMLDivElement | undefined = undefined;
   export let haveFadeTransition = true;
 
-  let imgElement: HTMLDivElement;
+  export let copyImage: (() => Promise<void>) | null = null;
-  let assetData: string;
+  export let zoomToggle: (() => void) | null = null;
-  let abortController: AbortController;
+
-  let hasZoomed = false;
+  const { slideshowState, slideshowLook } = slideshowStore;
-  let copyImageToClipboard: (source: string) => Promise<Blob>;
+
-  let canCopyImagesToClipboard: () => boolean;
+  let assetFileUrl: string = '';
   let imageLoaded: boolean = false;
+  let imageError: boolean = false;
+  let forceUseOriginal: boolean = false;
 
-  const loadOriginalByDefault = $alwaysLoadOriginalFile && isWebCompatibleImage(asset);
+  $: isWebCompatible = isWebCompatibleImage(asset);
+  $: useOriginalByDefault = isWebCompatible && $alwaysLoadOriginalFile;
+  $: useOriginalImage = useOriginalByDefault || forceUseOriginal;
+  // when true, will force loading of the original image
+  $: forceUseOriginal = forceUseOriginal || ($photoZoomState.currentZoom > 1 && isWebCompatible);
 
-  $: if (imgElement) {
+  $: preload(useOriginalImage, preloadAssets);
-    createZoomImageWheel(imgElement, {
+  $: imageLoaderUrl = getAssetUrl(asset.id, useOriginalImage, asset.checksum);
-      maxZoom: 10,
-      wheelZoomRatio: 0.2,
-    });
-  }
 
-  onMount(async () => {
+  photoZoomState.set({
-    // Import hack :( see https://github.com/vadimkorr/svelte-carousel/issues/27#issuecomment-851022295
+    currentRotation: 0,
-    // TODO: Move to regular import once the package correctly supports ESM.
+    currentZoom: 1,
-    const module = await import('copy-image-clipboard');
+    enable: true,
-    copyImageToClipboard = module.copyImageToClipboard;
+    currentPositionX: 0,
-    canCopyImagesToClipboard = module.canCopyImagesToClipboard;
+    currentPositionY: 0,
   });
-
+  $zoomed = false;
-  $: void loadAssetData({ loadOriginal: loadOriginalByDefault, checksum: asset.checksum });
 
   onDestroy(() => {
     $boundingBoxesArray = [];
-    abortController?.abort();
   });
 
-  const loadAssetData = async ({ loadOriginal, checksum }: { loadOriginal: boolean; checksum: string }) => {
+  const preload = (useOriginal: boolean, preloadAssets?: AssetResponseDto[]) => {
-    try {
+    for (const preloadAsset of preloadAssets || []) {
-      abortController?.abort();
+      if (preloadAsset.type === AssetTypeEnum.Image) {
-      abortController = new AbortController();
+        let img = new Image();
-
+        img.src = getAssetUrl(preloadAsset.id, useOriginal, preloadAsset.checksum);
-      // TODO: Use sdk once it supports signals
-      const res = await downloadRequest({
-        url: loadOriginal
-          ? getAssetOriginalUrl({ id: asset.id, checksum })
-          : getAssetThumbnailUrl({ id: asset.id, size: AssetMediaSize.Preview, checksum }),
-        signal: abortController.signal,
-      });
-
-      assetData = window.URL.createObjectURL(res.data);
-      imageLoaded = true;
-
-      if (!preloadAssets) {
-        return;
       }
-
-      for (const preloadAsset of preloadAssets) {
-        if (preloadAsset.type === AssetTypeEnum.Image) {
-          await downloadRequest({
-            url: loadOriginal
-              ? getAssetOriginalUrl(preloadAsset.id)
-              : getAssetThumbnailUrl({ id: preloadAsset.id, size: AssetMediaSize.Preview }),
-            signal: abortController.signal,
-          });
-        }
-      }
-    } catch {
-      imageLoaded = false;
     }
   };
 
-  const doCopy = async () => {
+  const getAssetUrl = (id: string, useOriginal: boolean, checksum: string) => {
+    return useOriginal
+      ? getAssetOriginalUrl({ id, checksum })
+      : getAssetThumbnailUrl({ id, size: AssetMediaSize.Preview, checksum });
+  };
+
+  copyImage = async () => {
     if (!canCopyImagesToClipboard()) {
       return;
     }
 
     try {
-      await copyImageToClipboard(assetData);
+      await copyImageToClipboard(assetFileUrl);
       notificationController.show({
         type: NotificationType.Info,
         message: $t('copied_image_to_clipboard'),
@@ -112,60 +92,46 @@
     }
   };
 
-  const doZoomImage = () => {
+  zoomToggle = () => {
-    setZoomImageWheelState({
+    $zoomed = $zoomed ? false : true;
-      currentZoom: $zoomImageWheelState.currentZoom === 1 ? 2 : 1,
-    });
   };
 
-  const {
-    createZoomImage: createZoomImageWheel,
-    zoomImageState: zoomImageWheelState,
-    setZoomImageState: setZoomImageWheelState,
-  } = useZoomImageWheel();
-
-  zoomImageWheelState.subscribe((state) => {
-    photoZoomState.set(state);
-
-    if (state.currentZoom > 1 && isWebCompatibleImage(asset) && !hasZoomed && !$alwaysLoadOriginalFile) {
-      hasZoomed = true;
-
-      handlePromiseError(loadAssetData({ loadOriginal: true, checksum: asset.checksum }));
-    }
-  });
-
   const onCopyShortcut = (event: KeyboardEvent) => {
     if (window.getSelection()?.type === $t('range')) {
       return;
     }
     event.preventDefault();
-    handlePromiseError(doCopy());
+    handlePromiseError(copyImage());
   };
 </script>
 
 <svelte:window
-  on:copyImage={doCopy}
+  on:wheel|preventDefault|nonpassive
-  on:zoomImage={doZoomImage}
   use:shortcuts={[
     { shortcut: { key: 'c', ctrl: true }, onShortcut: onCopyShortcut, preventDefault: false },
     { shortcut: { key: 'c', meta: true }, onShortcut: onCopyShortcut, preventDefault: false },
   ]}
 />
-
+{#if imageError}
-<div
+  <div class="h-full flex items-center justify-center">Error loading image</div>
-  bind:this={element}
+{/if}
-  transition:fade={{ duration: haveFadeTransition ? 150 : 0 }}
+<div bind:this={element} class="relative h-full select-none">
-  class="relative h-full select-none"
+  <img
->
+    style="display:none"
+    src={imageLoaderUrl}
+    alt={getAltText(asset)}
+    on:load={() => ((imageLoaded = true), (assetFileUrl = imageLoaderUrl))}
+    on:error={() => (imageError = imageLoaded = true)}
+  />
   {#if !imageLoaded}
     <div class="flex h-full items-center justify-center">
       <LoadingSpinner />
     </div>
-  {:else}
+  {:else if !imageError}
-    <div bind:this={imgElement} class="h-full w-full" transition:fade={{ duration: haveFadeTransition ? 150 : 0 }}>
+    <div use:zoomImageAction class="h-full w-full" transition:fade={{ duration: haveFadeTransition ? 150 : 0 }}>
       {#if $slideshowState !== SlideshowState.None && $slideshowLook === SlideshowLook.BlurredBackground}
         <img
-          src={assetData}
+          src={assetFileUrl}
           alt={getAltText(asset)}
           class="absolute top-0 left-0 -z-10 object-cover h-full w-full blur-lg"
           draggable="false"
@@ -173,7 +139,7 @@
       {/if}
       <img
         bind:this={$photoViewer}
-        src={assetData}
+        src={assetFileUrl}
         alt={getAltText(asset)}
         class="h-full w-full {$slideshowState === SlideshowState.None
           ? 'object-contain'

web/src/lib/stores/zoom-image.store.ts

@@ -2,3 +2,4 @@ import type { ZoomImageWheelState } from '@zoom-image/core';
 import { writable } from 'svelte/store';
 
 export const photoZoomState = writable<ZoomImageWheelState>();
+export const zoomed = writable<boolean>();