From 5764bf16f3970a796e63ba6c0e141620deb70ebf Mon Sep 17 00:00:00 2001
From: Sergey Kondrikov <sergey.kondrikov@gmail.com>
Date: Thu, 8 Jun 2023 18:22:45 +0300
Subject: [PATCH] feat(web, server): Implement justified layout for AssetGrid
 (#2666)

* Implement justified layout for timeline

* Add withoutThumbs field to GetTimelineLayotDto

* Back to rough estimation of initial buckets height

* Remove getTimelineLayout endpoint

* Estimate rough viewport height better

* Fix shift/jump issues while scrolling up

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
---
 .../immich/api-v1/asset/asset-repository.ts   |  1 +
 web/package-lock.json                         | 11 +++
 web/package.json                              |  1 +
 .../asset-viewer/intersection-observer.svelte | 13 +++-
 .../photos-page/asset-date-group.svelte       | 75 +++++++++++++++++--
 .../components/photos-page/asset-grid.svelte  | 18 +++--
 web/src/lib/models/asset-grid-state.ts        |  8 ++
 web/src/lib/stores/asset-interaction.store.ts |  4 +-
 web/src/lib/stores/assets.store.ts            | 57 ++++++++++----
 web/src/lib/utils/asset-utils.ts              | 15 ++++
 web/src/lib/utils/viewport-utils.ts           | 14 ----
 11 files changed, 172 insertions(+), 45 deletions(-)
 delete mode 100644 web/src/lib/utils/viewport-utils.ts

diff --git a/server/src/immich/api-v1/asset/asset-repository.ts b/server/src/immich/api-v1/asset/asset-repository.ts
index 6f2bf49527..def006c79e 100644
--- a/server/src/immich/api-v1/asset/asset-repository.ts
+++ b/server/src/immich/api-v1/asset/asset-repository.ts
@@ -104,6 +104,7 @@ export class AssetRepository implements IAssetRepository {
     // Get asset entity from a list of time buckets
     let builder = this.assetRepository
       .createQueryBuilder('asset')
+      .leftJoinAndSelect('asset.exifInfo', 'exifInfo')
       .where('asset.ownerId = :userId', { userId: userId })
       .andWhere(`date_trunc('month', "fileCreatedAt") IN (:...buckets)`, {
         buckets: [...dto.timeBucket],
diff --git a/web/package-lock.json b/web/package-lock.json
index 9c73fc4bdb..585e049a43 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -12,6 +12,7 @@
 				"axios": "^0.27.2",
 				"copy-image-clipboard": "^2.1.2",
 				"handlebars": "^4.7.7",
+				"justified-layout": "^4.1.0",
 				"leaflet": "^1.9.3",
 				"leaflet.markercluster": "^1.5.3",
 				"lodash-es": "^4.17.21",
@@ -9076,6 +9077,11 @@
 				"node": ">=6"
 			}
 		},
+		"node_modules/justified-layout": {
+			"version": "4.1.0",
+			"resolved": "https://registry.npmjs.org/justified-layout/-/justified-layout-4.1.0.tgz",
+			"integrity": "sha512-M5FimNMXgiOYerVRGsXZ2YK9YNCaTtwtYp7Hb2308U1Q9TXXHx5G0p08mcVR5O53qf8bWY4NJcPBxE6zuayXSg=="
+		},
 		"node_modules/kind-of": {
 			"version": "6.0.3",
 			"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
@@ -18186,6 +18192,11 @@
 			"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
 			"dev": true
 		},
+		"justified-layout": {
+			"version": "4.1.0",
+			"resolved": "https://registry.npmjs.org/justified-layout/-/justified-layout-4.1.0.tgz",
+			"integrity": "sha512-M5FimNMXgiOYerVRGsXZ2YK9YNCaTtwtYp7Hb2308U1Q9TXXHx5G0p08mcVR5O53qf8bWY4NJcPBxE6zuayXSg=="
+		},
 		"kind-of": {
 			"version": "6.0.3",
 			"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
diff --git a/web/package.json b/web/package.json
index 09df801a19..2b8bd617d0 100644
--- a/web/package.json
+++ b/web/package.json
@@ -62,6 +62,7 @@
 		"axios": "^0.27.2",
 		"copy-image-clipboard": "^2.1.2",
 		"handlebars": "^4.7.7",
+		"justified-layout": "^4.1.0",
 		"leaflet": "^1.9.3",
 		"leaflet.markercluster": "^1.5.3",
 		"lodash-es": "^4.17.21",
diff --git a/web/src/lib/components/asset-viewer/intersection-observer.svelte b/web/src/lib/components/asset-viewer/intersection-observer.svelte
index f08a244db1..b373c2d346 100644
--- a/web/src/lib/components/asset-viewer/intersection-observer.svelte
+++ b/web/src/lib/components/asset-viewer/intersection-observer.svelte
@@ -1,4 +1,5 @@
 <script lang="ts">
+	import { BucketPosition } from '$lib/models/asset-grid-state';
 	import { onMount } from 'svelte';
 	import { createEventDispatcher } from 'svelte';
 
@@ -28,7 +29,17 @@
 					}
 
 					if (intersecting) {
-						dispatch('intersected', container);
+						let position: BucketPosition = BucketPosition.Visible;
+						if (entries[0].boundingClientRect.top + 50 > entries[0].intersectionRect.bottom) {
+							position = BucketPosition.Below;
+						} else if (entries[0].boundingClientRect.bottom < 0) {
+							position = BucketPosition.Above;
+						}
+
+						dispatch('intersected', {
+							container,
+							position
+						});
 					}
 				},
 				{
diff --git a/web/src/lib/components/photos-page/asset-date-group.svelte b/web/src/lib/components/photos-page/asset-date-group.svelte
index a9e704ab15..e1caa31aa1 100644
--- a/web/src/lib/components/photos-page/asset-date-group.svelte
+++ b/web/src/lib/components/photos-page/asset-date-group.svelte
@@ -9,17 +9,20 @@
 	import { assetStore } from '$lib/stores/assets.store';
 	import { locale } from '$lib/stores/preferences.store';
 	import type { AssetResponseDto } from '@api';
+	import justifiedLayout from 'justified-layout';
 	import lodash from 'lodash-es';
 	import CheckCircle from 'svelte-material-icons/CheckCircle.svelte';
 	import CircleOutline from 'svelte-material-icons/CircleOutline.svelte';
-	import { flip } from 'svelte/animate';
 	import { fly } from 'svelte/transition';
+	import { getAssetRatio } from '$lib/utils/asset-utils';
 	import Thumbnail from '../assets/thumbnail/thumbnail.svelte';
+	import { createEventDispatcher } from 'svelte';
 
 	export let assets: AssetResponseDto[];
 	export let bucketDate: string;
 	export let bucketHeight: number;
 	export let isAlbumSelectionMode = false;
+	export let viewportWidth: number;
 
 	const groupDateFormat: Intl.DateTimeFormatOptions = {
 		weekday: 'short',
@@ -28,21 +31,66 @@
 		year: 'numeric'
 	};
 
+	const dispatch = createEventDispatcher();
+
 	let isMouseOverGroup = false;
 	let actualBucketHeight: number;
 	let hoveredDateGroup = '';
+
+	interface LayoutBox {
+		top: number;
+		left: number;
+		width: number;
+	}
+
 	$: assetsGroupByDate = lodash
 		.chain(assets)
 		.groupBy((a) => new Date(a.fileCreatedAt).toLocaleDateString($locale, groupDateFormat))
 		.sortBy((group) => assets.indexOf(group[0]))
 		.value();
 
+	$: geometry = (() => {
+		const geometry = [];
+		for (let group of assetsGroupByDate) {
+			geometry.push(
+				justifiedLayout(group.map(getAssetRatio), {
+					boxSpacing: 2,
+					containerWidth: Math.floor(viewportWidth),
+					containerPadding: 0,
+					targetRowHeightTolerance: 0.15,
+					targetRowHeight: 235
+				})
+			);
+		}
+		return geometry;
+	})();
+
 	$: {
 		if (actualBucketHeight && actualBucketHeight != 0 && actualBucketHeight != bucketHeight) {
-			assetStore.updateBucketHeight(bucketDate, actualBucketHeight);
+			const heightDelta = assetStore.updateBucketHeight(bucketDate, actualBucketHeight);
+			if (heightDelta !== 0) {
+				scrollTimeline(heightDelta);
+			}
 		}
 	}
 
+	function scrollTimeline(heightDelta: number) {
+		dispatch('shift', {
+			heightDelta
+		});
+	}
+
+	const calculateWidth = (boxes: LayoutBox[]): number => {
+		let width = 0;
+		for (const box of boxes) {
+			if (box.top < 100) {
+				width = box.left + box.width;
+			}
+		}
+
+		return width;
+	};
+
 	const assetClickHandler = (
 		asset: AssetResponseDto,
 		assetsInDateGroup: AssetResponseDto[],
@@ -112,8 +160,9 @@
 
 <section
 	id="asset-group-by-date"
-	class="flex flex-wrap gap-12 mt-5"
+	class="flex flex-wrap gap-x-12"
 	bind:clientHeight={actualBucketHeight}
+	bind:clientWidth={viewportWidth}
 >
 	{#each assetsGroupByDate as assetsInDateGroup, groupIndex (assetsInDateGroup[0].id)}
 		{@const dateGroupTitle = new Date(assetsInDateGroup[0].fileCreatedAt).toLocaleDateString(
@@ -123,8 +172,7 @@
 		<!-- Asset Group By Date -->
 
 		<div
-			animate:flip={{ duration: 300 }}
-			class="flex flex-col"
+			class="flex flex-col mt-5"
 			on:mouseenter={() => {
 				isMouseOverGroup = true;
 				assetMouseEventHandler(dateGroupTitle);
@@ -156,9 +204,18 @@
 			</p>
 
 			<!-- Image grid -->
-			<div class="flex flex-wrap gap-[2px]">
-				{#each assetsInDateGroup as asset (asset.id)}
-					<div animate:flip={{ duration: 300 }}>
+			<div
+				class="relative"
+				style="height: {geometry[groupIndex].containerHeight}px;width: {calculateWidth(
+					geometry[groupIndex].boxes
+				)}px"
+			>
+				{#each assetsInDateGroup as asset, index (asset.id)}
+					{@const box = geometry[groupIndex].boxes[index]}
+					<div
+						class="absolute"
+						style="width: {box.width}px; height: {box.height}px; top: {box.top}px; left: {box.left}px"
+					>
 						<Thumbnail
 							{asset}
 							{groupIndex}
@@ -168,6 +225,8 @@
 							selected={$selectedAssets.has(asset) ||
 								$assetsInAlbumStoreState.findIndex((a) => a.id == asset.id) != -1}
 							disabled={$assetsInAlbumStoreState.findIndex((a) => a.id == asset.id) != -1}
+							thumbnailWidth={box.width}
+							thumbnailHeight={box.height}
 						/>
 					</div>
 				{/each}
diff --git a/web/src/lib/components/photos-page/asset-grid.svelte b/web/src/lib/components/photos-page/asset-grid.svelte
index 65b0d09d56..1f549a6c7f 100644
--- a/web/src/lib/components/photos-page/asset-grid.svelte
+++ b/web/src/lib/components/photos-page/asset-grid.svelte
@@ -16,6 +16,7 @@
 		OnScrollbarDragDetail
 	} from '../shared-components/scrollbar/scrollbar.svelte';
 	import AssetDateGroup from './asset-date-group.svelte';
+	import { BucketPosition } from '$lib/models/asset-grid-state';
 
 	export let user: UserResponseDto | undefined = undefined;
 	export let isAlbumSelectionMode = false;
@@ -33,6 +34,7 @@
 				withoutThumbs: true
 			}
 		});
+
 		bucketInfo = assetCountByTimebucket;
 
 		assetStore.setInitialState(viewportHeight, viewportWidth, assetCountByTimebucket, user?.id);
@@ -51,7 +53,7 @@
 		});
 
 		bucketsToFetchInitially.forEach((bucketDate) => {
-			assetStore.getAssetsByBucket(bucketDate);
+			assetStore.getAssetsByBucket(bucketDate, BucketPosition.Visible);
 		});
 	});
 
@@ -60,15 +62,18 @@
 	});
 
 	function intersectedHandler(event: CustomEvent) {
-		const el = event.detail as HTMLElement;
+		const el = event.detail.container as HTMLElement;
 		const target = el.firstChild as HTMLElement;
-
 		if (target) {
 			const bucketDate = target.id.split('_')[1];
-			assetStore.getAssetsByBucket(bucketDate);
+			assetStore.getAssetsByBucket(bucketDate, event.detail.position);
 		}
 	}
 
+	function handleScrollTimeline(event: CustomEvent) {
+		assetGridElement.scrollBy(0, event.detail.heightDelta);
+	}
+
 	const navigateToPreviousAsset = () => {
 		assetInteractionStore.navigateAsset('previous');
 	};
@@ -115,9 +120,10 @@
 	/>
 {/if}
 
+<!-- Right margin MUST be equal to the width of immich-scrubbable-scrollbar -->
 <section
 	id="asset-grid"
-	class="overflow-y-auto pl-4 scrollbar-hidden"
+	class="overflow-y-auto ml-4 mb-4 mr-[60px] scrollbar-hidden"
 	bind:clientHeight={viewportHeight}
 	bind:clientWidth={viewportWidth}
 	bind:this={assetGridElement}
@@ -143,9 +149,11 @@
 						{#if intersecting}
 							<AssetDateGroup
 								{isAlbumSelectionMode}
+								on:shift={handleScrollTimeline}
 								assets={bucket.assets}
 								bucketDate={bucket.bucketDate}
 								bucketHeight={bucket.bucketHeight}
+								{viewportWidth}
 							/>
 						{/if}
 					</div>
diff --git a/web/src/lib/models/asset-grid-state.ts b/web/src/lib/models/asset-grid-state.ts
index 464fb55194..1f3eb26ab1 100644
--- a/web/src/lib/models/asset-grid-state.ts
+++ b/web/src/lib/models/asset-grid-state.ts
@@ -1,5 +1,12 @@
 import type { AssetResponseDto } from '@api';
 
+export enum BucketPosition {
+	Above = 'above',
+	Below = 'below',
+	Visible = 'visible',
+	Unknown = 'unknown'
+}
+
 export class AssetBucket {
 	/**
 	 * The DOM height of the bucket in pixel
@@ -9,6 +16,7 @@ export class AssetBucket {
 	bucketDate!: string;
 	assets!: AssetResponseDto[];
 	cancelToken!: AbortController;
+	position!: BucketPosition;
 }
 
 export class AssetGridState {
diff --git a/web/src/lib/stores/asset-interaction.store.ts b/web/src/lib/stores/asset-interaction.store.ts
index 3b05258fd1..ec7dbb169f 100644
--- a/web/src/lib/stores/asset-interaction.store.ts
+++ b/web/src/lib/stores/asset-interaction.store.ts
@@ -1,4 +1,4 @@
-import { AssetGridState } from '$lib/models/asset-grid-state';
+import { AssetGridState, BucketPosition } from '$lib/models/asset-grid-state';
 import { api, AssetResponseDto } from '@api';
 import { derived, writable } from 'svelte/store';
 import { assetGridState, assetStore } from './assets.store';
@@ -92,7 +92,7 @@ function createAssetInteractionStore() {
 			}
 
 			if (nextBucket !== '') {
-				await assetStore.getAssetsByBucket(nextBucket);
+				await assetStore.getAssetsByBucket(nextBucket, BucketPosition.Below);
 				navigateAsset(direction);
 			}
 			return;
diff --git a/web/src/lib/stores/assets.store.ts b/web/src/lib/stores/assets.store.ts
index 620626b647..eac4e18463 100644
--- a/web/src/lib/stores/assets.store.ts
+++ b/web/src/lib/stores/assets.store.ts
@@ -1,6 +1,5 @@
-import { AssetGridState } from '$lib/models/asset-grid-state';
-import { calculateViewportHeightByNumberOfAsset } from '$lib/utils/viewport-utils';
-import { api, AssetCountByTimeBucketResponseDto } from '@api';
+import { AssetGridState, BucketPosition } from '$lib/models/asset-grid-state';
+import { AssetCountByTimeBucketResponseDto, api } from '@api';
 import { sumBy, flatMap } from 'lodash-es';
 import { writable } from 'svelte/store';
 
@@ -20,6 +19,18 @@ function createAssetStore() {
 	loadingBucketState.subscribe((state) => {
 		_loadingBucketState = state;
 	});
+
+	const estimateViewportHeight = (assetCount: number, viewportWidth: number): number => {
+		// Ideally we would use the average aspect ratio for the photoset, however assume
+		// a normal landscape aspect ratio of 3:2, then discount for the likelihood we
+		// will be scaling down and coalescing.
+		const thumbnailHeight = 235;
+		const unwrappedWidth = (3 / 2) * assetCount * thumbnailHeight * (7 / 10);
+		const rows = Math.ceil(unwrappedWidth / viewportWidth);
+		const height = rows * thumbnailHeight;
+		return height;
+	};
+
 	/**
 	 * Set initial state
 	 * @param viewportHeight
@@ -36,11 +47,12 @@ function createAssetStore() {
 			viewportHeight,
 			viewportWidth,
 			timelineHeight: 0,
-			buckets: data.buckets.map((d) => ({
-				bucketDate: d.timeBucket,
-				bucketHeight: calculateViewportHeightByNumberOfAsset(d.count, viewportWidth),
+			buckets: data.buckets.map((bucket) => ({
+				bucketDate: bucket.timeBucket,
+				bucketHeight: estimateViewportHeight(bucket.count, viewportWidth),
 				assets: [],
-				cancelToken: new AbortController()
+				cancelToken: new AbortController(),
+				position: BucketPosition.Unknown
 			})),
 			assets: [],
 			userId
@@ -53,10 +65,15 @@ function createAssetStore() {
 		});
 	};
 
-	const getAssetsByBucket = async (bucket: string) => {
+	const getAssetsByBucket = async (bucket: string, position: BucketPosition) => {
 		try {
 			const currentBucketData = _assetGridState.buckets.find((b) => b.bucketDate === bucket);
 			if (currentBucketData?.assets && currentBucketData.assets.length > 0) {
+				assetGridState.update((state) => {
+					const bucketIndex = state.buckets.findIndex((b) => b.bucketDate === bucket);
+					state.buckets[bucketIndex].position = position;
+					return state;
+				});
 				return;
 			}
 
@@ -83,8 +100,8 @@ function createAssetStore() {
 			assetGridState.update((state) => {
 				const bucketIndex = state.buckets.findIndex((b) => b.bucketDate === bucket);
 				state.buckets[bucketIndex].assets = assets;
+				state.buckets[bucketIndex].position = position;
 				state.assets = flatMap(state.buckets, (b) => b.assets);
-
 				return state;
 			});
 			// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -120,21 +137,31 @@ function createAssetStore() {
 		});
 	};
 
-	const updateBucketHeight = (bucket: string, actualBucketHeight: number) => {
+	const updateBucketHeight = (bucket: string, actualBucketHeight: number): number => {
+		let scrollTimeline = false;
+		let heightDelta = 0;
+
 		assetGridState.update((state) => {
 			const bucketIndex = state.buckets.findIndex((b) => b.bucketDate === bucket);
 			// Update timeline height based on the new bucket height
 			const estimateBucketHeight = state.buckets[bucketIndex].bucketHeight;
 
-			if (actualBucketHeight >= estimateBucketHeight) {
-				state.timelineHeight += actualBucketHeight - estimateBucketHeight;
-			} else {
-				state.timelineHeight -= estimateBucketHeight - actualBucketHeight;
-			}
+			heightDelta = actualBucketHeight - estimateBucketHeight;
+			state.timelineHeight += heightDelta;
+
+			scrollTimeline = state.buckets[bucketIndex].position == BucketPosition.Above;
 
 			state.buckets[bucketIndex].bucketHeight = actualBucketHeight;
+			state.buckets[bucketIndex].position = BucketPosition.Unknown;
+
 			return state;
 		});
+
+		if (scrollTimeline) {
+			return heightDelta;
+		}
+
+		return 0;
 	};
 
 	const cancelBucketRequest = async (token: AbortController, bucketDate: string) => {
diff --git a/web/src/lib/utils/asset-utils.ts b/web/src/lib/utils/asset-utils.ts
index f4420248df..0363962dcc 100644
--- a/web/src/lib/utils/asset-utils.ts
+++ b/web/src/lib/utils/asset-utils.ts
@@ -150,3 +150,18 @@ export function getFileMimeType(file: File): string {
 			return '';
 	}
 }
+
+/**
+ * Returns aspect ratio for the asset
+ */
+export function getAssetRatio(asset: AssetResponseDto) {
+	let height = asset.exifInfo?.exifImageHeight || 235;
+	let width = asset.exifInfo?.exifImageWidth || 235;
+	const orientation = Number(asset.exifInfo?.orientation);
+	if (orientation) {
+		if (orientation == 6 || orientation == -90) {
+			[width, height] = [height, width];
+		}
+	}
+	return { width, height };
+}
diff --git a/web/src/lib/utils/viewport-utils.ts b/web/src/lib/utils/viewport-utils.ts
deleted file mode 100644
index 42293cc4a7..0000000000
--- a/web/src/lib/utils/viewport-utils.ts
+++ /dev/null
@@ -1,14 +0,0 @@
-/**
- * Glossary
- * 1. Section: Group of assets in a month
- */
-
-export function calculateViewportHeightByNumberOfAsset(assetCount: number, viewportWidth: number) {
-	const thumbnailHeight = 237;
-
-	// const unwrappedWidth = (3 / 2) * assetCount * thumbnailHeight * (7 / 10);
-	const unwrappedWidth = assetCount * thumbnailHeight;
-	const rows = Math.ceil(unwrappedWidth / viewportWidth);
-	const height = rows * thumbnailHeight;
-	return height;
-}