diff --git a/web/src/lib/components/photos-page/actions/select-all-assets.svelte b/web/src/lib/components/photos-page/actions/select-all-assets.svelte
index c14ea37882..2bbdec34fb 100644
--- a/web/src/lib/components/photos-page/actions/select-all-assets.svelte
+++ b/web/src/lib/components/photos-page/actions/select-all-assets.svelte
@@ -21,7 +21,7 @@
         if ($isSelectAllCancelled) {
           break;
         }
-        await assetStore.loadBucket(bucket.bucketDate, BucketPosition.Unknown);
+        await assetStore.loadBucket(bucket.date, BucketPosition.Unknown);
         for (const asset of bucket.assets) {
           assetInteractionStore.selectAsset(asset);
         }
diff --git a/web/src/lib/components/photos-page/asset-date-group.svelte b/web/src/lib/components/photos-page/asset-date-group.svelte
index 0c40c87108..6ac95d8d63 100644
--- a/web/src/lib/components/photos-page/asset-date-group.svelte
+++ b/web/src/lib/components/photos-page/asset-date-group.svelte
@@ -2,15 +2,9 @@
   import Icon from '$lib/components/elements/icon.svelte';
   import type { AssetInteractionStore } from '$lib/stores/asset-interaction.store';
   import { assetViewingStore } from '$lib/stores/asset-viewing.store';
-  import type { AssetStore, Viewport } from '$lib/stores/assets.store';
-  import { locale } from '$lib/stores/preferences.store';
+  import type { AssetBucket, AssetStore, Viewport } from '$lib/stores/assets.store';
   import { getAssetRatio } from '$lib/utils/asset-utils';
-  import {
-    calculateWidth,
-    formatGroupTitle,
-    fromLocalDateTime,
-    splitBucketIntoDateGroups,
-  } from '$lib/utils/timeline-util';
+  import { calculateWidth, formatGroupTitle, fromLocalDateTime } from '$lib/utils/timeline-util';
   import type { AssetResponseDto } from '@immich/sdk';
   import { mdiCheckCircle, mdiCircleOutline } from '@mdi/js';
   import justifiedLayout from 'justified-layout';
@@ -18,9 +12,7 @@
   import { fly } from 'svelte/transition';
   import Thumbnail from '../assets/thumbnail/thumbnail.svelte';
 
-  export let assets: AssetResponseDto[];
-  export let bucketDate: string;
-  export let bucketHeight: number;
+  export let bucket: AssetBucket;
   export let isSelectionMode = false;
   export let viewport: Viewport;
   export let singleSelect = false;
@@ -42,7 +34,7 @@
   let actualBucketHeight: number;
   let hoveredDateGroup = '';
 
-  $: assetsGroupByDate = splitBucketIntoDateGroups(assets, $locale);
+  $: assetsGroupByDate = [...bucket.dateGroups.values()];
 
   $: geometry = (() => {
     const geometry = [];
@@ -66,8 +58,8 @@
   })();
 
   $: {
-    if (actualBucketHeight && actualBucketHeight !== 0 && actualBucketHeight != bucketHeight) {
-      const heightDelta = assetStore.updateBucket(bucketDate, actualBucketHeight);
+    if (actualBucketHeight && actualBucketHeight !== 0 && actualBucketHeight != bucket.height) {
+      const heightDelta = assetStore.updateBucket(bucket.date, actualBucketHeight);
       if (heightDelta !== 0) {
         scrollTimeline(heightDelta);
       }
diff --git a/web/src/lib/components/photos-page/asset-grid.svelte b/web/src/lib/components/photos-page/asset-grid.svelte
index 10226a5ae4..1f01448f28 100644
--- a/web/src/lib/components/photos-page/asset-grid.svelte
+++ b/web/src/lib/components/photos-page/asset-grid.svelte
@@ -5,12 +5,12 @@
   import type { AssetInteractionStore } from '$lib/stores/asset-interaction.store';
   import { assetViewingStore } from '$lib/stores/asset-viewing.store';
   import { BucketPosition, type AssetStore, type Viewport } from '$lib/stores/assets.store';
-  import { locale, showDeleteModal } from '$lib/stores/preferences.store';
+  import { showDeleteModal } from '$lib/stores/preferences.store';
   import { isSearchEnabled } from '$lib/stores/search.store';
   import { featureFlags } from '$lib/stores/server-config.store';
   import { deleteAssets } from '$lib/utils/actions';
   import { shouldIgnoreShortcut } from '$lib/utils/shortcut';
-  import { formatGroupTitle, splitBucketIntoDateGroups } from '$lib/utils/timeline-util';
+  import { formatGroupTitle } from '$lib/utils/timeline-util';
   import type { AlbumResponseDto, AssetResponseDto } from '@immich/sdk';
   import { DateTime } from 'luxon';
   import { createEventDispatcher, onDestroy, onMount } from 'svelte';
@@ -22,7 +22,6 @@
   import AssetDateGroup from './asset-date-group.svelte';
   import DeleteAssetDialog from './delete-asset-dialog.svelte';
   import { handlePromiseError } from '$lib/utils';
-
   export let isSelectionMode = false;
   export let singleSelect = false;
   export let assetStore: AssetStore;
@@ -306,7 +305,7 @@
       // Select/deselect assets in all intermediate buckets
       for (let bucketIndex = startBucketIndex + 1; bucketIndex < endBucketIndex; bucketIndex++) {
         const bucket = $assetStore.buckets[bucketIndex];
-        await assetStore.loadBucket(bucket.bucketDate, BucketPosition.Unknown);
+        await assetStore.loadBucket(bucket.date, BucketPosition.Unknown);
         for (const asset of bucket.assets) {
           if (deselect) {
             assetInteractionStore.removeAssetFromMultiselectGroup(asset);
@@ -320,10 +319,7 @@
       for (let bucketIndex = startBucketIndex; bucketIndex <= endBucketIndex; bucketIndex++) {
         const bucket = $assetStore.buckets[bucketIndex];
 
-        // Split bucket into date groups and check each group
-        const assetsGroupByDate = splitBucketIntoDateGroups(bucket.assets, $locale);
-
-        for (const dateGroup of assetsGroupByDate) {
+        for (const dateGroup of bucket.dateGroups.values()) {
           const dateGroupTitle = formatGroupTitle(DateTime.fromISO(dateGroup[0].fileCreatedAt).startOf('day'));
           if (dateGroup.every((a) => $selectedAssets.has(a))) {
             assetInteractionStore.addGroupToMultiselectGroup(dateGroupTitle);
@@ -414,7 +410,7 @@
       <slot name="empty" />
     {/if}
     <section id="virtual-timeline" style:height={$assetStore.timelineHeight + 'px'}>
-      {#each $assetStore.buckets as bucket (bucket.bucketDate)}
+      {#each $assetStore.buckets as bucket (bucket.date)}
         <IntersectionObserver
           on:intersected={intersectedHandler}
           on:hidden={() => assetStore.cancelBucket(bucket)}
@@ -423,7 +419,7 @@
           bottom={750}
           root={element}
         >
-          <div id={'bucket_' + bucket.bucketDate} style:height={bucket.bucketHeight + 'px'}>
+          <div id={'bucket_' + bucket.date} style:height={bucket.height + 'px'}>
             {#if intersecting}
               <AssetDateGroup
                 {withStacked}
@@ -436,9 +432,7 @@
                 on:shift={handleScrollTimeline}
                 on:selectAssetCandidates={({ detail: asset }) => handleSelectAssetCandidates(asset)}
                 on:selectAssets={({ detail: asset }) => handleSelectAssets(asset)}
-                assets={bucket.assets}
-                bucketDate={bucket.bucketDate}
-                bucketHeight={bucket.bucketHeight}
+                {bucket}
                 {viewport}
               />
             {/if}
diff --git a/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte b/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte
index bef58fe250..c66fe0bc2a 100644
--- a/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte
+++ b/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte
@@ -38,8 +38,8 @@
     return buckets.map((bucket) => {
       const segment = new Segment();
       segment.count = bucket.assets.length;
-      segment.height = toScrollY(bucket.bucketHeight);
-      segment.timeGroup = bucket.bucketDate;
+      segment.height = toScrollY(bucket.height);
+      segment.timeGroup = bucket.date;
       segment.date = fromLocalDateTime(segment.timeGroup);
 
       if (previous?.date.year !== segment.date.year && height > MIN_YEAR_LABEL_DISTANCE) {
diff --git a/web/src/lib/stores/assets.store.ts b/web/src/lib/stores/assets.store.ts
index 418d530fee..b0176f5477 100644
--- a/web/src/lib/stores/assets.store.ts
+++ b/web/src/lib/stores/assets.store.ts
@@ -1,4 +1,5 @@
 import { getKey } from '$lib/utils';
+import { fromLocalDateTime, groupDateFormat } from '$lib/utils/timeline-util';
 import { TimeBucketSize, getTimeBucket, getTimeBuckets, type AssetResponseDto } from '@immich/sdk';
 import { throttle } from 'lodash-es';
 import { DateTime } from 'luxon';
@@ -26,17 +27,60 @@ interface AssetLookup {
   assetIndex: number;
 }
 
+type BucketConstructor = {
+  height: number;
+  date: string;
+  count: number;
+  assets?: AssetResponseDto[];
+  position?: BucketPosition;
+};
+
 export class AssetBucket {
   /**
    * The DOM height of the bucket in pixel
    * This value is first estimated by the number of asset and later is corrected as the user scroll
    */
-  bucketHeight!: number;
-  bucketDate!: string;
-  bucketCount!: number;
+  height!: number;
+  date!: string;
+  count!: number;
   assets!: AssetResponseDto[];
   cancelToken!: AbortController | null;
   position!: BucketPosition;
+  dateGroups!: Map<string, AssetResponseDto[]>;
+
+  constructor({ height, date, count, assets = [], position = BucketPosition.Unknown }: BucketConstructor) {
+    this.height = height;
+    this.date = date;
+    this.count = count;
+    this.assets = assets;
+    this.position = position;
+    this.cancelToken = null;
+    this.dateGroups = new Map();
+  }
+
+  public addAssets(assets: AssetResponseDto[]) {
+    if (this.assets.length === 0) {
+      this.assets = assets;
+    } else {
+      this.assets.push(...assets);
+    }
+    this.updateDateGroups();
+  }
+
+  public updateDateGroups() {
+    if (this.assets.length === 0) {
+      return;
+    }
+
+    for (const asset of this.assets) {
+      const curLocale = fromLocalDateTime(asset.localDateTime).toLocaleString(groupDateFormat);
+      if (this.dateGroups.has(curLocale)) {
+        this.dateGroups.get(curLocale)?.push(asset);
+      } else {
+        this.dateGroups.set(curLocale, [asset]);
+      }
+    }
+  }
 }
 
 const isMismatched = (option: boolean | undefined, value: boolean): boolean =>
@@ -165,37 +209,25 @@ export class AssetStore {
 
     this.initialized = true;
 
-    this.buckets = buckets.map((bucket) => {
-      const unwrappedWidth = (3 / 2) * bucket.count * THUMBNAIL_HEIGHT * (7 / 10);
-      const rows = Math.ceil(unwrappedWidth / viewport.width);
-      const height = rows * THUMBNAIL_HEIGHT;
-
-      return {
-        bucketDate: bucket.timeBucket,
-        bucketHeight: height,
-        bucketCount: bucket.count,
-        assets: [],
-        cancelToken: null,
-        position: BucketPosition.Unknown,
-      };
-    });
-
-    this.timelineHeight = this.buckets.reduce((accumulator, b) => accumulator + b.bucketHeight, 0);
-
-    this.emit(false);
-
-    let height = 0;
+    let viewHeight = 0;
     const loaders = [];
-    for (const bucket of this.buckets) {
-      if (height < viewport.height) {
-        height += bucket.bucketHeight;
-        loaders.push(this.loadBucket(bucket.bucketDate, BucketPosition.Visible));
-        continue;
-      }
+    for (const { timeBucket, count } of buckets) {
+      const unwrappedWidth = (3 / 2) * count * THUMBNAIL_HEIGHT * (7 / 10);
+      const rows = Math.ceil(unwrappedWidth / viewport.width);
+      const bucketHeight = rows * THUMBNAIL_HEIGHT;
 
-      break;
+      const bucket = new AssetBucket({ height: bucketHeight, date: timeBucket, count });
+      this.buckets.push(bucket);
+      if (viewHeight < viewport.height) {
+        viewHeight += bucket.height;
+        loaders.push(this.loadBucket(bucket.date, BucketPosition.Visible));
+      }
     }
+
+    this.timelineHeight = this.buckets.reduce((accumulator, b) => accumulator + b.height, 0);
+
     await Promise.all(loaders);
+    this.emit(false);
   }
 
   async loadBucket(bucketDate: string, position: BucketPosition): Promise<void> {
@@ -243,8 +275,7 @@ export class AssetStore {
         return;
       }
 
-      bucket.assets = assets;
-
+      bucket.addAssets(assets);
       this.emit(true);
     } catch (error) {
       handleError(error, 'Failed to load assets');
@@ -261,10 +292,10 @@ export class AssetStore {
       return 0;
     }
 
-    const delta = height - bucket.bucketHeight;
+    const delta = height - bucket.height;
     const scrollTimeline = bucket.position == BucketPosition.Above;
 
-    bucket.bucketHeight = height;
+    bucket.height = height;
     bucket.position = BucketPosition.Unknown;
 
     this.timelineHeight += delta;
@@ -297,29 +328,17 @@ export class AssetStore {
     let bucket = this.getBucketByDate(timeBucket);
 
     if (!bucket) {
-      bucket = {
-        bucketDate: timeBucket,
-        bucketHeight: THUMBNAIL_HEIGHT,
-        bucketCount: 0,
-        assets: [],
-        cancelToken: null,
-        position: BucketPosition.Unknown,
-      };
+      bucket = new AssetBucket({ height: 0, date: timeBucket, count: 0 });
 
       this.buckets.push(bucket);
       this.buckets = this.buckets.sort((a, b) => {
-        const aDate = DateTime.fromISO(a.bucketDate).toUTC();
-        const bDate = DateTime.fromISO(b.bucketDate).toUTC();
+        const aDate = DateTime.fromISO(a.date).toUTC();
+        const bDate = DateTime.fromISO(b.date).toUTC();
         return bDate.diff(aDate).milliseconds;
       });
     }
 
-    bucket.assets.push(asset);
-    bucket.assets.sort((a, b) => {
-      const aDate = DateTime.fromISO(a.fileCreatedAt).toUTC();
-      const bDate = DateTime.fromISO(b.fileCreatedAt).toUTC();
-      return bDate.diff(aDate).milliseconds;
-    });
+    bucket.addAssets([asset]);
 
     // If we added an asset to the store, we need to recalculate
     // asset store containers
@@ -328,7 +347,7 @@ export class AssetStore {
   }
 
   getBucketByDate(bucketDate: string): AssetBucket | null {
-    return this.buckets.find((bucket) => bucket.bucketDate === bucketDate) || null;
+    return this.buckets.find((bucket) => bucket.date === bucketDate) || null;
   }
 
   getBucketInfoForAssetId(assetId: string) {
@@ -340,16 +359,14 @@ export class AssetStore {
   }
 
   async getRandomAsset(): Promise<AssetResponseDto | null> {
-    let index = Math.floor(
-      Math.random() * this.buckets.reduce((accumulator, bucket) => accumulator + bucket.bucketCount, 0),
-    );
+    let index = Math.floor(Math.random() * this.buckets.reduce((accumulator, bucket) => accumulator + bucket.count, 0));
     for (const bucket of this.buckets) {
-      if (index < bucket.bucketCount) {
-        await this.loadBucket(bucket.bucketDate, BucketPosition.Unknown);
+      if (index < bucket.count) {
+        await this.loadBucket(bucket.date, BucketPosition.Unknown);
         return bucket.assets[index] || null;
       }
 
-      index -= bucket.bucketCount;
+      index -= bucket.count;
     }
 
     return null;
@@ -386,8 +403,8 @@ export class AssetStore {
         }
 
         bucket.assets.splice(index_, 1);
-        bucket.bucketCount = bucket.assets.length;
-        if (bucket.bucketCount === 0) {
+        bucket.count = bucket.assets.length;
+        if (bucket.count === 0) {
           this.buckets.splice(index, 1);
         }
 
@@ -415,7 +432,7 @@ export class AssetStore {
     }
 
     const previousBucket = this.buckets[bucketIndex - 1];
-    await this.loadBucket(previousBucket.bucketDate, BucketPosition.Unknown);
+    await this.loadBucket(previousBucket.date, BucketPosition.Unknown);
     return previousBucket.assets.at(-1)?.id || null;
   }
 
@@ -436,7 +453,7 @@ export class AssetStore {
     }
 
     const nextBucket = this.buckets[bucketIndex + 1];
-    await this.loadBucket(nextBucket.bucketDate, BucketPosition.Unknown);
+    await this.loadBucket(nextBucket.date, BucketPosition.Unknown);
     return nextBucket.assets[0]?.id || null;
   }
 
@@ -452,7 +469,7 @@ export class AssetStore {
       for (let index = 0; index < this.buckets.length; index++) {
         const bucket = this.buckets[index];
         if (bucket.assets.length > 0) {
-          bucket.bucketCount = bucket.assets.length;
+          bucket.count = bucket.assets.length;
         }
         for (let index_ = 0; index_ < bucket.assets.length; index_++) {
           const asset = bucket.assets[index_];
diff --git a/web/src/lib/utils/timeline-util.ts b/web/src/lib/utils/timeline-util.ts
index 83756a4064..e8a5712098 100644
--- a/web/src/lib/utils/timeline-util.ts
+++ b/web/src/lib/utils/timeline-util.ts
@@ -1,6 +1,4 @@
 import { locale } from '$lib/stores/preferences.store';
-import type { AssetResponseDto } from '@immich/sdk';
-import { groupBy, sortBy } from 'lodash-es';
 import { DateTime, Interval } from 'luxon';
 import { get } from 'svelte/store';
 
@@ -44,20 +42,11 @@ export function formatGroupTitle(date: DateTime): string {
   return date.toLocaleString(groupDateFormat);
 }
 
-export function splitBucketIntoDateGroups(
-  assets: AssetResponseDto[],
-  locale: string | undefined,
-): AssetResponseDto[][] {
-  const grouped = groupBy(assets, (asset) =>
-    fromLocalDateTime(asset.localDateTime).toLocaleString(groupDateFormat, { locale }),
-  );
-  return sortBy(grouped, (group) => assets.indexOf(group[0]));
-}
-
 export type LayoutBox = {
   top: number;
   left: number;
   width: number;
+  height: number;
 };
 
 export function calculateWidth(boxes: LayoutBox[]): number {