diff --git a/web/src/lib/components/photos-page/actions/select-all-assets.svelte b/web/src/lib/components/photos-page/actions/select-all-assets.svelte
index 2bbdec34fb..c14ea37882 100644
--- a/web/src/lib/components/photos-page/actions/select-all-assets.svelte
+++ b/web/src/lib/components/photos-page/actions/select-all-assets.svelte
@@ -21,7 +21,7 @@
         if ($isSelectAllCancelled) {
           break;
         }
-        await assetStore.loadBucket(bucket.date, BucketPosition.Unknown);
+        await assetStore.loadBucket(bucket.bucketDate, BucketPosition.Unknown);
         for (const asset of bucket.assets) {
           assetInteractionStore.selectAsset(asset);
         }
diff --git a/web/src/lib/components/photos-page/asset-date-group.svelte b/web/src/lib/components/photos-page/asset-date-group.svelte
index 6ac95d8d63..0c40c87108 100644
--- a/web/src/lib/components/photos-page/asset-date-group.svelte
+++ b/web/src/lib/components/photos-page/asset-date-group.svelte
@@ -2,9 +2,15 @@
   import Icon from '$lib/components/elements/icon.svelte';
   import type { AssetInteractionStore } from '$lib/stores/asset-interaction.store';
   import { assetViewingStore } from '$lib/stores/asset-viewing.store';
-  import type { AssetBucket, AssetStore, Viewport } from '$lib/stores/assets.store';
+  import type { AssetStore, Viewport } from '$lib/stores/assets.store';
+  import { locale } from '$lib/stores/preferences.store';
   import { getAssetRatio } from '$lib/utils/asset-utils';
-  import { calculateWidth, formatGroupTitle, fromLocalDateTime } from '$lib/utils/timeline-util';
+  import {
+    calculateWidth,
+    formatGroupTitle,
+    fromLocalDateTime,
+    splitBucketIntoDateGroups,
+  } from '$lib/utils/timeline-util';
   import type { AssetResponseDto } from '@immich/sdk';
   import { mdiCheckCircle, mdiCircleOutline } from '@mdi/js';
   import justifiedLayout from 'justified-layout';
@@ -12,7 +18,9 @@
   import { fly } from 'svelte/transition';
   import Thumbnail from '../assets/thumbnail/thumbnail.svelte';
 
-  export let bucket: AssetBucket;
+  export let assets: AssetResponseDto[];
+  export let bucketDate: string;
+  export let bucketHeight: number;
   export let isSelectionMode = false;
   export let viewport: Viewport;
   export let singleSelect = false;
@@ -34,7 +42,7 @@
   let actualBucketHeight: number;
   let hoveredDateGroup = '';
 
-  $: assetsGroupByDate = [...bucket.dateGroups.values()];
+  $: assetsGroupByDate = splitBucketIntoDateGroups(assets, $locale);
 
   $: geometry = (() => {
     const geometry = [];
@@ -58,8 +66,8 @@
   })();
 
   $: {
-    if (actualBucketHeight && actualBucketHeight !== 0 && actualBucketHeight != bucket.height) {
-      const heightDelta = assetStore.updateBucket(bucket.date, actualBucketHeight);
+    if (actualBucketHeight && actualBucketHeight !== 0 && actualBucketHeight != bucketHeight) {
+      const heightDelta = assetStore.updateBucket(bucketDate, actualBucketHeight);
       if (heightDelta !== 0) {
         scrollTimeline(heightDelta);
       }
diff --git a/web/src/lib/components/photos-page/asset-grid.svelte b/web/src/lib/components/photos-page/asset-grid.svelte
index 1f01448f28..10226a5ae4 100644
--- a/web/src/lib/components/photos-page/asset-grid.svelte
+++ b/web/src/lib/components/photos-page/asset-grid.svelte
@@ -5,12 +5,12 @@
   import type { AssetInteractionStore } from '$lib/stores/asset-interaction.store';
   import { assetViewingStore } from '$lib/stores/asset-viewing.store';
   import { BucketPosition, type AssetStore, type Viewport } from '$lib/stores/assets.store';
-  import { showDeleteModal } from '$lib/stores/preferences.store';
+  import { locale, showDeleteModal } from '$lib/stores/preferences.store';
   import { isSearchEnabled } from '$lib/stores/search.store';
   import { featureFlags } from '$lib/stores/server-config.store';
   import { deleteAssets } from '$lib/utils/actions';
   import { shouldIgnoreShortcut } from '$lib/utils/shortcut';
-  import { formatGroupTitle } from '$lib/utils/timeline-util';
+  import { formatGroupTitle, splitBucketIntoDateGroups } from '$lib/utils/timeline-util';
   import type { AlbumResponseDto, AssetResponseDto } from '@immich/sdk';
   import { DateTime } from 'luxon';
   import { createEventDispatcher, onDestroy, onMount } from 'svelte';
@@ -22,6 +22,7 @@
   import AssetDateGroup from './asset-date-group.svelte';
   import DeleteAssetDialog from './delete-asset-dialog.svelte';
   import { handlePromiseError } from '$lib/utils';
+
   export let isSelectionMode = false;
   export let singleSelect = false;
   export let assetStore: AssetStore;
@@ -305,7 +306,7 @@
       // Select/deselect assets in all intermediate buckets
       for (let bucketIndex = startBucketIndex + 1; bucketIndex < endBucketIndex; bucketIndex++) {
         const bucket = $assetStore.buckets[bucketIndex];
-        await assetStore.loadBucket(bucket.date, BucketPosition.Unknown);
+        await assetStore.loadBucket(bucket.bucketDate, BucketPosition.Unknown);
         for (const asset of bucket.assets) {
           if (deselect) {
             assetInteractionStore.removeAssetFromMultiselectGroup(asset);
@@ -319,7 +320,10 @@
       for (let bucketIndex = startBucketIndex; bucketIndex <= endBucketIndex; bucketIndex++) {
         const bucket = $assetStore.buckets[bucketIndex];
 
-        for (const dateGroup of bucket.dateGroups.values()) {
+        // Split bucket into date groups and check each group
+        const assetsGroupByDate = splitBucketIntoDateGroups(bucket.assets, $locale);
+
+        for (const dateGroup of assetsGroupByDate) {
           const dateGroupTitle = formatGroupTitle(DateTime.fromISO(dateGroup[0].fileCreatedAt).startOf('day'));
           if (dateGroup.every((a) => $selectedAssets.has(a))) {
             assetInteractionStore.addGroupToMultiselectGroup(dateGroupTitle);
@@ -410,7 +414,7 @@
       <slot name="empty" />
     {/if}
     <section id="virtual-timeline" style:height={$assetStore.timelineHeight + 'px'}>
-      {#each $assetStore.buckets as bucket (bucket.date)}
+      {#each $assetStore.buckets as bucket (bucket.bucketDate)}
         <IntersectionObserver
           on:intersected={intersectedHandler}
           on:hidden={() => assetStore.cancelBucket(bucket)}
@@ -419,7 +423,7 @@
           bottom={750}
           root={element}
         >
-          <div id={'bucket_' + bucket.date} style:height={bucket.height + 'px'}>
+          <div id={'bucket_' + bucket.bucketDate} style:height={bucket.bucketHeight + 'px'}>
             {#if intersecting}
               <AssetDateGroup
                 {withStacked}
@@ -432,7 +436,9 @@
                 on:shift={handleScrollTimeline}
                 on:selectAssetCandidates={({ detail: asset }) => handleSelectAssetCandidates(asset)}
                 on:selectAssets={({ detail: asset }) => handleSelectAssets(asset)}
-                {bucket}
+                assets={bucket.assets}
+                bucketDate={bucket.bucketDate}
+                bucketHeight={bucket.bucketHeight}
                 {viewport}
               />
             {/if}
diff --git a/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte b/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte
index c66fe0bc2a..bef58fe250 100644
--- a/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte
+++ b/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte
@@ -38,8 +38,8 @@
     return buckets.map((bucket) => {
       const segment = new Segment();
       segment.count = bucket.assets.length;
-      segment.height = toScrollY(bucket.height);
-      segment.timeGroup = bucket.date;
+      segment.height = toScrollY(bucket.bucketHeight);
+      segment.timeGroup = bucket.bucketDate;
       segment.date = fromLocalDateTime(segment.timeGroup);
 
       if (previous?.date.year !== segment.date.year && height > MIN_YEAR_LABEL_DISTANCE) {
diff --git a/web/src/lib/stores/assets.store.ts b/web/src/lib/stores/assets.store.ts
index b0176f5477..418d530fee 100644
--- a/web/src/lib/stores/assets.store.ts
+++ b/web/src/lib/stores/assets.store.ts
@@ -1,5 +1,4 @@
 import { getKey } from '$lib/utils';
-import { fromLocalDateTime, groupDateFormat } from '$lib/utils/timeline-util';
 import { TimeBucketSize, getTimeBucket, getTimeBuckets, type AssetResponseDto } from '@immich/sdk';
 import { throttle } from 'lodash-es';
 import { DateTime } from 'luxon';
@@ -27,60 +26,17 @@ interface AssetLookup {
   assetIndex: number;
 }
 
-type BucketConstructor = {
-  height: number;
-  date: string;
-  count: number;
-  assets?: AssetResponseDto[];
-  position?: BucketPosition;
-};
-
 export class AssetBucket {
   /**
    * The DOM height of the bucket in pixel
    * This value is first estimated by the number of asset and later is corrected as the user scroll
    */
-  height!: number;
-  date!: string;
-  count!: number;
+  bucketHeight!: number;
+  bucketDate!: string;
+  bucketCount!: number;
   assets!: AssetResponseDto[];
   cancelToken!: AbortController | null;
   position!: BucketPosition;
-  dateGroups!: Map<string, AssetResponseDto[]>;
-
-  constructor({ height, date, count, assets = [], position = BucketPosition.Unknown }: BucketConstructor) {
-    this.height = height;
-    this.date = date;
-    this.count = count;
-    this.assets = assets;
-    this.position = position;
-    this.cancelToken = null;
-    this.dateGroups = new Map();
-  }
-
-  public addAssets(assets: AssetResponseDto[]) {
-    if (this.assets.length === 0) {
-      this.assets = assets;
-    } else {
-      this.assets.push(...assets);
-    }
-    this.updateDateGroups();
-  }
-
-  public updateDateGroups() {
-    if (this.assets.length === 0) {
-      return;
-    }
-
-    for (const asset of this.assets) {
-      const curLocale = fromLocalDateTime(asset.localDateTime).toLocaleString(groupDateFormat);
-      if (this.dateGroups.has(curLocale)) {
-        this.dateGroups.get(curLocale)?.push(asset);
-      } else {
-        this.dateGroups.set(curLocale, [asset]);
-      }
-    }
-  }
 }
 
 const isMismatched = (option: boolean | undefined, value: boolean): boolean =>
@@ -209,25 +165,37 @@ export class AssetStore {
 
     this.initialized = true;
 
-    let viewHeight = 0;
-    const loaders = [];
-    for (const { timeBucket, count } of buckets) {
-      const unwrappedWidth = (3 / 2) * count * THUMBNAIL_HEIGHT * (7 / 10);
+    this.buckets = buckets.map((bucket) => {
+      const unwrappedWidth = (3 / 2) * bucket.count * THUMBNAIL_HEIGHT * (7 / 10);
       const rows = Math.ceil(unwrappedWidth / viewport.width);
-      const bucketHeight = rows * THUMBNAIL_HEIGHT;
+      const height = rows * THUMBNAIL_HEIGHT;
 
-      const bucket = new AssetBucket({ height: bucketHeight, date: timeBucket, count });
-      this.buckets.push(bucket);
-      if (viewHeight < viewport.height) {
-        viewHeight += bucket.height;
-        loaders.push(this.loadBucket(bucket.date, BucketPosition.Visible));
-      }
-    }
+      return {
+        bucketDate: bucket.timeBucket,
+        bucketHeight: height,
+        bucketCount: bucket.count,
+        assets: [],
+        cancelToken: null,
+        position: BucketPosition.Unknown,
+      };
+    });
 
-    this.timelineHeight = this.buckets.reduce((accumulator, b) => accumulator + b.height, 0);
+    this.timelineHeight = this.buckets.reduce((accumulator, b) => accumulator + b.bucketHeight, 0);
 
-    await Promise.all(loaders);
     this.emit(false);
+
+    let height = 0;
+    const loaders = [];
+    for (const bucket of this.buckets) {
+      if (height < viewport.height) {
+        height += bucket.bucketHeight;
+        loaders.push(this.loadBucket(bucket.bucketDate, BucketPosition.Visible));
+        continue;
+      }
+
+      break;
+    }
+    await Promise.all(loaders);
   }
 
   async loadBucket(bucketDate: string, position: BucketPosition): Promise<void> {
@@ -275,7 +243,8 @@ export class AssetStore {
         return;
       }
 
-      bucket.addAssets(assets);
+      bucket.assets = assets;
+
       this.emit(true);
     } catch (error) {
       handleError(error, 'Failed to load assets');
@@ -292,10 +261,10 @@ export class AssetStore {
       return 0;
     }
 
-    const delta = height - bucket.height;
+    const delta = height - bucket.bucketHeight;
     const scrollTimeline = bucket.position == BucketPosition.Above;
 
-    bucket.height = height;
+    bucket.bucketHeight = height;
     bucket.position = BucketPosition.Unknown;
 
     this.timelineHeight += delta;
@@ -328,17 +297,29 @@ export class AssetStore {
     let bucket = this.getBucketByDate(timeBucket);
 
     if (!bucket) {
-      bucket = new AssetBucket({ height: 0, date: timeBucket, count: 0 });
+      bucket = {
+        bucketDate: timeBucket,
+        bucketHeight: THUMBNAIL_HEIGHT,
+        bucketCount: 0,
+        assets: [],
+        cancelToken: null,
+        position: BucketPosition.Unknown,
+      };
 
       this.buckets.push(bucket);
       this.buckets = this.buckets.sort((a, b) => {
-        const aDate = DateTime.fromISO(a.date).toUTC();
-        const bDate = DateTime.fromISO(b.date).toUTC();
+        const aDate = DateTime.fromISO(a.bucketDate).toUTC();
+        const bDate = DateTime.fromISO(b.bucketDate).toUTC();
         return bDate.diff(aDate).milliseconds;
       });
     }
 
-    bucket.addAssets([asset]);
+    bucket.assets.push(asset);
+    bucket.assets.sort((a, b) => {
+      const aDate = DateTime.fromISO(a.fileCreatedAt).toUTC();
+      const bDate = DateTime.fromISO(b.fileCreatedAt).toUTC();
+      return bDate.diff(aDate).milliseconds;
+    });
 
     // If we added an asset to the store, we need to recalculate
     // asset store containers
@@ -347,7 +328,7 @@ export class AssetStore {
   }
 
   getBucketByDate(bucketDate: string): AssetBucket | null {
-    return this.buckets.find((bucket) => bucket.date === bucketDate) || null;
+    return this.buckets.find((bucket) => bucket.bucketDate === bucketDate) || null;
   }
 
   getBucketInfoForAssetId(assetId: string) {
@@ -359,14 +340,16 @@ export class AssetStore {
   }
 
   async getRandomAsset(): Promise<AssetResponseDto | null> {
-    let index = Math.floor(Math.random() * this.buckets.reduce((accumulator, bucket) => accumulator + bucket.count, 0));
+    let index = Math.floor(
+      Math.random() * this.buckets.reduce((accumulator, bucket) => accumulator + bucket.bucketCount, 0),
+    );
     for (const bucket of this.buckets) {
-      if (index < bucket.count) {
-        await this.loadBucket(bucket.date, BucketPosition.Unknown);
+      if (index < bucket.bucketCount) {
+        await this.loadBucket(bucket.bucketDate, BucketPosition.Unknown);
         return bucket.assets[index] || null;
       }
 
-      index -= bucket.count;
+      index -= bucket.bucketCount;
     }
 
     return null;
@@ -403,8 +386,8 @@ export class AssetStore {
         }
 
         bucket.assets.splice(index_, 1);
-        bucket.count = bucket.assets.length;
-        if (bucket.count === 0) {
+        bucket.bucketCount = bucket.assets.length;
+        if (bucket.bucketCount === 0) {
           this.buckets.splice(index, 1);
         }
 
@@ -432,7 +415,7 @@ export class AssetStore {
     }
 
     const previousBucket = this.buckets[bucketIndex - 1];
-    await this.loadBucket(previousBucket.date, BucketPosition.Unknown);
+    await this.loadBucket(previousBucket.bucketDate, BucketPosition.Unknown);
     return previousBucket.assets.at(-1)?.id || null;
   }
 
@@ -453,7 +436,7 @@ export class AssetStore {
     }
 
     const nextBucket = this.buckets[bucketIndex + 1];
-    await this.loadBucket(nextBucket.date, BucketPosition.Unknown);
+    await this.loadBucket(nextBucket.bucketDate, BucketPosition.Unknown);
     return nextBucket.assets[0]?.id || null;
   }
 
@@ -469,7 +452,7 @@ export class AssetStore {
       for (let index = 0; index < this.buckets.length; index++) {
         const bucket = this.buckets[index];
         if (bucket.assets.length > 0) {
-          bucket.count = bucket.assets.length;
+          bucket.bucketCount = bucket.assets.length;
         }
         for (let index_ = 0; index_ < bucket.assets.length; index_++) {
           const asset = bucket.assets[index_];
diff --git a/web/src/lib/utils/timeline-util.ts b/web/src/lib/utils/timeline-util.ts
index e8a5712098..83756a4064 100644
--- a/web/src/lib/utils/timeline-util.ts
+++ b/web/src/lib/utils/timeline-util.ts
@@ -1,4 +1,6 @@
 import { locale } from '$lib/stores/preferences.store';
+import type { AssetResponseDto } from '@immich/sdk';
+import { groupBy, sortBy } from 'lodash-es';
 import { DateTime, Interval } from 'luxon';
 import { get } from 'svelte/store';
 
@@ -42,11 +44,20 @@ export function formatGroupTitle(date: DateTime): string {
   return date.toLocaleString(groupDateFormat);
 }
 
+export function splitBucketIntoDateGroups(
+  assets: AssetResponseDto[],
+  locale: string | undefined,
+): AssetResponseDto[][] {
+  const grouped = groupBy(assets, (asset) =>
+    fromLocalDateTime(asset.localDateTime).toLocaleString(groupDateFormat, { locale }),
+  );
+  return sortBy(grouped, (group) => assets.indexOf(group[0]));
+}
+
 export type LayoutBox = {
   top: number;
   left: number;
   width: number;
-  height: number;
 };
 
 export function calculateWidth(boxes: LayoutBox[]): number {