feat(web): remove upload file limit with rxjs and improve import size (#1743)

* feat(web): remove upload file limit with rxjs

* refactor: remove exif

* refactor: remove unused code

* fix: import lodash-es instead of lodash

* refactor: optimize import

web/package-lock.json

@@ -16,6 +16,7 @@
 				"leaflet": "^1.8.0",
 				"lodash-es": "^4.17.21",
 				"luxon": "^3.1.1",
+				"rxjs": "^7.8.0",
 				"socket.io-client": "^4.5.1",
 				"svelte-material-icons": "^2.0.2"
 			},
@@ -10148,6 +10149,14 @@
 				"queue-microtask": "^1.2.2"
 			}
 		},
+		"node_modules/rxjs": {
+			"version": "7.8.0",
+			"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.0.tgz",
+			"integrity": "sha512-F2+gxDshqmIub1KdvZkaEfGDwLNpPvk9Fs6LD/MyQxNgMds/WH9OdDDXOmxUZpME+iSK3rQCctkL0DYyytUqMg==",
+			"dependencies": {
+				"tslib": "^2.1.0"
+			}
+		},
 		"node_modules/sade": {
 			"version": "1.8.1",
 			"resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz",
@@ -10842,8 +10851,7 @@
 		"node_modules/tslib": {
 			"version": "2.4.1",
 			"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz",
-			"integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==",
-			"dev": true
+			"integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA=="
 		},
 		"node_modules/tsutils": {
 			"version": "3.21.0",
@@ -18691,6 +18699,14 @@
 				"queue-microtask": "^1.2.2"
 			}
 		},
+		"rxjs": {
+			"version": "7.8.0",
+			"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.0.tgz",
+			"integrity": "sha512-F2+gxDshqmIub1KdvZkaEfGDwLNpPvk9Fs6LD/MyQxNgMds/WH9OdDDXOmxUZpME+iSK3rQCctkL0DYyytUqMg==",
+			"requires": {
+				"tslib": "^2.1.0"
+			}
+		},
 		"sade": {
 			"version": "1.8.1",
 			"resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz",
@@ -19193,8 +19209,7 @@
 		"tslib": {
 			"version": "2.4.1",
 			"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz",
-			"integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==",
-			"dev": true
+			"integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA=="
 		},
 		"tsutils": {
 			"version": "3.21.0",

web/package.json

@@ -63,11 +63,11 @@
 		"axios": "^0.27.2",
 		"cookie": "^0.4.2",
 		"copy-image-clipboard": "^2.1.2",
-		"exifr": "^7.1.3",
 		"handlebars": "^4.7.7",
 		"leaflet": "^1.8.0",
 		"lodash-es": "^4.17.21",
 		"luxon": "^3.1.1",
+		"rxjs": "^7.8.0",
 		"socket.io-client": "^4.5.1",
 		"svelte-material-icons": "^2.0.2"
 	}

web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte

@@ -8,7 +8,7 @@
 	import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
 	import SettingSelect from '../setting-select.svelte';
 	import SettingSwitch from '../setting-switch.svelte';
-	import _ from 'lodash';
+	import { isEqual } from 'lodash-es';
 	import { fade } from 'svelte/transition';
 
 	export let ffmpegConfig: SystemConfigFFmpegDto; // this is the config that is being edited
@@ -130,7 +130,7 @@
 						on:reset={reset}
 						on:save={saveSetting}
 						on:reset-to-default={resetToDefault}
-						showResetToDefault={!_.isEqual(savedConfig, defaultConfig)}
+						showResetToDefault={!isEqual(savedConfig, defaultConfig)}
 					/>
 				</div>
 			</form>

web/src/lib/components/admin-page/settings/oauth/oauth-settings.svelte

@@ -5,7 +5,7 @@
 	} from '$lib/components/shared-components/notification/notification';
 	import { handleError } from '$lib/utils/handle-error';
 	import { api, SystemConfigOAuthDto } from '@api';
-	import _ from 'lodash';
+	import { isEqual } from 'lodash-es';
 	import { fade } from 'svelte/transition';
 	import ConfirmDisableLogin from '../confirm-disable-login.svelte';
 	import SettingButtonsRow from '../setting-buttons-row.svelte';
@@ -202,7 +202,7 @@
 					on:reset={reset}
 					on:save={saveSetting}
 					on:reset-to-default={resetToDefault}
-					showResetToDefault={!_.isEqual(savedConfig, defaultConfig)}
+					showResetToDefault={!isEqual(savedConfig, defaultConfig)}
 				/>
 			</form>
 		</div>

web/src/lib/components/admin-page/settings/password-login/password-login-settings.svelte

@@ -5,7 +5,7 @@
 	} from '$lib/components/shared-components/notification/notification';
 	import { handleError } from '$lib/utils/handle-error';
 	import { api, SystemConfigPasswordLoginDto } from '@api';
-	import _ from 'lodash';
+	import { isEqual } from 'lodash-es';
 	import { fade } from 'svelte/transition';
 	import ConfirmDisableLogin from '../confirm-disable-login.svelte';
 	import SettingButtonsRow from '../setting-buttons-row.svelte';
@@ -109,7 +109,7 @@
 							on:reset={reset}
 							on:save={saveSetting}
 							on:reset-to-default={resetToDefault}
-							showResetToDefault={!_.isEqual(savedConfig, defaultConfig)}
+							showResetToDefault={!isEqual(savedConfig, defaultConfig)}
 						/>
 					</div>
 				</div>

web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte

@@ -12,7 +12,7 @@
 	import SupportedDatetimePanel from './supported-datetime-panel.svelte';
 	import SupportedVariablesPanel from './supported-variables-panel.svelte';
 	import SettingButtonsRow from '../setting-buttons-row.svelte';
-	import _ from 'lodash';
+	import { isEqual } from 'lodash-es';
 	import {
 		notificationController,
 		NotificationType
@@ -230,7 +230,7 @@
 						on:reset={reset}
 						on:save={saveSetting}
 						on:reset-to-default={resetToDefault}
-						showResetToDefault={!_.isEqual(savedConfig, defaultConfig)}
+						showResetToDefault={!isEqual(savedConfig, defaultConfig)}
 					/>
 				</form>
 			</div>

web/src/lib/components/photos-page/asset-date-group.svelte

@@ -4,7 +4,7 @@
 	import CircleOutline from 'svelte-material-icons/CircleOutline.svelte';
 	import { fly } from 'svelte/transition';
 	import { AssetResponseDto } from '@api';
-	import lodash from 'lodash-es';
+	import { chain } from 'lodash-es';
 	import ImmichThumbnail from '../shared-components/immich-thumbnail.svelte';
 	import {
 		assetInteractionStore,
@@ -29,8 +29,7 @@
 	let isMouseOverGroup = false;
 	let actualBucketHeight: number;
 	let hoveredDateGroup = '';
-	$: assetsGroupByDate = lodash
-		.chain(assets)
+	$: assetsGroupByDate = chain(assets)
 		.groupBy((a) => new Date(a.createdAt).toLocaleDateString(locale, groupDateFormat))
 		.sortBy((group) => assets.indexOf(group[0]))
 		.value();

web/src/lib/stores/asset-interaction.store.ts

@@ -2,7 +2,7 @@ import { AssetGridState } from '$lib/models/asset-grid-state';
 import { api, AssetResponseDto } from '@api';
 import { derived, writable } from 'svelte/store';
 import { assetGridState, assetStore } from './assets.store';
-import _ from 'lodash-es';
+import { sortBy } from 'lodash-es';
 
 // Asset Viewer
 export const viewingAssetStoreState = writable<AssetResponseDto>();
@@ -65,7 +65,7 @@ function createAssetInteractionStore() {
 	const navigateAsset = async (direction: 'next' | 'previous') => {
 		// Flatten and sort the asset by date if there are new assets
 		if (assetSortedByDate.length === 0 || savedAssetLength !== _assetGridState.assets.length) {
-			assetSortedByDate = _.sortBy(_assetGridState.assets, (a) => a.createdAt);
+			assetSortedByDate = sortBy(_assetGridState.assets, (a) => a.createdAt);
 			savedAssetLength = _assetGridState.assets.length;
 		}
 

web/src/lib/stores/assets.store.ts

@@ -1,7 +1,7 @@
 import { AssetGridState } from '$lib/models/asset-grid-state';
 import { calculateViewportHeightByNumberOfAsset } from '$lib/utils/viewport-utils';
 import { api, AssetCountByTimeBucketResponseDto } from '@api';
-import lodash from 'lodash-es';
+import { sumBy, flatMap } from 'lodash-es';
 import { writable } from 'svelte/store';
 
 /**
@@ -46,7 +46,7 @@ function createAssetStore() {
 
 		// Update timeline height based on calculated bucket height
 		assetGridState.update((state) => {
-			state.timelineHeight = lodash.sumBy(state.buckets, (d) => d.bucketHeight);
+			state.timelineHeight = sumBy(state.buckets, (d) => d.bucketHeight);
 			return state;
 		});
 	};
@@ -77,7 +77,7 @@ function createAssetStore() {
 			assetGridState.update((state) => {
 				const bucketIndex = state.buckets.findIndex((b) => b.bucketDate === bucket);
 				state.buckets[bucketIndex].assets = assets;
-				state.assets = lodash.flatMap(state.buckets, (b) => b.assets);
+				state.assets = flatMap(state.buckets, (b) => b.assets);
 
 				return state;
 			});
@@ -100,7 +100,7 @@ function createAssetStore() {
 			if (state.buckets[bucketIndex].assets.length === 0) {
 				_removeBucket(state.buckets[bucketIndex].bucketDate);
 			}
-			state.assets = lodash.flatMap(state.buckets, (b) => b.assets);
+			state.assets = flatMap(state.buckets, (b) => b.assets);
 			return state;
 		});
 	};
@@ -109,7 +109,7 @@ function createAssetStore() {
 		assetGridState.update((state) => {
 			const bucketIndex = state.buckets.findIndex((b) => b.bucketDate === bucketDate);
 			state.buckets.splice(bucketIndex, 1);
-			state.assets = lodash.flatMap(state.buckets, (b) => b.assets);
+			state.assets = flatMap(state.buckets, (b) => b.assets);
 			return state;
 		});
 	};
@@ -147,7 +147,7 @@ function createAssetStore() {
 			const assetIndex = state.buckets[bucketIndex].assets.findIndex((a) => a.id === assetId);
 			state.buckets[bucketIndex].assets[assetIndex].isFavorite = isFavorite;
 
-			state.assets = lodash.flatMap(state.buckets, (b) => b.assets);
+			state.assets = flatMap(state.buckets, (b) => b.assets);
 			return state;
 		});
 	};

web/src/lib/utils/file-uploader.ts

@@ -2,12 +2,11 @@ import {
 	notificationController,
 	NotificationType
 } from './../components/shared-components/notification/notification';
-/* @vite-ignore */
-import * as exifr from 'exifr';
 import { uploadAssetsStore } from '$lib/stores/upload';
 import type { UploadAsset } from '../models/upload-asset';
 import { api, AssetFileUploadResponseDto } from '@api';
 import { addAssetsToAlbum, getFileMimeType, getFilenameExtension } from '$lib/utils/asset-utils';
+import { Subject, mergeMap } from 'rxjs';
 
 export const openFileUploadDialog = (
 	albumId: string | undefined = undefined,
@@ -46,25 +45,20 @@ export const fileUploadHandler = async (
 	sharedKey: string | undefined = undefined,
 	onDone?: (id: string) => void
 ) => {
-	if (files.length > 50) {
-		notificationController.show({
-			type: NotificationType.Error,
-			message: `Cannot upload more than 50 files at a time - you are uploading ${files.length} files. 
-			Please check out <u>the bulk upload documentation</u> if you need to upload more than 50 files.`,
-			timeout: 10000,
-			action: { type: 'link', target: 'https://immich.app/docs/features/bulk-upload' }
-		});
-
-		return;
-	}
-
+	const files$ = new Subject<File>();
+	files$
+		.pipe(
+			mergeMap(async (file) => {
+				await fileUploader(file, albumId, sharedKey, onDone);
+			}, 2)
+		)
+		.subscribe();
 	const acceptedFile = files.filter((file) => {
 		const assetType = getFileMimeType(file).split('/')[0];
 		return assetType === 'video' || assetType === 'image';
 	});
-
-	for (const asset of acceptedFile) {
-		await fileUploader(asset, albumId, sharedKey, onDone);
+	for (const file of acceptedFile) {
+		files$.next(file);
 	}
 };
 
@@ -75,25 +69,15 @@ async function fileUploader(
 	sharedKey: string | undefined = undefined,
 	onDone?: (id: string) => void
 ) {
+	console.log('uploading', asset.name);
 	const mimeType = getFileMimeType(asset);
 	const assetType = mimeType.split('/')[0].toUpperCase();
 	const fileExtension = getFilenameExtension(asset.name);
 	const formData = new FormData();
+	const createdAt = new Date(asset.lastModified).toISOString();
+	const deviceAssetId = 'web' + '-' + asset.name + '-' + asset.lastModified;
 
 	try {
-		let exifData = null;
-
-		if (assetType !== 'VIDEO') {
-			exifData = await exifr.parse(asset).catch((e) => console.log('error parsing exif', e));
-		}
-
-		const createdAt =
-			exifData && exifData.DateTimeOriginal != null
-				? new Date(exifData.DateTimeOriginal).toISOString()
-				: new Date(asset.lastModified).toISOString();
-
-		const deviceAssetId = 'web' + '-' + asset.name + '-' + asset.lastModified;
-
 		// Create and add Unique ID of asset on the device
 		formData.append('deviceAssetId', deviceAssetId);
 
@@ -160,20 +144,18 @@ async function fileUploader(
 		};
 
 		request.upload.onload = () => {
-			setTimeout(() => {
-				uploadAssetsStore.removeUploadAsset(deviceAssetId);
-				const res: AssetFileUploadResponseDto = JSON.parse(request.response || '{}');
-				if (albumId) {
-					try {
-						if (res.id) {
-							addAssetsToAlbum(albumId, [res.id], sharedKey);
-						}
-					} catch (e) {
-						console.error('ERROR parsing data JSON in upload onload');
+			uploadAssetsStore.removeUploadAsset(deviceAssetId);
+			const res: AssetFileUploadResponseDto = JSON.parse(request.response || '{}');
+			if (albumId) {
+				try {
+					if (res.id) {
+						addAssetsToAlbum(albumId, [res.id], sharedKey);
 					}
+				} catch (e) {
+					console.error('ERROR parsing data JSON in upload onload');
 				}
-				onDone && onDone(res.id);
-			}, 1000);
+			}
+			onDone && onDone(res.id);
 		};
 
 		// listen for `error` event