From 2c1aab154aeb5d4ba4413eb08c2ab3161d333412 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 13 Feb 2023 13:18:11 -0600 Subject: [PATCH] feat(web): remove upload file limit with rxjs and improve import size (#1743) * feat(web): remove upload file limit with rxjs * refactor: remove exif * refactor: remove unused code * fix: import lodash-es instead of lodash * refactor: optimize import --- web/package-lock.json | 23 +++++-- web/package.json | 2 +- .../settings/ffmpeg/ffmpeg-settings.svelte | 4 +- .../settings/oauth/oauth-settings.svelte | 4 +- .../password-login-settings.svelte | 4 +- .../storage-template-settings.svelte | 4 +- .../photos-page/asset-date-group.svelte | 5 +- web/src/lib/stores/asset-interaction.store.ts | 4 +- web/src/lib/stores/assets.store.ts | 12 ++-- web/src/lib/utils/file-uploader.ts | 66 +++++++------------ 10 files changed, 62 insertions(+), 66 deletions(-) diff --git a/web/package-lock.json b/web/package-lock.json index 32bcd44cbd..82ff896da0 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -16,6 +16,7 @@ "leaflet": "^1.8.0", "lodash-es": "^4.17.21", "luxon": "^3.1.1", + "rxjs": "^7.8.0", "socket.io-client": "^4.5.1", "svelte-material-icons": "^2.0.2" }, @@ -10148,6 +10149,14 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/rxjs": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.0.tgz", + "integrity": "sha512-F2+gxDshqmIub1KdvZkaEfGDwLNpPvk9Fs6LD/MyQxNgMds/WH9OdDDXOmxUZpME+iSK3rQCctkL0DYyytUqMg==", + "dependencies": { + "tslib": "^2.1.0" + } + }, "node_modules/sade": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", @@ -10842,8 +10851,7 @@ "node_modules/tslib": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==", - "dev": true + "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" }, "node_modules/tsutils": { "version": "3.21.0", @@ -18691,6 +18699,14 @@ "queue-microtask": "^1.2.2" } }, + "rxjs": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.0.tgz", + "integrity": "sha512-F2+gxDshqmIub1KdvZkaEfGDwLNpPvk9Fs6LD/MyQxNgMds/WH9OdDDXOmxUZpME+iSK3rQCctkL0DYyytUqMg==", + "requires": { + "tslib": "^2.1.0" + } + }, "sade": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", @@ -19193,8 +19209,7 @@ "tslib": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==", - "dev": true + "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" }, "tsutils": { "version": "3.21.0", diff --git a/web/package.json b/web/package.json index f26ccc9ba4..30261e1baa 100644 --- a/web/package.json +++ b/web/package.json @@ -63,11 +63,11 @@ "axios": "^0.27.2", "cookie": "^0.4.2", "copy-image-clipboard": "^2.1.2", - "exifr": "^7.1.3", "handlebars": "^4.7.7", "leaflet": "^1.8.0", "lodash-es": "^4.17.21", "luxon": "^3.1.1", + "rxjs": "^7.8.0", "socket.io-client": "^4.5.1", "svelte-material-icons": "^2.0.2" } diff --git a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte index 03bbf9a90c..e6746f5ae8 100644 --- a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte +++ b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte @@ -8,7 +8,7 @@ import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte'; import SettingSelect from '../setting-select.svelte'; import SettingSwitch from '../setting-switch.svelte'; - import _ from 'lodash'; + import { isEqual } from 'lodash-es'; import { fade } from 'svelte/transition'; export let ffmpegConfig: SystemConfigFFmpegDto; // this is the config that is being edited @@ -130,7 +130,7 @@ on:reset={reset} on:save={saveSetting} on:reset-to-default={resetToDefault} - showResetToDefault={!_.isEqual(savedConfig, defaultConfig)} + showResetToDefault={!isEqual(savedConfig, defaultConfig)} /> diff --git a/web/src/lib/components/admin-page/settings/oauth/oauth-settings.svelte b/web/src/lib/components/admin-page/settings/oauth/oauth-settings.svelte index c191e516df..17599f2ff6 100644 --- a/web/src/lib/components/admin-page/settings/oauth/oauth-settings.svelte +++ b/web/src/lib/components/admin-page/settings/oauth/oauth-settings.svelte @@ -5,7 +5,7 @@ } from '$lib/components/shared-components/notification/notification'; import { handleError } from '$lib/utils/handle-error'; import { api, SystemConfigOAuthDto } from '@api'; - import _ from 'lodash'; + import { isEqual } from 'lodash-es'; import { fade } from 'svelte/transition'; import ConfirmDisableLogin from '../confirm-disable-login.svelte'; import SettingButtonsRow from '../setting-buttons-row.svelte'; @@ -202,7 +202,7 @@ on:reset={reset} on:save={saveSetting} on:reset-to-default={resetToDefault} - showResetToDefault={!_.isEqual(savedConfig, defaultConfig)} + showResetToDefault={!isEqual(savedConfig, defaultConfig)} /> diff --git a/web/src/lib/components/admin-page/settings/password-login/password-login-settings.svelte b/web/src/lib/components/admin-page/settings/password-login/password-login-settings.svelte index 98269ec7d2..9650f4ebab 100644 --- a/web/src/lib/components/admin-page/settings/password-login/password-login-settings.svelte +++ b/web/src/lib/components/admin-page/settings/password-login/password-login-settings.svelte @@ -5,7 +5,7 @@ } from '$lib/components/shared-components/notification/notification'; import { handleError } from '$lib/utils/handle-error'; import { api, SystemConfigPasswordLoginDto } from '@api'; - import _ from 'lodash'; + import { isEqual } from 'lodash-es'; import { fade } from 'svelte/transition'; import ConfirmDisableLogin from '../confirm-disable-login.svelte'; import SettingButtonsRow from '../setting-buttons-row.svelte'; @@ -109,7 +109,7 @@ on:reset={reset} on:save={saveSetting} on:reset-to-default={resetToDefault} - showResetToDefault={!_.isEqual(savedConfig, defaultConfig)} + showResetToDefault={!isEqual(savedConfig, defaultConfig)} /> diff --git a/web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte b/web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte index 4a698d7108..929c800959 100644 --- a/web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte +++ b/web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte @@ -12,7 +12,7 @@ import SupportedDatetimePanel from './supported-datetime-panel.svelte'; import SupportedVariablesPanel from './supported-variables-panel.svelte'; import SettingButtonsRow from '../setting-buttons-row.svelte'; - import _ from 'lodash'; + import { isEqual } from 'lodash-es'; import { notificationController, NotificationType @@ -230,7 +230,7 @@ on:reset={reset} on:save={saveSetting} on:reset-to-default={resetToDefault} - showResetToDefault={!_.isEqual(savedConfig, defaultConfig)} + showResetToDefault={!isEqual(savedConfig, defaultConfig)} /> diff --git a/web/src/lib/components/photos-page/asset-date-group.svelte b/web/src/lib/components/photos-page/asset-date-group.svelte index 93ee84ffc1..31dbc944b4 100644 --- a/web/src/lib/components/photos-page/asset-date-group.svelte +++ b/web/src/lib/components/photos-page/asset-date-group.svelte @@ -4,7 +4,7 @@ import CircleOutline from 'svelte-material-icons/CircleOutline.svelte'; import { fly } from 'svelte/transition'; import { AssetResponseDto } from '@api'; - import lodash from 'lodash-es'; + import { chain } from 'lodash-es'; import ImmichThumbnail from '../shared-components/immich-thumbnail.svelte'; import { assetInteractionStore, @@ -29,8 +29,7 @@ let isMouseOverGroup = false; let actualBucketHeight: number; let hoveredDateGroup = ''; - $: assetsGroupByDate = lodash - .chain(assets) + $: assetsGroupByDate = chain(assets) .groupBy((a) => new Date(a.createdAt).toLocaleDateString(locale, groupDateFormat)) .sortBy((group) => assets.indexOf(group[0])) .value(); diff --git a/web/src/lib/stores/asset-interaction.store.ts b/web/src/lib/stores/asset-interaction.store.ts index d9407cf6ce..7ed28e1258 100644 --- a/web/src/lib/stores/asset-interaction.store.ts +++ b/web/src/lib/stores/asset-interaction.store.ts @@ -2,7 +2,7 @@ import { AssetGridState } from '$lib/models/asset-grid-state'; import { api, AssetResponseDto } from '@api'; import { derived, writable } from 'svelte/store'; import { assetGridState, assetStore } from './assets.store'; -import _ from 'lodash-es'; +import { sortBy } from 'lodash-es'; // Asset Viewer export const viewingAssetStoreState = writable(); @@ -65,7 +65,7 @@ function createAssetInteractionStore() { const navigateAsset = async (direction: 'next' | 'previous') => { // Flatten and sort the asset by date if there are new assets if (assetSortedByDate.length === 0 || savedAssetLength !== _assetGridState.assets.length) { - assetSortedByDate = _.sortBy(_assetGridState.assets, (a) => a.createdAt); + assetSortedByDate = sortBy(_assetGridState.assets, (a) => a.createdAt); savedAssetLength = _assetGridState.assets.length; } diff --git a/web/src/lib/stores/assets.store.ts b/web/src/lib/stores/assets.store.ts index 6ec402236a..ce8dd7fda4 100644 --- a/web/src/lib/stores/assets.store.ts +++ b/web/src/lib/stores/assets.store.ts @@ -1,7 +1,7 @@ import { AssetGridState } from '$lib/models/asset-grid-state'; import { calculateViewportHeightByNumberOfAsset } from '$lib/utils/viewport-utils'; import { api, AssetCountByTimeBucketResponseDto } from '@api'; -import lodash from 'lodash-es'; +import { sumBy, flatMap } from 'lodash-es'; import { writable } from 'svelte/store'; /** @@ -46,7 +46,7 @@ function createAssetStore() { // Update timeline height based on calculated bucket height assetGridState.update((state) => { - state.timelineHeight = lodash.sumBy(state.buckets, (d) => d.bucketHeight); + state.timelineHeight = sumBy(state.buckets, (d) => d.bucketHeight); return state; }); }; @@ -77,7 +77,7 @@ function createAssetStore() { assetGridState.update((state) => { const bucketIndex = state.buckets.findIndex((b) => b.bucketDate === bucket); state.buckets[bucketIndex].assets = assets; - state.assets = lodash.flatMap(state.buckets, (b) => b.assets); + state.assets = flatMap(state.buckets, (b) => b.assets); return state; }); @@ -100,7 +100,7 @@ function createAssetStore() { if (state.buckets[bucketIndex].assets.length === 0) { _removeBucket(state.buckets[bucketIndex].bucketDate); } - state.assets = lodash.flatMap(state.buckets, (b) => b.assets); + state.assets = flatMap(state.buckets, (b) => b.assets); return state; }); }; @@ -109,7 +109,7 @@ function createAssetStore() { assetGridState.update((state) => { const bucketIndex = state.buckets.findIndex((b) => b.bucketDate === bucketDate); state.buckets.splice(bucketIndex, 1); - state.assets = lodash.flatMap(state.buckets, (b) => b.assets); + state.assets = flatMap(state.buckets, (b) => b.assets); return state; }); }; @@ -147,7 +147,7 @@ function createAssetStore() { const assetIndex = state.buckets[bucketIndex].assets.findIndex((a) => a.id === assetId); state.buckets[bucketIndex].assets[assetIndex].isFavorite = isFavorite; - state.assets = lodash.flatMap(state.buckets, (b) => b.assets); + state.assets = flatMap(state.buckets, (b) => b.assets); return state; }); }; diff --git a/web/src/lib/utils/file-uploader.ts b/web/src/lib/utils/file-uploader.ts index cffcfa55b3..a96865b1df 100644 --- a/web/src/lib/utils/file-uploader.ts +++ b/web/src/lib/utils/file-uploader.ts @@ -2,12 +2,11 @@ import { notificationController, NotificationType } from './../components/shared-components/notification/notification'; -/* @vite-ignore */ -import * as exifr from 'exifr'; import { uploadAssetsStore } from '$lib/stores/upload'; import type { UploadAsset } from '../models/upload-asset'; import { api, AssetFileUploadResponseDto } from '@api'; import { addAssetsToAlbum, getFileMimeType, getFilenameExtension } from '$lib/utils/asset-utils'; +import { Subject, mergeMap } from 'rxjs'; export const openFileUploadDialog = ( albumId: string | undefined = undefined, @@ -46,25 +45,20 @@ export const fileUploadHandler = async ( sharedKey: string | undefined = undefined, onDone?: (id: string) => void ) => { - if (files.length > 50) { - notificationController.show({ - type: NotificationType.Error, - message: `Cannot upload more than 50 files at a time - you are uploading ${files.length} files. - Please check out the bulk upload documentation if you need to upload more than 50 files.`, - timeout: 10000, - action: { type: 'link', target: 'https://immich.app/docs/features/bulk-upload' } - }); - - return; - } - + const files$ = new Subject(); + files$ + .pipe( + mergeMap(async (file) => { + await fileUploader(file, albumId, sharedKey, onDone); + }, 2) + ) + .subscribe(); const acceptedFile = files.filter((file) => { const assetType = getFileMimeType(file).split('/')[0]; return assetType === 'video' || assetType === 'image'; }); - - for (const asset of acceptedFile) { - await fileUploader(asset, albumId, sharedKey, onDone); + for (const file of acceptedFile) { + files$.next(file); } }; @@ -75,25 +69,15 @@ async function fileUploader( sharedKey: string | undefined = undefined, onDone?: (id: string) => void ) { + console.log('uploading', asset.name); const mimeType = getFileMimeType(asset); const assetType = mimeType.split('/')[0].toUpperCase(); const fileExtension = getFilenameExtension(asset.name); const formData = new FormData(); + const createdAt = new Date(asset.lastModified).toISOString(); + const deviceAssetId = 'web' + '-' + asset.name + '-' + asset.lastModified; try { - let exifData = null; - - if (assetType !== 'VIDEO') { - exifData = await exifr.parse(asset).catch((e) => console.log('error parsing exif', e)); - } - - const createdAt = - exifData && exifData.DateTimeOriginal != null - ? new Date(exifData.DateTimeOriginal).toISOString() - : new Date(asset.lastModified).toISOString(); - - const deviceAssetId = 'web' + '-' + asset.name + '-' + asset.lastModified; - // Create and add Unique ID of asset on the device formData.append('deviceAssetId', deviceAssetId); @@ -160,20 +144,18 @@ async function fileUploader( }; request.upload.onload = () => { - setTimeout(() => { - uploadAssetsStore.removeUploadAsset(deviceAssetId); - const res: AssetFileUploadResponseDto = JSON.parse(request.response || '{}'); - if (albumId) { - try { - if (res.id) { - addAssetsToAlbum(albumId, [res.id], sharedKey); - } - } catch (e) { - console.error('ERROR parsing data JSON in upload onload'); + uploadAssetsStore.removeUploadAsset(deviceAssetId); + const res: AssetFileUploadResponseDto = JSON.parse(request.response || '{}'); + if (albumId) { + try { + if (res.id) { + addAssetsToAlbum(albumId, [res.id], sharedKey); } + } catch (e) { + console.error('ERROR parsing data JSON in upload onload'); } - onDone && onDone(res.id); - }, 1000); + } + onDone && onDone(res.id); }; // listen for `error` event