From 1b54c4f8e7ffac7b8ae53203e5608327b051400d Mon Sep 17 00:00:00 2001 From: Jonathan Jogenfors Date: Wed, 24 May 2023 23:08:21 +0200 Subject: [PATCH] feat(server): Add support for client-side hashing (#2072) * Modify controller DTOs * Can check duplicates on server side * Remove deviceassetid and deviceid * Remove device ids from file uploader * Add db migration for removed device ids * Don't sanitize checksum * Convert asset checksum to string * Make checksum not optional for asset * Use enums when rejecting duplicates * Cleanup * Return of the device id, but optional * Don't use deviceId for upload folder * Use checksum in thumb path * Only use asset id in thumb path * Openapi generation * Put deviceAssetId back in asset response dto * Add missing checksum in test fixture * Add another missing checksum in test fixture * Cleanup asset repository * Add back previous /exists endpoint * Require checksum to not be null * Correctly set deviceId in db * Remove index * Fix compilation errors * Make device id nullabel in asset response dto * Reduce PR scope * Revert asset service * Reorder imports * Reorder imports * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Reduce PR scope * Update openapi * Reduce PR scope * refactor: asset bulk upload check * chore: regenreate open-api * chore: fix tests * chore: tests * update migrations and regenerate api * Feat: use checksum in web file uploader * Change to wasm-crypto * Use crypto api for checksumming in web uploader * Minor cleanup of file upload * feat(web): pause and resume jobs * Make device asset id not nullable again * Cleanup * Device id not nullable in response dto * Update API specs * Bump api specs * Remove old TODO comment * Remove NOT NULL constraint on checksum index * Fix requested pubspec changes * Remove unneeded import * Update server/apps/immich/src/api-v1/asset/asset.service.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove unneeded check * Update server/apps/immich/src/api-v1/asset/asset-repository.ts Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> * Remove hashing in the web uploader * Cleanup file uploader * Remove varchar from asset entity fields * Return 200 from bulk upload check * Put device asset id back into asset repository * Merge migrations * Revert pubspec lock * Update openapi specs * Merge upstream changes * Fix failing asset service tests * Fix formatting issue * Cleanup migrations * Remove newline from pubspec * Revert newline * Checkout main version * Revert again * Only return AssetCheck --------- Co-authored-by: Jason Rasmussen Co-authored-by: Michel Heusschen <59014050+michelheusschen@users.noreply.github.com> --- mobile/openapi/.openapi-generator/FILES | 12 ++ mobile/openapi/README.md | Bin 16861 -> 17241 bytes mobile/openapi/doc/AssetApi.md | Bin 55292 -> 57539 bytes mobile/openapi/doc/AssetBulkUploadCheckDto.md | Bin 0 -> 495 bytes .../openapi/doc/AssetBulkUploadCheckItem.md | Bin 0 -> 448 bytes .../doc/AssetBulkUploadCheckResponseDto.md | Bin 0 -> 508 bytes .../openapi/doc/AssetBulkUploadCheckResult.md | Bin 0 -> 531 bytes mobile/openapi/lib/api.dart | Bin 5220 -> 5421 bytes mobile/openapi/lib/api/asset_api.dart | Bin 49337 -> 51195 bytes mobile/openapi/lib/api_client.dart | Bin 16998 -> 17414 bytes .../model/asset_bulk_upload_check_dto.dart | Bin 0 -> 3471 bytes .../model/asset_bulk_upload_check_item.dart | Bin 0 -> 3651 bytes .../asset_bulk_upload_check_response_dto.dart | Bin 0 -> 3647 bytes .../model/asset_bulk_upload_check_result.dart | Bin 0 -> 10600 bytes mobile/openapi/test/asset_api_test.dart | Bin 5272 -> 5506 bytes .../asset_bulk_upload_check_dto_test.dart | Bin 0 -> 639 bytes .../asset_bulk_upload_check_item_test.dart | Bin 0 -> 685 bytes ...t_bulk_upload_check_response_dto_test.dart | Bin 0 -> 667 bytes .../asset_bulk_upload_check_result_test.dart | Bin 0 -> 883 bytes .../src/api-v1/asset/asset-repository.ts | 45 +++-- .../src/api-v1/asset/asset.controller.ts | 15 ++ .../immich/src/api-v1/asset/asset.core.ts | 2 +- .../src/api-v1/asset/asset.service.spec.ts | 4 +- .../immich/src/api-v1/asset/asset.service.ts | 48 ++++- .../src/api-v1/asset/dto/asset-check.dto.ts | 19 ++ .../response-dto/asset-check-response.dto.ts | 20 +++ .../check-existing-assets-response.dto.ts | 5 +- server/immich-openapi-specs.json | 115 ++++++++++++ server/libs/domain/test/fixtures.ts | 5 + .../libs/infra/src/entities/asset.entity.ts | 6 +- .../1684328185099-RequireChecksumNotNull.ts | 19 ++ web/src/api/open-api/api.ts | 164 ++++++++++++++++++ web/src/lib/utils/file-uploader.ts | 21 +-- 33 files changed, 442 insertions(+), 58 deletions(-) create mode 100644 mobile/openapi/doc/AssetBulkUploadCheckDto.md create mode 100644 mobile/openapi/doc/AssetBulkUploadCheckItem.md create mode 100644 mobile/openapi/doc/AssetBulkUploadCheckResponseDto.md create mode 100644 mobile/openapi/doc/AssetBulkUploadCheckResult.md create mode 100644 mobile/openapi/lib/model/asset_bulk_upload_check_dto.dart create mode 100644 mobile/openapi/lib/model/asset_bulk_upload_check_item.dart create mode 100644 mobile/openapi/lib/model/asset_bulk_upload_check_response_dto.dart create mode 100644 mobile/openapi/lib/model/asset_bulk_upload_check_result.dart create mode 100644 mobile/openapi/test/asset_bulk_upload_check_dto_test.dart create mode 100644 mobile/openapi/test/asset_bulk_upload_check_item_test.dart create mode 100644 mobile/openapi/test/asset_bulk_upload_check_response_dto_test.dart create mode 100644 mobile/openapi/test/asset_bulk_upload_check_result_test.dart create mode 100644 server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts create mode 100644 server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts create mode 100644 server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 094bcbe40a..eafeb6851f 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -17,6 +17,10 @@ doc/AlbumCountResponseDto.md doc/AlbumResponseDto.md doc/AllJobStatusResponseDto.md doc/AssetApi.md +doc/AssetBulkUploadCheckDto.md +doc/AssetBulkUploadCheckItem.md +doc/AssetBulkUploadCheckResponseDto.md +doc/AssetBulkUploadCheckResult.md doc/AssetCountByTimeBucket.md doc/AssetCountByTimeBucketResponseDto.md doc/AssetCountByUserIdResponseDto.md @@ -142,6 +146,10 @@ lib/model/api_key_create_dto.dart lib/model/api_key_create_response_dto.dart lib/model/api_key_response_dto.dart lib/model/api_key_update_dto.dart +lib/model/asset_bulk_upload_check_dto.dart +lib/model/asset_bulk_upload_check_item.dart +lib/model/asset_bulk_upload_check_response_dto.dart +lib/model/asset_bulk_upload_check_result.dart lib/model/asset_count_by_time_bucket.dart lib/model/asset_count_by_time_bucket_response_dto.dart lib/model/asset_count_by_user_id_response_dto.dart @@ -236,6 +244,10 @@ test/api_key_create_response_dto_test.dart test/api_key_response_dto_test.dart test/api_key_update_dto_test.dart test/asset_api_test.dart +test/asset_bulk_upload_check_dto_test.dart +test/asset_bulk_upload_check_item_test.dart +test/asset_bulk_upload_check_response_dto_test.dart +test/asset_bulk_upload_check_result_test.dart test/asset_count_by_time_bucket_response_dto_test.dart test/asset_count_by_time_bucket_test.dart test/asset_count_by_user_id_response_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 2c317643255b024500277996baa05928f5ba8100..b6b3ea18df60ade0306ec7246a5d6ef2d4e6557b 100644 GIT binary patch delta 321 zcmccH%y_enal;IL{-n~J?9hUo{KORJjMU`p$%R6qCLpm=uvjukOjDsoK}##ZKR857 zOF=)exHz>$AEZPVqDB|2XmX&C$Rq*Q&2#zV73!T(4RtBWkJU)YPuACW1Zl>U)yqxM z|LwO+O#1^^@-zi1VXaI=Rq5(hN;3D7CmWrv#5x5Cxmx>Ax2Q E0EAO@3;+NC delta 19 bcmccF#(1}xal;J$%`5^b3Y#ko9|{5hQqBjm diff --git a/mobile/openapi/doc/AssetApi.md b/mobile/openapi/doc/AssetApi.md index 2e5a4641fdd183813eccb5175773a439594ebe5e..fbe6d02a70cda1c6ac511693b5a3c604faf86842 100644 GIT binary patch delta 494 zcmeyfp84=W<_*@2dP$`@*`WnF`H3mc8L7$HT3WFhj>W~PC5{D|dbugeAn8)DbTUYK zav`ILAV^FXBB%=%-E6|xB~XuH5|^C<*eEAdD}qvs3-a@dQ(a2(70~o(B!a9)Qx8<9 z$pr*p_Z2H-rYV3`6)U7xWEPhwBvmSa9l=sunp?biVyG0$!}dq(8=`%!F;F+Cr?=4HaXss)f3Y#T3QM<3ej3x*n$Y?Ab3Qe h>6|QoQk@@N%<-BCAF9|Ut@(_T7f7*gJ}hW?3;-aSwdMc- delta 19 bcmX?nkonJg<_*@2o2N213v6zcvO5L42AFh6@nbv7@Xd>F-Vsl3T-K4JB2_@tga@BZEWQ**pHu`71oW7%@?Dm zr%&i*U>zFibTL|mLs5o+w4R3k*$q~zhckA4MV7HFgIyGW&Iq#=oa}n}f~;*@B|R}c zI_n7*7yae}UL|-kEN+u^CLMijdeRu!L5>J-@bv1^S7GS9cBHL|)xlD9h^rUHt4fH! zYR0)t7X{9vwdQ=M1Ks^}ErhTS|!W)Fft-efIUMbKQr_n2=U z4TOkZMJrt}ILJ06{ z?84t(k{30-8fJIZIwkE>5ffR=Y^NN;J3PO|`mHeQWFy(&3)%%L4srDgd4+`d3p38G z;k?*5f1+glX#xcoMc3`}=ziDP;jkk`*FJV9N`jGr}wZr`k@vp=g>$ zD^J1<&U%8y1;4ultYSzcEbb$9b~<=odNLTqUJVEz@bd2BS7K=Ugto)im!EVSGbK~>j6RMnbm ziQq2w3O&cw^)k;_$4yc0Hf)e++fD(hrNT Kz+d615aJW0`J+hy literal 0 HcmV?d00001 diff --git a/mobile/openapi/doc/AssetBulkUploadCheckResult.md b/mobile/openapi/doc/AssetBulkUploadCheckResult.md new file mode 100644 index 0000000000000000000000000000000000000000..670d1d9fa4151ff27dc971daf63de47557c11455 GIT binary patch literal 531 zcma)2L2AP=5WMRZ3qGU~$ho&Cr^zAEgpjtUV6d<@iLfjQX?-a4@s->_-O|wNB4%fH zW|;z!E~?QIH?{l*mehugCWL>0-n+g=z=h+!OOOjXJk#&C>e<9 z-n&3ByUy>g-iz#?1+#}N9i8?ek%2Tuwv#==JG{L4{H-vyL3`59CbV;u6k_tyaDs&R z3p38O;ylSXf2N@Ac?1b3Mz%TU78_8;8|=}I7fMr)Ho3LU?m-H~-vC)eE%BJ8kKk#b@A;@Hu1b E1Nc~@WdHyG literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index e22e9ac694e103318a75cee5fb7cbf271858df21..833628d62e01cbd45031d5c7b4c50fb9c7176c90 100644 GIT binary patch delta 128 zcmaE&u~ut?4Vz$6X-;;0X+chYVoH2+Mrv~QT%)a2~q(%j<7`)raY+plHS za4OBo4lT&ZPfT$J$+?u|$3k?WNla!GQuatfRUDpKlHpNOQs9}FmapLmvIWI z1$x5XP65s9q1;mesmElBCV}KGbsbG$XenTG{p1M}CY#@I HoZSupEmX+% delta 14 Wcmey}&%CpddBaDJ&Bg*JwgUh)PzLD$ diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index e159f6057899c3bd5fc6a82aeb4174af2832b1f5..7b46ea15f1ec2b71e592d61cc116ccd9fc3c702d 100644 GIT binary patch delta 220 zcmaFX!r0cqxM8|1zf);Wc4$FPeqxGqMrv~QWJ4JVq2l7y5>)ZY7nM{b95KbSJxfw^ zCo3w7h+vbP{7z2=yF{b3f-n}-CU2C{k;Eb%lv-SxQv$L_5}Q3>>B$#m%{Q-*-KPQo DkH}Lj delta 18 ZcmZqcV0_lXxM8~NWL^dK&35u9Q~*R-2BH7} diff --git a/mobile/openapi/lib/model/asset_bulk_upload_check_dto.dart b/mobile/openapi/lib/model/asset_bulk_upload_check_dto.dart new file mode 100644 index 0000000000000000000000000000000000000000..6d5fdf5142b9c458cb8282bdc9ba277b815479e2 GIT binary patch literal 3471 zcmbVOZExB-6#mYyIGv`FN;KT{)6_0(y(_(Hw-u@DK3pLbnZy*Y!OpdvA=L5T?>Tl1 zVIx~tBnYw3`+3e|_xqiGAI`trjgJ2r-VATg?}is}dG&GFgNqT|jPBsm=;G?~-vbmQ z$#*%^HaUr3opk6`+)Jf-I#W8GiGrU(B`d@8lox!#rR{}xu_%?cdo@_GWGj=Fl}$DO zuM!&7C0pQc$uvGK*9MJivtK<^(pV;KDsprvW`b*{ZuUAWgybeIl)OhVJ7+R|{a2dJ znKqpcy0f5Wph_-TDPsJdbvjwWjDd4wxc$2-9=?=?V)^?y&mJzUg2xU(AM0?-ZKb6F z29hsf+#)^!;eyS0jCw6X5!nQ=b75k1&8>k1T$V7N^l)!|gkb@E6vn>UXXDE9#Q?mj zlYFPXajmgFs5MoppbRy$N<)%Fqv=0X2p>M3HxPh5gi`wMa zEg>XlD9|P7jxpIu4`fv!LMBDJnUvs_;t*Pqr8uv1l~aTK zLL188x=>sh>$3U~LrpQN$OTAv!??maz)rQDNSrYog&(^Vc|6c8Vf;gBD$TLr%;FD+ z1uJQ&EW?R(3xRv!fv*gWXRUanEQeR8PNps*;jn8VBNE^_Ao)@3A{TZAQ@&DI`Y9*i zNhOb|nm?L5A#<2lMxe*1L7bK)ni3K;QYSMQAnqO1VGv@pjW&Rz&_cW)C|nF-$g+79 zVes2Zkyx!t;EY4QsNw{B(7ml0m$HTz=lfZ1{1SI3FDTN5ForTC9KrSH(M{aNGLs|! zX}BAc5*#EBPk-tPXPUT_kXWNY{_Kv3FgUIU$LflQ#dUPFg+RUEl3}#5LkU=Ox)-)Y ztZ{*~dx_aQ=u%l56MaPVOrJWiwR^2yQ!<;Gu7{(Q!;y~$u;VIB3AyaSqvP>=t(I(B zAaTNs6^8fLq4|DDG&faYiIiIM9*ifxS~PBCnR-k?!Qr7Jp;#CkfZ?LF4O|i6quPU7 z5@<;^xTgI5rte&`x0aY-Odz^O%?Q(Iv~oN8HF%nq3HX#$%<|k5L_^cpn`o1Q*r|T4 zWff%?g~FXe_~)!Rx~*H&B>uMQ5sPfF&}>kM*w=x$rg1Q|Sqle6vlke}FI(YymiSh+ zu^`c(zo(tT=1P?!&~cG2Y5p|Drfu7bt9lh08`GQMCM5NY20MGP!$YH8ziBjTQx13=GU%2jh57So3kiQ^56U0# VnBKN*@aU=%u7d7Xf7U_z`5%WhXG#D7 literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/model/asset_bulk_upload_check_item.dart b/mobile/openapi/lib/model/asset_bulk_upload_check_item.dart new file mode 100644 index 0000000000000000000000000000000000000000..89ee3cf341e66446838a9864cc6e8b5fafeb2eb7 GIT binary patch literal 3651 zcmbVPZExE)5dQ98aVdgY!5n4Hry-fWW{o?fFENm&0|vtoXoma~z5VY1 z#YpmX#Zx z&A*mHqq<~E{4AKpZ_Bko$Y2HKj;Qxum_#5ylJf$!a0zAhPCL zkaKIV)i#tF_oLDD3s12ScER-#^ap5&3Y)S#qBweC=lgg61mY*< zdHK@C>?JV;bb$6?cR9pHo8kFEoI+Zda|;V*7H?F>F*0Ym1|eZj@{}9H^qR1PQ7^a= z#)(BDi-YPB{N~q!v+d3Uw^cE%@(+=#NVlzVW0Ca_u~Bsz{lnm6(si~!vf@u~st4H` zaW2vUSoO{*iMD;;XF{?ZzTgt{;equ1=G@oRNsM(@WZel0Rv`IdGq~T1gcUES9d4Z| zTG$YPo`q*%G6PR@1Hu~k#`2OIm}#}FY>wR$QecJxUE<0Z%Q5MJEOW#l0d6IXV@9K^ zh2}FPtK+#YQ$(pr2)_x^9$}f_gjlGIy6hKPS@oBT#Z|hhypJ)I9m9%TgoJ1eEUW|U zG{lL-DKqv#Ya&lT;w~mIl%~`i`@$@Ka9FYeFbm6Y1h|Eo-*?KM zDiWHz*29Wtfb*7O7n!hYnDVv4+D|#LUMhJ^<^0jyDVf8(Qlbui8ypfvfu@ASIg@i{ zFhJZpsKQ`4@U)F;Oj2ke-uDzPhA?F5B8o8hZKX)8QYHMxAzxH+f<5ToR6`z65ni-1 z=@7RkFDTTdFoseiB*D8+qwBbfbtX^#ui+oYL{!EeIF=tB_!EHK2Npl77X-Uo z?J9$fb6>?sS=R%gBGLze7;g)H06A{I#h&ai%7=UJPL;Vlt>RKtwrhW9oBD4tN7 zn=-dF<651JC%&-6O)^b*ps?@+aWg9x2H7%P7PbLGVnsRrLvJ-_Lj+;1Hi zF(J-nhlw=Wz8w@5(we4;CX`mpvdoW@hNdq$jaiCfrva6gRg_*7N_Pg~$yssZx#|v1 zsCodk){juNw?`plUkMsbjiaX3TsV80?Z7yG+6~vXzVBAs)TD9guaaj_-}(}Uyq`YuIhE@ZA@>1n~>FWdh86w4ik-zJ<@9|+}JC+$DI@I`@wPE!HM)gG(4I( literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/model/asset_bulk_upload_check_response_dto.dart b/mobile/openapi/lib/model/asset_bulk_upload_check_response_dto.dart new file mode 100644 index 0000000000000000000000000000000000000000..9a0ed965f40c86e65083f7b5b458946a0157b8c5 GIT binary patch literal 3647 zcmbtXZExC05dO}um{wJAB;s82sY(jhTtiMx+aRUshZ929ViwrO?D}@sMJVOJ-^{*% zX`#-wNML(C@AJ&F%js$B^b{^X-w#gy?%($BF7NwSaQ*H>zXMkTxEwli1 z7)idFFm1bM>C3YgJ&KuBnrBm`v#BWfIaIPTe3J2kFSxXw_$(Hs(sr)~E0%0!va+(7 z=Koegqq<}Z{4JTrtL56Dac%ai=SmvOq|L+x9g3;o+Nqnp&I%#9$qFTBC}!tOX0QIp z@;TF{)k1d`)D%?7B`ZaW--}i&FPJfKX$-f2RmH>SvQTXDcFyyMJ8nvJ&abS3#}>dT z*5i)bN=pL_Bwxa?L2U%W1)uX2^&5l}@&RDy!las;s;~yS;L?ZTsDm%p#~3fb2Vv~% z1IAFR2SM8jzfoVg)>u1K94l2&_L^Cxq1#Oc<1akNn%Ftl$I$J8uW`{=H{ZVb zI}pv4Iy!P`c!g~u-GfmHX<^1K%$b?LRTGYJI13x}5$Gh(xiL(y2sDU;IXA*Mu_R=h zMtxtTdfuO`7IFqw`J2S0+CEUh*z#ep+kFvkMr~KnkF0#{?Q~$WLLiI0ho*0gl4#iX za4IA#;0rEc2Y--$TwnMSIfr$!t6sOK1R#!P<%w&YV4CmL1 zJi}3+s|j_@FEkCrpDrX<*idL6YOFFwm$*O)mKa}L2iU2<6G?N1L-T##5|0%cJItdm zO{F<@fLZ+EuwW&PpJh12ZXwbaJn)sFi4Kal%5r#l?qotS38ZZcIq?Xe0P-irE+)dR zV9ZwvTQKGXN~z=tRrPywXJiiZ%4s6-Y7oO^iKc|aNh&xq7$ClT7Q$diVA>`d2vTS< z-uDzPhR|pEJV`M4ZKX&os1jV$m@le0!49#!!ZYE4cYI zxJ}zwZgS{9HMc`jf@fWauU~bE^G~!)2o0!^Ke{CfjRAzgvby4Nah)7*Aram;WSrFO zInq8qglpvHv_S5@#)3FhQ^o7!e@rA#pAh8QJ%%tSvCaI40s7fa5P7TzJ3iwik?#&X zx-DU*)sl@1WL}uE!tl;I0>lrJ=O?1FhFbK7BVS1xP4YlEpdjJs$Prj73=T_wQQ8^? zX*iPLNi7NBq#8q@dia;VtKCE8jR~g&xEr*PU|O|yZ|Ncct?q$_g}Rrr$;6XNP1E;* zsF#A-=}OSDin2^%Aq0zL+h&@q#_gxxBaG^& zEpol=`W7ByQle0QTs()($u46^0y9ZxMZaBRGq>%=RlSN0Pw7c;6O;Rm#yh*I!%MA= z4FJG04!xpdqpoL>$PBUX@w$ToRNagwRskQ0id@l@KVmI~o#PUXA{{{+dUWEo&`8*w z@qmHabTlX92*W*SA_O)LXpi(saxbo)N(gA9j<~0J{uSIk5mHZqXQ0;qgFZ}i7ZD9@ z+hz9}2s;}-*fK+BgUmg&f}U%DI)!^YjzsLlUEQ5R@Q{F~l|_ZH8gsxCm_fHRQOxgs eTd4Ry5>fd*9tpNB>^-{bY!0IC&&EXydFwx%ys=6E literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/model/asset_bulk_upload_check_result.dart b/mobile/openapi/lib/model/asset_bulk_upload_check_result.dart new file mode 100644 index 0000000000000000000000000000000000000000..016342de0a293a529de48d3b6a777393466e9ac9 GIT binary patch literal 10600 zcmeHNZEqXL5&o`Uu`+~^Lg`cDPlZxSQEj(P+Y6B(zy{*9R2St>V z)I@30Tk}-LWm@F?#xB!JnuQ10geotr@Y5l-CB(;4r}>>o6Z@^*jWetJJg=sL&1_kJ zZB5hqQ}|A9SOJ6xMk!D~W_YSZUX-Yp8*ECDU>1Ah;f`%i9$LOHI|g9&oZl&=vIcdO-f$5 zH^;AP4-R``Qaq^7O84xNJC!RfOO-hFrl>Srhzn?*R;CnVnPuWu#`mHu$Zyh(3UDrq zaz3+JrhUIfrJ_i+kV(SJ);IvaTQ+D7Kpw9VGKwCR4~La=c`96%ZNR@PunP?Vthz0V zjOrR&DYWSKyH~eAsTkFcFz7?kKS3GL>4|05CG|c3=I5XJlYq(g5)og01ta)l^`S;# z0veYdbM??G_i0gzYlU9BPm8bJ)~qkQGEoG>GV%@57ir^_%u`)*L zIl&9|JW(b!c3C$~A}@L<{=m=J)z#<0TgtUK6<>>E=lK5W>W1@;WO3y7*6986yEnZ z#jwqQG){AwiJ$OD^u@~R$ER<(7VL_F?xaa)h=mrG2yASj_?PTnSis+TQ92g37$XNLx?isR@-H@T^AbBX4t(CmfeFbrki5R zwzX{{)=ARxL_kg0@q*?eWS$7_yGEp?Vf-^7jR#XuA@BCHvq4+BC*XxWn@p)sdo?$N zuTJn1a4Wt!wri|41PkcVDMd_FKqUech8zh8*h zg+0-^cDzeKP`o-LL|Dsp1_xL$d_~P*ni|_jshY(1AFnQ>4kAsq_)o*)fQ0-ZdVw`y zvTLcXg(E&I7Kzk&+wKsRx>Ek(Q}-|wf_vY-&u*DW|F>*T70FzDB9RsXm|1%En=#Q`~h%iU&H0BUnDsC?W%2Ly`keO@S6Jq`^s*1#PhF!*?wFC#$8IBVcI_ zmF;^tq1s=YDIXDz2Ek~93T#m5*+DZY@MU98Z1Vb?vo?$;aOb-y`jfvljwCSZE~kqq zGdyH3chVUlifzXWFWaY;831wk6{T-hc%yPWBn(JaG9$`+pbG4pCa? zSOO)BkkZDhV1oaUmJ>Xp6GvCu_RPFy0mwU|K-Uf2o#;*sy|Cg=_lHL1P_=D3g0_BW z(b^j7iQP^+%r~oRtI?O#*pB71ODWpZ`MgTQ4i`(HJx>v(vJPjX;Cl68568Y&SK%7W zfYCN@eXf(QW-BhV##+r3r9{tV30!I9jCLv2_}xWGw5;!YlIV5nVkFY?Wf zzTwV*BHlV{37xZ^x?J$|C&e8d{?Oz_(WJuGJ>Bs!TKJ5+Z6X}kB@)JNw#PV_iAH<_ z@kbiM#KtL9RgSBCHYb9|%>c*C(m88?s;0L}lQBjScaO6xce~_;rB1fKh2gdyUb<7s zTat0CX644Vw6HfK93;WiZ9$b*c0jPH!!4;|2~y>vL`A_6qz%&aNKFvI>3LF8u?pq@kbSpo^EX9#Zb6H+lvbFO{;z=CP!~ z!2D%Lfd-PJIu@s={bxY-k~2VXMx@8%cmSOE&g>$ zCFfcfnm0MYH7j2X1-0oVhcKp^V?LZGbMZjCfyT2RFEGTB_(cf0!0BnQ2%8VuEUsgq z5V$*NHN~jQ{)|Eqq4oH^OJfDbN~TN^8~%L(iVOUbA!oC|Md$$?b@FnT#GblrbbWnL zVR+VD=NRptGF)Gg>AGPL{L1~(fEC3Pe%rk0;8LY9$Z%YM#@O^r)=n-)EJQ?-|2SQa z3_MR43k6;N=b2i$ph;zSzutOt6J)-mA`oZ#uA)1iEsk#IQ+}VLG?*81MuC}J=f~oJ zLbW)$Vy?XQ!nH8&paQ@zQg@juFk+UU8~RNxq#z5~`dAHVr_9AZrECO`0svD$R## Tz~qktI+OdE**4cO$p`@eP&`2b delta 12 TcmZqDo}sy+hG}yevy2b`9k>JM diff --git a/mobile/openapi/test/asset_bulk_upload_check_dto_test.dart b/mobile/openapi/test/asset_bulk_upload_check_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..830cf2e29e5514f12e7ced34fa84d2f5bbb33b94 GIT binary patch literal 639 zcmZ`$O;6)65WVMD%%0jxg#ahSm*{Q^s>Birl>=I>%qW?(Rvp{e6IRuV|IXNj6XoFf zklvd&Z(No|S;FjbSs&ca@8=J*<-CHM#oc@gRSoy`5`Nd!;^ygxz!LJh#lV**hsP&H zYW2#xK&o>=b=uGw40edrDl*iQ<>_ayd+!48WuW14hO^Vie`}S z%-4lKe4!1GQOg{ArKTlsa15R33>Fj{Y*E<3vFELl+MXm>^TBq8s?Ji4ZynbT7p!$vG3{=P+PLA` zoW^r0jAg=7UN}Qua?}AgC(Rlz(9lMT%E@fa1iiVUdCjyT9ASDE?)4|cA(fV$>P6n{tTY?k~#Kp|gBn|lf5aZ|I-0x?)_RsN0;r~qBxL~#?Ux0qymjD0& literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/asset_bulk_upload_check_response_dto_test.dart b/mobile/openapi/test/asset_bulk_upload_check_response_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..1af1fede086cb3ce9da556ba25f9476de90593bc GIT binary patch literal 667 zcmah`O-~~+486~3mkf`NidI3{?%+^&GD1>f-$FKwt^^uf-5gjt+kv z6{*!r>jJ6H1=VRo$I#oJsa0fXLpF|gz20~i;$8-tXWX*#eN=%SdmV&saU;=VkQ^zn z3=h1Y-Z;wBT%lFJf2wAMfs5jOM;Lf)*0`i&=gFei zrM$!BdNBC}iNvu?VFyT}sb`Rrz?7NdX>ZoI-Z*T}R@AI-h`p>#RdmIZ(=BXJ+u;aa z3IHK@+M)rSJv9U;-_00~CWSDU!S#MPNYdXi24nb;%Yv7hD7~&Qd9}^bu%E(*WWV?wcmgjX<=NYfbi&Qzk F7H_^|;FsIK-TErQinDQFjJJ$Q=4?sPjeO=6N&iuAvm>{bvgwRqTU z2HyAH%p^^dG=<^wJUhJ~Psg+2d_01?$-}q{qYS3m93HdLz>uL6X*_eD}NNwb9n^B~X6g#tBn-Vd<@68*qi|a2mAX@5#Dg zZry%7*UEWRUT_ftdBJ3B-0cmMtjGzeRT*S{iAr3ai+qXJB}vmGv<&7eEUB*zDKwmh z?<`K;8vK%Yaz2g6w;+bb+Cf+e44or?Q_0n{ks6EJCFQFbxk~!vU2_3zWYzA#X95to z1uK-m^*A}f)d9TS86*KJt*%VF6$1~5TV2?N)G<%qvMPa#U;uTq!9mv-@Jj+?!{F9$ z;FXKlhDEngdEI72LaXy^l-2mQY2>Fd=AO0s)4Y-WC%_WABLW)Pe*!Q*+vM;4>~zOP KgoV`0+9qGFuNGJU literal 0 HcmV?d00001 diff --git a/server/apps/immich/src/api-v1/asset/asset-repository.ts b/server/apps/immich/src/api-v1/asset/asset-repository.ts index a5500a020c..cf033c783a 100644 --- a/server/apps/immich/src/api-v1/asset/asset-repository.ts +++ b/server/apps/immich/src/api-v1/asset/asset-repository.ts @@ -10,13 +10,17 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto'; import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto'; import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto'; import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto'; -import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-assets-response.dto'; import { In } from 'typeorm/find-options/operator/In'; import { UpdateAssetDto } from './dto/update-asset.dto'; import { ITagRepository } from '../tag/tag.repository'; import { IsNull, Not } from 'typeorm'; import { AssetSearchDto } from './dto/asset-search.dto'; +export interface AssetCheck { + id: string; + checksum: Buffer; +} + export interface IAssetRepository { get(id: string): Promise; create( @@ -38,11 +42,8 @@ export interface IAssetRepository { getAssetCountByUserId(userId: string): Promise; getArchivedAssetCountByUserId(userId: string): Promise; getAssetByTimeBucket(userId: string, getAssetByTimeBucketDto: GetAssetByTimeBucketDto): Promise; - getAssetByChecksum(userId: string, checksum: Buffer): Promise; - getExistingAssets( - userId: string, - checkDuplicateAssetDto: CheckExistingAssetsDto, - ): Promise; + getAssetsByChecksums(userId: string, checksums: Buffer[]): Promise; + getExistingAssets(userId: string, checkDuplicateAssetDto: CheckExistingAssetsDto): Promise; countByIdAndUser(assetId: string, userId: string): Promise; } @@ -310,41 +311,39 @@ export class AssetRepository implements IAssetRepository { * @returns Promise - Array of assetIds belong to the device */ async getAllByDeviceId(ownerId: string, deviceId: string): Promise { - const rows = await this.assetRepository.find({ + const items = await this.assetRepository.find({ + select: { deviceAssetId: true }, where: { ownerId, deviceId, isVisible: true, }, - select: ['deviceAssetId'], }); - const res: string[] = []; - rows.forEach((v) => res.push(v.deviceAssetId)); - return res; + return items.map((asset) => asset.deviceAssetId); } /** - * Get asset by checksum on the database + * Get assets by checksums on the database * @param ownerId - * @param checksum + * @param checksums * */ - getAssetByChecksum(ownerId: string, checksum: Buffer): Promise { - return this.assetRepository.findOneOrFail({ + async getAssetsByChecksums(ownerId: string, checksums: Buffer[]): Promise { + return this.assetRepository.find({ + select: { + id: true, + checksum: true, + }, where: { ownerId, - checksum, + checksum: In(checksums), }, - relations: ['exifInfo'], }); } - async getExistingAssets( - ownerId: string, - checkDuplicateAssetDto: CheckExistingAssetsDto, - ): Promise { - const existingAssets = await this.assetRepository.find({ + async getExistingAssets(ownerId: string, checkDuplicateAssetDto: CheckExistingAssetsDto): Promise { + const assets = await this.assetRepository.find({ select: { deviceAssetId: true }, where: { deviceAssetId: In(checkDuplicateAssetDto.deviceAssetIds), @@ -352,7 +351,7 @@ export class AssetRepository implements IAssetRepository { ownerId, }, }); - return new CheckExistingAssetsResponseDto(existingAssets.map((a) => a.deviceAssetId)); + return assets.map((asset) => asset.deviceAssetId); } async countByIdAndUser(assetId: string, ownerId: string): Promise { diff --git a/server/apps/immich/src/api-v1/asset/asset.controller.ts b/server/apps/immich/src/api-v1/asset/asset.controller.ts index 6088b81222..774a72ea9b 100644 --- a/server/apps/immich/src/api-v1/asset/asset.controller.ts +++ b/server/apps/immich/src/api-v1/asset/asset.controller.ts @@ -57,6 +57,8 @@ import { AssetSearchDto } from './dto/asset-search.dto'; import { assetUploadOption, ImmichFile } from '../../config/asset-upload.config'; import FileNotEmptyValidator from '../validation/file-not-empty-validator'; import { RemoveAssetsDto } from '../album/dto/remove-assets.dto'; +import { AssetBulkUploadCheckDto } from './dto/asset-check.dto'; +import { AssetBulkUploadCheckResponseDto } from './response-dto/asset-check-response.dto'; import { AssetIdDto } from './dto/asset-id.dto'; import { DeviceIdDto } from './dto/device-id.dto'; @@ -332,6 +334,19 @@ export class AssetController { return await this.assetService.checkExistingAssets(authUser, checkExistingAssetsDto); } + /** + * Checks if assets exist by checksums + */ + @Authenticated() + @Post('/bulk-upload-check') + @HttpCode(200) + bulkUploadCheck( + @GetAuthUser() authUser: AuthUserDto, + @Body(ValidationPipe) dto: AssetBulkUploadCheckDto, + ): Promise { + return this.assetService.bulkUploadCheck(authUser, dto); + } + @Authenticated() @Post('/shared-link') async createAssetsSharedLink( diff --git a/server/apps/immich/src/api-v1/asset/asset.core.ts b/server/apps/immich/src/api-v1/asset/asset.core.ts index 8d3992d389..34e014fc72 100644 --- a/server/apps/immich/src/api-v1/asset/asset.core.ts +++ b/server/apps/immich/src/api-v1/asset/asset.core.ts @@ -17,7 +17,7 @@ export class AssetCore { owner: { id: authUser.id } as UserEntity, mimeType: file.mimeType, - checksum: file.checksum || null, + checksum: file.checksum, originalPath: file.originalPath, deviceAssetId: dto.deviceAssetId, diff --git a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts index 80d5cc91da..cacacc606a 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts @@ -157,7 +157,7 @@ describe('AssetService', () => { getLocationsByUserId: jest.fn(), getSearchPropertiesByUserId: jest.fn(), getAssetByTimeBucket: jest.fn(), - getAssetByChecksum: jest.fn(), + getAssetsByChecksums: jest.fn(), getAssetCountByUserId: jest.fn(), getArchivedAssetCountByUserId: jest.fn(), getExistingAssets: jest.fn(), @@ -299,7 +299,7 @@ describe('AssetService', () => { (error as any).constraint = 'UQ_userid_checksum'; assetRepositoryMock.create.mockRejectedValue(error); - assetRepositoryMock.getAssetByChecksum.mockResolvedValue(_getAsset_1()); + assetRepositoryMock.getAssetsByChecksums.mockResolvedValue([_getAsset_1()]); await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: true, id: 'id_1' }); diff --git a/server/apps/immich/src/api-v1/asset/asset.service.ts b/server/apps/immich/src/api-v1/asset/asset.service.ts index e130b6b07b..145cd717de 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.ts @@ -63,6 +63,12 @@ import { mapSharedLink, SharedLinkResponseDto } from '@app/domain'; import { AssetSearchDto } from './dto/asset-search.dto'; import { AddAssetsDto } from '../album/dto/add-assets.dto'; import { RemoveAssetsDto } from '../album/dto/remove-assets.dto'; +import { AssetBulkUploadCheckDto } from './dto/asset-check.dto'; +import { + AssetUploadAction, + AssetRejectReason, + AssetBulkUploadCheckResponseDto, +} from './response-dto/asset-check-response.dto'; const fileInfo = promisify(stat); @@ -128,7 +134,8 @@ export class AssetService { // handle duplicates with a success response if (error instanceof QueryFailedError && (error as any).constraint === 'UQ_userid_checksum') { - const duplicate = await this.getAssetByChecksum(authUser.id, file.checksum); + const checksums = [file.checksum, livePhotoFile?.checksum].filter((checksum): checksum is Buffer => !!checksum); + const [duplicate] = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums); return { id: duplicate.id, duplicate: true }; } @@ -463,7 +470,40 @@ export class AssetService { authUser: AuthUserDto, checkExistingAssetsDto: CheckExistingAssetsDto, ): Promise { - return this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto); + return { + existingIds: await this._assetRepository.getExistingAssets(authUser.id, checkExistingAssetsDto), + }; + } + + async bulkUploadCheck(authUser: AuthUserDto, dto: AssetBulkUploadCheckDto): Promise { + const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex')); + const results = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums); + const resultsMap: Record = {}; + + for (const { id, checksum } of results) { + resultsMap[checksum.toString('hex')] = id; + } + + return { + results: dto.assets.map(({ id, checksum }) => { + const duplicate = resultsMap[checksum]; + if (duplicate) { + return { + id, + assetId: duplicate, + action: AssetUploadAction.REJECT, + reason: AssetRejectReason.DUPLICATE, + }; + } + + // TODO mime-check + + return { + id, + action: AssetUploadAction.ACCEPT, + }; + }), + }; } async getAssetCountByTimeBucket( @@ -482,10 +522,6 @@ export class AssetService { return mapAssetCountByTimeBucket(result); } - getAssetByChecksum(userId: string, checksum: Buffer) { - return this._assetRepository.getAssetByChecksum(userId, checksum); - } - getAssetCountByUserId(authUser: AuthUserDto): Promise { return this._assetRepository.getAssetCountByUserId(authUser.id); } diff --git a/server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts b/server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts new file mode 100644 index 0000000000..6fab46d631 --- /dev/null +++ b/server/apps/immich/src/api-v1/asset/dto/asset-check.dto.ts @@ -0,0 +1,19 @@ +import { Type } from 'class-transformer'; +import { IsArray, IsNotEmpty, IsString, ValidateNested } from 'class-validator'; + +export class AssetBulkUploadCheckItem { + @IsString() + @IsNotEmpty() + id!: string; + + @IsString() + @IsNotEmpty() + checksum!: string; +} + +export class AssetBulkUploadCheckDto { + @IsArray() + @ValidateNested({ each: true }) + @Type(() => AssetBulkUploadCheckItem) + assets!: AssetBulkUploadCheckItem[]; +} diff --git a/server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts b/server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts new file mode 100644 index 0000000000..1a51dc53f2 --- /dev/null +++ b/server/apps/immich/src/api-v1/asset/response-dto/asset-check-response.dto.ts @@ -0,0 +1,20 @@ +export class AssetBulkUploadCheckResult { + id!: string; + action!: AssetUploadAction; + reason?: AssetRejectReason; + assetId?: string; +} + +export class AssetBulkUploadCheckResponseDto { + results!: AssetBulkUploadCheckResult[]; +} + +export enum AssetUploadAction { + ACCEPT = 'accept', + REJECT = 'reject', +} + +export enum AssetRejectReason { + DUPLICATE = 'duplicate', + UNSUPPORTED_FORMAT = 'unsupported-format', +} diff --git a/server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts b/server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts index 9a159c308a..c39a79606b 100644 --- a/server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts +++ b/server/apps/immich/src/api-v1/asset/response-dto/check-existing-assets-response.dto.ts @@ -1,6 +1,3 @@ export class CheckExistingAssetsResponseDto { - constructor(existingIds: string[]) { - this.existingIds = existingIds; - } - existingIds: string[]; + existingIds!: string[]; } diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index e0c021f4b8..10704b9bd3 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -3251,6 +3251,49 @@ ] } }, + "/asset/bulk-upload-check": { + "post": { + "operationId": "bulkUploadCheck", + "description": "Checks if assets exist by checksums", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssetBulkUploadCheckDto" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssetBulkUploadCheckResponseDto" + } + } + } + } + }, + "tags": [ + "Asset" + ], + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ] + } + }, "/asset/shared-link": { "post": { "operationId": "createAssetsSharedLink", @@ -6046,6 +6089,78 @@ "existingIds" ] }, + "AssetBulkUploadCheckItem": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "checksum": { + "type": "string" + } + }, + "required": [ + "id", + "checksum" + ] + }, + "AssetBulkUploadCheckDto": { + "type": "object", + "properties": { + "assets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AssetBulkUploadCheckItem" + } + } + }, + "required": [ + "assets" + ] + }, + "AssetBulkUploadCheckResult": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "action": { + "type": "string", + "enum": [ + "accept", + "reject" + ] + }, + "reason": { + "type": "string", + "enum": [ + "duplicate", + "unsupported-format" + ] + }, + "assetId": { + "type": "string" + } + }, + "required": [ + "id", + "action" + ] + }, + "AssetBulkUploadCheckResponseDto": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AssetBulkUploadCheckResult" + } + } + }, + "required": [ + "results" + ] + }, "CreateAssetsShareLinkDto": { "type": "object", "properties": { diff --git a/server/libs/domain/test/fixtures.ts b/server/libs/domain/test/fixtures.ts index c963709e7f..664347da0c 100644 --- a/server/libs/domain/test/fixtures.ts +++ b/server/libs/domain/test/fixtures.ts @@ -147,6 +147,7 @@ export const assetEntityStub = { deviceId: 'device-id', originalPath: 'upload/upload/path.ext', resizePath: null, + checksum: Buffer.from('file hash', 'utf8'), type: AssetType.IMAGE, webpPath: null, encodedVideoPath: null, @@ -173,6 +174,7 @@ export const assetEntityStub = { deviceId: 'device-id', originalPath: '/original/path.ext', resizePath: '/uploads/user-id/thumbs/path.ext', + checksum: Buffer.from('file hash', 'utf8'), type: AssetType.IMAGE, webpPath: null, encodedVideoPath: null, @@ -201,6 +203,7 @@ export const assetEntityStub = { deviceId: 'device-id', originalPath: '/original/path.ext', resizePath: '/uploads/user-id/thumbs/path.ext', + checksum: Buffer.from('file hash', 'utf8'), type: AssetType.VIDEO, webpPath: null, encodedVideoPath: null, @@ -246,6 +249,7 @@ export const assetEntityStub = { owner: userEntityStub.user1, ownerId: 'user-id', deviceId: 'device-id', + checksum: Buffer.from('file hash', 'utf8'), originalPath: '/original/path.ext', resizePath: '/uploads/user-id/thumbs/path.ext', type: AssetType.IMAGE, @@ -663,6 +667,7 @@ export const sharedLinkStub = { type: AssetType.VIDEO, originalPath: 'fake_path/jpeg', resizePath: '', + checksum: Buffer.from('file hash', 'utf8'), fileModifiedAt: today.toISOString(), fileCreatedAt: today.toISOString(), createdAt: today.toISOString(), diff --git a/server/libs/infra/src/entities/asset.entity.ts b/server/libs/infra/src/entities/asset.entity.ts index cba5518ea9..3e6356e2c6 100644 --- a/server/libs/infra/src/entities/asset.entity.ts +++ b/server/libs/infra/src/entities/asset.entity.ts @@ -75,9 +75,9 @@ export class AssetEntity { @Column({ type: 'varchar', nullable: true }) mimeType!: string | null; - @Column({ type: 'bytea', nullable: true, select: false }) - @Index({ where: `'checksum' IS NOT NULL` }) // avoid null index - checksum?: Buffer | null; // sha1 checksum + @Column({ type: 'bytea' }) + @Index() + checksum!: Buffer; // sha1 checksum @Column({ type: 'varchar', nullable: true }) duration!: string | null; diff --git a/server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts b/server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts new file mode 100644 index 0000000000..6da8f32622 --- /dev/null +++ b/server/libs/infra/src/migrations/1684328185099-RequireChecksumNotNull.ts @@ -0,0 +1,19 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class RequireChecksumNotNull1684328185099 implements MigrationInterface { + name = 'removeNotNullFromChecksumIndex1684328185099'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_64c507300988dd1764f9a6530c"`); + await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "checksum" SET NOT NULL`); + await queryRunner.query(`CREATE INDEX "IDX_8d3efe36c0755849395e6ea866" ON "assets" ("checksum") `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_8d3efe36c0755849395e6ea866"`); + await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "checksum" DROP NOT NULL`); + await queryRunner.query( + `CREATE INDEX "IDX_64c507300988dd1764f9a6530c" ON "assets" ("checksum") WHERE ('checksum' IS NOT NULL)`, + ); + } +} diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index 685964b291..37d1ce4491 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -346,6 +346,96 @@ export interface AllJobStatusResponseDto { */ 'recognize-faces-queue': JobStatusDto; } +/** + * + * @export + * @interface AssetBulkUploadCheckDto + */ +export interface AssetBulkUploadCheckDto { + /** + * + * @type {Array} + * @memberof AssetBulkUploadCheckDto + */ + 'assets': Array; +} +/** + * + * @export + * @interface AssetBulkUploadCheckItem + */ +export interface AssetBulkUploadCheckItem { + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckItem + */ + 'id': string; + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckItem + */ + 'checksum': string; +} +/** + * + * @export + * @interface AssetBulkUploadCheckResponseDto + */ +export interface AssetBulkUploadCheckResponseDto { + /** + * + * @type {Array} + * @memberof AssetBulkUploadCheckResponseDto + */ + 'results': Array; +} +/** + * + * @export + * @interface AssetBulkUploadCheckResult + */ +export interface AssetBulkUploadCheckResult { + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckResult + */ + 'id': string; + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckResult + */ + 'action': AssetBulkUploadCheckResultActionEnum; + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckResult + */ + 'reason'?: AssetBulkUploadCheckResultReasonEnum; + /** + * + * @type {string} + * @memberof AssetBulkUploadCheckResult + */ + 'assetId'?: string; +} + +export const AssetBulkUploadCheckResultActionEnum = { + Accept: 'accept', + Reject: 'reject' +} as const; + +export type AssetBulkUploadCheckResultActionEnum = typeof AssetBulkUploadCheckResultActionEnum[keyof typeof AssetBulkUploadCheckResultActionEnum]; +export const AssetBulkUploadCheckResultReasonEnum = { + Duplicate: 'duplicate', + UnsupportedFormat: 'unsupported-format' +} as const; + +export type AssetBulkUploadCheckResultReasonEnum = typeof AssetBulkUploadCheckResultReasonEnum[keyof typeof AssetBulkUploadCheckResultReasonEnum]; + /** * * @export @@ -4120,6 +4210,50 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration options: localVarRequestOptions, }; }, + /** + * Checks if assets exist by checksums + * @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + bulkUploadCheck: async (assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options: AxiosRequestConfig = {}): Promise => { + // verify required parameter 'assetBulkUploadCheckDto' is not null or undefined + assertParamExists('bulkUploadCheck', 'assetBulkUploadCheckDto', assetBulkUploadCheckDto) + const localVarPath = `/asset/bulk-upload-check`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + localVarHeaderParameter['Content-Type'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(assetBulkUploadCheckDto, localVarRequestOptions, configuration) + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, /** * Check duplicated asset before uploading - for Web upload used * @param {CheckDuplicateAssetDto} checkDuplicateAssetDto @@ -5312,6 +5446,16 @@ export const AssetApiFp = function(configuration?: Configuration) { const localVarAxiosArgs = await localVarAxiosParamCreator.addAssetsToSharedLink(addAssetsDto, key, options); return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); }, + /** + * Checks if assets exist by checksums + * @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async bulkUploadCheck(assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.bulkUploadCheck(assetBulkUploadCheckDto, options); + return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); + }, /** * Check duplicated asset before uploading - for Web upload used * @param {CheckDuplicateAssetDto} checkDuplicateAssetDto @@ -5595,6 +5739,15 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath addAssetsToSharedLink(addAssetsDto: AddAssetsDto, key?: string, options?: any): AxiosPromise { return localVarFp.addAssetsToSharedLink(addAssetsDto, key, options).then((request) => request(axios, basePath)); }, + /** + * Checks if assets exist by checksums + * @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + bulkUploadCheck(assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options?: any): AxiosPromise { + return localVarFp.bulkUploadCheck(assetBulkUploadCheckDto, options).then((request) => request(axios, basePath)); + }, /** * Check duplicated asset before uploading - for Web upload used * @param {CheckDuplicateAssetDto} checkDuplicateAssetDto @@ -5856,6 +6009,17 @@ export class AssetApi extends BaseAPI { return AssetApiFp(this.configuration).addAssetsToSharedLink(addAssetsDto, key, options).then((request) => request(this.axios, this.basePath)); } + /** + * Checks if assets exist by checksums + * @param {AssetBulkUploadCheckDto} assetBulkUploadCheckDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof AssetApi + */ + public bulkUploadCheck(assetBulkUploadCheckDto: AssetBulkUploadCheckDto, options?: AxiosRequestConfig) { + return AssetApiFp(this.configuration).bulkUploadCheck(assetBulkUploadCheckDto, options).then((request) => request(this.axios, this.basePath)); + } + /** * Check duplicated asset before uploading - for Web upload used * @param {CheckDuplicateAssetDto} checkDuplicateAssetDto diff --git a/web/src/lib/utils/file-uploader.ts b/web/src/lib/utils/file-uploader.ts index bad793e5da..e825d0eb74 100644 --- a/web/src/lib/utils/file-uploader.ts +++ b/web/src/lib/utils/file-uploader.ts @@ -4,7 +4,7 @@ import { } from './../components/shared-components/notification/notification'; import { uploadAssetsStore } from '$lib/stores/upload'; import type { UploadAsset } from '../models/upload-asset'; -import { api, AssetFileUploadResponseDto } from '@api'; +import { AssetFileUploadResponseDto } from '@api'; import { addAssetsToAlbum, getFileMimeType, getFilenameExtension } from '$lib/utils/asset-utils'; import { mergeMap, filter, firstValueFrom, from, of, combineLatestAll } from 'rxjs'; import axios from 'axios'; @@ -73,7 +73,7 @@ async function fileUploader( const deviceAssetId = 'web' + '-' + asset.name + '-' + asset.lastModified; try { - // Create and add Unique ID of asset on the device + // Create and add pseudo-unique ID of asset on the device formData.append('deviceAssetId', deviceAssetId); // Get device id - for web -> use WEB @@ -102,23 +102,6 @@ async function fileUploader( // failed uploads. formData.append('assetData', new File([asset], asset.name, { type: mimeType })); - // Check if asset upload on server before performing upload - const { data, status } = await api.assetApi.checkDuplicateAsset( - { - deviceAssetId: String(deviceAssetId), - deviceId: 'WEB' - }, - sharedKey - ); - - if (status === 200 && data.isExist && data.id) { - if (albumId) { - await addAssetsToAlbum(albumId, [data.id], sharedKey); - } - - return data.id; - } - const newUploadAsset: UploadAsset = { id: deviceAssetId, file: asset,