From 2493dfaba3527213877c35f22dc2949c02586400 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 1 Jun 2023 06:32:51 -0400 Subject: [PATCH] feat(server): dynamic job concurrency (#2622) * feat(server): dynamic job concurrency * styling and add setting info to top of the job list * regenerate api * remove DETECT_OBJECT job --------- Co-authored-by: Alex Tran --- mobile/openapi/.openapi-generator/FILES | 6 + mobile/openapi/README.md | Bin 17625 -> 17721 bytes mobile/openapi/doc/AllJobStatusResponseDto.md | Bin 1071 -> 1021 bytes mobile/openapi/doc/JobSettingsDto.md | Bin 0 -> 412 bytes mobile/openapi/doc/SystemConfigDto.md | Bin 718 -> 782 bytes mobile/openapi/doc/SystemConfigJobDto.md | Bin 0 -> 1056 bytes mobile/openapi/lib/api.dart | Bin 5496 -> 5573 bytes mobile/openapi/lib/api_client.dart | Bin 17578 -> 17746 bytes .../model/all_job_status_response_dto.dart | Bin 7138 -> 6481 bytes mobile/openapi/lib/model/job_name.dart | Bin 4020 -> 3732 bytes .../openapi/lib/model/job_settings_dto.dart | Bin 0 -> 3305 bytes .../openapi/lib/model/system_config_dto.dart | Bin 4245 -> 4450 bytes .../lib/model/system_config_job_dto.dart | Bin 0 -> 6421 bytes .../all_job_status_response_dto_test.dart | Bin 1786 -> 1686 bytes .../openapi/test/job_settings_dto_test.dart | Bin 0 -> 569 bytes .../openapi/test/system_config_dto_test.dart | Bin 961 -> 1064 bytes .../test/system_config_job_dto_test.dart | Bin 0 -> 1691 bytes .../immich/src/controllers/job.controller.ts | 2 +- server/apps/microservices/src/app.service.ts | 75 +++++++ server/apps/microservices/src/main.ts | 4 +- .../microservices/src/microservices.module.ts | 4 +- .../microservices/src/processor.service.ts | 113 ---------- server/immich-openapi-specs.json | 124 ++++++++--- server/libs/domain/src/job/job.constants.ts | 34 +-- server/libs/domain/src/job/job.repository.ts | 18 +- .../libs/domain/src/job/job.service.spec.ts | 65 ++++-- server/libs/domain/src/job/job.service.ts | 40 +++- .../dto/system-config-job.dto.ts | 73 +++++++ .../system-config/dto/system-config.dto.ts | 6 + .../src/system-config/system-config.core.ts | 25 ++- .../system-config.service.spec.ts | 18 +- server/libs/domain/test/fixtures.ts | 13 ++ .../libs/domain/test/job.repository.mock.ts | 2 + .../src/entities/system-config.entity.ts | 20 +- server/libs/infra/src/infra.config.ts | 9 +- server/libs/infra/src/infra.module.ts | 2 +- .../infra/src/repositories/job.repository.ts | 41 +++- .../repositories/system-config.repository.ts | 2 +- server/package-lock.json | 204 ++++++++++-------- server/package.json | 10 +- web/src/api/api.ts | 20 +- web/src/api/open-api/api.ts | 126 +++++++++-- .../admin-page/jobs/job-tile.svelte | 2 +- .../admin-page/jobs/jobs-panel.svelte | 51 +++-- .../settings/job-settings/job-settings.svelte | 103 +++++++++ .../settings/setting-input-field.svelte | 3 + web/src/lib/utils/handle-error.ts | 2 +- .../routes/admin/system-settings/+page.svelte | 9 + 48 files changed, 870 insertions(+), 356 deletions(-) create mode 100644 mobile/openapi/doc/JobSettingsDto.md create mode 100644 mobile/openapi/doc/SystemConfigJobDto.md create mode 100644 mobile/openapi/lib/model/job_settings_dto.dart create mode 100644 mobile/openapi/lib/model/system_config_job_dto.dart create mode 100644 mobile/openapi/test/job_settings_dto_test.dart create mode 100644 mobile/openapi/test/system_config_job_dto_test.dart create mode 100644 server/apps/microservices/src/app.service.ts delete mode 100644 server/apps/microservices/src/processor.service.ts create mode 100644 server/libs/domain/src/system-config/dto/system-config-job.dto.ts create mode 100644 web/src/lib/components/admin-page/settings/job-settings/job-settings.svelte diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 9335f934b3..329dee833e 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -57,6 +57,7 @@ doc/JobCommand.md doc/JobCommandDto.md doc/JobCountsDto.md doc/JobName.md +doc/JobSettingsDto.md doc/JobStatusDto.md doc/LoginCredentialDto.md doc/LoginResponseDto.md @@ -95,6 +96,7 @@ doc/SmartInfoResponseDto.md doc/SystemConfigApi.md doc/SystemConfigDto.md doc/SystemConfigFFmpegDto.md +doc/SystemConfigJobDto.md doc/SystemConfigOAuthDto.md doc/SystemConfigPasswordLoginDto.md doc/SystemConfigStorageTemplateDto.md @@ -186,6 +188,7 @@ lib/model/job_command.dart lib/model/job_command_dto.dart lib/model/job_counts_dto.dart lib/model/job_name.dart +lib/model/job_settings_dto.dart lib/model/job_status_dto.dart lib/model/login_credential_dto.dart lib/model/login_response_dto.dart @@ -217,6 +220,7 @@ lib/model/sign_up_dto.dart lib/model/smart_info_response_dto.dart lib/model/system_config_dto.dart lib/model/system_config_f_fmpeg_dto.dart +lib/model/system_config_job_dto.dart lib/model/system_config_o_auth_dto.dart lib/model/system_config_password_login_dto.dart lib/model/system_config_storage_template_dto.dart @@ -288,6 +292,7 @@ test/job_command_dto_test.dart test/job_command_test.dart test/job_counts_dto_test.dart test/job_name_test.dart +test/job_settings_dto_test.dart test/job_status_dto_test.dart test/login_credential_dto_test.dart test/login_response_dto_test.dart @@ -326,6 +331,7 @@ test/smart_info_response_dto_test.dart test/system_config_api_test.dart test/system_config_dto_test.dart test/system_config_f_fmpeg_dto_test.dart +test/system_config_job_dto_test.dart test/system_config_o_auth_dto_test.dart test/system_config_password_login_dto_test.dart test/system_config_storage_template_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 99d6f2da260bf3393eff1adc23aefb69722fe066..fe38a406bdcdec6db33a5a9dda935f6766e2ab4d 100644 GIT binary patch delta 67 zcmccF$+)wNal>y*`P7n<%)Insmy-NgjgL=69B*Y|KEh$@&grU|P^Y KVso-X84CcCFBtIv delta 19 bcmdnl#dx!mal>!R&FWSeY?}`|l(7H+S;GiE diff --git a/mobile/openapi/doc/AllJobStatusResponseDto.md b/mobile/openapi/doc/AllJobStatusResponseDto.md index 5578b44a0e66457647aa85e9bfa26a19a856c163..7ab4eaf9d434fb0b6cd0393f5b904c265f7b3997 100644 GIT binary patch delta 90 zcmZ3_@t1wV*U9sk3@2+d3V~?;vdom!eCPbUvecsD%=|pC+zv)wF#Vg62S^)Dwqp_p b^Shb&fV4T7&CSd=S&rEP$kw0S#k>>%<{2MT delta 161 zcmey%zMf;kSJuGN)Y8<+ADIj%t1${8@iH0tk$5W^c@aFpntUK5}LDzNOW`WQr&VrLm{PyBE ztDY5@+*NdR+T`+?CC~tC+Y^4^%Q@AnrNQNpBycQh%~A+`bxQC`Y5kW5r5s0h%9)}& zD^+-CHWzkd7x literal 0 HcmV?d00001 diff --git a/mobile/openapi/doc/SystemConfigDto.md b/mobile/openapi/doc/SystemConfigDto.md index 8ad2bfb9a3e8331a32148b8228b98909c546ae4c..908fc46da4d8eabc180255ce860a1b8b0df03a80 100644 GIT binary patch delta 32 kcmX@d+Q+s*lZjPJD=R-~;sX(8ul%IRj*P-!TAFDy0HtXO;{X5v delta 11 ScmeBUJIA^~lWB4$(^3Ez#{=d7 diff --git a/mobile/openapi/doc/SystemConfigJobDto.md b/mobile/openapi/doc/SystemConfigJobDto.md new file mode 100644 index 0000000000000000000000000000000000000000..bdff864764b9d2fa855914d704ec8bb4bcfb729c GIT binary patch literal 1056 zcmbtT!Ab)$5WVLs0zKF+?DnoFwN(^biuP6tohIWpc9RLoR8;(UXRBcAMOi~2%;de7 zykw@QDPn-=Do+-h>ftpr`b!FruD+%9h)JnlPu~$(%^TEvl#XDwP@`I{Ru&Q~=OZQ7 zCtvx~FU?`A+X7C`TIgPBmSi)r!>p#@QNQc?YiyShW|LfGM?s<~u^`HWA<2WNEPsnt zQSfHx*8yB}jR6z1(nTS>MNx>i2V{1k&f>8smi^;QGX=Z-Ob&C?85Fqkc41m zth9Ksq0-Vy5<3bFe}ajnjg}-xRh;8BpKD@|HmD$oBc%PcQUh3fTqa1@Y^8#e5x-u9|*2M!&WVS8g($QwxmJIW{=tYaEkAjah2oSlD;YSZ@!9Vd9C%;QtBnqhIxB(W&Z7Ca?`!gue+tu3 zDNezi;vD!zGa_hcso#mvw`8Gt|1vYxo470 zQZYfKE=fpBlwR9*87x~&-3`wNM_^wm!H2=xkwh28>=qF75-Cx-3r=DQB+8zG7o&!_ zAhGzFr0}T06CY16AFEIrnr1#lMst?GV(caBa|9GP)F=I`G)ZgdGrcF%TS*X_?PdrW zaob+y8Fo-eoDbFGcBzLeB2Sc-foft(g!4O0VR`=dXSouWUKqYK9zUHDf!}M&D5UDW z9Jcbz#w{ax&imLlU6CA`rbQummEMU{AiK5eX&?>8Y1}_N68(@eQzJ0Y;gWW!Y5xJ3 CXe+}2 literal 7138 zcmbVRZEq7f5dPj@;i8D7x=niU=~QleAetiHT3zXVI7LlDdL{~`X`!cl4hQlDXVR#v;9)s0n40FISb~^d z(PH|?@6&8Wr8+o(<1`TqA~vj`8$N=+FAff}oGL|L<@q}?`=qHZ)fH14IFy}fK^_hW zIR-miFyB{upqnh!8ul6vR+cl#UR@fd#k)C7?gj7A!q z8udKchKRTw5n@~003kY{c5r=L)#nH*23EJ#Jldd^zqqZmlSl}#TVNY3Kr@60EpID2 z)H7$cd7$H#01Z(gwA?wtNebo&=h>qwFEm*)O;%K`UW+*cavFLy#S&{|!5|4KD(|tz zLD^a{#g!2Q`i3+8s0KVDzgfu$mwdqxZG0)8BfgBs2w!BU&wcTLe|;gx#r|f%f4-2L zUD{pGd7rd|+C{XmzYI)j=|>9nB<8BL7u!i<2_Wr7yqN@pKJA5k3R%yq(`nCNOZ>=A zd!B2P-Qk<|c6vGqMQ+*)xEs!CLl#imha?o6X)oY#z;I#`K%A$WzH#$My7|+3H6inR zXfSzp3XXoKL;-HLJ=cO4G$)^7iKN7net3Lg3x$!G$bRGH;j-z)!yt&c>|+ zReOKPnx?IbREu=lx(<-x#Sog}s-^3qgm}||u!I?E+wFA$d^&a1kw`;UwSir2k7XU@ zic>>fwTHuX-5++5cCxR7+OiJbO5Jv$+U=&p>;PC&cJ&;pcH|q14j0&Nb`_iLwi)dJ zR0QrSy1cd;(hMGkk+|@P-RiFlB!m`oVy84kxK`vV%}b`pLW*?-IWbEx!z@6ckF(AM zg6bwAMVUi)hOO8gS7x+virq%C1{hOL%XE%kZP4jFDv|foh7C*ZX8;kxj zS25vQg|EG09lLO=fw>^IdT_6Losg}lHX@^pD(%Jh!1nq$e8K(4g(@Y3Af_7rVPs7= z1YSZ@7~#x9GoanFdxi5{rFbDUBhOBZOl3r@@qJCQ70YhfoWQhwiWiV0b4C?h5cUqMYrx3Lq#iVh4X;+;_dQ-0 zt}v$AYA}Gyx0XV&N)?mfsD58iF@g!{UsWSI0Y&m8RvCM+d-DWuy5>sZn<0VvDm(XwRq;^W|`r9ibGdBZ`2 z(-eWuWPY+WG>-874SCV9duqqgp&je-s*$X}2ffw`loJ;`R1EeY$Lfm<>z9Z}YsyM{RZVFsgDL!z>;*kJ7B{}S#*HKrO42-^nJecnZ@KNcG&T^R0 z$b#mIC69F`v=u`#RpuHy!Pps-8=E??_8>AQJiHVPo~9y?xPodhUT?IMWHI{>suRQ= zW^h!kkH(m{n>LN5v3l5M6nG6MueEis%A_tbRsnpe5uML%wRAMCy|IVDOKe%c($WIo zZcrQUgw($o6w`ZDu5r8VnDwxv1|3x2*xCULuhWgk(CY5o^#$Bwrz?!lQtPT%OMtzA z{mOkpnh(-bC|j_IpSbPV4o+*`3aj$IcI*hBI8*gY{epuH-pKI5an~M4gV`ol(UevA zc!UjIUFvH8)<3tr4XRD3!P~dOIo2khi*DF}U5W_F#5uXgLG$>c?B+Of&h5v(q zpg^+#TaaPVfICG!lnC_b QtrhGy4@U#OS3pkwA6iy)m;e9( diff --git a/mobile/openapi/lib/model/job_name.dart b/mobile/openapi/lib/model/job_name.dart index 5226e967ddf560dc24262259ac750c9176cd984d..984e51ba171c73840c6753d9951e163fb82f2074 100644 GIT binary patch delta 789 zcmZXS%}T>S6ooO31#6Px54LG-O+l%s=vr$*kSe%v)r~8ePKOw4XJnEVse)?-SJ$^t zuuxyX7w{R}_!cfqGHppFS%q^xZtl4!pKGtBVNHP}C7(GZlyKxZjKY_?1P{t$#j0Cp ztaXmMWcPBDSG&YRZRDZGoyQTLlHW?UERlVuP1rG`eZt+zz(Pu2l`z)2CfI8t+jgiO ze07@>%xvlm$O*#4mGHP3Zuy?liV}ryoZAM)tUa08Lbzjd)}!qvayt?oH~pE*P*-zV ziGo8lA9w|VIKGlhHxzxl06*#q{HcS-os8u_$$JT1x{eTIw-8c1uh63Dgc45>YEV;6 zQ@CQe@w?v&MtpRUzMqZO{X4$_S}@Qi6%b1zRt}vtCq#8eyng3kf{zDQ#@+B^+^w(mx>7L$Cc2{vNx#GbR&uGRK+s-uJ#Y@8|2g=IwTQeSZWaKs)FY>wP8>tRHrG zTMkB1FDBs-2M_}%GGsCSmI0H4atqri5lGS}l3+NMoKS=DyS)6v#yAQ4z%XU;32-qU z&|cYohLDiyOp4M+Arb3+1uDBF2*Nn9DJcM!3ZaZn=_G5;%Sz5nJ~#~@e<2ja<3die zZ4;%#@EAi(SwEesvDePhiVfQ%^eEtL6!%jT;z*1D> zEQrH1_?Ts85VCysbUeoOB#LgDME39$lv48CZFCHzv84hCW1By>mGeriSig5(Ya`dl zTA~gm0uXP{F5q|XjZWj2SFaTsm;CdVR_&U$VqN)7t=9BkO;_o?k5oMh=4@p;_ZPL$ zjlbZT#rf-_O}il>87BeEEK^ddt}f4@=B3gFDOF!|CM7G_)ju%-d*;_%R_99p0m#&A A6aWAK diff --git a/mobile/openapi/lib/model/job_settings_dto.dart b/mobile/openapi/lib/model/job_settings_dto.dart new file mode 100644 index 0000000000000000000000000000000000000000..dd9f82f3a8da43c6b5cdeb7d09f6c34f9e66e5c9 GIT binary patch literal 3305 zcmbVOU2oeq6n*!vxG9QQ0bF^_Q<2nOi^Un5wlR=o0|vtoXqk@L%A!Y7HH_5%edm&r zA~{}{&Ol;|s8NcwbsTRv=G@wY;CeG zTw>(kg)%VD#R~t|!r{dFb@_T#XG(B{P`Y8WKy29j^1m$u= zllG}8DW6&@V;pbqMeKp7Dwy0-*%+myg)uTuw>@m{P7t2p#8G3%uf5aXq%o)%`!lUG z&e;g34Gjj-c=ko6sKhO$IibOjLT6HuMI5GA?R@_pevx`o-|?|eFYGt-g8v^T!^z%D zM@wOszv;O|c<-sE3Ns{^skBy@Eh7xjmeMNgk423_9L$l;tXMhAb9ezZ<%5J;E zlZLCo%f`dB>s$C)tltLn9?iEPoJxoAcx81Ce*vC_%0)(ButWoTa{6I+9=haDOiZ^C zq&tPhKB(ci@tpBCys%@8e}YRS@;L=LSyIl@H<1<6(!%If>1pDZK)(eJbeWwKR6Ffa zUSyyO^J1f{XJrGIwUG;m0!3J7iWFqB3DYcryqroCC~>Lhj8jasr)r^31?{f+_=ST{&}l-t5c^jJ9jle7|RW~LJmonNT6^;K>y zE3h=Y)=tvvGk>P+9kZ|tl`UoZRn7_1jB{6W<+e1FTaAv-Br`78`73VXzPnR)hq_Xx zJigIEwO+%O%AxU03X24>cT}cU!sLxg$9lsAWYa1lR~t41 zE`vQ4$4m%c36gH$^QhjE2u+8~T`nOO^LfyC!!k6L=N!jwwYao$nv066Hl0!R%X{(j zvFty_Mm-hwdc~+8M4?JFRLxVOG0OLxR1>|~aK!v!zdbO6fXBa$EuL55rgTR2r6z&K zu*lb>@bUvPkJw?=Y%BC)ep1p_wLBk1?3Tad!E z77dEVBM$=KBOR_K8|9W*V%6|vtBt^`a0oW#dkt@79OgSG`vQdyFoD^#FP$MD=Z%A$ z=+X0(2_H?i91Qe-IP*+8JcnB41p680psF^!Z46?Drfyq zPj`4tvvbD1dfV}Pir?3Gi?0f7%_QL>ViA_fFTz{n5qEz(sf|CfZ7t{FgKtLb8s_iO J=N$}@e*kMjD)ayV literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/model/system_config_dto.dart b/mobile/openapi/lib/model/system_config_dto.dart index 41ae4cee9911a7096a4855827b328616ecf1ca9c..1cef06bd17edce799874bcabdad00629ae8c70c5 100644 GIT binary patch delta 220 zcmbQL_(*BPH%4Ivg`(8L(#)dN6orzE%woN){G`dB7}tsgR~DC~<~rx+rDdjjp|S@cM{nV06j?~ar# zk)q~i0g_0(7as4qH|q5C==2oc{xQCO@%!j%^yTe%bOD#|KaCQ&xQ46iF?_zhcz^ll zF|rZon>kY^eKmaf>WE&&vXF{T7g9|ZBIjqYEw-A^r#$CtUYMkM7we5wCMv=5d$!Zl z&DKm6|7$B0iWh8+e>Y6wf5VkV zZnmn>z@XswFllq%g0LoLe25}#`hYY6REGz%5_)J>+x4tqBLAHiTrneLkwnHG_{|#N)p-9fY^306caw&>sY^N+2n^BR;c?Vq^so0W_`FfKx!#|5< zx79P2-7S^e7V|OFcd^|xXDVC8D2iutS%`o5CCj*uacDek#uSkrPY^)>MC^tegO+PT zF5=r5TidrWuE4i37SFdahNf?0^e*4Vczxf-IGk@|6yLWoj?%Z5^hRPlDRee^cqU~| ziG!F+1!+3Cw($hgthhRX^c-AzW-K32d?yv<`*-&QBAR5+pWC2!Jh7y}y<&KRFW82X zj8jeb%l3AhAxGufmTD5)4*+b{8a8zfHx!p?;(g zTd)#nb17-nxyM!niNe5=8(1;DdMD=`W5z9LQPnD# z!%Ei){N$M0h_`tHgxd@rrfqT)#x}JJtv06*Pn+%#YOni1wCQCE*Mt@nH%4Kp1S8$U zz~*ni-MA(ZpQpXZViHNUwAThB3DYa>1!!R1Xf~>}=e;Ig4yC;&6iH{Iq`fXsk|0OY zo(~G@w8?=QT)zskAno~JAis-#6mSo>ee3<`CuUm zmcv(Ef)rZZ509^1rL-Iq)sMi?zeUNe=^_wsgHublX@PJ6S}hq60x=KETF6@tM4AZK z63x{>pbP3+xQioD_Z~3$F;g@9>w!=iPc?VB;MkO%QEh0@!Xphu zPn=zW6?1T7yar)3d}H~RYgj0`F11bU5{|nIWa!gGJi)-eO`zE3I33U^a4)o-f*VzR zqxb@caXZJWZHDuMp3-D7#j)OqY>Lsgl5?u^ZXq&3|7NYQ*-~bYv#v@VG-zOLqbUPT zs@4I#q_Ql-jA_$KuYnsRX--GeM}^)hj;X{9{^77@8$kOF!$IFJ>}UBqzSA`Imx5Q) zaCmuUc}o9LAL|>)Ry@1obHJMv*)Hb7>|n-s61_g-RE`T-ydX!0>RypLJQt4UoX)4Q zdT%zUN)%X*a?Ugwh_vI<47|>8Wd{4YLZ(&rek60zgb~YDg8>>}dy3dfQK~`1>bxjo z858J#DX(vUg77TTFM3G0eMUyEh1PT^iXQlIbA2`Jqlak^_tV&ELPBty+9-O$F4+9( zPDY_uVqdg(q6hM8*0l%-ztIr2jdECCDX4yWdr^)QA=_=p{Rdxyb>iq(LP^r)E zt8(T4po*$%G!gWh!9f*1dtikdMGorNx0u?FGJ76v0+jhd8MkjTWy$^|sHV2Qp`1gG zgv%-+s@hG7k-&q!Q{rcWdp67QIsglnYo0t-CCQa;#q~Bfb|Px4pWM1crut7xsq&C1 zc$u-ia40lZrPKCuQ7&3 z6jc*x)A_jffBN2nlsas>`Y<(M_u26b_Fo96*f4OBzB#)4Afi;`h^uN>AvvTs!F6?> zry*)p4K4N!d=4Os&YCzu+sp7P0?kfUlc@l${(JDE=sub7;P^^i&HBkzQn(jpg-eu4 z^!2ih#0$rWa^=v5HwzS?54ofdLn;y(6iu;=Jtq+&Mq}hhvlewUqo;GT{}V$$7#=?o zYGq%4N+jXQe5l)a3<;{e`x*h2?31=m-((+Uk=)148o_Z2W1K}vImJOW($O$RLn)4` z>n)OI9B|^+xGg0Fb0bk519lAG3x6cpeVU&?ScgQfmTp|>M+T3Q5{?k%vOf-#i(YRp g*E*YXNe}vLC1vmB{qim7f)$VJf0hYv(O4`01;d(GyZ`_I literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/all_job_status_response_dto_test.dart b/mobile/openapi/test/all_job_status_response_dto_test.dart index 437ed3d1dac5bef9ec0db5382452e96d92dfd030..657191ec976f77157ac0ee0576af7bfcadeedd47 100644 GIT binary patch delta 192 zcmeyxJB@b(C(~p@R>jHsOu~~dFj)!Zrj{h8B$gz)R+JPaCYNO9=YgenG6?~tJ;7{y zW`2;;$-T_HVEzFh4dU~#@PPStKpMpFWRU>#j{|8CUyhXz$kzw6=d-eb_9QK$sgFb050M^<^TWy delta 295 zcmbQn`-^u2ClhO6X=-WeleL(Hp*(q3tI6d|LP&C};5?ArHzs~0IW=Y&4xYgtXtb9mv@8LXoR%`Cel+@(JBA8ngC&#m~!*!}n?qTBs E0L{{3y8r+H diff --git a/mobile/openapi/test/job_settings_dto_test.dart b/mobile/openapi/test/job_settings_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..e06900185a8ecc989a13f70096adfdd16fce83bf GIT binary patch literal 569 zcmZvY(M!WH5XRs0SDa7XV6M6+*%&xzClg&k>w`~OY_DCgO=6NNL-xOW=^V)1Ly|lA zecyeTIA(DSiV(;`n3Sgsz@5hNL`vm73?WVL*~AS{r#rLcZJ8DCFWXl1Rm zMP6ykE7{-_I@LKWIX2j$@}pxfTcfQ%s}RkX*g0-G&nmTn^~y&kP2bL-B_ z3$2_N%5z!LKv_w&z1*E0)<}hpH(J$%`K?g==86|vVI5uB~5@#uUUt|JhJhTZ{v3EbWmr(FlIcdON8+(1#SAwWx|6NuiHJTHP1mw+P14YS!;y^M~CWtc2UC WqE8pJlrYbdtoPZ6EL8RHp8Wv8V7sXR literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/system_config_dto_test.dart b/mobile/openapi/test/system_config_dto_test.dart index e44406f7ec6a6b4f853ffad3a87977e2de142aa8..7ba7608efcedc92f5ddaa16c19cfc953437b4bef 100644 GIT binary patch delta 34 ncmX@ezJg=JcIL@C%)(q=`AIG%`3hP2Ns|+q6`4WwL}mj3$VCeb delta 11 ScmZ3%agcq(cIL@)EL;E@bpxUR diff --git a/mobile/openapi/test/system_config_job_dto_test.dart b/mobile/openapi/test/system_config_job_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..3fd36d1846b3285a69c4b2710dc3b0fdffccb661 GIT binary patch literal 1691 zcmbu9L2sKt5QXpjitWjcv~^4~k)M2Z_K zBXIz<(tGcl+1()wgD_N=U(@9HD!z@sT&8iPme-%-g^Ch&o22R{iLRI5-iWMIe#pV` z#p%hr)1X_LlB~n5ur4c1g=eaz)?=Pwg*6f{p4X;stm8okR6k+sv!>LD$>;AgnTkKC3L1BI7ks_Wo_A*Wi2*>}piA2J{1z zcxD?UXiO;2N!Ru4{FQ{!bJ)F}1s$|odnH?{%EN%>%1Gf3mc_8A_*|wR_I>3Nvwop%@eXIcQkl9e_1*~}8)++*2T_K?Z zHkKq&_0>wG2)a~WqiURV?s%*22R!#X3ivJQzJV@7nn;r<>GK|Ur4E%KX9+nl#J|{q z9#z3>%aj5(a!0Hj$40KynEwCKPeI$&chsChS;~PN-1t596s)dH6O*?280t&$6sTwG zWPnnvn+h0jOgZwt{m(e1n3YU(H#d~0;2#d|{Jwk&*dsW-J_y{&rhpu3Tax*WOVH>K a0{7)pz+T2lgF8rk(v3q&TK@}q4E_Q;Y!yZT literal 0 HcmV?d00001 diff --git a/server/apps/immich/src/controllers/job.controller.ts b/server/apps/immich/src/controllers/job.controller.ts index b0d441dd02..310e3ed6e4 100644 --- a/server/apps/immich/src/controllers/job.controller.ts +++ b/server/apps/immich/src/controllers/job.controller.ts @@ -19,6 +19,6 @@ export class JobController { @Put('/:jobId') async sendJobCommand(@Param() { jobId }: JobIdDto, @Body() dto: JobCommandDto): Promise { await this.service.handleCommand(jobId, dto); - return await this.service.getJobStatus(jobId); + return this.service.getJobStatus(jobId); } } diff --git a/server/apps/microservices/src/app.service.ts b/server/apps/microservices/src/app.service.ts new file mode 100644 index 0000000000..b657b1f5b3 --- /dev/null +++ b/server/apps/microservices/src/app.service.ts @@ -0,0 +1,75 @@ +import { + FacialRecognitionService, + IDeleteFilesJob, + JobName, + JobService, + MediaService, + MetadataService, + PersonService, + SearchService, + SmartInfoService, + StorageService, + StorageTemplateService, + SystemConfigService, + UserService, +} from '@app/domain'; +import { Injectable } from '@nestjs/common'; +import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor'; + +@Injectable() +export class AppService { + constructor( + // TODO refactor to domain + private metadataProcessor: MetadataExtractionProcessor, + + private facialRecognitionService: FacialRecognitionService, + private jobService: JobService, + private mediaService: MediaService, + private metadataService: MetadataService, + private personService: PersonService, + private searchService: SearchService, + private smartInfoService: SmartInfoService, + private storageTemplateService: StorageTemplateService, + private storageService: StorageService, + private systemConfigService: SystemConfigService, + private userService: UserService, + ) {} + + async init() { + await this.jobService.registerHandlers({ + [JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data), + [JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(), + [JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data), + [JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data), + [JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data), + [JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data), + [JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data), + [JobName.SEARCH_INDEX_ALBUMS]: () => this.searchService.handleIndexAlbums(), + [JobName.SEARCH_INDEX_ASSETS]: () => this.searchService.handleIndexAssets(), + [JobName.SEARCH_INDEX_FACES]: () => this.searchService.handleIndexFaces(), + [JobName.SEARCH_INDEX_ALBUM]: (data) => this.searchService.handleIndexAlbum(data), + [JobName.SEARCH_INDEX_ASSET]: (data) => this.searchService.handleIndexAsset(data), + [JobName.SEARCH_INDEX_FACE]: (data) => this.searchService.handleIndexFace(data), + [JobName.SEARCH_REMOVE_ALBUM]: (data) => this.searchService.handleRemoveAlbum(data), + [JobName.SEARCH_REMOVE_ASSET]: (data) => this.searchService.handleRemoveAsset(data), + [JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data), + [JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(), + [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data), + [JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(), + [JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data), + [JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data), + [JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data), + [JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data), + [JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data), + [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data), + [JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data), + [JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data), + [JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data), + [JobName.GENERATE_FACE_THUMBNAIL]: (data) => this.facialRecognitionService.handleGenerateFaceThumbnail(data), + [JobName.PERSON_CLEANUP]: () => this.personService.handlePersonCleanup(), + [JobName.QUEUE_SIDECAR]: (data) => this.metadataService.handleQueueSidecar(data), + [JobName.SIDECAR_DISCOVERY]: (data) => this.metadataService.handleSidecarDiscovery(data), + [JobName.SIDECAR_SYNC]: () => this.metadataService.handleSidecarSync(), + }); + } +} diff --git a/server/apps/microservices/src/main.ts b/server/apps/microservices/src/main.ts index e4d54859ef..0dfdecfc81 100644 --- a/server/apps/microservices/src/main.ts +++ b/server/apps/microservices/src/main.ts @@ -2,8 +2,8 @@ import { getLogLevels, SERVER_VERSION } from '@app/domain'; import { RedisIoAdapter } from '@app/infra'; import { Logger } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; +import { AppService } from './app.service'; import { MicroservicesModule } from './microservices.module'; -import { ProcessorService } from './processor.service'; import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor'; const logger = new Logger('ImmichMicroservice'); @@ -15,7 +15,7 @@ async function bootstrap() { const listeningPort = Number(process.env.MICROSERVICES_PORT) || 3002; - await app.get(ProcessorService).init(); + await app.get(AppService).init(); app.useWebSocketAdapter(new RedisIoAdapter(app)); diff --git a/server/apps/microservices/src/microservices.module.ts b/server/apps/microservices/src/microservices.module.ts index 6df9496190..7dabbc145d 100644 --- a/server/apps/microservices/src/microservices.module.ts +++ b/server/apps/microservices/src/microservices.module.ts @@ -3,7 +3,7 @@ import { InfraModule } from '@app/infra'; import { ExifEntity } from '@app/infra/entities'; import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { ProcessorService } from './processor.service'; +import { AppService } from './app.service'; import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor'; @Module({ @@ -12,6 +12,6 @@ import { MetadataExtractionProcessor } from './processors/metadata-extraction.pr DomainModule.register({ imports: [InfraModule] }), TypeOrmModule.forFeature([ExifEntity]), ], - providers: [MetadataExtractionProcessor, ProcessorService], + providers: [MetadataExtractionProcessor, AppService], }) export class MicroservicesModule {} diff --git a/server/apps/microservices/src/processor.service.ts b/server/apps/microservices/src/processor.service.ts deleted file mode 100644 index 082b9adcdf..0000000000 --- a/server/apps/microservices/src/processor.service.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { - FacialRecognitionService, - IDeleteFilesJob, - JobItem, - JobName, - JobService, - JOBS_TO_QUEUE, - MediaService, - MetadataService, - PersonService, - QueueName, - QUEUE_TO_CONCURRENCY, - SearchService, - SmartInfoService, - StorageService, - StorageTemplateService, - SystemConfigService, - UserService, -} from '@app/domain'; -import { getQueueToken } from '@nestjs/bull'; -import { Injectable, Logger } from '@nestjs/common'; -import { ModuleRef } from '@nestjs/core'; -import { Queue } from 'bull'; -import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor'; - -type JobHandler = (data: T) => boolean | Promise; - -@Injectable() -export class ProcessorService { - constructor( - private moduleRef: ModuleRef, - // TODO refactor to domain - private metadataProcessor: MetadataExtractionProcessor, - - private facialRecognitionService: FacialRecognitionService, - private jobService: JobService, - private mediaService: MediaService, - private metadataService: MetadataService, - private personService: PersonService, - private searchService: SearchService, - private smartInfoService: SmartInfoService, - private storageTemplateService: StorageTemplateService, - private storageService: StorageService, - private systemConfigService: SystemConfigService, - private userService: UserService, - ) {} - - private logger = new Logger(ProcessorService.name); - - private handlers: Record = { - [JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data), - [JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(), - [JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data), - [JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data), - [JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data), - [JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data), - [JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data), - [JobName.SEARCH_INDEX_ALBUMS]: () => this.searchService.handleIndexAlbums(), - [JobName.SEARCH_INDEX_ASSETS]: () => this.searchService.handleIndexAssets(), - [JobName.SEARCH_INDEX_FACES]: () => this.searchService.handleIndexFaces(), - [JobName.SEARCH_INDEX_ALBUM]: (data) => this.searchService.handleIndexAlbum(data), - [JobName.SEARCH_INDEX_ASSET]: (data) => this.searchService.handleIndexAsset(data), - [JobName.SEARCH_INDEX_FACE]: (data) => this.searchService.handleIndexFace(data), - [JobName.SEARCH_REMOVE_ALBUM]: (data) => this.searchService.handleRemoveAlbum(data), - [JobName.SEARCH_REMOVE_ASSET]: (data) => this.searchService.handleRemoveAsset(data), - [JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data), - [JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(), - [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data), - [JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(), - [JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data), - [JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data), - [JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data), - [JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data), - [JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data), - [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data), - [JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data), - [JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data), - [JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data), - [JobName.GENERATE_FACE_THUMBNAIL]: (data) => this.facialRecognitionService.handleGenerateFaceThumbnail(data), - [JobName.PERSON_CLEANUP]: () => this.personService.handlePersonCleanup(), - [JobName.QUEUE_SIDECAR]: (data) => this.metadataService.handleQueueSidecar(data), - [JobName.SIDECAR_DISCOVERY]: (data) => this.metadataService.handleSidecarDiscovery(data), - [JobName.SIDECAR_SYNC]: () => this.metadataService.handleSidecarSync(), - }; - - async init() { - const queueSeen: Partial> = {}; - - for (const jobName of Object.values(JobName)) { - const handler = this.handlers[jobName]; - const queueName = JOBS_TO_QUEUE[jobName]; - const queue = this.moduleRef.get(getQueueToken(queueName), { strict: false }); - - // only set concurrency on the first job for a queue, since concurrency stacks - const seen = queueSeen[queueName]; - const concurrency = seen ? 0 : QUEUE_TO_CONCURRENCY[queueName]; - queueSeen[queueName] = true; - - await queue.isReady(); - - queue.process(jobName, concurrency, async (job): Promise => { - try { - const success = await handler(job.data); - if (success) { - await this.jobService.onDone({ name: jobName, data: job.data } as JobItem); - } - } catch (error: Error | any) { - this.logger.error(`Unable to run job handler: ${error}`, error?.stack, job.data); - } - }); - } - } -} diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index a09c0e4937..63f194df81 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -5106,63 +5106,63 @@ "AllJobStatusResponseDto": { "type": "object", "properties": { - "thumbnail-generation-queue": { + "thumbnailGeneration": { "$ref": "#/components/schemas/JobStatusDto" }, - "metadata-extraction-queue": { + "metadataExtraction": { "$ref": "#/components/schemas/JobStatusDto" }, - "video-conversion-queue": { + "videoConversion": { "$ref": "#/components/schemas/JobStatusDto" }, - "object-tagging-queue": { + "objectTagging": { "$ref": "#/components/schemas/JobStatusDto" }, - "clip-encoding-queue": { + "clipEncoding": { "$ref": "#/components/schemas/JobStatusDto" }, - "storage-template-migration-queue": { + "storageTemplateMigration": { "$ref": "#/components/schemas/JobStatusDto" }, - "background-task-queue": { + "backgroundTask": { "$ref": "#/components/schemas/JobStatusDto" }, - "search-queue": { + "search": { "$ref": "#/components/schemas/JobStatusDto" }, - "recognize-faces-queue": { + "recognizeFaces": { "$ref": "#/components/schemas/JobStatusDto" }, - "sidecar-queue": { + "sidecar": { "$ref": "#/components/schemas/JobStatusDto" } }, "required": [ - "thumbnail-generation-queue", - "metadata-extraction-queue", - "video-conversion-queue", - "object-tagging-queue", - "clip-encoding-queue", - "storage-template-migration-queue", - "background-task-queue", - "search-queue", - "recognize-faces-queue", - "sidecar-queue" + "thumbnailGeneration", + "metadataExtraction", + "videoConversion", + "objectTagging", + "clipEncoding", + "storageTemplateMigration", + "backgroundTask", + "search", + "recognizeFaces", + "sidecar" ] }, "JobName": { "type": "string", "enum": [ - "thumbnail-generation-queue", - "metadata-extraction-queue", - "video-conversion-queue", - "object-tagging-queue", - "recognize-faces-queue", - "clip-encoding-queue", - "background-task-queue", - "storage-template-migration-queue", - "search-queue", - "sidecar-queue" + "thumbnailGeneration", + "metadataExtraction", + "videoConversion", + "objectTagging", + "recognizeFaces", + "clipEncoding", + "backgroundTask", + "storageTemplateMigration", + "search", + "sidecar" ] }, "JobCommand": { @@ -5733,6 +5733,64 @@ "template" ] }, + "JobSettingsDto": { + "type": "object", + "properties": { + "concurrency": { + "type": "integer" + } + }, + "required": [ + "concurrency" + ] + }, + "SystemConfigJobDto": { + "type": "object", + "properties": { + "thumbnailGeneration": { + "$ref": "#/components/schemas/JobSettingsDto" + }, + "metadataExtraction": { + "$ref": "#/components/schemas/JobSettingsDto" + }, + "videoConversion": { + "$ref": "#/components/schemas/JobSettingsDto" + }, + "objectTagging": { + "$ref": "#/components/schemas/JobSettingsDto" + }, + "clipEncoding": { + "$ref": "#/components/schemas/JobSettingsDto" + }, + "storageTemplateMigration": { + "$ref": "#/components/schemas/JobSettingsDto" + }, + "backgroundTask": { + "$ref": "#/components/schemas/JobSettingsDto" + }, + "search": { + "$ref": "#/components/schemas/JobSettingsDto" + }, + "recognizeFaces": { + "$ref": "#/components/schemas/JobSettingsDto" + }, + "sidecar": { + "$ref": "#/components/schemas/JobSettingsDto" + } + }, + "required": [ + "thumbnailGeneration", + "metadataExtraction", + "videoConversion", + "objectTagging", + "clipEncoding", + "storageTemplateMigration", + "backgroundTask", + "search", + "recognizeFaces", + "sidecar" + ] + }, "SystemConfigDto": { "type": "object", "properties": { @@ -5747,13 +5805,17 @@ }, "storageTemplate": { "$ref": "#/components/schemas/SystemConfigStorageTemplateDto" + }, + "job": { + "$ref": "#/components/schemas/SystemConfigJobDto" } }, "required": [ "ffmpeg", "oauth", "passwordLogin", - "storageTemplate" + "storageTemplate", + "job" ] }, "SystemConfigTemplateStorageOptionDto": { diff --git a/server/libs/domain/src/job/job.constants.ts b/server/libs/domain/src/job/job.constants.ts index 88f7a853b6..1d3b55b536 100644 --- a/server/libs/domain/src/job/job.constants.ts +++ b/server/libs/domain/src/job/job.constants.ts @@ -1,14 +1,14 @@ export enum QueueName { - THUMBNAIL_GENERATION = 'thumbnail-generation-queue', - METADATA_EXTRACTION = 'metadata-extraction-queue', - VIDEO_CONVERSION = 'video-conversion-queue', - OBJECT_TAGGING = 'object-tagging-queue', - RECOGNIZE_FACES = 'recognize-faces-queue', - CLIP_ENCODING = 'clip-encoding-queue', - BACKGROUND_TASK = 'background-task-queue', - STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration-queue', - SEARCH = 'search-queue', - SIDECAR = 'sidecar-queue', + THUMBNAIL_GENERATION = 'thumbnailGeneration', + METADATA_EXTRACTION = 'metadataExtraction', + VIDEO_CONVERSION = 'videoConversion', + OBJECT_TAGGING = 'objectTagging', + RECOGNIZE_FACES = 'recognizeFaces', + CLIP_ENCODING = 'clipEncoding', + BACKGROUND_TASK = 'backgroundTask', + STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration', + SEARCH = 'search', + SIDECAR = 'sidecar', } export enum JobCommand { @@ -135,17 +135,3 @@ export const JOBS_TO_QUEUE: Record = { [JobName.SIDECAR_DISCOVERY]: QueueName.SIDECAR, [JobName.SIDECAR_SYNC]: QueueName.SIDECAR, }; - -// max concurrency for each queue (total concurrency across all jobs) -export const QUEUE_TO_CONCURRENCY: Record = { - [QueueName.BACKGROUND_TASK]: 5, - [QueueName.CLIP_ENCODING]: 2, - [QueueName.METADATA_EXTRACTION]: 5, - [QueueName.OBJECT_TAGGING]: 2, - [QueueName.RECOGNIZE_FACES]: 2, - [QueueName.SEARCH]: 5, - [QueueName.SIDECAR]: 5, - [QueueName.STORAGE_TEMPLATE_MIGRATION]: 5, - [QueueName.THUMBNAIL_GENERATION]: 5, - [QueueName.VIDEO_CONVERSION]: 1, -}; diff --git a/server/libs/domain/src/job/job.repository.ts b/server/libs/domain/src/job/job.repository.ts index d88123ed50..bd1d002543 100644 --- a/server/libs/domain/src/job/job.repository.ts +++ b/server/libs/domain/src/job/job.repository.ts @@ -33,13 +33,13 @@ export type JobItem = | { name: JobName.GENERATE_WEBP_THUMBNAIL; data: IEntityJob } // User Deletion - | { name: JobName.USER_DELETE_CHECK } + | { name: JobName.USER_DELETE_CHECK; data?: IBaseJob } | { name: JobName.USER_DELETION; data: IEntityJob } // Storage Template - | { name: JobName.STORAGE_TEMPLATE_MIGRATION } + | { name: JobName.STORAGE_TEMPLATE_MIGRATION; data?: IBaseJob } | { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IEntityJob } - | { name: JobName.SYSTEM_CONFIG_CHANGE } + | { name: JobName.SYSTEM_CONFIG_CHANGE; data?: IBaseJob } // Metadata Extraction | { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob } @@ -67,22 +67,26 @@ export type JobItem = | { name: JobName.DELETE_FILES; data: IDeleteFilesJob } // Asset Deletion - | { name: JobName.PERSON_CLEANUP } + | { name: JobName.PERSON_CLEANUP; data?: IBaseJob } // Search - | { name: JobName.SEARCH_INDEX_ASSETS } + | { name: JobName.SEARCH_INDEX_ASSETS; data?: IBaseJob } | { name: JobName.SEARCH_INDEX_ASSET; data: IBulkEntityJob } - | { name: JobName.SEARCH_INDEX_FACES } + | { name: JobName.SEARCH_INDEX_FACES; data?: IBaseJob } | { name: JobName.SEARCH_INDEX_FACE; data: IAssetFaceJob } - | { name: JobName.SEARCH_INDEX_ALBUMS } + | { name: JobName.SEARCH_INDEX_ALBUMS; data?: IBaseJob } | { name: JobName.SEARCH_INDEX_ALBUM; data: IBulkEntityJob } | { name: JobName.SEARCH_REMOVE_ASSET; data: IBulkEntityJob } | { name: JobName.SEARCH_REMOVE_ALBUM; data: IBulkEntityJob } | { name: JobName.SEARCH_REMOVE_FACE; data: IAssetFaceJob }; +export type JobHandler = (data: T) => boolean | Promise; + export const IJobRepository = 'IJobRepository'; export interface IJobRepository { + addHandler(queueName: QueueName, concurrency: number, handler: (job: JobItem) => Promise): void; + setConcurrency(queueName: QueueName, concurrency: number): void; queue(item: JobItem): Promise; pause(name: QueueName): Promise; resume(name: QueueName): Promise; diff --git a/server/libs/domain/src/job/job.service.spec.ts b/server/libs/domain/src/job/job.service.spec.ts index a0b5c174b0..8b79e30d2c 100644 --- a/server/libs/domain/src/job/job.service.spec.ts +++ b/server/libs/domain/src/job/job.service.spec.ts @@ -1,20 +1,28 @@ import { BadRequestException } from '@nestjs/common'; -import { newAssetRepositoryMock, newCommunicationRepositoryMock, newJobRepositoryMock } from '../../test'; +import { + newAssetRepositoryMock, + newCommunicationRepositoryMock, + newJobRepositoryMock, + newSystemConfigRepositoryMock, +} from '../../test'; import { IAssetRepository } from '../asset'; import { ICommunicationRepository } from '../communication'; -import { IJobRepository, JobCommand, JobName, JobService, QueueName } from '../job'; +import { IJobRepository, JobCommand, JobHandler, JobName, JobService, QueueName } from '../job'; +import { ISystemConfigRepository } from '../system-config'; describe(JobService.name, () => { let sut: JobService; let assetMock: jest.Mocked; + let configMock: jest.Mocked; let communicationMock: jest.Mocked; let jobMock: jest.Mocked; beforeEach(async () => { assetMock = newAssetRepositoryMock(); + configMock = newSystemConfigRepositoryMock(); communicationMock = newCommunicationRepositoryMock(); jobMock = newJobRepositoryMock(); - sut = new JobService(assetMock, communicationMock, jobMock); + sut = new JobService(assetMock, communicationMock, jobMock, configMock); }); it('should work', () => { @@ -64,16 +72,16 @@ describe(JobService.name, () => { }; await expect(sut.getAllJobsStatus()).resolves.toEqual({ - 'background-task-queue': expectedJobStatus, - 'clip-encoding-queue': expectedJobStatus, - 'metadata-extraction-queue': expectedJobStatus, - 'object-tagging-queue': expectedJobStatus, - 'search-queue': expectedJobStatus, - 'storage-template-migration-queue': expectedJobStatus, - 'thumbnail-generation-queue': expectedJobStatus, - 'video-conversion-queue': expectedJobStatus, - 'recognize-faces-queue': expectedJobStatus, - 'sidecar-queue': expectedJobStatus, + [QueueName.BACKGROUND_TASK]: expectedJobStatus, + [QueueName.CLIP_ENCODING]: expectedJobStatus, + [QueueName.METADATA_EXTRACTION]: expectedJobStatus, + [QueueName.OBJECT_TAGGING]: expectedJobStatus, + [QueueName.SEARCH]: expectedJobStatus, + [QueueName.STORAGE_TEMPLATE_MIGRATION]: expectedJobStatus, + [QueueName.THUMBNAIL_GENERATION]: expectedJobStatus, + [QueueName.VIDEO_CONVERSION]: expectedJobStatus, + [QueueName.RECOGNIZE_FACES]: expectedJobStatus, + [QueueName.SIDECAR]: expectedJobStatus, }); }); }); @@ -147,6 +155,14 @@ describe(JobService.name, () => { expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force: false } }); }); + it('should handle a start sidecar command', async () => { + jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false }); + + await sut.handleCommand(QueueName.SIDECAR, { command: JobCommand.START, force: false }); + + expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_SIDECAR, data: { force: false } }); + }); + it('should handle a start thumbnail generation command', async () => { jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false }); @@ -155,6 +171,14 @@ describe(JobService.name, () => { expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }); }); + it('should handle a start recognize faces command', async () => { + jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false }); + + await sut.handleCommand(QueueName.RECOGNIZE_FACES, { command: JobCommand.START, force: false }); + + expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force: false } }); + }); + it('should throw a bad request when an invalid queue is used', async () => { jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false }); @@ -165,4 +189,19 @@ describe(JobService.name, () => { expect(jobMock.queue).not.toHaveBeenCalled(); }); }); + + describe('registerHandlers', () => { + it('should register a handler for each queue', async () => { + const mock = jest.fn(); + const handlers = Object.values(JobName).reduce((map, jobName) => ({ ...map, [jobName]: mock }), {}) as Record< + JobName, + JobHandler + >; + + await sut.registerHandlers(handlers); + + expect(configMock.load).toHaveBeenCalled(); + expect(jobMock.addHandler).toHaveBeenCalledTimes(Object.keys(QueueName).length); + }); + }); }); diff --git a/server/libs/domain/src/job/job.service.ts b/server/libs/domain/src/job/job.service.ts index 3d09f27b3e..c52904b9bb 100644 --- a/server/libs/domain/src/job/job.service.ts +++ b/server/libs/domain/src/job/job.service.ts @@ -2,20 +2,26 @@ import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common' import { IAssetRepository, mapAsset } from '../asset'; import { CommunicationEvent, ICommunicationRepository } from '../communication'; import { assertMachineLearningEnabled } from '../domain.constant'; +import { ISystemConfigRepository } from '../system-config'; +import { SystemConfigCore } from '../system-config/system-config.core'; import { JobCommandDto } from './dto'; import { JobCommand, JobName, QueueName } from './job.constants'; -import { IJobRepository, JobItem } from './job.repository'; +import { IJobRepository, JobHandler, JobItem } from './job.repository'; import { AllJobStatusResponseDto, JobStatusDto } from './response-dto'; @Injectable() export class JobService { private logger = new Logger(JobService.name); + private configCore: SystemConfigCore; constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @Inject(ICommunicationRepository) private communicationRepository: ICommunicationRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, - ) {} + @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, + ) { + this.configCore = new SystemConfigCore(configRepository); + } handleCommand(queueName: QueueName, dto: JobCommandDto): Promise { this.logger.debug(`Handling command: queue=${queueName},force=${dto.force}`); @@ -90,6 +96,36 @@ export class JobService { } } + async registerHandlers(jobHandlers: Record) { + const config = await this.configCore.getConfig(); + for (const queueName of Object.values(QueueName)) { + const concurrency = config.job[queueName].concurrency; + this.logger.debug(`Registering ${queueName} with a concurrency of ${concurrency}`); + this.jobRepository.addHandler(queueName, concurrency, async (item: JobItem): Promise => { + const { name, data } = item; + + try { + const handler = jobHandlers[name]; + const success = await handler(data); + if (success) { + await this.onDone(item); + } + } catch (error: Error | any) { + this.logger.error(`Unable to run job handler: ${error}`, error?.stack, data); + } + }); + } + + this.configCore.config$.subscribe((config) => { + this.logger.log(`Updating queue concurrency settings`); + for (const queueName of Object.values(QueueName)) { + const concurrency = config.job[queueName].concurrency; + this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`); + this.jobRepository.setConcurrency(queueName, concurrency); + } + }); + } + async handleNightlyJobs() { await this.jobRepository.queue({ name: JobName.USER_DELETE_CHECK }); await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP }); diff --git a/server/libs/domain/src/system-config/dto/system-config-job.dto.ts b/server/libs/domain/src/system-config/dto/system-config-job.dto.ts new file mode 100644 index 0000000000..ce9bcb7e77 --- /dev/null +++ b/server/libs/domain/src/system-config/dto/system-config-job.dto.ts @@ -0,0 +1,73 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { Type } from 'class-transformer'; +import { IsInt, IsObject, IsPositive, ValidateNested } from 'class-validator'; +import { QueueName } from '../../job'; + +export class JobSettingsDto { + @IsInt() + @IsPositive() + @ApiProperty({ type: 'integer' }) + concurrency!: number; +} + +export class SystemConfigJobDto implements Record { + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.THUMBNAIL_GENERATION]!: JobSettingsDto; + + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.METADATA_EXTRACTION]!: JobSettingsDto; + + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.VIDEO_CONVERSION]!: JobSettingsDto; + + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.OBJECT_TAGGING]!: JobSettingsDto; + + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.CLIP_ENCODING]!: JobSettingsDto; + + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto; + + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.BACKGROUND_TASK]!: JobSettingsDto; + + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.SEARCH]!: JobSettingsDto; + + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.RECOGNIZE_FACES]!: JobSettingsDto; + + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.SIDECAR]!: JobSettingsDto; +} diff --git a/server/libs/domain/src/system-config/dto/system-config.dto.ts b/server/libs/domain/src/system-config/dto/system-config.dto.ts index 92dc51bef1..a350a721aa 100644 --- a/server/libs/domain/src/system-config/dto/system-config.dto.ts +++ b/server/libs/domain/src/system-config/dto/system-config.dto.ts @@ -1,6 +1,7 @@ import { SystemConfig } from '@app/infra/entities'; import { Type } from 'class-transformer'; import { IsObject, ValidateNested } from 'class-validator'; +import { SystemConfigJobDto } from './system-config-job.dto'; import { SystemConfigFFmpegDto } from './system-config-ffmpeg.dto'; import { SystemConfigOAuthDto } from './system-config-oauth.dto'; import { SystemConfigPasswordLoginDto } from './system-config-password-login.dto'; @@ -26,6 +27,11 @@ export class SystemConfigDto { @ValidateNested() @IsObject() storageTemplate!: SystemConfigStorageTemplateDto; + + @Type(() => SystemConfigJobDto) + @ValidateNested() + @IsObject() + job!: SystemConfigJobDto; } export function mapConfig(config: SystemConfig): SystemConfigDto { diff --git a/server/libs/domain/src/system-config/system-config.core.ts b/server/libs/domain/src/system-config/system-config.core.ts index 53937cc083..dcec26690f 100644 --- a/server/libs/domain/src/system-config/system-config.core.ts +++ b/server/libs/domain/src/system-config/system-config.core.ts @@ -1,13 +1,20 @@ -import { SystemConfig, SystemConfigEntity, SystemConfigKey, TranscodePreset } from '@app/infra/entities'; +import { + SystemConfig, + SystemConfigEntity, + SystemConfigKey, + SystemConfigValue, + TranscodePreset, +} from '@app/infra/entities'; import { BadRequestException, Injectable, Logger } from '@nestjs/common'; import * as _ from 'lodash'; import { Subject } from 'rxjs'; import { DeepPartial } from 'typeorm'; +import { QueueName } from '../job/job.constants'; import { ISystemConfigRepository } from './system-config.repository'; export type SystemConfigValidator = (config: SystemConfig) => void | Promise; -const defaults: SystemConfig = Object.freeze({ +const defaults = Object.freeze({ ffmpeg: { crf: 23, threads: 0, @@ -19,6 +26,18 @@ const defaults: SystemConfig = Object.freeze({ twoPass: false, transcode: TranscodePreset.REQUIRED, }, + job: { + [QueueName.BACKGROUND_TASK]: { concurrency: 5 }, + [QueueName.CLIP_ENCODING]: { concurrency: 2 }, + [QueueName.METADATA_EXTRACTION]: { concurrency: 5 }, + [QueueName.OBJECT_TAGGING]: { concurrency: 2 }, + [QueueName.RECOGNIZE_FACES]: { concurrency: 2 }, + [QueueName.SEARCH]: { concurrency: 5 }, + [QueueName.SIDECAR]: { concurrency: 5 }, + [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 }, + [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 }, + [QueueName.VIDEO_CONVERSION]: { concurrency: 1 }, + }, oauth: { enabled: false, issuerUrl: '', @@ -85,7 +104,7 @@ export class SystemConfigCore { for (const key of Object.values(SystemConfigKey)) { // get via dot notation - const item = { key, value: _.get(config, key) }; + const item = { key, value: _.get(config, key) as SystemConfigValue }; const defaultValue = _.get(defaults, key); const isMissing = !_.has(config, key); diff --git a/server/libs/domain/src/system-config/system-config.service.spec.ts b/server/libs/domain/src/system-config/system-config.service.spec.ts index 6b4120e8c8..4038506dd2 100644 --- a/server/libs/domain/src/system-config/system-config.service.spec.ts +++ b/server/libs/domain/src/system-config/system-config.service.spec.ts @@ -1,7 +1,7 @@ -import { SystemConfigEntity, SystemConfigKey, TranscodePreset } from '@app/infra/entities'; +import { SystemConfig, SystemConfigEntity, SystemConfigKey, TranscodePreset } from '@app/infra/entities'; import { BadRequestException } from '@nestjs/common'; import { newJobRepositoryMock, newSystemConfigRepositoryMock, systemConfigStub } from '../../test'; -import { IJobRepository, JobName } from '../job'; +import { IJobRepository, JobName, QueueName } from '../job'; import { SystemConfigValidator } from './system-config.core'; import { ISystemConfigRepository } from './system-config.repository'; import { SystemConfigService } from './system-config.service'; @@ -11,7 +11,19 @@ const updates: SystemConfigEntity[] = [ { key: SystemConfigKey.OAUTH_AUTO_LAUNCH, value: true }, ]; -const updatedConfig = Object.freeze({ +const updatedConfig = Object.freeze({ + job: { + [QueueName.BACKGROUND_TASK]: { concurrency: 5 }, + [QueueName.CLIP_ENCODING]: { concurrency: 2 }, + [QueueName.METADATA_EXTRACTION]: { concurrency: 5 }, + [QueueName.OBJECT_TAGGING]: { concurrency: 2 }, + [QueueName.RECOGNIZE_FACES]: { concurrency: 2 }, + [QueueName.SEARCH]: { concurrency: 5 }, + [QueueName.SIDECAR]: { concurrency: 5 }, + [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 }, + [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 }, + [QueueName.VIDEO_CONVERSION]: { concurrency: 1 }, + }, ffmpeg: { crf: 30, threads: 0, diff --git a/server/libs/domain/test/fixtures.ts b/server/libs/domain/test/fixtures.ts index 50cf697cb8..11abf265c5 100644 --- a/server/libs/domain/test/fixtures.ts +++ b/server/libs/domain/test/fixtures.ts @@ -23,6 +23,7 @@ import { AuthUserDto, ExifResponseDto, mapUser, + QueueName, SearchResult, SharedLinkResponseDto, TagResponseDto, @@ -531,6 +532,18 @@ export const systemConfigStub = { twoPass: false, transcode: TranscodePreset.REQUIRED, }, + job: { + [QueueName.BACKGROUND_TASK]: { concurrency: 5 }, + [QueueName.CLIP_ENCODING]: { concurrency: 2 }, + [QueueName.METADATA_EXTRACTION]: { concurrency: 5 }, + [QueueName.OBJECT_TAGGING]: { concurrency: 2 }, + [QueueName.RECOGNIZE_FACES]: { concurrency: 2 }, + [QueueName.SEARCH]: { concurrency: 5 }, + [QueueName.SIDECAR]: { concurrency: 5 }, + [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 }, + [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 }, + [QueueName.VIDEO_CONVERSION]: { concurrency: 1 }, + }, oauth: { autoLaunch: false, autoRegister: true, diff --git a/server/libs/domain/test/job.repository.mock.ts b/server/libs/domain/test/job.repository.mock.ts index cb347bb09b..073478e3be 100644 --- a/server/libs/domain/test/job.repository.mock.ts +++ b/server/libs/domain/test/job.repository.mock.ts @@ -2,6 +2,8 @@ import { IJobRepository } from '../src'; export const newJobRepositoryMock = (): jest.Mocked => { return { + addHandler: jest.fn(), + setConcurrency: jest.fn(), empty: jest.fn(), pause: jest.fn(), resume: jest.fn(), diff --git a/server/libs/infra/src/entities/system-config.entity.ts b/server/libs/infra/src/entities/system-config.entity.ts index 3d4c5d1571..81c7df1802 100644 --- a/server/libs/infra/src/entities/system-config.entity.ts +++ b/server/libs/infra/src/entities/system-config.entity.ts @@ -1,7 +1,8 @@ import { Column, Entity, PrimaryColumn } from 'typeorm'; +import { QueueName } from '../../../domain/src'; @Entity('system_config') -export class SystemConfigEntity { +export class SystemConfigEntity { @PrimaryColumn() key!: SystemConfigKey; @@ -9,7 +10,7 @@ export class SystemConfigEntity { value!: T; } -export type SystemConfigValue = any; +export type SystemConfigValue = string | number | boolean; // dot notation matches path in `SystemConfig` export enum SystemConfigKey { @@ -22,6 +23,18 @@ export enum SystemConfigKey { FFMPEG_MAX_BITRATE = 'ffmpeg.maxBitrate', FFMPEG_TWO_PASS = 'ffmpeg.twoPass', FFMPEG_TRANSCODE = 'ffmpeg.transcode', + + JOB_THUMBNAIL_GENERATION_CONCURRENCY = 'job.thumbnailGeneration.concurrency', + JOB_METADATA_EXTRACTION_CONCURRENCY = 'job.metadataExtraction.concurrency', + JOB_VIDEO_CONVERSION_CONCURRENCY = 'job.videoConversion.concurrency', + JOB_OBJECT_TAGGING_CONCURRENCY = 'job.objectTagging.concurrency', + JOB_RECOGNIZE_FACES_CONCURRENCY = 'job.recognizeFaces.concurrency', + JOB_CLIP_ENCODING_CONCURRENCY = 'job.clipEncoding.concurrency', + JOB_BACKGROUND_TASK_CONCURRENCY = 'job.backgroundTask.concurrency', + JOB_STORAGE_TEMPLATE_MIGRATION_CONCURRENCY = 'job.storageTemplateMigration.concurrency', + JOB_SEARCH_CONCURRENCY = 'job.search.concurrency', + JOB_SIDECAR_CONCURRENCY = 'job.sidecar.concurrency', + OAUTH_ENABLED = 'oauth.enabled', OAUTH_ISSUER_URL = 'oauth.issuerUrl', OAUTH_CLIENT_ID = 'oauth.clientId', @@ -32,7 +45,9 @@ export enum SystemConfigKey { OAUTH_AUTO_REGISTER = 'oauth.autoRegister', OAUTH_MOBILE_OVERRIDE_ENABLED = 'oauth.mobileOverrideEnabled', OAUTH_MOBILE_REDIRECT_URI = 'oauth.mobileRedirectUri', + PASSWORD_LOGIN_ENABLED = 'passwordLogin.enabled', + STORAGE_TEMPLATE = 'storageTemplate.template', } @@ -55,6 +70,7 @@ export interface SystemConfig { twoPass: boolean; transcode: TranscodePreset; }; + job: Record; oauth: { enabled: boolean; issuerUrl: string; diff --git a/server/libs/infra/src/infra.config.ts b/server/libs/infra/src/infra.config.ts index 5a7b70884b..81bb61bf1e 100644 --- a/server/libs/infra/src/infra.config.ts +++ b/server/libs/infra/src/infra.config.ts @@ -1,5 +1,6 @@ import { QueueName } from '@app/domain'; -import { BullModuleOptions } from '@nestjs/bull'; +import { RegisterQueueOptions } from '@nestjs/bullmq'; +import { QueueOptions } from 'bullmq'; import { RedisOptions } from 'ioredis'; import { InitOptions } from 'local-reverse-geocoder'; import { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration'; @@ -26,9 +27,9 @@ function parseRedisConfig(): RedisOptions { export const redisConfig: RedisOptions = parseRedisConfig(); -export const bullConfig: BullModuleOptions = { +export const bullConfig: QueueOptions = { prefix: 'immich_bull', - redis: redisConfig, + connection: redisConfig, defaultJobOptions: { attempts: 3, removeOnComplete: true, @@ -36,7 +37,7 @@ export const bullConfig: BullModuleOptions = { }, }; -export const bullQueues: BullModuleOptions[] = Object.values(QueueName).map((name) => ({ name })); +export const bullQueues: RegisterQueueOptions[] = Object.values(QueueName).map((name) => ({ name })); function parseTypeSenseConfig(): ConfigurationOptions { const typesenseURL = process.env.TYPESENSE_URL; diff --git a/server/libs/infra/src/infra.module.ts b/server/libs/infra/src/infra.module.ts index 70b462db11..420ef583ef 100644 --- a/server/libs/infra/src/infra.module.ts +++ b/server/libs/infra/src/infra.module.ts @@ -21,7 +21,7 @@ import { IUserRepository, IUserTokenRepository, } from '@app/domain'; -import { BullModule } from '@nestjs/bull'; +import { BullModule } from '@nestjs/bullmq'; import { Global, Module, Provider } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { TypeOrmModule } from '@nestjs/typeorm'; diff --git a/server/libs/infra/src/repositories/job.repository.ts b/server/libs/infra/src/repositories/job.repository.ts index 69a39444e0..1d24a917e1 100644 --- a/server/libs/infra/src/repositories/job.repository.ts +++ b/server/libs/infra/src/repositories/job.repository.ts @@ -1,13 +1,33 @@ import { IJobRepository, JobCounts, JobItem, JobName, JOBS_TO_QUEUE, QueueName, QueueStatus } from '@app/domain'; -import { getQueueToken } from '@nestjs/bull'; -import { Injectable } from '@nestjs/common'; +import { getQueueToken } from '@nestjs/bullmq'; +import { Injectable, Logger } from '@nestjs/common'; import { ModuleRef } from '@nestjs/core'; -import { JobOptions, Queue, type JobCounts as BullJobCounts } from 'bull'; +import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullmq'; +import { bullConfig } from '../infra.config'; @Injectable() export class JobRepository implements IJobRepository { + private workers: Partial> = {}; + private logger = new Logger(JobRepository.name); + constructor(private moduleRef: ModuleRef) {} + addHandler(queueName: QueueName, concurrency: number, handler: (item: JobItem) => Promise) { + const workerHandler: Processor = async (job: Job) => handler(job as JobItem); + const workerOptions: WorkerOptions = { ...bullConfig, concurrency }; + this.workers[queueName] = new Worker(queueName, workerHandler, workerOptions); + } + + setConcurrency(queueName: QueueName, concurrency: number) { + const worker = this.workers[queueName]; + if (!worker) { + this.logger.warn(`Unable to set queue concurrency, worker not found: '${queueName}'`); + return; + } + + worker.concurrency = concurrency; + } + async getQueueStatus(name: QueueName): Promise { const queue = this.getQueue(name); @@ -26,13 +46,18 @@ export class JobRepository implements IJobRepository { } empty(name: QueueName) { - return this.getQueue(name).empty(); + return this.getQueue(name).drain(); } getJobCounts(name: QueueName): Promise { - // Typecast needed because the `paused` key is missing from Bull's - // type definition. Can be removed once fixed upstream. - return this.getQueue(name).getJobCounts() as Promise; + return this.getQueue(name).getJobCounts( + 'active', + 'completed', + 'failed', + 'delayed', + 'waiting', + 'paused', + ) as unknown as Promise; } async queue(item: JobItem): Promise { @@ -43,7 +68,7 @@ export class JobRepository implements IJobRepository { await this.getQueue(JOBS_TO_QUEUE[jobName]).add(jobName, jobData, jobOptions); } - private getJobOptions(item: JobItem): JobOptions | null { + private getJobOptions(item: JobItem): JobsOptions | null { switch (item.name) { case JobName.GENERATE_FACE_THUMBNAIL: return { priority: 1 }; diff --git a/server/libs/infra/src/repositories/system-config.repository.ts b/server/libs/infra/src/repositories/system-config.repository.ts index 4ffd3d6e28..0ce7c07a56 100644 --- a/server/libs/infra/src/repositories/system-config.repository.ts +++ b/server/libs/infra/src/repositories/system-config.repository.ts @@ -9,7 +9,7 @@ export class SystemConfigRepository implements ISystemConfigRepository { private repository: Repository, ) {} - load(): Promise[]> { + load(): Promise { return this.repository.find(); } diff --git a/server/package-lock.json b/server/package-lock.json index 53208bdcdd..138889bd6b 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -10,7 +10,7 @@ "license": "UNLICENSED", "dependencies": { "@babel/runtime": "^7.20.13", - "@nestjs/bull": "^0.6.2", + "@nestjs/bullmq": "^1.1.0", "@nestjs/common": "^9.2.1", "@nestjs/config": "^2.2.0", "@nestjs/core": "^9.2.1", @@ -24,7 +24,7 @@ "archiver": "^5.3.1", "axios": "^0.26.0", "bcrypt": "^5.0.1", - "bull": "^4.10.2", + "bullmq": "^3.14.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", @@ -1507,20 +1507,6 @@ "win32" ] }, - "node_modules/@nestjs/bull": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/@nestjs/bull/-/bull-0.6.3.tgz", - "integrity": "sha512-CckH9O3t9qSiO4RCzdYvtFSaaMfIhTXMYagV/rtmVvI1SX5XNnxEaQXvtjxDBXF9DB1JE/5AejIl6ICym+MJIw==", - "dependencies": { - "@nestjs/bull-shared": "^0.1.3", - "tslib": "2.5.0" - }, - "peerDependencies": { - "@nestjs/common": "^6.10.11 || ^7.0.0 || ^8.0.0 || ^9.0.0", - "@nestjs/core": "^6.10.11 || ^7.0.0 || ^8.0.0 || ^9.0.0", - "bull": "^3.3 || ^4.0.0" - } - }, "node_modules/@nestjs/bull-shared": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-0.1.3.tgz", @@ -1533,6 +1519,20 @@ "@nestjs/core": "^6.10.11 || ^7.0.0 || ^8.0.0 || ^9.0.0" } }, + "node_modules/@nestjs/bullmq": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-1.1.0.tgz", + "integrity": "sha512-XloO39ACm9TuB8XOX53iMUaCW5BTQAnZADtX4a9kczJZU/5/cQVBfSCyfzq9L6dfi5EX6w/1Ayyv+5qBQ5yrzw==", + "dependencies": { + "@nestjs/bull-shared": "^0.1.3", + "tslib": "2.5.0" + }, + "peerDependencies": { + "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0", + "@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0", + "bullmq": "^3.0.0" + } + }, "node_modules/@nestjs/cli": { "version": "9.4.2", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-9.4.2.tgz", @@ -4232,30 +4232,56 @@ "node": ">=0.2.0" } }, - "node_modules/bull": { - "version": "4.10.4", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.10.4.tgz", - "integrity": "sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==", + "node_modules/bullmq": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.14.1.tgz", + "integrity": "sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==", "dependencies": { - "cron-parser": "^4.2.1", - "debuglog": "^1.0.0", - "get-port": "^5.1.1", - "ioredis": "^5.0.0", + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.2", "lodash": "^4.17.21", - "msgpackr": "^1.5.2", - "semver": "^7.3.2", - "uuid": "^8.3.0" + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bullmq/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/bullmq/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" }, "engines": { "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/bull/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" + "node_modules/bullmq/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" } }, "node_modules/busboy": { @@ -5013,14 +5039,6 @@ } } }, - "node_modules/debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==", - "engines": { - "node": "*" - } - }, "node_modules/decimal.js": { "version": "10.4.3", "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", @@ -6422,17 +6440,6 @@ "node": ">=8.0.0" } }, - "node_modules/get-port": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", - "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -8429,9 +8436,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/msgpackr": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.1.tgz", - "integrity": "sha512-jJdrNH8tzfCtT0rjPFryBXjRDQE7rqfLkah4/8B4gYa7NNZYFBcGxqWBtfQpGC+oYyBwlkj3fARk4aooKNPHxg==", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.2.tgz", + "integrity": "sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==", "optionalDependencies": { "msgpackr-extract": "^3.0.2" } @@ -13122,15 +13129,6 @@ "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", "optional": true }, - "@nestjs/bull": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/@nestjs/bull/-/bull-0.6.3.tgz", - "integrity": "sha512-CckH9O3t9qSiO4RCzdYvtFSaaMfIhTXMYagV/rtmVvI1SX5XNnxEaQXvtjxDBXF9DB1JE/5AejIl6ICym+MJIw==", - "requires": { - "@nestjs/bull-shared": "^0.1.3", - "tslib": "2.5.0" - } - }, "@nestjs/bull-shared": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-0.1.3.tgz", @@ -13139,6 +13137,15 @@ "tslib": "2.5.0" } }, + "@nestjs/bullmq": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-1.1.0.tgz", + "integrity": "sha512-XloO39ACm9TuB8XOX53iMUaCW5BTQAnZADtX4a9kczJZU/5/cQVBfSCyfzq9L6dfi5EX6w/1Ayyv+5qBQ5yrzw==", + "requires": { + "@nestjs/bull-shared": "^0.1.3", + "tslib": "2.5.0" + } + }, "@nestjs/cli": { "version": "9.4.2", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-9.4.2.tgz", @@ -15212,25 +15219,48 @@ "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==" }, - "bull": { - "version": "4.10.4", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.10.4.tgz", - "integrity": "sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==", + "bullmq": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.14.1.tgz", + "integrity": "sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==", "requires": { - "cron-parser": "^4.2.1", - "debuglog": "^1.0.0", - "get-port": "^5.1.1", - "ioredis": "^5.0.0", + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.2", "lodash": "^4.17.21", - "msgpackr": "^1.5.2", - "semver": "^7.3.2", - "uuid": "^8.3.0" + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" }, "dependencies": { - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + } + }, + "minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "requires": { + "brace-expansion": "^2.0.1" + } } } }, @@ -15800,11 +15830,6 @@ "ms": "2.1.2" } }, - "debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==" - }, "decimal.js": { "version": "10.4.3", "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", @@ -16867,11 +16892,6 @@ "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", "dev": true }, - "get-port": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", - "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==" - }, "get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -18386,9 +18406,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "msgpackr": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.1.tgz", - "integrity": "sha512-jJdrNH8tzfCtT0rjPFryBXjRDQE7rqfLkah4/8B4gYa7NNZYFBcGxqWBtfQpGC+oYyBwlkj3fARk4aooKNPHxg==", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.2.tgz", + "integrity": "sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==", "requires": { "msgpackr-extract": "^3.0.2" } diff --git a/server/package.json b/server/package.json index a1fa4c9d37..7ed25630b9 100644 --- a/server/package.json +++ b/server/package.json @@ -41,7 +41,7 @@ }, "dependencies": { "@babel/runtime": "^7.20.13", - "@nestjs/bull": "^0.6.2", + "@nestjs/bullmq": "^1.1.0", "@nestjs/common": "^9.2.1", "@nestjs/config": "^2.2.0", "@nestjs/core": "^9.2.1", @@ -55,7 +55,7 @@ "archiver": "^5.3.1", "axios": "^0.26.0", "bcrypt": "^5.0.1", - "bull": "^4.10.2", + "bullmq": "^3.14.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", @@ -140,9 +140,9 @@ "coverageThreshold": { "./libs/domain/": { "branches": 80, - "functions": 85, - "lines": 93, - "statements": 93 + "functions": 80, + "lines": 90, + "statements": 90 } }, "setupFilesAfterEnv": [ diff --git a/web/src/api/api.ts b/web/src/api/api.ts index 79397a9003..3361f5e187 100644 --- a/web/src/api/api.ts +++ b/web/src/api/api.ts @@ -15,7 +15,8 @@ import { ShareApi, SystemConfigApi, UserApi, - UserApiFp + UserApiFp, + JobName } from './open-api'; import { BASE_PATH } from './open-api/base'; import { DUMMY_BASE_URL, toPathString } from './open-api/common'; @@ -106,6 +107,23 @@ export class ImmichApi { const path = `/person/${personId}/thumbnail`; return this.createUrl(path); } + + public getJobName(jobName: JobName) { + const names: Record = { + [JobName.ThumbnailGeneration]: 'Generate Thumbnails', + [JobName.MetadataExtraction]: 'Extract Metadata', + [JobName.Sidecar]: 'Sidecar Metadata', + [JobName.ObjectTagging]: 'Tag Objects', + [JobName.ClipEncoding]: 'Encode Clip', + [JobName.RecognizeFaces]: 'Recognize Faces', + [JobName.VideoConversion]: 'Transcode Videos', + [JobName.StorageTemplateMigration]: 'Storage Template Migration', + [JobName.BackgroundTask]: 'Background Tasks', + [JobName.Search]: 'Search' + }; + + return names[jobName]; + } } export const api = new ImmichApi({ basePath: '/api' }); diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index 5fbe6bd5d6..2d51a853cc 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -296,61 +296,61 @@ export interface AllJobStatusResponseDto { * @type {JobStatusDto} * @memberof AllJobStatusResponseDto */ - 'thumbnail-generation-queue': JobStatusDto; + 'thumbnailGeneration': JobStatusDto; /** * * @type {JobStatusDto} * @memberof AllJobStatusResponseDto */ - 'metadata-extraction-queue': JobStatusDto; + 'metadataExtraction': JobStatusDto; /** * * @type {JobStatusDto} * @memberof AllJobStatusResponseDto */ - 'video-conversion-queue': JobStatusDto; + 'videoConversion': JobStatusDto; /** * * @type {JobStatusDto} * @memberof AllJobStatusResponseDto */ - 'object-tagging-queue': JobStatusDto; + 'objectTagging': JobStatusDto; /** * * @type {JobStatusDto} * @memberof AllJobStatusResponseDto */ - 'clip-encoding-queue': JobStatusDto; + 'clipEncoding': JobStatusDto; /** * * @type {JobStatusDto} * @memberof AllJobStatusResponseDto */ - 'storage-template-migration-queue': JobStatusDto; + 'storageTemplateMigration': JobStatusDto; /** * * @type {JobStatusDto} * @memberof AllJobStatusResponseDto */ - 'background-task-queue': JobStatusDto; + 'backgroundTask': JobStatusDto; /** * * @type {JobStatusDto} * @memberof AllJobStatusResponseDto */ - 'search-queue': JobStatusDto; + 'search': JobStatusDto; /** * * @type {JobStatusDto} * @memberof AllJobStatusResponseDto */ - 'recognize-faces-queue': JobStatusDto; + 'recognizeFaces': JobStatusDto; /** * * @type {JobStatusDto} * @memberof AllJobStatusResponseDto */ - 'sidecar-queue': JobStatusDto; + 'sidecar': JobStatusDto; } /** * @@ -1486,21 +1486,34 @@ export interface JobCountsDto { */ export const JobName = { - ThumbnailGenerationQueue: 'thumbnail-generation-queue', - MetadataExtractionQueue: 'metadata-extraction-queue', - VideoConversionQueue: 'video-conversion-queue', - ObjectTaggingQueue: 'object-tagging-queue', - RecognizeFacesQueue: 'recognize-faces-queue', - ClipEncodingQueue: 'clip-encoding-queue', - BackgroundTaskQueue: 'background-task-queue', - StorageTemplateMigrationQueue: 'storage-template-migration-queue', - SearchQueue: 'search-queue', - SidecarQueue: 'sidecar-queue' + ThumbnailGeneration: 'thumbnailGeneration', + MetadataExtraction: 'metadataExtraction', + VideoConversion: 'videoConversion', + ObjectTagging: 'objectTagging', + RecognizeFaces: 'recognizeFaces', + ClipEncoding: 'clipEncoding', + BackgroundTask: 'backgroundTask', + StorageTemplateMigration: 'storageTemplateMigration', + Search: 'search', + Sidecar: 'sidecar' } as const; export type JobName = typeof JobName[keyof typeof JobName]; +/** + * + * @export + * @interface JobSettingsDto + */ +export interface JobSettingsDto { + /** + * + * @type {number} + * @memberof JobSettingsDto + */ + 'concurrency': number; +} /** * * @export @@ -2247,6 +2260,12 @@ export interface SystemConfigDto { * @memberof SystemConfigDto */ 'storageTemplate': SystemConfigStorageTemplateDto; + /** + * + * @type {SystemConfigJobDto} + * @memberof SystemConfigDto + */ + 'job': SystemConfigJobDto; } /** * @@ -2319,6 +2338,73 @@ export const SystemConfigFFmpegDtoTranscodeEnum = { export type SystemConfigFFmpegDtoTranscodeEnum = typeof SystemConfigFFmpegDtoTranscodeEnum[keyof typeof SystemConfigFFmpegDtoTranscodeEnum]; +/** + * + * @export + * @interface SystemConfigJobDto + */ +export interface SystemConfigJobDto { + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'thumbnailGeneration': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'metadataExtraction': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'videoConversion': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'objectTagging': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'clipEncoding': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'storageTemplateMigration': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'backgroundTask': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'search': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'recognizeFaces': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'sidecar': JobSettingsDto; +} /** * * @export diff --git a/web/src/lib/components/admin-page/jobs/job-tile.svelte b/web/src/lib/components/admin-page/jobs/job-tile.svelte index d7bea7ddef..12644f68c2 100644 --- a/web/src/lib/components/admin-page/jobs/job-tile.svelte +++ b/web/src/lib/components/admin-page/jobs/job-tile.svelte @@ -30,7 +30,7 @@
{#if queueStatus.isPaused} diff --git a/web/src/lib/components/admin-page/jobs/jobs-panel.svelte b/web/src/lib/components/admin-page/jobs/jobs-panel.svelte index 041266cbef..06a3d05a5a 100644 --- a/web/src/lib/components/admin-page/jobs/jobs-panel.svelte +++ b/web/src/lib/components/admin-page/jobs/jobs-panel.svelte @@ -9,15 +9,17 @@ import Icon from 'svelte-material-icons/DotsVertical.svelte'; import FaceRecognition from 'svelte-material-icons/FaceRecognition.svelte'; import FileJpgBox from 'svelte-material-icons/FileJpgBox.svelte'; - import FolderMove from 'svelte-material-icons/FolderMove.svelte'; - import Table from 'svelte-material-icons/Table.svelte'; import FileXmlBox from 'svelte-material-icons/FileXmlBox.svelte'; + import FolderMove from 'svelte-material-icons/FolderMove.svelte'; + import Information from 'svelte-material-icons/Information.svelte'; + import Table from 'svelte-material-icons/Table.svelte'; import TagMultiple from 'svelte-material-icons/TagMultiple.svelte'; import VectorCircle from 'svelte-material-icons/VectorCircle.svelte'; import Video from 'svelte-material-icons/Video.svelte'; import ConfirmDialogue from '../../shared-components/confirm-dialogue.svelte'; import JobTile from './job-tile.svelte'; import StorageMigrationDescription from './storage-migration-description.svelte'; + import { AppRoute } from '$lib/constants'; export let jobs: AllJobStatusResponseDto; @@ -45,52 +47,52 @@ const onFaceConfirm = () => { faceConfirm = false; - handleCommand(JobName.RecognizeFacesQueue, { command: JobCommand.Start, force: true }); + handleCommand(JobName.RecognizeFaces, { command: JobCommand.Start, force: true }); }; const jobDetails: Partial> = { - [JobName.ThumbnailGenerationQueue]: { + [JobName.ThumbnailGeneration]: { icon: FileJpgBox, - title: 'Generate Thumbnails', + title: api.getJobName(JobName.ThumbnailGeneration), subtitle: 'Regenerate JPEG and WebP thumbnails' }, - [JobName.MetadataExtractionQueue]: { + [JobName.MetadataExtraction]: { icon: Table, - title: 'Extract Metadata', + title: api.getJobName(JobName.MetadataExtraction), subtitle: 'Extract metadata information i.e. GPS, resolution...etc' }, - [JobName.SidecarQueue]: { - title: 'Sidecar Metadata', + [JobName.Sidecar]: { + title: api.getJobName(JobName.Sidecar), icon: FileXmlBox, subtitle: 'Discover or synchronize sidecar metadata from the filesystem', allText: 'SYNC', missingText: 'DISCOVER' }, - [JobName.ObjectTaggingQueue]: { + [JobName.ObjectTagging]: { icon: TagMultiple, - title: 'Tag Objects', + title: api.getJobName(JobName.ObjectTagging), subtitle: 'Run machine learning to tag objects\nNote that some assets may not have any objects detected' }, - [JobName.ClipEncodingQueue]: { + [JobName.ClipEncoding]: { icon: VectorCircle, - title: 'Encode Clip', + title: api.getJobName(JobName.ClipEncoding), subtitle: 'Run machine learning to generate clip embeddings' }, - [JobName.RecognizeFacesQueue]: { + [JobName.RecognizeFaces]: { icon: FaceRecognition, - title: 'Recognize Faces', + title: api.getJobName(JobName.RecognizeFaces), subtitle: 'Run machine learning to recognize faces', handleCommand: handleFaceCommand }, - [JobName.VideoConversionQueue]: { + [JobName.VideoConversion]: { icon: Video, - title: 'Transcode Videos', + title: api.getJobName(JobName.VideoConversion), subtitle: 'Transcode videos not in the desired format' }, - [JobName.StorageTemplateMigrationQueue]: { + [JobName.StorageTemplateMigration]: { icon: FolderMove, - title: 'Storage Template Migration', + title: api.getJobName(JobName.StorageTemplateMigration), allowForceCommand: false, component: StorageMigrationDescription } @@ -128,6 +130,17 @@ {/if}
+
+ +

+ MANAGE JOB CURRENCENCY LEVEL IN + JOB SETTINGS +

+
+ {#each jobDetailsArray as [jobName, { title, subtitle, allText, missingText, allowForceCommand, icon, component, handleCommand: handleCommandOverride }]} {@const { jobCounts, queueStatus } = jobs[jobName]} + import { + notificationController, + NotificationType + } from '$lib/components/shared-components/notification/notification'; + import { api, JobName, SystemConfigJobDto } from '@api'; + import { isEqual } from 'lodash-es'; + import { fade } from 'svelte/transition'; + import { handleError } from '../../../../utils/handle-error'; + import SettingButtonsRow from '../setting-buttons-row.svelte'; + import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte'; + + export let jobConfig: SystemConfigJobDto; // this is the config that is being edited + + let savedConfig: SystemConfigJobDto; + let defaultConfig: SystemConfigJobDto; + + const ignoredJobs = [JobName.BackgroundTask, JobName.Search] as JobName[]; + const jobNames = Object.values(JobName).filter( + (jobName) => !ignoredJobs.includes(jobName as JobName) + ); + + async function getConfigs() { + [savedConfig, defaultConfig] = await Promise.all([ + api.systemConfigApi.getConfig().then((res) => res.data.job), + api.systemConfigApi.getDefaults().then((res) => res.data.job) + ]); + } + + async function saveSetting() { + try { + const { data: configs } = await api.systemConfigApi.getConfig(); + + const result = await api.systemConfigApi.updateConfig({ + systemConfigDto: { + ...configs, + job: jobConfig + } + }); + + jobConfig = { ...result.data.job }; + savedConfig = { ...result.data.job }; + + notificationController.show({ message: 'Job settings saved', type: NotificationType.Info }); + } catch (error) { + handleError(error, 'Unable to save settings'); + } + } + + async function reset() { + const { data: resetConfig } = await api.systemConfigApi.getConfig(); + + jobConfig = { ...resetConfig.job }; + savedConfig = { ...resetConfig.job }; + + notificationController.show({ + message: 'Reset Job settings to the recent saved settings', + type: NotificationType.Info + }); + } + + async function resetToDefault() { + const { data: configs } = await api.systemConfigApi.getDefaults(); + + jobConfig = { ...configs.job }; + defaultConfig = { ...configs.job }; + + notificationController.show({ + message: 'Reset Job settings to default', + type: NotificationType.Info + }); + } + + +
+ {#await getConfigs() then} +
+
+ {#each jobNames as jobName} +
+ +
+ {/each} + +
+ +
+
+
+ {/await} +
diff --git a/web/src/lib/components/admin-page/settings/setting-input-field.svelte b/web/src/lib/components/admin-page/settings/setting-input-field.svelte index 9c463b9eee..28ac4ca003 100644 --- a/web/src/lib/components/admin-page/settings/setting-input-field.svelte +++ b/web/src/lib/components/admin-page/settings/setting-input-field.svelte @@ -21,6 +21,9 @@ const handleInput = (e: Event) => { value = (e.target as HTMLInputElement).value; + if (inputType === SettingInputFieldType.NUMBER) { + value = Number(value) || 0; + } }; diff --git a/web/src/lib/utils/handle-error.ts b/web/src/lib/utils/handle-error.ts index 77c1d6b2f1..5751ef9096 100644 --- a/web/src/lib/utils/handle-error.ts +++ b/web/src/lib/utils/handle-error.ts @@ -9,7 +9,7 @@ export function handleError(error: unknown, message: string) { let serverMessage = (error as ApiError)?.response?.data?.message; if (serverMessage) { - serverMessage = `${String(serverMessage).slice(0, 50)}\n(Immich Server Error)`; + serverMessage = `${String(serverMessage).slice(0, 75)}\n(Immich Server Error)`; } notificationController.show({ diff --git a/web/src/routes/admin/system-settings/+page.svelte b/web/src/routes/admin/system-settings/+page.svelte index ef47a9bde3..e6274900df 100644 --- a/web/src/routes/admin/system-settings/+page.svelte +++ b/web/src/routes/admin/system-settings/+page.svelte @@ -1,6 +1,7 @@