From 4cfac476745cfe839eac8dc4f585e05416107678 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Sat, 21 Jan 2023 23:13:36 -0500 Subject: [PATCH] refactor(server): job repository (#1382) * refactor(server): job repository * refactor: job repository * chore: generate open-api * fix: job panel * Remove incorrect subtitle Co-authored-by: Alex Tran --- mobile/openapi/.openapi-generator/FILES | 3 - mobile/openapi/README.md | Bin 13908 -> 13766 bytes mobile/openapi/doc/AllJobStatusResponseDto.md | Bin 970 -> 695 bytes mobile/openapi/doc/JobApi.md | Bin 4234 -> 2895 bytes mobile/openapi/doc/JobStatusResponseDto.md | Bin 457 -> 0 bytes mobile/openapi/lib/api.dart | Bin 4747 -> 4704 bytes mobile/openapi/lib/api/job_api.dart | Bin 5122 -> 3547 bytes mobile/openapi/lib/api_client.dart | Bin 16426 -> 16334 bytes .../model/all_job_status_response_dto.dart | Bin 7839 -> 5137 bytes .../lib/model/job_status_response_dto.dart | Bin 3728 -> 0 bytes .../all_job_status_response_dto_test.dart | Bin 1859 -> 1114 bytes mobile/openapi/test/job_api_test.dart | Bin 900 -> 754 bytes .../test/job_status_response_dto_test.dart | Bin 687 -> 0 bytes .../src/api-v1/asset/asset.controller.ts | 2 +- .../src/api-v1/asset/asset.service.spec.ts | 12 +- .../immich/src/api-v1/asset/asset.service.ts | 27 +- .../immich/src/api-v1/job/job.controller.ts | 15 +- .../apps/immich/src/api-v1/job/job.service.ts | 277 ++++++------------ .../all-job-status-response.dto.ts | 38 +-- .../response-dto/job-status-response.dto.ts | 6 - .../background-task/background-task.module.ts | 5 +- .../background-task.processor.ts | 4 +- .../background-task.service.ts | 17 +- .../schedule-tasks/schedule-tasks.service.ts | 37 +-- .../src/microservices.service.ts | 17 +- .../processors/generate-checksum.processor.ts | 4 +- server/immich-openapi-specs.json | 89 +----- server/libs/domain/src/job/job.constants.ts | 1 + server/libs/domain/src/job/job.repository.ts | 16 +- .../libs/domain/test/job.repository.mock.ts | 3 + server/libs/infra/src/job/job.repository.ts | 99 ++++++- web/src/api/open-api/api.ts | 145 +-------- .../admin-page/jobs/job-tile.svelte | 27 +- .../admin-page/jobs/jobs-panel.svelte | 244 +++++---------- 34 files changed, 367 insertions(+), 721 deletions(-) delete mode 100644 mobile/openapi/doc/JobStatusResponseDto.md delete mode 100644 mobile/openapi/lib/model/job_status_response_dto.dart delete mode 100644 mobile/openapi/test/job_status_response_dto_test.dart delete mode 100644 server/apps/immich/src/api-v1/job/response-dto/job-status-response.dto.ts diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 62b938b3a9..56a94fb339 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -53,7 +53,6 @@ doc/JobCommand.md doc/JobCommandDto.md doc/JobCounts.md doc/JobId.md -doc/JobStatusResponseDto.md doc/LoginCredentialDto.md doc/LoginResponseDto.md doc/LogoutResponseDto.md @@ -162,7 +161,6 @@ lib/model/job_command.dart lib/model/job_command_dto.dart lib/model/job_counts.dart lib/model/job_id.dart -lib/model/job_status_response_dto.dart lib/model/login_credential_dto.dart lib/model/login_response_dto.dart lib/model/logout_response_dto.dart @@ -250,7 +248,6 @@ test/job_command_dto_test.dart test/job_command_test.dart test/job_counts_test.dart test/job_id_test.dart -test/job_status_response_dto_test.dart test/login_credential_dto_test.dart test/login_response_dto_test.dart test/logout_response_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index d2a1a0c83b278890523455cb1da2b9ff06dbee5b..30bcbc8b3f7737ad6f17330d77b8ec20106135dc 100644 GIT binary patch delta 17 ZcmcbTb1ZveI;juuLN6qYKd2V(qu*n8KtcJq~emqlG0*Lg&GAdEqB)tEiDCokf?q&kn~Kc t-E1wfS5z6OJ{YVnD7CmCKd(5|r6fOABPBmsUmqqlnNeGG^AFu~>;P(TB>(^b diff --git a/mobile/openapi/doc/AllJobStatusResponseDto.md b/mobile/openapi/doc/AllJobStatusResponseDto.md index 606375b8a6826886de39aa742b34f812637c3aed..66695ac2e1bf05adc2c6fb5bf47095b3035e516b 100644 GIT binary patch delta 70 zcmX@bzMXZ#*NOjaC;r#wFUw3x&3DevD@!dZ&dkr7_+Mx8Hby?6j6`useoxs&8g`QDO;LiRNTg=A{5@gC4>F literal 970 zcmbu8Uu(iJ6vf~3DFS^^1Fi4dQ>QXEtT>guh+u5)G_1`H$*pX#55Kg^<{##UD~0AH zCnwD>xe-STMp~VesGvyFqIksBlq6Mt0^XwW&?jUpiue$v;s zypfan)iGPG0DEY*;QZCUBJ+(lFa+t0HtVal;|X0jy-ctF!9@GnIo&kC*+N5Bh!tYd zG^PYC?%X7l-7xa^#eXLXhELVH>)+DX!#4AOo!2p1(% sm{%p3P&wgP;}>*sQzpK{zh|f8hd<5@MH7Qj7PXv)}^>XijgfnwZtnwY2rO`rL6p<;*!LY(qc`88U-yach?XtEd_m$sD3q&^h~MU zEX??bMG&GR7_3xFi_1=7ay_F2KhO|}kOo-oW_`AAjFS^M)P<0hL*yo};_!poJDHzF oZL%6?>*Uv*LekM%T5wZ>Qi}`n^NLelO7gX|Vl_5TX4%6A0BeaoWdHyG diff --git a/mobile/openapi/doc/JobStatusResponseDto.md b/mobile/openapi/doc/JobStatusResponseDto.md deleted file mode 100644 index 13325a5152d58a74153f27d66a95969c21321b5a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 457 zcma)&F;BxV5QTUD3QHMEEG6Ax3Zw&wma4EtmEyz)7!secFGxuIc+Q~`r3-km-}$?D z_RA!CaN65)L<2*4gWWx8n&J+kM;D2ChM`Vi2@Y|mn4>y7Ha@mo+%1hAab%cw;Ee|Y6;XNR^U`Ptvjx1ps1r2g(2d diff --git a/mobile/openapi/lib/model/all_job_status_response_dto.dart b/mobile/openapi/lib/model/all_job_status_response_dto.dart index 9bf95e3702455b4e58cb4ee4d3bef9cd211dc8e1..60f788a30207b06234d6da80755ec58cc3250e5f 100644 GIT binary patch delta 1056 zcmbPlJ5gi9F2>1bu9%FRox~t5A|rnwylD zn3?0AnwJVSa|WxTJ4h}!wInemu_V#8qNFG>8DyP;ni|k}AjmJtNG;MU%S=hlch1i% zOD!q}$pZDD2~E~ulM4fzm6)87nV0I5npl*VnU{`c5H4qebmEem%*l~5xtv2~@=^}P z$rph5H;1GkW&r)+j1jd}!RF0IE|Gd$6--a+pnJkr10UCoUpo7!dwkk*- z;zfPX53vW~y5PO_sX3uylzq6v?+Jp545G ze-`uP^CB{n3xqkjk$gS5S6BfgE)4Stk~b&67nTLd37~sovbP9WVsf7dyFT27>am)N zI?%9Fz!@Vz1C>Rw`eE};u^P5|B)6$UOv4%jP$_hm!L^`;9!wk>KG^IBnOJMh!d1(~ F1pv#$bff?P literal 7839 zcmds6ZBH9H5dO}u7%J2R-8NkN=|me^RMM(15FK!pP7#WnjYHgSc5}8DE~5PRo3Zyj zw$tmElde*Mte<&2_B@^$Z;p=kj*j5;kL&TX_oIu^<>~e44BmbCJc{6K3>V{TI3J&V zc=zW%!U*|h&ZHi`7(9QmM<2y)nn^xgWOBL?2|tEB%@vzArQkG4K4!Bk&2+9Vxmx3-{7h%? zum^C2`*6v1E>i^zQhpB;5Bvs%aW>`ywDW-a#0gOD1?Z(v1HH^wvy_SCJx{r0T4d?( zInVjqEXQ4T7~sCKIn!+Px0Wp4jlaX~%id<`9q#sSauzRfBR_K{Q<2_=JyAN7?3SO4 z+aK{;sO#;ep5g-TxNDh0oo_8Pg#OZAW(a!TUStS*)mdVVeLO@70ibqFq!dOo5XcTT z6vGbG8@>*1)m~H`xem@BGy|MvSptDjH9@`rH8IYxGy%?FGy(1q7{r?lMJ+M#7zpDy z9&nUmCSf@2k7r+bjK^Ltxjcm73D`6ev!qW4FDggAf46_(s(l9+o<21P(E`Iq7N-tN zp+gpu0&xY$+kSbaLOlM#Hi=h9#u=Z;I*am)j93DrZC?U9B%aW4m&m78%i|sG(m3U8 zIov~C0;k*yC)%59Q%z@Mg>oMU%itc{Iw~U7aLY9;nOeTh<{S@)k#|+~so1AH=1MWS zp<<2ocgdAd28PEckm&{`>mmGN;UNWYqrE!ahI*yC4fRWI8yk?73+&U53+vQ~3+R-I z3+N7cjob?Um6ll8L1*@lzDbQePscI|hI)7q9FBsyYj_aSMdZ&{!vn8GaK`o-s;eMQ z=NpOw32AuXS56d4K*NK8jL6a_8#45bk)2aQWaJz$Gqo3P^NN|qH9!__QIUyz4553@z)RDP4m z3D}}JcO8c&jT5j%<31uy9yh`gNpsya@}~!nbjSFaYbdaE4)$)YKxhSbEXlcoh0In3 z<-|O~o9hArJ-Xpe@C4sPkmgBZTitu1%!Rhm8LTB=;0@bc&Sf5Bv#qAohEMUXu0=e> zgR{)$)G62pzMlHEamD0@B3D=!%S3@*^^Jp=j{yBDp98c~h=B}Zru0@a>DwC`UAfW! zQL1vuFw$zbg%0A_e&Ghb zsNhoft1L>cmHvJrchQ9r%D3M~=j)v!SYb*TcTlbuO$;!C-evJ(2G9_m_}YL29KCr$ zp00#av@+5We7qW840^cDG?D$+m~KKwus<|<`nOYI7fG*NNQs3C((UQ!4qi@N92_lk zUOs&6AJ!5m#yu`gp)qSHUilTSoWIx?(f}p zPT(td9e!sm+ez-U>M%RY+fHVuRfpMK@ODzW%^KXr@qaXjw>aRGGOEg1l%UT7%Y*0S zpS*?yhu@W_xhzID@(<><(yh$y*(||}GAvl4c=TB2SX+xES9zkTq4iBNxv@!!sv0p< zDkZP*dT(m|Kq#y;qt#kB(l^zWelbIKPql$e%c&`m*g=~hhrUW%XDVDqBV*PTw#d;< zW~wi=r~#YLZ5?h{+6|}fgq&z?e_^FFT3x@~xML{)GAyR6DPn`Ay{g>4-J`SV3{dH@ z-3Kb^jZ1}(t-NG3V}V`-Lt$bzw4w4f8`NdkZ%vM&`hsJM1rZ<8S0?uD2?nhWD?XK* zGOz>sB)BRU^%DU$d;_D0M!0qa8njS21LKLwGmA#!qEW?bBU;fnQbdX}bEt(B<~?vHWJd_6ZX)=xgN+w_Kr2ZB3-YwYC7OVU=^5K}8lEj>2+s~m@B9PrYj(0@8< z%(juWq|8mGGX6xiRT4RQFd1&gYY5xe^@815f$_IkB}w}dTED>R6@WmmQy@aUN(J$D Z8x_LcGE~Fvx?c@@YB!DJJ02_m{{VGjh_wI! diff --git a/mobile/openapi/lib/model/job_status_response_dto.dart b/mobile/openapi/lib/model/job_status_response_dto.dart deleted file mode 100644 index 2e6edb29380c00ddb7cffb19d2104248be0eed88..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3728 zcmds4ZExE)5dQ98aVdgY!4!GTry;4`28}y3ZDSzG1`LKF&@vsfl}VMPVi>9ad+$h5 zQev%JJ`Gra*b;f~_uNBzFz5{iaQ?^bd(I*9MJi)2*H-Dmn83UJUeq)&}%{4a#dgK>Y!9x#VfJM0Gw$#!9 z1Iafq?UJ2=aA9UVL8UHSWMG9pV#_CQ1@&-u7Y^(!w3LuwrKQPAxb_>0H&2BqU0nabuX?5|R)c zD{h2wVo@kj5)2%{FMg+S+U)JQzp+}#JFv<>M6OExuC`8tr{Q6Eid%^d4;z28zVq{$ z6|cS77#3Tc1d)x<)1^@o{dkltg=9H=!6g{Np7g`+%-7vXOm#na?$1#C5wM;JHo-Ka z&6*e14(-k~E$j$(&z=Hcasi$c3nWwFaEFPYf4IRFmyNqwt%k+wl841zbksDz+V9FOs*Qy4JtbiLNrN(3-fj#2n%I66QTbwMP5IG* zKQAbVW_tIuJo=6C6E-Or+)%AuSDv-6!(sWQd30o{Yi?lR-orzT`f=5-a zw5+1kqENC^2v5$6W8dF)Y+F1uCD;GQR|fHG!>4Vb_>w|PdngRHS`_& zJCY=J^moitsIP-5&N4321=Anr*sG?SxT?3Ip9#GQZbC-S?W;4EI1IG9^bh^Tf=#`m zBbuS-iZK77t>ODolTo#vEKD6=iEvz{l)uPrgk9kh%@kc$ySj1gMNzXlSMmCQDs*!v ziwLa_gb=E_$7@7NwB3o782;T1qm4F-e&X(bh9&A$UAY#X;(BtM8IJ!0kPYA#uN)L5 z;VGQlbmr3C18*ej61V0Y@M>i+ei|+2PrU|7?!Hwke4Op22YnS>`#VAGu}&%!mil}QLFW(?dh(5QgvkitWjc za4`1D^UOTEyx{wu?~D15IQn=M-iCMcI1I$)^-VYvK_qUYSS+I8`ttDum(`VDGtl<@ zQtGtbpnJ7Wh z^hi8=LNK(F1ci{zUQ!h2UF`Je%wr zsKH7_-yJ$^{^s7I*#S3K%@3|eQE5I6ZMWj-ApNu(3T8`bv$MJz?=FslsOkK&(|>2g zHyJXp5dN_m(t$#U;E_HX&XT%NwBhItJDpB)q)7?snj?FIph;3b;68PdBTbA|nxBe= z%m+uN8y)1bREnPv6Ys}&&Y9WZf1`VV9V4>XOKoqDE_95>!`ra-c4+0s2rS-aw6{Pj XH%8$8T{L?ew1fQ=IBm_#?8W;FI*3`= diff --git a/mobile/openapi/test/job_api_test.dart b/mobile/openapi/test/job_api_test.dart index 5ff6fd7745350b4b4205f98b6ea2d147ba0cdbac..cfdfa6479c77c3065ec34d814669da930b688ec8 100644 GIT binary patch delta 11 ScmZo+|HQf>g=z9Prak~0nFO)` delta 69 zcmeyw+QPmeg-OpVKPk8*v81#(D7CmCKd(5|r6k`@Aw9JOE};Qrd8R02fyl`Vn8ev2 Kl9Oebt^xps5*bAR diff --git a/mobile/openapi/test/job_status_response_dto_test.dart b/mobile/openapi/test/job_status_response_dto_test.dart deleted file mode 100644 index 09ea08df58cb68d01d58956f4d83f9ec80ae436d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 687 zcma)&K~KU!5QXpk72^p=kg6vGA(2=#8l?%=gQsOF1FV+aWp`FX4FBEfBJp4oJ#@R% zyzjj?P17VzVf0+&r?=zDcseS^8Qk1GjJuHKFv$ye%(J_jmlKxN%BzKRG`#4YUnKRY zR@yieOXI{+RoI81gU5xy3OA^!dz{tA8b^(UtUlz<3mZr{+y>m4pXpLYUuj*(Av!if! uaFKqe%UH4oxoDARc{JG{%EXWP{1r=rZNPxnCTRLI=12O!AU2GF`{WDO#NSr{ diff --git a/server/apps/immich/src/api-v1/asset/asset.controller.ts b/server/apps/immich/src/api-v1/asset/asset.controller.ts index 4f2d24a7c8..223fac9e78 100644 --- a/server/apps/immich/src/api-v1/asset/asset.controller.ts +++ b/server/apps/immich/src/api-v1/asset/asset.controller.ts @@ -295,7 +295,7 @@ export class AssetController { deleteAssetList.filter((a) => a.id == res.id && res.status == DeleteAssetStatusEnum.SUCCESS); }); - await this.backgroundTaskService.deleteFileOnDisk(deleteAssetList); + await this.backgroundTaskService.deleteFileOnDisk(deleteAssetList as any[]); return result; } diff --git a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts index a07873d815..7a46efb6eb 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts @@ -9,11 +9,11 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto'; import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto'; import { DownloadService } from '../../modules/download/download.service'; import { BackgroundTaskService } from '../../modules/background-task/background-task.service'; -import { IAssetUploadedJob, IVideoTranscodeJob } from '@app/domain'; -import { Queue } from 'bull'; import { IAlbumRepository } from '../album/album-repository'; import { StorageService } from '@app/storage'; import { ISharedLinkRepository } from '../share/shared-link.repository'; +import { IJobRepository } from '@app/domain'; +import { newJobRepositoryMock } from '@app/domain/../test'; describe('AssetService', () => { let sui: AssetService; @@ -22,10 +22,9 @@ describe('AssetService', () => { let albumRepositoryMock: jest.Mocked; let downloadServiceMock: jest.Mocked>; let backgroundTaskServiceMock: jest.Mocked; - let assetUploadedQueueMock: jest.Mocked>; - let videoConversionQueueMock: jest.Mocked>; let storageSeriveMock: jest.Mocked; let sharedLinkRepositoryMock: jest.Mocked; + let jobMock: jest.Mocked; const authUser: AuthUserDto = Object.freeze({ id: 'user_id_1', email: 'auth@test.com', @@ -148,16 +147,17 @@ describe('AssetService', () => { getByIdAndUserId: jest.fn(), }; + jobMock = newJobRepositoryMock(); + sui = new AssetService( assetRepositoryMock, albumRepositoryMock, a, backgroundTaskServiceMock, - assetUploadedQueueMock, - videoConversionQueueMock, downloadServiceMock as DownloadService, storageSeriveMock, sharedLinkRepositoryMock, + jobMock, ); }); diff --git a/server/apps/immich/src/api-v1/asset/asset.service.ts b/server/apps/immich/src/api-v1/asset/asset.service.ts index c4977cd5bf..0f6438f0b0 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.ts @@ -43,9 +43,7 @@ import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-as import { UpdateAssetDto } from './dto/update-asset.dto'; import { AssetFileUploadResponseDto } from './response-dto/asset-file-upload-response.dto'; import { BackgroundTaskService } from '../../modules/background-task/background-task.service'; -import { IAssetUploadedJob, IVideoTranscodeJob, JobName, QueueName } from '@app/domain'; -import { InjectQueue } from '@nestjs/bull'; -import { Queue } from 'bull'; +import { IJobRepository, JobName } from '@app/domain'; import { DownloadService } from '../../modules/download/download.service'; import { DownloadDto } from './dto/download-library.dto'; import { IAlbumRepository } from '../album/album-repository'; @@ -66,24 +64,14 @@ export class AssetService { constructor( @Inject(IAssetRepository) private _assetRepository: IAssetRepository, - @Inject(IAlbumRepository) private _albumRepository: IAlbumRepository, - @InjectRepository(AssetEntity) private assetRepository: Repository, - private backgroundTaskService: BackgroundTaskService, - - @InjectQueue(QueueName.ASSET_UPLOADED) - private assetUploadedQueue: Queue, - - @InjectQueue(QueueName.VIDEO_CONVERSION) - private videoConversionQueue: Queue, - private downloadService: DownloadService, - private storageService: StorageService, @Inject(ISharedLinkRepository) sharedLinkRepository: ISharedLinkRepository, + @Inject(IJobRepository) private jobRepository: IJobRepository, ) { this.shareCore = new ShareCore(sharedLinkRepository); } @@ -122,7 +110,7 @@ export class AssetService { await this.storageService.moveAsset(livePhotoAssetEntity, originalAssetData.originalname); - await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset: livePhotoAssetEntity }); + await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset: livePhotoAssetEntity } }); } const assetEntity = await this.createUserAsset( @@ -146,11 +134,10 @@ export class AssetService { const movedAsset = await this.storageService.moveAsset(assetEntity, originalAssetData.originalname); - await this.assetUploadedQueue.add( - JobName.ASSET_UPLOADED, - { asset: movedAsset, fileName: originalAssetData.originalname }, - { jobId: movedAsset.id }, - ); + await this.jobRepository.add({ + name: JobName.ASSET_UPLOADED, + data: { asset: movedAsset, fileName: originalAssetData.originalname }, + }); return new AssetFileUploadResponseDto(movedAsset.id); } catch (err) { diff --git a/server/apps/immich/src/api-v1/job/job.controller.ts b/server/apps/immich/src/api-v1/job/job.controller.ts index 5766ada3cf..5dcb3e7a0c 100644 --- a/server/apps/immich/src/api-v1/job/job.controller.ts +++ b/server/apps/immich/src/api-v1/job/job.controller.ts @@ -1,11 +1,9 @@ -import { Controller, Get, Body, ValidationPipe, Put, Param } from '@nestjs/common'; -import { JobService } from './job.service'; +import { Body, Controller, Get, Param, Put, ValidationPipe } from '@nestjs/common'; import { ApiBearerAuth, ApiTags } from '@nestjs/swagger'; import { Authenticated } from '../../decorators/authenticated.decorator'; import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto'; import { GetJobDto } from './dto/get-job.dto'; -import { JobStatusResponseDto } from './response-dto/job-status-response.dto'; - +import { JobService } from './job.service'; import { JobCommandDto } from './dto/job-command.dto'; @Authenticated({ admin: true }) @@ -20,21 +18,16 @@ export class JobController { return this.jobService.getAllJobsStatus(); } - @Get('/:jobId') - getJobStatus(@Param(ValidationPipe) params: GetJobDto): Promise { - return this.jobService.getJobStatus(params); - } - @Put('/:jobId') async sendJobCommand( @Param(ValidationPipe) params: GetJobDto, @Body(ValidationPipe) body: JobCommandDto, ): Promise { if (body.command === 'start') { - return await this.jobService.startJob(params); + return await this.jobService.start(params.jobId); } if (body.command === 'stop') { - return await this.jobService.stopJob(params); + return await this.jobService.stop(params.jobId); } return 0; } diff --git a/server/apps/immich/src/api-v1/job/job.service.ts b/server/apps/immich/src/api-v1/job/job.service.ts index 4bba764bc4..ea45f7aca8 100644 --- a/server/apps/immich/src/api-v1/job/job.service.ts +++ b/server/apps/immich/src/api-v1/job/job.service.ts @@ -1,217 +1,118 @@ -import { - IMachineLearningJob, - IMetadataExtractionJob, - IThumbnailGenerationJob, - IVideoTranscodeJob, - JobName, - QueueName, -} from '@app/domain'; -import { InjectQueue } from '@nestjs/bull'; -import { Queue } from 'bull'; +import { JobName, IJobRepository, QueueName } from '@app/domain'; import { BadRequestException, Inject, Injectable } from '@nestjs/common'; import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto'; import { IAssetRepository } from '../asset/asset-repository'; import { AssetType } from '@app/infra'; -import { GetJobDto, JobId } from './dto/get-job.dto'; -import { JobStatusResponseDto } from './response-dto/job-status-response.dto'; -import { StorageService } from '@app/storage'; +import { JobId } from './dto/get-job.dto'; import { MACHINE_LEARNING_ENABLED } from '@app/common'; +const jobIds = Object.values(JobId) as JobId[]; + @Injectable() export class JobService { constructor( - @InjectQueue(QueueName.THUMBNAIL_GENERATION) - private thumbnailGeneratorQueue: Queue, - - @InjectQueue(QueueName.METADATA_EXTRACTION) - private metadataExtractionQueue: Queue, - - @InjectQueue(QueueName.VIDEO_CONVERSION) - private videoConversionQueue: Queue, - - @InjectQueue(QueueName.MACHINE_LEARNING) - private machineLearningQueue: Queue, - - @InjectQueue(QueueName.CONFIG) - private configQueue: Queue, - - @Inject(IAssetRepository) - private _assetRepository: IAssetRepository, - - private storageService: StorageService, + @Inject(IAssetRepository) private _assetRepository: IAssetRepository, + @Inject(IJobRepository) private jobRepository: IJobRepository, ) { - this.thumbnailGeneratorQueue.empty(); - this.metadataExtractionQueue.empty(); - this.videoConversionQueue.empty(); - this.configQueue.empty(); + for (const jobId of jobIds) { + this.jobRepository.empty(this.asQueueName(jobId)); + } } - async startJob(jobDto: GetJobDto): Promise { - switch (jobDto.jobId) { - case JobId.THUMBNAIL_GENERATION: - return this.runThumbnailGenerationJob(); - case JobId.METADATA_EXTRACTION: - return this.runMetadataExtractionJob(); - case JobId.VIDEO_CONVERSION: - return this.runVideoConversionJob(); - case JobId.MACHINE_LEARNING: - return this.runMachineLearningPipeline(); - case JobId.STORAGE_TEMPLATE_MIGRATION: - return this.runStorageMigration(); - default: - throw new BadRequestException('Invalid job id'); - } + start(jobId: JobId): Promise { + return this.run(this.asQueueName(jobId)); + } + + async stop(jobId: JobId): Promise { + await this.jobRepository.empty(this.asQueueName(jobId)); + return 0; } async getAllJobsStatus(): Promise { - const thumbnailGeneratorJobCount = await this.thumbnailGeneratorQueue.getJobCounts(); - const metadataExtractionJobCount = await this.metadataExtractionQueue.getJobCounts(); - const videoConversionJobCount = await this.videoConversionQueue.getJobCounts(); - const machineLearningJobCount = await this.machineLearningQueue.getJobCounts(); - const storageMigrationJobCount = await this.configQueue.getJobCounts(); - const response = new AllJobStatusResponseDto(); - response.isThumbnailGenerationActive = Boolean(thumbnailGeneratorJobCount.waiting); - response.thumbnailGenerationQueueCount = thumbnailGeneratorJobCount; - response.isMetadataExtractionActive = Boolean(metadataExtractionJobCount.waiting); - response.metadataExtractionQueueCount = metadataExtractionJobCount; - response.isVideoConversionActive = Boolean(videoConversionJobCount.waiting); - response.videoConversionQueueCount = videoConversionJobCount; - response.isMachineLearningActive = Boolean(machineLearningJobCount.waiting); - response.machineLearningQueueCount = machineLearningJobCount; - response.isStorageMigrationActive = Boolean(storageMigrationJobCount.active); - response.storageMigrationQueueCount = storageMigrationJobCount; - + for (const jobId of jobIds) { + response[jobId] = await this.jobRepository.getJobCounts(this.asQueueName(jobId)); + } return response; } - async getJobStatus(query: GetJobDto): Promise { - const response = new JobStatusResponseDto(); - if (query.jobId === JobId.THUMBNAIL_GENERATION) { - response.isActive = Boolean((await this.thumbnailGeneratorQueue.getJobCounts()).waiting); - response.queueCount = await this.thumbnailGeneratorQueue.getJobCounts(); + private async run(name: QueueName): Promise { + const isActive = await this.jobRepository.isActive(name); + if (isActive) { + throw new BadRequestException(`Job is already running`); } - if (query.jobId === JobId.METADATA_EXTRACTION) { - response.isActive = Boolean((await this.metadataExtractionQueue.getJobCounts()).waiting); - response.queueCount = await this.metadataExtractionQueue.getJobCounts(); - } + switch (name) { + case QueueName.VIDEO_CONVERSION: { + const assets = await this._assetRepository.getAssetWithNoEncodedVideo(); + for (const asset of assets) { + await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } }); + } - if (query.jobId === JobId.VIDEO_CONVERSION) { - response.isActive = Boolean((await this.videoConversionQueue.getJobCounts()).waiting); - response.queueCount = await this.videoConversionQueue.getJobCounts(); - } - - if (query.jobId === JobId.STORAGE_TEMPLATE_MIGRATION) { - response.isActive = Boolean((await this.configQueue.getJobCounts()).waiting); - response.queueCount = await this.configQueue.getJobCounts(); - } - - return response; - } - - async stopJob(query: GetJobDto): Promise { - switch (query.jobId) { - case JobId.THUMBNAIL_GENERATION: - this.thumbnailGeneratorQueue.empty(); - return 0; - case JobId.METADATA_EXTRACTION: - this.metadataExtractionQueue.empty(); - return 0; - case JobId.VIDEO_CONVERSION: - this.videoConversionQueue.empty(); - return 0; - case JobId.MACHINE_LEARNING: - this.machineLearningQueue.empty(); - return 0; - case JobId.STORAGE_TEMPLATE_MIGRATION: - this.configQueue.empty(); - return 0; - default: - throw new BadRequestException('Invalid job id'); - } - } - - private async runThumbnailGenerationJob(): Promise { - const jobCount = await this.thumbnailGeneratorQueue.getJobCounts(); - - if (jobCount.waiting > 0) { - throw new BadRequestException('Thumbnail generation job is already running'); - } - - const assetsWithNoThumbnail = await this._assetRepository.getAssetWithNoThumbnail(); - - for (const asset of assetsWithNoThumbnail) { - await this.thumbnailGeneratorQueue.add(JobName.GENERATE_JPEG_THUMBNAIL, { asset }); - } - - return assetsWithNoThumbnail.length; - } - - private async runMetadataExtractionJob(): Promise { - const jobCount = await this.metadataExtractionQueue.getJobCounts(); - - if (jobCount.waiting > 0) { - throw new BadRequestException('Metadata extraction job is already running'); - } - - const assetsWithNoExif = await this._assetRepository.getAssetWithNoEXIF(); - for (const asset of assetsWithNoExif) { - if (asset.type === AssetType.VIDEO) { - await this.metadataExtractionQueue.add(JobName.EXTRACT_VIDEO_METADATA, { asset, fileName: asset.id }); - } else { - await this.metadataExtractionQueue.add(JobName.EXIF_EXTRACTION, { asset, fileName: asset.id }); + return assets.length; } + + case QueueName.CONFIG: + await this.jobRepository.add({ name: JobName.TEMPLATE_MIGRATION }); + return 1; + + case QueueName.MACHINE_LEARNING: { + if (!MACHINE_LEARNING_ENABLED) { + throw new BadRequestException('Machine learning is not enabled.'); + } + + const assets = await this._assetRepository.getAssetWithNoSmartInfo(); + for (const asset of assets) { + await this.jobRepository.add({ name: JobName.IMAGE_TAGGING, data: { asset } }); + await this.jobRepository.add({ name: JobName.OBJECT_DETECTION, data: { asset } }); + } + return assets.length; + } + + case QueueName.METADATA_EXTRACTION: { + const assets = await this._assetRepository.getAssetWithNoEXIF(); + for (const asset of assets) { + if (asset.type === AssetType.VIDEO) { + await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } }); + } else { + await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } }); + } + } + return assets.length; + } + + case QueueName.THUMBNAIL_GENERATION: { + const assets = await this._assetRepository.getAssetWithNoThumbnail(); + for (const asset of assets) { + await this.jobRepository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } }); + } + return assets.length; + } + + default: + return 0; } - return assetsWithNoExif.length; } - private async runMachineLearningPipeline(): Promise { - if (!MACHINE_LEARNING_ENABLED) { - throw new BadRequestException('Machine learning is not enabled.'); + private asQueueName(jobId: JobId) { + switch (jobId) { + case JobId.THUMBNAIL_GENERATION: + return QueueName.THUMBNAIL_GENERATION; + + case JobId.METADATA_EXTRACTION: + return QueueName.METADATA_EXTRACTION; + + case JobId.VIDEO_CONVERSION: + return QueueName.VIDEO_CONVERSION; + + case JobId.STORAGE_TEMPLATE_MIGRATION: + return QueueName.CONFIG; + + case JobId.MACHINE_LEARNING: + return QueueName.MACHINE_LEARNING; + + default: + throw new BadRequestException(`Invalid job id: ${jobId}`); } - - const jobCount = await this.machineLearningQueue.getJobCounts(); - - if (jobCount.waiting > 0) { - throw new BadRequestException('Metadata extraction job is already running'); - } - - const assetWithNoSmartInfo = await this._assetRepository.getAssetWithNoSmartInfo(); - - for (const asset of assetWithNoSmartInfo) { - await this.machineLearningQueue.add(JobName.IMAGE_TAGGING, { asset }); - await this.machineLearningQueue.add(JobName.OBJECT_DETECTION, { asset }); - } - - return assetWithNoSmartInfo.length; - } - - private async runVideoConversionJob(): Promise { - const jobCount = await this.videoConversionQueue.getJobCounts(); - - if (jobCount.waiting > 0) { - throw new BadRequestException('Video conversion job is already running'); - } - - const assetsWithNoConvertedVideo = await this._assetRepository.getAssetWithNoEncodedVideo(); - - for (const asset of assetsWithNoConvertedVideo) { - await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset }); - } - - return assetsWithNoConvertedVideo.length; - } - - async runStorageMigration() { - const jobCount = await this.configQueue.getJobCounts(); - - if (jobCount.active > 0) { - throw new BadRequestException('Storage migration job is already running'); - } - - await this.configQueue.add(JobName.TEMPLATE_MIGRATION, {}); - - return 1; } } diff --git a/server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts b/server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts index 06cbbdf2a5..9b26936ce6 100644 --- a/server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts +++ b/server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts @@ -1,4 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; +import { JobId } from '../dto/get-job.dto'; export class JobCounts { @ApiProperty({ type: 'integer' }) @@ -12,35 +13,20 @@ export class JobCounts { @ApiProperty({ type: 'integer' }) waiting!: number; } + export class AllJobStatusResponseDto { - isThumbnailGenerationActive!: boolean; - isMetadataExtractionActive!: boolean; - isVideoConversionActive!: boolean; - isMachineLearningActive!: boolean; - isStorageMigrationActive!: boolean; + @ApiProperty({ type: JobCounts }) + [JobId.THUMBNAIL_GENERATION]!: JobCounts; - @ApiProperty({ - type: JobCounts, - }) - thumbnailGenerationQueueCount!: JobCounts; + @ApiProperty({ type: JobCounts }) + [JobId.METADATA_EXTRACTION]!: JobCounts; - @ApiProperty({ - type: JobCounts, - }) - metadataExtractionQueueCount!: JobCounts; + @ApiProperty({ type: JobCounts }) + [JobId.VIDEO_CONVERSION]!: JobCounts; - @ApiProperty({ - type: JobCounts, - }) - videoConversionQueueCount!: JobCounts; + @ApiProperty({ type: JobCounts }) + [JobId.MACHINE_LEARNING]!: JobCounts; - @ApiProperty({ - type: JobCounts, - }) - machineLearningQueueCount!: JobCounts; - - @ApiProperty({ - type: JobCounts, - }) - storageMigrationQueueCount!: JobCounts; + @ApiProperty({ type: JobCounts }) + [JobId.STORAGE_TEMPLATE_MIGRATION]!: JobCounts; } diff --git a/server/apps/immich/src/api-v1/job/response-dto/job-status-response.dto.ts b/server/apps/immich/src/api-v1/job/response-dto/job-status-response.dto.ts deleted file mode 100644 index fe411fa2ef..0000000000 --- a/server/apps/immich/src/api-v1/job/response-dto/job-status-response.dto.ts +++ /dev/null @@ -1,6 +0,0 @@ -import Bull from 'bull'; - -export class JobStatusResponseDto { - isActive!: boolean; - queueCount!: Bull.JobCounts; -} diff --git a/server/apps/immich/src/modules/background-task/background-task.module.ts b/server/apps/immich/src/modules/background-task/background-task.module.ts index fc199abc54..fa52b6b44d 100644 --- a/server/apps/immich/src/modules/background-task/background-task.module.ts +++ b/server/apps/immich/src/modules/background-task/background-task.module.ts @@ -1,12 +1,9 @@ -import { BullModule } from '@nestjs/bull'; import { Module } from '@nestjs/common'; -import { QueueName } from '@app/domain'; import { BackgroundTaskProcessor } from './background-task.processor'; import { BackgroundTaskService } from './background-task.service'; @Module({ - imports: [BullModule.registerQueue({ name: QueueName.BACKGROUND_TASK })], providers: [BackgroundTaskService, BackgroundTaskProcessor], - exports: [BackgroundTaskService, BullModule], + exports: [BackgroundTaskService], }) export class BackgroundTaskModule {} diff --git a/server/apps/immich/src/modules/background-task/background-task.processor.ts b/server/apps/immich/src/modules/background-task/background-task.processor.ts index 7e68185689..0df0d0ada6 100644 --- a/server/apps/immich/src/modules/background-task/background-task.processor.ts +++ b/server/apps/immich/src/modules/background-task/background-task.processor.ts @@ -2,12 +2,12 @@ import { assetUtils } from '@app/common/utils'; import { Process, Processor } from '@nestjs/bull'; import { Job } from 'bull'; import { JobName, QueueName } from '@app/domain'; -import { AssetResponseDto } from '../../api-v1/asset/response-dto/asset-response.dto'; +import { AssetEntity } from '@app/infra'; @Processor(QueueName.BACKGROUND_TASK) export class BackgroundTaskProcessor { @Process(JobName.DELETE_FILE_ON_DISK) - async deleteFileOnDisk(job: Job<{ assets: AssetResponseDto[] }>) { + async deleteFileOnDisk(job: Job<{ assets: AssetEntity[] }>) { const { assets } = job.data; for (const asset of assets) { diff --git a/server/apps/immich/src/modules/background-task/background-task.service.ts b/server/apps/immich/src/modules/background-task/background-task.service.ts index a7124c8807..b32a89b266 100644 --- a/server/apps/immich/src/modules/background-task/background-task.service.ts +++ b/server/apps/immich/src/modules/background-task/background-task.service.ts @@ -1,17 +1,12 @@ -import { InjectQueue } from '@nestjs/bull/dist/decorators'; -import { Injectable } from '@nestjs/common'; -import { Queue } from 'bull'; -import { JobName, QueueName } from '@app/domain'; -import { AssetResponseDto } from '../../api-v1/asset/response-dto/asset-response.dto'; +import { IJobRepository, JobName } from '@app/domain'; +import { AssetEntity } from '@app/infra'; +import { Inject, Injectable } from '@nestjs/common'; @Injectable() export class BackgroundTaskService { - constructor( - @InjectQueue(QueueName.BACKGROUND_TASK) - private backgroundTaskQueue: Queue, - ) {} + constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {} - async deleteFileOnDisk(assets: AssetResponseDto[]) { - await this.backgroundTaskQueue.add(JobName.DELETE_FILE_ON_DISK, { assets }); + async deleteFileOnDisk(assets: AssetEntity[]) { + await this.jobRepository.add({ name: JobName.DELETE_FILE_ON_DISK, data: { assets } }); } } diff --git a/server/apps/immich/src/modules/schedule-tasks/schedule-tasks.service.ts b/server/apps/immich/src/modules/schedule-tasks/schedule-tasks.service.ts index e1e92dcd2b..e77163bf02 100644 --- a/server/apps/immich/src/modules/schedule-tasks/schedule-tasks.service.ts +++ b/server/apps/immich/src/modules/schedule-tasks/schedule-tasks.service.ts @@ -1,14 +1,11 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { Inject, Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { InjectRepository } from '@nestjs/typeorm'; import { IsNull, Not, Repository } from 'typeorm'; import { AssetEntity, AssetType, ExifEntity, UserEntity } from '@app/infra'; -import { InjectQueue } from '@nestjs/bull'; -import { Queue } from 'bull'; -import { IMetadataExtractionJob, IVideoTranscodeJob, QueueName, JobName } from '@app/domain'; import { ConfigService } from '@nestjs/config'; -import { IUserDeletionJob } from '@app/domain'; import { userUtils } from '@app/common'; +import { IJobRepository, JobName } from '@app/domain'; @Injectable() export class ScheduleTasksService { @@ -22,17 +19,7 @@ export class ScheduleTasksService { @InjectRepository(ExifEntity) private exifRepository: Repository, - @InjectQueue(QueueName.THUMBNAIL_GENERATION) - private thumbnailGeneratorQueue: Queue, - - @InjectQueue(QueueName.VIDEO_CONVERSION) - private videoConversionQueue: Queue, - - @InjectQueue(QueueName.METADATA_EXTRACTION) - private metadataExtractionQueue: Queue, - - @InjectQueue(QueueName.USER_DELETION) - private userDeletionQueue: Queue, + @Inject(IJobRepository) private jobRepository: IJobRepository, private configService: ConfigService, ) {} @@ -51,7 +38,7 @@ export class ScheduleTasksService { } for (const asset of assets) { - await this.thumbnailGeneratorQueue.add(JobName.GENERATE_WEBP_THUMBNAIL, { asset: asset }); + await this.jobRepository.add({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { asset } }); } } @@ -69,7 +56,7 @@ export class ScheduleTasksService { }); for (const asset of assets) { - await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset }); + await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } }); } } @@ -87,11 +74,11 @@ export class ScheduleTasksService { }); for (const exif of exifInfo) { - await this.metadataExtractionQueue.add( - JobName.REVERSE_GEOCODING, + await this.jobRepository.add({ + name: JobName.REVERSE_GEOCODING, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - { exifId: exif.id, latitude: exif.latitude!, longitude: exif.longitude! }, - ); + data: { exifId: exif.id, latitude: exif.latitude!, longitude: exif.longitude! }, + }); } } } @@ -106,9 +93,9 @@ export class ScheduleTasksService { for (const asset of exifAssets) { if (asset.type === AssetType.VIDEO) { - await this.metadataExtractionQueue.add(JobName.EXTRACT_VIDEO_METADATA, { asset, fileName: asset.id }); + await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } }); } else { - await this.metadataExtractionQueue.add(JobName.EXIF_EXTRACTION, { asset, fileName: asset.id }); + await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } }); } } } @@ -118,7 +105,7 @@ export class ScheduleTasksService { const usersToDelete = await this.userRepository.find({ withDeleted: true, where: { deletedAt: Not(IsNull()) } }); for (const user of usersToDelete) { if (userUtils.isReadyForDeletion(user)) { - await this.userDeletionQueue.add(JobName.USER_DELETION, { user }); + await this.jobRepository.add({ name: JobName.USER_DELETION, data: { user } }); } } } diff --git a/server/apps/microservices/src/microservices.service.ts b/server/apps/microservices/src/microservices.service.ts index 329f2b5d4d..a52928e3ab 100644 --- a/server/apps/microservices/src/microservices.service.ts +++ b/server/apps/microservices/src/microservices.service.ts @@ -1,17 +1,16 @@ -import { QueueName } from '@app/domain'; -import { InjectQueue } from '@nestjs/bull'; -import { Injectable, OnModuleInit } from '@nestjs/common'; -import { Queue } from 'bull'; +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { IJobRepository, JobName } from '@app/domain'; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(() => resolve(), ms)); @Injectable() export class MicroservicesService implements OnModuleInit { - constructor( - @InjectQueue(QueueName.CHECKSUM_GENERATION) - private generateChecksumQueue: Queue, - ) {} + constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {} async onModuleInit() { // wait for migration - await this.generateChecksumQueue.add({}, { delay: 10000 }); + await sleep(10_000); + + await this.jobRepository.add({ name: JobName.CHECKSUM_GENERATION }); } } diff --git a/server/apps/microservices/src/processors/generate-checksum.processor.ts b/server/apps/microservices/src/processors/generate-checksum.processor.ts index 32c2035151..fc9fafbfcf 100644 --- a/server/apps/microservices/src/processors/generate-checksum.processor.ts +++ b/server/apps/microservices/src/processors/generate-checksum.processor.ts @@ -1,5 +1,5 @@ import { AssetEntity } from '@app/infra'; -import { QueueName } from '@app/domain'; +import { JobName, QueueName } from '@app/domain'; import { Process, Processor } from '@nestjs/bull'; import { Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; @@ -15,7 +15,7 @@ export class GenerateChecksumProcessor { private assetRepository: Repository, ) {} - @Process() + @Process(JobName.CHECKSUM_GENERATION) async generateChecksum() { const pageSize = 200; let hasNext = true; diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index d9f5f23ece..a1b42eb5ec 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -2721,40 +2721,6 @@ } }, "/jobs/{jobId}": { - "get": { - "operationId": "getJobStatus", - "description": "", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "$ref": "#/components/schemas/JobId" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JobStatusResponseDto" - } - } - } - } - }, - "tags": [ - "Job" - ], - "security": [ - { - "bearer": [] - } - ] - }, "put": { "operationId": "sendJobCommand", "description": "", @@ -4569,48 +4535,28 @@ "AllJobStatusResponseDto": { "type": "object", "properties": { - "thumbnailGenerationQueueCount": { + "thumbnail-generation": { "$ref": "#/components/schemas/JobCounts" }, - "metadataExtractionQueueCount": { + "metadata-extraction": { "$ref": "#/components/schemas/JobCounts" }, - "videoConversionQueueCount": { + "video-conversion": { "$ref": "#/components/schemas/JobCounts" }, - "machineLearningQueueCount": { + "machine-learning": { "$ref": "#/components/schemas/JobCounts" }, - "storageMigrationQueueCount": { + "storage-template-migration": { "$ref": "#/components/schemas/JobCounts" - }, - "isThumbnailGenerationActive": { - "type": "boolean" - }, - "isMetadataExtractionActive": { - "type": "boolean" - }, - "isVideoConversionActive": { - "type": "boolean" - }, - "isMachineLearningActive": { - "type": "boolean" - }, - "isStorageMigrationActive": { - "type": "boolean" } }, "required": [ - "thumbnailGenerationQueueCount", - "metadataExtractionQueueCount", - "videoConversionQueueCount", - "machineLearningQueueCount", - "storageMigrationQueueCount", - "isThumbnailGenerationActive", - "isMetadataExtractionActive", - "isVideoConversionActive", - "isMachineLearningActive", - "isStorageMigrationActive" + "thumbnail-generation", + "metadata-extraction", + "video-conversion", + "machine-learning", + "storage-template-migration" ] }, "JobId": { @@ -4623,21 +4569,6 @@ "storage-template-migration" ] }, - "JobStatusResponseDto": { - "type": "object", - "properties": { - "isActive": { - "type": "boolean" - }, - "queueCount": { - "type": "object" - } - }, - "required": [ - "isActive", - "queueCount" - ] - }, "JobCommand": { "type": "string", "enum": [ diff --git a/server/libs/domain/src/job/job.constants.ts b/server/libs/domain/src/job/job.constants.ts index dd5d060c44..992faf19f0 100644 --- a/server/libs/domain/src/job/job.constants.ts +++ b/server/libs/domain/src/job/job.constants.ts @@ -24,4 +24,5 @@ export enum JobName { OBJECT_DETECTION = 'detect-object', IMAGE_TAGGING = 'tag-image', DELETE_FILE_ON_DISK = 'delete-file-on-disk', + CHECKSUM_GENERATION = 'checksum-generation', } diff --git a/server/libs/domain/src/job/job.repository.ts b/server/libs/domain/src/job/job.repository.ts index fe129b7748..3e5bc7c18a 100644 --- a/server/libs/domain/src/job/job.repository.ts +++ b/server/libs/domain/src/job/job.repository.ts @@ -6,10 +6,19 @@ import { IVideoConversionProcessor, IReverseGeocodingProcessor, IUserDeletionJob, + IVideoLengthExtractionProcessor, JpegGeneratorProcessor, WebpGeneratorProcessor, } from './interfaces'; -import { JobName } from './job.constants'; +import { JobName, QueueName } from './job.constants'; + +export interface JobCounts { + active: number; + completed: number; + failed: number; + delayed: number; + waiting: number; +} export type JobItem = | { name: JobName.ASSET_UPLOADED; data: IAssetUploadedJob } @@ -21,6 +30,8 @@ export type JobItem = | { name: JobName.USER_DELETION; data: IUserDeletionJob } | { name: JobName.TEMPLATE_MIGRATION } | { name: JobName.CONFIG_CHANGE } + | { name: JobName.CHECKSUM_GENERATION } + | { name: JobName.EXTRACT_VIDEO_METADATA; data: IVideoLengthExtractionProcessor } | { name: JobName.OBJECT_DETECTION; data: IMachineLearningJob } | { name: JobName.IMAGE_TAGGING; data: IMachineLearningJob } | { name: JobName.DELETE_FILE_ON_DISK; data: IDeleteFileOnDiskJob }; @@ -28,5 +39,8 @@ export type JobItem = export const IJobRepository = 'IJobRepository'; export interface IJobRepository { + empty(name: QueueName): Promise; add(item: JobItem): Promise; + isActive(name: QueueName): Promise; + getJobCounts(name: QueueName): Promise; } diff --git a/server/libs/domain/test/job.repository.mock.ts b/server/libs/domain/test/job.repository.mock.ts index d75beed223..623f10fbf1 100644 --- a/server/libs/domain/test/job.repository.mock.ts +++ b/server/libs/domain/test/job.repository.mock.ts @@ -2,6 +2,9 @@ import { IJobRepository } from '../src'; export const newJobRepositoryMock = (): jest.Mocked => { return { + empty: jest.fn(), add: jest.fn().mockImplementation(() => Promise.resolve()), + isActive: jest.fn(), + getJobCounts: jest.fn(), }; }; diff --git a/server/libs/infra/src/job/job.repository.ts b/server/libs/infra/src/job/job.repository.ts index cdac756c54..dea3438de9 100644 --- a/server/libs/infra/src/job/job.repository.ts +++ b/server/libs/infra/src/job/job.repository.ts @@ -1,21 +1,110 @@ -import { IJobRepository, JobItem, JobName, QueueName } from '@app/domain'; +import { + IAssetUploadedJob, + IJobRepository, + IMachineLearningJob, + IMetadataExtractionJob, + IUserDeletionJob, + IVideoTranscodeJob, + JobCounts, + JobItem, + JobName, + QueueName, +} from '@app/domain'; import { InjectQueue } from '@nestjs/bull'; -import { Logger } from '@nestjs/common'; +import { BadRequestException, Logger } from '@nestjs/common'; import { Queue } from 'bull'; export class JobRepository implements IJobRepository { private logger = new Logger(JobRepository.name); - constructor(@InjectQueue(QueueName.CONFIG) private configQueue: Queue) {} + constructor( + @InjectQueue(QueueName.ASSET_UPLOADED) private assetUploaded: Queue, + @InjectQueue(QueueName.BACKGROUND_TASK) private backgroundTask: Queue, + @InjectQueue(QueueName.CHECKSUM_GENERATION) private generateChecksum: Queue, + @InjectQueue(QueueName.MACHINE_LEARNING) private machineLearning: Queue, + @InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue, + @InjectQueue(QueueName.CONFIG) private storageMigration: Queue, + @InjectQueue(QueueName.THUMBNAIL_GENERATION) private thumbnail: Queue, + @InjectQueue(QueueName.USER_DELETION) private userDeletion: Queue, + @InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue, + ) {} + + async isActive(name: QueueName): Promise { + const counts = await this.getJobCounts(name); + return !!counts.active; + } + + empty(name: QueueName) { + return this.getQueue(name).empty(); + } + + getJobCounts(name: QueueName): Promise { + return this.getQueue(name).getJobCounts(); + } async add(item: JobItem): Promise { switch (item.name) { - case JobName.CONFIG_CHANGE: - await this.configQueue.add(JobName.CONFIG_CHANGE, {}); + case JobName.ASSET_UPLOADED: + await this.assetUploaded.add(item.name, item.data, { jobId: item.data.asset.id }); break; + + case JobName.DELETE_FILE_ON_DISK: + await this.backgroundTask.add(item.name, item.data); + break; + + case JobName.CHECKSUM_GENERATION: + await this.generateChecksum.add(item.name, {}); + break; + + case JobName.OBJECT_DETECTION: + case JobName.IMAGE_TAGGING: + await this.machineLearning.add(item.name, item.data); + break; + + case JobName.EXIF_EXTRACTION: + case JobName.EXTRACT_VIDEO_METADATA: + case JobName.REVERSE_GEOCODING: + await this.metadataExtraction.add(item.name, item.data); + break; + + case JobName.TEMPLATE_MIGRATION: + case JobName.CONFIG_CHANGE: + await this.storageMigration.add(item.name, {}); + break; + + case JobName.GENERATE_JPEG_THUMBNAIL: + case JobName.GENERATE_WEBP_THUMBNAIL: + await this.thumbnail.add(item.name, item.data); + break; + + case JobName.USER_DELETION: + await this.userDeletion.add(item.name, item.data); + break; + + case JobName.VIDEO_CONVERSION: + await this.videoTranscode.add(item.name, item.data); + break; + default: // TODO inject remaining queues and map job to queue this.logger.error('Invalid job', item); } } + + private getQueue(name: QueueName) { + switch (name) { + case QueueName.THUMBNAIL_GENERATION: + return this.thumbnail; + case QueueName.METADATA_EXTRACTION: + return this.metadataExtraction; + case QueueName.VIDEO_CONVERSION: + return this.videoTranscode; + case QueueName.CONFIG: + return this.storageMigration; + case QueueName.MACHINE_LEARNING: + return this.machineLearning; + default: + throw new BadRequestException('Invalid job name'); + } + } } diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index ba8b0e9f9c..a2e80b807e 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -13,24 +13,13 @@ */ -import {Configuration} from './configuration'; -import globalAxios, {AxiosInstance, AxiosPromise, AxiosRequestConfig} from 'axios'; +import { Configuration } from './configuration'; +import globalAxios, { AxiosPromise, AxiosInstance, AxiosRequestConfig } from 'axios'; // Some imports not used depending on template conditions // @ts-ignore -import { - assertParamExists, - createRequestFunction, - DUMMY_BASE_URL, - serializeDataIfNeeded, - setApiKeyToObject, - setBasicAuthToObject, - setBearerAuthToObject, - setOAuthToObject, - setSearchParams, - toPathString -} from './common'; +import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction } from './common'; // @ts-ignore -import {BASE_PATH, BaseAPI, COLLECTION_FORMATS, RequestArgs, RequiredError} from './base'; +import { BASE_PATH, COLLECTION_FORMATS, RequestArgs, BaseAPI, RequiredError } from './base'; /** * @@ -293,61 +282,31 @@ export interface AllJobStatusResponseDto { * @type {JobCounts} * @memberof AllJobStatusResponseDto */ - 'thumbnailGenerationQueueCount': JobCounts; + 'thumbnail-generation': JobCounts; /** * * @type {JobCounts} * @memberof AllJobStatusResponseDto */ - 'metadataExtractionQueueCount': JobCounts; + 'metadata-extraction': JobCounts; /** * * @type {JobCounts} * @memberof AllJobStatusResponseDto */ - 'videoConversionQueueCount': JobCounts; + 'video-conversion': JobCounts; /** * * @type {JobCounts} * @memberof AllJobStatusResponseDto */ - 'machineLearningQueueCount': JobCounts; + 'machine-learning': JobCounts; /** * * @type {JobCounts} * @memberof AllJobStatusResponseDto */ - 'storageMigrationQueueCount': JobCounts; - /** - * - * @type {boolean} - * @memberof AllJobStatusResponseDto - */ - 'isThumbnailGenerationActive': boolean; - /** - * - * @type {boolean} - * @memberof AllJobStatusResponseDto - */ - 'isMetadataExtractionActive': boolean; - /** - * - * @type {boolean} - * @memberof AllJobStatusResponseDto - */ - 'isVideoConversionActive': boolean; - /** - * - * @type {boolean} - * @memberof AllJobStatusResponseDto - */ - 'isMachineLearningActive': boolean; - /** - * - * @type {boolean} - * @memberof AllJobStatusResponseDto - */ - 'isStorageMigrationActive': boolean; + 'storage-template-migration': JobCounts; } /** * @@ -1269,25 +1228,6 @@ export const JobId = { export type JobId = typeof JobId[keyof typeof JobId]; -/** - * - * @export - * @interface JobStatusResponseDto - */ -export interface JobStatusResponseDto { - /** - * - * @type {boolean} - * @memberof JobStatusResponseDto - */ - 'isActive': boolean; - /** - * - * @type {object} - * @memberof JobStatusResponseDto - */ - 'queueCount': object; -} /** * * @export @@ -5772,43 +5712,6 @@ export const JobApiAxiosParamCreator = function (configuration?: Configuration) - setSearchParams(localVarUrlObj, localVarQueryParameter); - let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; - localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; - - return { - url: toPathString(localVarUrlObj), - options: localVarRequestOptions, - }; - }, - /** - * - * @param {JobId} jobId - * @param {*} [options] Override http request option. - * @throws {RequiredError} - */ - getJobStatus: async (jobId: JobId, options: AxiosRequestConfig = {}): Promise => { - // verify required parameter 'jobId' is not null or undefined - assertParamExists('getJobStatus', 'jobId', jobId) - const localVarPath = `/jobs/{jobId}` - .replace(`{${"jobId"}}`, encodeURIComponent(String(jobId))); - // use dummy base URL string because the URL constructor only accepts absolute URLs. - const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); - let baseOptions; - if (configuration) { - baseOptions = configuration.baseOptions; - } - - const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; - const localVarHeaderParameter = {} as any; - const localVarQueryParameter = {} as any; - - // authentication bearer required - // http bearer authentication required - await setBearerAuthToObject(localVarHeaderParameter, configuration) - - - setSearchParams(localVarUrlObj, localVarQueryParameter); let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; @@ -5880,16 +5783,6 @@ export const JobApiFp = function(configuration?: Configuration) { const localVarAxiosArgs = await localVarAxiosParamCreator.getAllJobsStatus(options); return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); }, - /** - * - * @param {JobId} jobId - * @param {*} [options] Override http request option. - * @throws {RequiredError} - */ - async getJobStatus(jobId: JobId, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { - const localVarAxiosArgs = await localVarAxiosParamCreator.getJobStatus(jobId, options); - return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); - }, /** * * @param {JobId} jobId @@ -5919,15 +5812,6 @@ export const JobApiFactory = function (configuration?: Configuration, basePath?: getAllJobsStatus(options?: any): AxiosPromise { return localVarFp.getAllJobsStatus(options).then((request) => request(axios, basePath)); }, - /** - * - * @param {JobId} jobId - * @param {*} [options] Override http request option. - * @throws {RequiredError} - */ - getJobStatus(jobId: JobId, options?: any): AxiosPromise { - return localVarFp.getJobStatus(jobId, options).then((request) => request(axios, basePath)); - }, /** * * @param {JobId} jobId @@ -5958,17 +5842,6 @@ export class JobApi extends BaseAPI { return JobApiFp(this.configuration).getAllJobsStatus(options).then((request) => request(this.axios, this.basePath)); } - /** - * - * @param {JobId} jobId - * @param {*} [options] Override http request option. - * @throws {RequiredError} - * @memberof JobApi - */ - public getJobStatus(jobId: JobId, options?: AxiosRequestConfig) { - return JobApiFp(this.configuration).getJobStatus(jobId, options).then((request) => request(this.axios, this.basePath)); - } - /** * * @param {JobId} jobId diff --git a/web/src/lib/components/admin-page/jobs/job-tile.svelte b/web/src/lib/components/admin-page/jobs/job-tile.svelte index e23e33a525..134ad611fe 100644 --- a/web/src/lib/components/admin-page/jobs/job-tile.svelte +++ b/web/src/lib/components/admin-page/jobs/job-tile.svelte @@ -1,13 +1,12 @@ @@ -36,17 +35,23 @@ class="overflow-y-auto rounded-md w-full max-h-[320px] block border bg-white dark:border-immich-dark-gray dark:bg-immich-dark-gray/75 dark:text-immich-dark-fg" > - {jobStatus ? 'Active' : 'Idle'} - - {#if activeJobCount !== undefined} - {activeJobCount} + + {#if jobCounts} + {jobCounts.active > 0 || jobCounts.waiting > 0 ? 'Active' : 'Idle'} {:else} {/if} - {#if waitingJobCount !== undefined} - {waitingJobCount} + {#if jobCounts.active !== undefined} + {jobCounts.active} + {:else} + + {/if} + + + {#if jobCounts.waiting !== undefined} + {jobCounts.waiting} {:else} {/if} @@ -59,9 +64,9 @@