From 8193416230de434ebdc33c99a1a26f96395697db Mon Sep 17 00:00:00 2001 From: Mert <101130780+mertalev@users.noreply.github.com> Date: Sun, 14 Jul 2024 18:53:42 -0400 Subject: [PATCH] feat(server): conditionally run facial recognition nightly (#11080) * only run nightly if new person * add tests * use string instead of date * update sql * update tests * simplify condition --- server/src/entities/system-metadata.entity.ts | 2 + server/src/interfaces/job.interface.ts | 6 +- server/src/interfaces/person.interface.ts | 1 + server/src/queries/person.repository.sql | 6 ++ server/src/repositories/person.repository.ts | 11 ++++ server/src/services/job.service.spec.ts | 2 +- server/src/services/job.service.ts | 2 +- server/src/services/person.service.spec.ts | 59 +++++++++++++++++++ server/src/services/person.service.ts | 22 ++++++- .../repositories/person.repository.mock.ts | 1 + 10 files changed, 107 insertions(+), 5 deletions(-) diff --git a/server/src/entities/system-metadata.entity.ts b/server/src/entities/system-metadata.entity.ts index c72c20f2a1..72aca4c72b 100644 --- a/server/src/entities/system-metadata.entity.ts +++ b/server/src/entities/system-metadata.entity.ts @@ -12,6 +12,7 @@ export class SystemMetadataEntity> { [SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string }; + [SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string }; [SystemMetadataKey.ADMIN_ONBOARDING]: { isOnboarded: boolean }; [SystemMetadataKey.SYSTEM_CONFIG]: DeepPartial; [SystemMetadataKey.VERSION_CHECK_STATE]: VersionCheckMetadata; diff --git a/server/src/interfaces/job.interface.ts b/server/src/interfaces/job.interface.ts index 69e5226cfe..0fd35167af 100644 --- a/server/src/interfaces/job.interface.ts +++ b/server/src/interfaces/job.interface.ts @@ -153,6 +153,10 @@ export interface IDeferrableJob extends IEntityJob { deferred?: boolean; } +export interface INightlyJob extends IBaseJob { + nightly?: boolean; +} + export interface IEmailJob { to: string; subject: string; @@ -229,7 +233,7 @@ export type JobItem = // Facial Recognition | { name: JobName.QUEUE_FACE_DETECTION; data: IBaseJob } | { name: JobName.FACE_DETECTION; data: IEntityJob } - | { name: JobName.QUEUE_FACIAL_RECOGNITION; data: IBaseJob } + | { name: JobName.QUEUE_FACIAL_RECOGNITION; data: INightlyJob } | { name: JobName.FACIAL_RECOGNITION; data: IDeferrableJob } | { name: JobName.GENERATE_PERSON_THUMBNAIL; data: IEntityJob } diff --git a/server/src/interfaces/person.interface.ts b/server/src/interfaces/person.interface.ts index 382bbda22f..b68bd534b8 100644 --- a/server/src/interfaces/person.interface.ts +++ b/server/src/interfaces/person.interface.ts @@ -64,4 +64,5 @@ export interface IPersonRepository { getNumberOfPeople(userId: string): Promise; reassignFaces(data: UpdateFacesData): Promise; update(entity: Partial): Promise; + getLatestFaceDate(): Promise; } diff --git a/server/src/queries/person.repository.sql b/server/src/queries/person.repository.sql index 7e22a30ecd..588606628c 100644 --- a/server/src/queries/person.repository.sql +++ b/server/src/queries/person.repository.sql @@ -434,3 +434,9 @@ WHERE (("AssetFaceEntity"."personId" = $1)) LIMIT 1 + +-- PersonRepository.getLatestFaceDate +SELECT + MAX("jobStatus"."facesRecognizedAt")::text AS "latestDate" +FROM + "asset_job_status" "jobStatus" diff --git a/server/src/repositories/person.repository.ts b/server/src/repositories/person.repository.ts index 36d742f8dc..f0344e8ab8 100644 --- a/server/src/repositories/person.repository.ts +++ b/server/src/repositories/person.repository.ts @@ -3,6 +3,7 @@ import { InjectRepository } from '@nestjs/typeorm'; import _ from 'lodash'; import { ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; +import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; import { AssetEntity } from 'src/entities/asset.entity'; import { PersonEntity } from 'src/entities/person.entity'; import { @@ -25,6 +26,7 @@ export class PersonRepository implements IPersonRepository { @InjectRepository(AssetEntity) private assetRepository: Repository, @InjectRepository(PersonEntity) private personRepository: Repository, @InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository, + @InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository, ) {} @GenerateSql({ params: [{ oldPersonId: DummyValue.UUID, newPersonId: DummyValue.UUID }] }) @@ -267,4 +269,13 @@ export class PersonRepository implements IPersonRepository { async getRandomFace(personId: string): Promise { return this.assetFaceRepository.findOneBy({ personId }); } + + @GenerateSql() + async getLatestFaceDate(): Promise { + const result: { latestDate?: string } | undefined = await this.jobStatusRepository + .createQueryBuilder('jobStatus') + .select('MAX(jobStatus.facesRecognizedAt)::text', 'latestDate') + .getRawOne(); + return result?.latestDate; + } } diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.ts index 20e52ac28e..1c810facb4 100644 --- a/server/src/services/job.service.spec.ts +++ b/server/src/services/job.service.spec.ts @@ -71,7 +71,7 @@ describe(JobService.name, () => { { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }, { name: JobName.CLEAN_OLD_AUDIT_LOGS }, { name: JobName.USER_SYNC_USAGE }, - { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } }, + { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } }, { name: JobName.CLEAN_OLD_SESSION_TOKENS }, ]); }); diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index 25c22ebd04..f232c4ac77 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -210,7 +210,7 @@ export class JobService { { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }, { name: JobName.CLEAN_OLD_AUDIT_LOGS }, { name: JobName.USER_SYNC_USAGE }, - { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } }, + { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } }, { name: JobName.CLEAN_OLD_SESSION_TOKENS }, ]); } diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts index 2aea5c2798..46087436ab 100644 --- a/server/src/services/person.service.spec.ts +++ b/server/src/services/person.service.spec.ts @@ -3,6 +3,7 @@ import { Colorspace } from 'src/config'; import { BulkIdErrorReason } from 'src/dtos/asset-ids.response.dto'; import { PersonResponseDto, mapFaces, mapPerson } from 'src/dtos/person.dto'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; +import { SystemMetadataKey } from 'src/entities/system-metadata.entity'; import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface'; import { ICryptoRepository } from 'src/interfaces/crypto.interface'; import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface'; @@ -539,6 +540,7 @@ describe(PersonService.name, () => { await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(JobStatus.SKIPPED); expect(jobMock.queueAll).not.toHaveBeenCalled(); expect(systemMock.get).toHaveBeenCalled(); + expect(systemMock.set).not.toHaveBeenCalled(); }); it('should skip if recognition jobs are already queued', async () => { @@ -546,6 +548,7 @@ describe(PersonService.name, () => { await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(JobStatus.SKIPPED); expect(jobMock.queueAll).not.toHaveBeenCalled(); + expect(systemMock.set).not.toHaveBeenCalled(); }); it('should queue missing assets', async () => { @@ -564,6 +567,9 @@ describe(PersonService.name, () => { data: { id: faceStub.face1.id, deferred: false }, }, ]); + expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE, { + lastRun: expect.any(String), + }); }); it('should queue all assets', async () => { @@ -586,6 +592,59 @@ describe(PersonService.name, () => { data: { id: faceStub.face1.id, deferred: false }, }, ]); + expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE, { + lastRun: expect.any(String), + }); + }); + + it('should run nightly if new face has been added since last run', async () => { + personMock.getLatestFaceDate.mockResolvedValue(new Date().toISOString()); + personMock.getAllFaces.mockResolvedValue({ + items: [faceStub.face1], + hasNextPage: false, + }); + jobMock.getJobCounts.mockResolvedValue({ active: 1, waiting: 0, paused: 0, completed: 0, failed: 0, delayed: 0 }); + personMock.getAll.mockResolvedValue({ + items: [], + hasNextPage: false, + }); + personMock.getAllFaces.mockResolvedValue({ + items: [faceStub.face1], + hasNextPage: false, + }); + await sut.handleQueueRecognizeFaces({ force: true, nightly: true }); + + expect(systemMock.get).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE); + expect(personMock.getLatestFaceDate).toHaveBeenCalledOnce(); + expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {}); + expect(jobMock.queueAll).toHaveBeenCalledWith([ + { + name: JobName.FACIAL_RECOGNITION, + data: { id: faceStub.face1.id, deferred: false }, + }, + ]); + expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE, { + lastRun: expect.any(String), + }); + }); + + it('should skip nightly if no new face has been added since last run', async () => { + const lastRun = new Date(); + + systemMock.get.mockResolvedValue({ lastRun: lastRun.toISOString() }); + personMock.getLatestFaceDate.mockResolvedValue(new Date(lastRun.getTime() - 1).toISOString()); + personMock.getAllFaces.mockResolvedValue({ + items: [faceStub.face1], + hasNextPage: false, + }); + + await sut.handleQueueRecognizeFaces({ force: true, nightly: true }); + + expect(systemMock.get).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE); + expect(personMock.getLatestFaceDate).toHaveBeenCalledOnce(); + expect(personMock.getAllFaces).not.toHaveBeenCalled(); + expect(jobMock.queueAll).not.toHaveBeenCalled(); + expect(systemMock.set).not.toHaveBeenCalled(); }); it('should delete existing people and faces if forced', async () => { diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index 96982b10ad..18b7ce6315 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -26,6 +26,7 @@ import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetEntity, AssetType } from 'src/entities/asset.entity'; import { PersonPathType } from 'src/entities/move.entity'; import { PersonEntity } from 'src/entities/person.entity'; +import { SystemMetadataKey } from 'src/entities/system-metadata.entity'; import { IAccessRepository } from 'src/interfaces/access.interface'; import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface'; import { ICryptoRepository } from 'src/interfaces/crypto.interface'; @@ -34,6 +35,7 @@ import { IDeferrableJob, IEntityJob, IJobRepository, + INightlyJob, JOBS_ASSET_PAGINATION_SIZE, JobItem, JobName, @@ -67,7 +69,7 @@ export class PersonService { @Inject(IMoveRepository) moveRepository: IMoveRepository, @Inject(IMediaRepository) private mediaRepository: IMediaRepository, @Inject(IPersonRepository) private repository: IPersonRepository, - @Inject(ISystemMetadataRepository) systemMetadataRepository: ISystemMetadataRepository, + @Inject(ISystemMetadataRepository) private systemMetadataRepository: ISystemMetadataRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(ISearchRepository) private smartInfoRepository: ISearchRepository, @@ -376,13 +378,26 @@ export class PersonService { return JobStatus.SUCCESS; } - async handleQueueRecognizeFaces({ force }: IBaseJob): Promise { + async handleQueueRecognizeFaces({ force, nightly }: INightlyJob): Promise { const { machineLearning } = await this.configCore.getConfig({ withCache: false }); if (!isFacialRecognitionEnabled(machineLearning)) { return JobStatus.SKIPPED; } await this.jobRepository.waitForQueueCompletion(QueueName.THUMBNAIL_GENERATION, QueueName.FACE_DETECTION); + + if (nightly) { + const [state, latestFaceDate] = await Promise.all([ + this.systemMetadataRepository.get(SystemMetadataKey.FACIAL_RECOGNITION_STATE), + this.repository.getLatestFaceDate(), + ]); + + if (state?.lastRun && latestFaceDate && state.lastRun > latestFaceDate) { + this.logger.debug('Skipping facial recognition nightly since no face has been added since the last run'); + return JobStatus.SKIPPED; + } + } + const { waiting } = await this.jobRepository.getJobCounts(QueueName.FACIAL_RECOGNITION); if (force) { @@ -394,6 +409,7 @@ export class PersonService { return JobStatus.SKIPPED; } + const lastRun = new Date().toISOString(); const facePagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => this.repository.getAllFaces(pagination, { where: force ? undefined : { personId: IsNull() } }), ); @@ -404,6 +420,8 @@ export class PersonService { ); } + await this.systemMetadataRepository.set(SystemMetadataKey.FACIAL_RECOGNITION_STATE, { lastRun }); + return JobStatus.SUCCESS; } diff --git a/server/test/repositories/person.repository.mock.ts b/server/test/repositories/person.repository.mock.ts index 5909e3c967..94a4486c81 100644 --- a/server/test/repositories/person.repository.mock.ts +++ b/server/test/repositories/person.repository.mock.ts @@ -29,5 +29,6 @@ export const newPersonRepositoryMock = (): Mocked => { getFaceById: vitest.fn(), getFaceByIdWithAssets: vitest.fn(), getNumberOfPeople: vitest.fn(), + getLatestFaceDate: vitest.fn(), }; };