1
0
mirror of https://github.com/immich-app/immich.git synced 2025-01-13 15:35:15 +02:00

refactor: library watching (#7071)

This commit is contained in:
Jason Rasmussen 2024-02-13 08:48:47 -05:00 committed by GitHub
parent 3d7a7bcb7a
commit b648025e2f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 143 additions and 248 deletions

View File

@ -11,7 +11,8 @@ describe(`${LibraryController.name} (e2e)`, () => {
let admin: LoginResponseDto; let admin: LoginResponseDto;
beforeAll(async () => { beforeAll(async () => {
server = (await testApp.create()).getHttpServer(); const app = await testApp.create();
server = app.getHttpServer();
}); });
afterAll(async () => { afterAll(async () => {

View File

@ -2,7 +2,7 @@ import { LibraryResponseDto, LoginResponseDto } from '@app/domain';
import { LibraryController } from '@app/immich'; import { LibraryController } from '@app/immich';
import { AssetType, LibraryType } from '@app/infra/entities'; import { AssetType, LibraryType } from '@app/infra/entities';
import { errorStub, uuidStub } from '@test/fixtures'; import { errorStub, uuidStub } from '@test/fixtures';
import * as fs from 'fs'; import * as fs from 'node:fs';
import request from 'supertest'; import request from 'supertest';
import { utimes } from 'utimes'; import { utimes } from 'utimes';
import { import {
@ -18,7 +18,8 @@ describe(`${LibraryController.name} (e2e)`, () => {
let admin: LoginResponseDto; let admin: LoginResponseDto;
beforeAll(async () => { beforeAll(async () => {
server = (await testApp.create()).getHttpServer(); const app = await testApp.create();
server = app.getHttpServer();
}); });
beforeEach(async () => { beforeEach(async () => {
@ -264,7 +265,7 @@ describe(`${LibraryController.name} (e2e)`, () => {
`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, `${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`,
); );
await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447775200000); await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447_775_200_000);
await api.libraryApi.scanLibrary(server, admin.accessToken, library.id); await api.libraryApi.scanLibrary(server, admin.accessToken, library.id);
@ -273,7 +274,7 @@ describe(`${LibraryController.name} (e2e)`, () => {
`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, `${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`,
); );
await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447775200001); await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447_775_200_001);
await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, { refreshModifiedFiles: true }); await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, { refreshModifiedFiles: true });
@ -289,7 +290,7 @@ describe(`${LibraryController.name} (e2e)`, () => {
exifImageWidth: 800, exifImageWidth: 800,
exposureTime: '1/15', exposureTime: '1/15',
fNumber: 22, fNumber: 22,
fileSizeInByte: 114225, fileSizeInByte: 114_225,
focalLength: 35, focalLength: 35,
iso: 1000, iso: 1000,
make: 'NIKON CORPORATION', make: 'NIKON CORPORATION',
@ -311,7 +312,7 @@ describe(`${LibraryController.name} (e2e)`, () => {
`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, `${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`,
); );
await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447775200000); await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447_775_200_000);
await api.libraryApi.scanLibrary(server, admin.accessToken, library.id); await api.libraryApi.scanLibrary(server, admin.accessToken, library.id);
@ -320,7 +321,7 @@ describe(`${LibraryController.name} (e2e)`, () => {
`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, `${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`,
); );
await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447775200000); await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447_775_200_000);
await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, { refreshModifiedFiles: true }); await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, { refreshModifiedFiles: true });
@ -351,7 +352,7 @@ describe(`${LibraryController.name} (e2e)`, () => {
`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, `${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`,
); );
await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447775200000); await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447_775_200_000);
await api.libraryApi.scanLibrary(server, admin.accessToken, library.id); await api.libraryApi.scanLibrary(server, admin.accessToken, library.id);
@ -360,7 +361,7 @@ describe(`${LibraryController.name} (e2e)`, () => {
`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, `${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`,
); );
await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447775200000); await utimes(`${IMMICH_TEST_ASSET_TEMP_PATH}/el_torcal_rocks.jpg`, 447_775_200_000);
await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, { refreshAllFiles: true }); await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, { refreshAllFiles: true });
@ -375,7 +376,7 @@ describe(`${LibraryController.name} (e2e)`, () => {
exifImageWidth: 800, exifImageWidth: 800,
exposureTime: '1/15', exposureTime: '1/15',
fNumber: 22, fNumber: 22,
fileSizeInByte: 114225, fileSizeInByte: 114_225,
focalLength: 35, focalLength: 35,
iso: 1000, iso: 1000,
make: 'NIKON CORPORATION', make: 'NIKON CORPORATION',

View File

@ -1,11 +1,11 @@
import { AssetType, LibraryType, SystemConfig, SystemConfigKey, UserEntity } from '@app/infra/entities'; import { AssetType, LibraryType, SystemConfig, SystemConfigKey, UserEntity } from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common'; import { BadRequestException } from '@nestjs/common';
import { import {
IAccessRepositoryMock, IAccessRepositoryMock,
assetStub, assetStub,
authStub, authStub,
libraryStub, libraryStub,
makeMockWatcher,
newAccessRepositoryMock, newAccessRepositoryMock,
newAssetRepositoryMock, newAssetRepositoryMock,
newCryptoRepositoryMock, newCryptoRepositoryMock,
@ -17,8 +17,6 @@ import {
systemConfigStub, systemConfigStub,
userStub, userStub,
} from '@test'; } from '@test';
import { newFSWatcherMock } from '@test/mocks';
import { Stats } from 'node:fs'; import { Stats } from 'node:fs';
import { ILibraryFileJob, ILibraryRefreshJob, JobName } from '../job'; import { ILibraryFileJob, ILibraryRefreshJob, JobName } from '../job';
import { import {
@ -128,16 +126,6 @@ describe(LibraryService.name, () => {
} }
}); });
const mockWatcher = newFSWatcherMock();
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
callback();
}
});
storageMock.watch.mockReturnValue(mockWatcher);
await sut.init(); await sut.init();
expect(storageMock.watch.mock.calls).toEqual( expect(storageMock.watch.mock.calls).toEqual(
@ -718,21 +706,13 @@ describe(LibraryService.name, () => {
configMock.load.mockResolvedValue(systemConfigStub.libraryWatchEnabled); configMock.load.mockResolvedValue(systemConfigStub.libraryWatchEnabled);
const mockWatcher = newFSWatcherMock(); const mockClose = jest.fn();
storageMock.watch.mockImplementation(makeMockWatcher({ close: mockClose }));
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
callback();
}
});
storageMock.watch.mockReturnValue(mockWatcher);
await sut.init(); await sut.init();
await sut.delete(authStub.admin, libraryStub.externalLibraryWithImportPaths1.id); await sut.delete(authStub.admin, libraryStub.externalLibraryWithImportPaths1.id);
expect(mockWatcher.close).toHaveBeenCalled(); expect(mockClose).toHaveBeenCalled();
}); });
}); });
@ -940,16 +920,6 @@ describe(LibraryService.name, () => {
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
libraryMock.getAll.mockResolvedValue([]); libraryMock.getAll.mockResolvedValue([]);
const mockWatcher = newFSWatcherMock();
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
callback();
}
});
storageMock.watch.mockReturnValue(mockWatcher);
await sut.init(); await sut.init();
await sut.create(authStub.admin, { await sut.create(authStub.admin, {
type: LibraryType.EXTERNAL, type: LibraryType.EXTERNAL,
@ -959,6 +929,7 @@ describe(LibraryService.name, () => {
expect(storageMock.watch).toHaveBeenCalledWith( expect(storageMock.watch).toHaveBeenCalledWith(
libraryStub.externalLibraryWithImportPaths1.importPaths, libraryStub.externalLibraryWithImportPaths1.importPaths,
expect.anything(), expect.anything(),
expect.anything(),
); );
}); });
@ -1133,16 +1104,6 @@ describe(LibraryService.name, () => {
libraryMock.update.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.update.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
const mockWatcher = newFSWatcherMock();
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
callback();
}
});
storageMock.watch.mockReturnValue(mockWatcher);
await expect(sut.update(authStub.admin, authStub.admin.user.id, { importPaths: ['/foo'] })).resolves.toEqual( await expect(sut.update(authStub.admin, authStub.admin.user.id, { importPaths: ['/foo'] })).resolves.toEqual(
mapLibrary(libraryStub.externalLibraryWithImportPaths1), mapLibrary(libraryStub.externalLibraryWithImportPaths1),
); );
@ -1155,6 +1116,7 @@ describe(LibraryService.name, () => {
expect(storageMock.watch).toHaveBeenCalledWith( expect(storageMock.watch).toHaveBeenCalledWith(
libraryStub.externalLibraryWithImportPaths1.importPaths, libraryStub.externalLibraryWithImportPaths1.importPaths,
expect.anything(), expect.anything(),
expect.anything(),
); );
}); });
@ -1163,16 +1125,6 @@ describe(LibraryService.name, () => {
configMock.load.mockResolvedValue(systemConfigStub.libraryWatchEnabled); configMock.load.mockResolvedValue(systemConfigStub.libraryWatchEnabled);
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
const mockWatcher = newFSWatcherMock();
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
callback();
}
});
storageMock.watch.mockReturnValue(mockWatcher);
await expect(sut.update(authStub.admin, authStub.admin.user.id, { exclusionPatterns: ['bar'] })).resolves.toEqual( await expect(sut.update(authStub.admin, authStub.admin.user.id, { exclusionPatterns: ['bar'] })).resolves.toEqual(
mapLibrary(libraryStub.externalLibraryWithImportPaths1), mapLibrary(libraryStub.externalLibraryWithImportPaths1),
); );
@ -1182,7 +1134,11 @@ describe(LibraryService.name, () => {
id: authStub.admin.user.id, id: authStub.admin.user.id,
}), }),
); );
expect(storageMock.watch).toHaveBeenCalledWith(expect.arrayContaining([expect.any(String)]), expect.anything()); expect(storageMock.watch).toHaveBeenCalledWith(
expect.arrayContaining([expect.any(String)]),
expect.anything(),
expect.anything(),
);
}); });
}); });
@ -1204,56 +1160,35 @@ describe(LibraryService.name, () => {
}); });
describe('watching enabled', () => { describe('watching enabled', () => {
const mockWatcher = newFSWatcherMock();
beforeEach(async () => { beforeEach(async () => {
configMock.load.mockResolvedValue(systemConfigStub.libraryWatchEnabled); configMock.load.mockResolvedValue(systemConfigStub.libraryWatchEnabled);
libraryMock.getAll.mockResolvedValue([]); libraryMock.getAll.mockResolvedValue([]);
await sut.init(); await sut.init();
storageMock.watch.mockReturnValue(mockWatcher);
}); });
it('should watch library', async () => { it('should watch library', async () => {
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]); libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
const mockWatcher = newFSWatcherMock();
let isReady = false;
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
isReady = true;
callback();
}
});
storageMock.watch.mockReturnValue(mockWatcher);
await sut.watchAll(); await sut.watchAll();
expect(storageMock.watch).toHaveBeenCalledWith( expect(storageMock.watch).toHaveBeenCalledWith(
libraryStub.externalLibraryWithImportPaths1.importPaths, libraryStub.externalLibraryWithImportPaths1.importPaths,
expect.anything(), expect.anything(),
expect.anything(),
); );
expect(isReady).toBe(true);
}); });
it('should watch and unwatch library', async () => { it('should watch and unwatch library', async () => {
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]); libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
const mockClose = jest.fn();
mockWatcher.on.mockImplementation((event, callback) => { storageMock.watch.mockImplementation(makeMockWatcher({ close: mockClose }));
if (event === 'ready') {
callback();
}
});
await sut.watchAll(); await sut.watchAll();
await sut.unwatch(libraryStub.externalLibraryWithImportPaths1.id); await sut.unwatch(libraryStub.externalLibraryWithImportPaths1.id);
expect(mockWatcher.close).toHaveBeenCalled(); expect(mockClose).toHaveBeenCalled();
}); });
it('should not watch library without import paths', async () => { it('should not watch library without import paths', async () => {
@ -1277,14 +1212,7 @@ describe(LibraryService.name, () => {
it('should handle a new file event', async () => { it('should handle a new file event', async () => {
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]); libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
storageMock.watch.mockImplementation(makeMockWatcher({ items: [{ event: 'add', value: '/foo/photo.jpg' }] }));
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
callback();
} else if (event === 'add') {
callback('/foo/photo.jpg');
}
});
await sut.watchAll(); await sut.watchAll();
@ -1304,14 +1232,9 @@ describe(LibraryService.name, () => {
it('should handle a file change event', async () => { it('should handle a file change event', async () => {
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]); libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
storageMock.watch.mockImplementation(
mockWatcher.on.mockImplementation((event, callback) => { makeMockWatcher({ items: [{ event: 'change', value: '/foo/photo.jpg' }] }),
if (event === 'ready') { );
callback();
} else if (event === 'change') {
callback('/foo/photo.jpg');
}
});
await sut.watchAll(); await sut.watchAll();
@ -1331,16 +1254,10 @@ describe(LibraryService.name, () => {
it('should handle a file unlink event', async () => { it('should handle a file unlink event', async () => {
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]); libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.external); assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.external);
storageMock.watch.mockImplementation(
mockWatcher.on.mockImplementation((event, callback) => { makeMockWatcher({ items: [{ event: 'unlink', value: '/foo/photo.jpg' }] }),
if (event === 'ready') { );
callback();
} else if (event === 'unlink') {
callback('/foo/photo.jpg');
}
});
await sut.watchAll(); await sut.watchAll();
@ -1351,34 +1268,19 @@ describe(LibraryService.name, () => {
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.external); assetMock.getByLibraryIdAndOriginalPath.mockResolvedValue(assetStub.external);
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]); libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
storageMock.watch.mockImplementation(
let didError = false; makeMockWatcher({
items: [{ event: 'error', value: 'Error!' }],
mockWatcher.on.mockImplementation((event, callback) => { }),
if (event === 'ready') { );
callback();
} else if (event === 'error') {
didError = true;
callback('Error!');
}
});
await sut.watchAll(); await sut.watchAll();
expect(didError).toBe(true);
}); });
it('should ignore unknown extensions', async () => { it('should ignore unknown extensions', async () => {
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]); libraryMock.getAll.mockResolvedValue([libraryStub.externalLibraryWithImportPaths1]);
storageMock.watch.mockImplementation(makeMockWatcher({ items: [{ event: 'add', value: '/foo/photo.jpg' }] }));
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
callback();
} else if (event === 'add') {
callback('/foo/photo.txt');
}
});
await sut.watchAll(); await sut.watchAll();
@ -1388,14 +1290,7 @@ describe(LibraryService.name, () => {
it('should ignore excluded paths', async () => { it('should ignore excluded paths', async () => {
libraryMock.get.mockResolvedValue(libraryStub.patternPath); libraryMock.get.mockResolvedValue(libraryStub.patternPath);
libraryMock.getAll.mockResolvedValue([libraryStub.patternPath]); libraryMock.getAll.mockResolvedValue([libraryStub.patternPath]);
storageMock.watch.mockImplementation(makeMockWatcher({ items: [{ event: 'add', value: '/dir1/photo.txt' }] }));
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
callback();
} else if (event === 'add') {
callback('/dir1/photo.txt');
}
});
await sut.watchAll(); await sut.watchAll();
@ -1405,14 +1300,7 @@ describe(LibraryService.name, () => {
it('should ignore excluded paths without case sensitivity', async () => { it('should ignore excluded paths without case sensitivity', async () => {
libraryMock.get.mockResolvedValue(libraryStub.patternPath); libraryMock.get.mockResolvedValue(libraryStub.patternPath);
libraryMock.getAll.mockResolvedValue([libraryStub.patternPath]); libraryMock.getAll.mockResolvedValue([libraryStub.patternPath]);
storageMock.watch.mockImplementation(makeMockWatcher({ items: [{ event: 'add', value: '/DIR1/photo.txt' }] }));
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
callback();
} else if (event === 'add') {
callback('/DIR1/photo.txt');
}
});
await sut.watchAll(); await sut.watchAll();
@ -1445,20 +1333,13 @@ describe(LibraryService.name, () => {
} }
}); });
const mockWatcher = newFSWatcherMock(); const mockClose = jest.fn();
storageMock.watch.mockImplementation(makeMockWatcher({ close: mockClose }));
mockWatcher.on.mockImplementation((event, callback) => {
if (event === 'ready') {
callback();
}
});
storageMock.watch.mockReturnValue(mockWatcher);
await sut.init(); await sut.init();
await sut.unwatchAll(); await sut.unwatchAll();
expect(mockWatcher.close).toHaveBeenCalledTimes(2); expect(mockClose).toHaveBeenCalledTimes(2);
}); });
}); });

View File

@ -38,10 +38,8 @@ export class LibraryService extends EventEmitter {
readonly logger = new ImmichLogger(LibraryService.name); readonly logger = new ImmichLogger(LibraryService.name);
private access: AccessCore; private access: AccessCore;
private configCore: SystemConfigCore; private configCore: SystemConfigCore;
private watchLibraries = false; private watchLibraries = false;
private watchers: Record<string, () => void> = {};
private watchers: Record<string, () => Promise<void>> = {};
constructor( constructor(
@Inject(IAccessRepository) accessRepository: IAccessRepository, @Inject(IAccessRepository) accessRepository: IAccessRepository,
@ -116,63 +114,57 @@ export class LibraryService extends EventEmitter {
this.logger.debug(`Settings for watcher: usePolling: ${usePolling}, interval: ${interval}`); this.logger.debug(`Settings for watcher: usePolling: ${usePolling}, interval: ${interval}`);
const watcher = this.storageRepository.watch(library.importPaths, { let _resolve: () => void;
usePolling, const ready$ = new Promise<void>((resolve) => (_resolve = resolve));
interval,
binaryInterval: interval,
ignoreInitial: true,
});
this.watchers[id] = async () => { this.watchers[id] = this.storageRepository.watch(
await watcher.close(); library.importPaths,
}; {
usePolling,
watcher.on('add', async (path) => { interval,
this.logger.debug(`File add event received for ${path} in library ${library.id}}`); binaryInterval: interval,
if (matcher(path)) { ignoreInitial: true,
await this.scanAssets(library.id, [path], library.ownerId, false); },
} {
this.emit('add', path); onReady: () => _resolve(),
}); onAdd: async (path) => {
this.logger.debug(`File add event received for ${path} in library ${library.id}}`);
watcher.on('change', async (path) => { if (matcher(path)) {
this.logger.debug(`Detected file change for ${path} in library ${library.id}`); await this.scanAssets(library.id, [path], library.ownerId, false);
}
if (matcher(path)) { this.emit('add', path);
// Note: if the changed file was not previously imported, it will be imported now. },
await this.scanAssets(library.id, [path], library.ownerId, false); onChange: async (path) => {
} this.logger.debug(`Detected file change for ${path} in library ${library.id}`);
this.emit('change', path); if (matcher(path)) {
}); // Note: if the changed file was not previously imported, it will be imported now.
await this.scanAssets(library.id, [path], library.ownerId, false);
watcher.on('unlink', async (path) => { }
this.logger.debug(`Detected deleted file at ${path} in library ${library.id}`); this.emit('change', path);
const existingAssetEntity = await this.assetRepository.getByLibraryIdAndOriginalPath(library.id, path); },
onUnlink: async (path) => {
if (existingAssetEntity && matcher(path)) { this.logger.debug(`Detected deleted file at ${path} in library ${library.id}`);
await this.assetRepository.save({ id: existingAssetEntity.id, isOffline: true }); const asset = await this.assetRepository.getByLibraryIdAndOriginalPath(library.id, path);
} if (asset && matcher(path)) {
await this.assetRepository.save({ id: asset.id, isOffline: true });
this.emit('unlink', path); }
}); this.emit('unlink', path);
},
watcher.on('error', async (error) => { onError: (error) => {
// TODO: should we log, or throw an exception? // TODO: should we log, or throw an exception?
this.logger.error(`Library watcher for library ${library.id} encountered error: ${error}`); this.logger.error(`Library watcher for library ${library.id} encountered error: ${error}`);
}); },
},
);
// Wait for the watcher to initialize before returning // Wait for the watcher to initialize before returning
await new Promise<void>((resolve) => { await ready$;
watcher.on('ready', async () => {
resolve();
});
});
return true; return true;
} }
async unwatch(id: string) { async unwatch(id: string) {
if (this.watchers.hasOwnProperty(id)) { if (this.watchers[id]) {
await this.watchers[id](); await this.watchers[id]();
delete this.watchers[id]; delete this.watchers[id];
} }

View File

@ -1,4 +1,4 @@
import { FSWatcher, WatchOptions } from 'chokidar'; import { WatchOptions } from 'chokidar';
import { Stats } from 'node:fs'; import { Stats } from 'node:fs';
import { FileReadOptions } from 'node:fs/promises'; import { FileReadOptions } from 'node:fs/promises';
import { Readable } from 'node:stream'; import { Readable } from 'node:stream';
@ -23,7 +23,13 @@ export interface DiskUsage {
export const IStorageRepository = 'IStorageRepository'; export const IStorageRepository = 'IStorageRepository';
export interface ImmichWatcher extends FSWatcher {} export interface WatchEvents {
onReady(): void;
onAdd(path: string): void;
onChange(path: string): void;
onUnlink(path: string): void;
onError(error: Error): void;
}
export interface IStorageRepository { export interface IStorageRepository {
createZipStream(): ImmichZipStream; createZipStream(): ImmichZipStream;
@ -41,6 +47,6 @@ export interface IStorageRepository {
crawl(crawlOptions: CrawlOptionsDto): Promise<string[]>; crawl(crawlOptions: CrawlOptionsDto): Promise<string[]>;
copyFile(source: string, target: string): Promise<void>; copyFile(source: string, target: string): Promise<void>;
rename(source: string, target: string): Promise<void>; rename(source: string, target: string): Promise<void>;
watch(paths: string[], options: WatchOptions): ImmichWatcher; watch(paths: string[], options: WatchOptions, events: Partial<WatchEvents>): () => void;
utimes(filepath: string, atime: Date, mtime: Date): Promise<void>; utimes(filepath: string, atime: Date, mtime: Date): Promise<void>;
} }

View File

@ -2,10 +2,10 @@ import {
CrawlOptionsDto, CrawlOptionsDto,
DiskUsage, DiskUsage,
ImmichReadStream, ImmichReadStream,
ImmichWatcher,
ImmichZipStream, ImmichZipStream,
IStorageRepository, IStorageRepository,
mimeTypes, mimeTypes,
WatchEvents,
} from '@app/domain'; } from '@app/domain';
import { ImmichLogger } from '@app/infra/logger'; import { ImmichLogger } from '@app/infra/logger';
import archiver from 'archiver'; import archiver from 'archiver';
@ -136,8 +136,15 @@ export class FilesystemProvider implements IStorageRepository {
}); });
} }
watch(paths: string[], options: WatchOptions): ImmichWatcher { watch(paths: string[], options: WatchOptions, events: Partial<WatchEvents>) {
return chokidar.watch(paths, options); const watcher = chokidar.watch(paths, options);
watcher.on('ready', () => events.onReady?.());
watcher.on('add', (path) => events.onAdd?.(path));
watcher.on('change', (path) => events.onChange?.(path));
watcher.on('unlink', (path) => events.onUnlink?.(path));
return () => watcher.close();
} }
readdir = readdir; readdir = readdir;

View File

@ -1,24 +0,0 @@
export const newFSWatcherMock = () => {
return {
options: {},
on: jest.fn(),
add: jest.fn(),
unwatch: jest.fn(),
getWatched: jest.fn(),
close: jest.fn(),
addListener: jest.fn(),
removeListener: jest.fn(),
removeAllListeners: jest.fn(),
eventNames: jest.fn(),
rawListeners: jest.fn(),
listeners: jest.fn(),
emit: jest.fn(),
listenerCount: jest.fn(),
off: jest.fn(),
once: jest.fn(),
prependListener: jest.fn(),
prependOnceListener: jest.fn(),
setMaxListeners: jest.fn(),
getMaxListeners: jest.fn(),
};
};

View File

@ -1 +0,0 @@
export * from './fswatcher.mock';

View File

@ -1,4 +1,36 @@
import { IStorageRepository, StorageCore } from '@app/domain'; import { IStorageRepository, StorageCore, WatchEvents } from '@app/domain';
import { WatchOptions } from 'chokidar';
interface MockWatcherOptions {
items?: Array<{ event: 'change' | 'add' | 'unlink' | 'error'; value: string }>;
close?: () => void;
}
export const makeMockWatcher =
({ items, close }: MockWatcherOptions) =>
(paths: string[], options: WatchOptions, events: Partial<WatchEvents>) => {
events.onReady?.();
for (const item of items || []) {
switch (item.event) {
case 'add': {
events.onAdd?.(item.value);
break;
}
case 'change': {
events.onChange?.(item.value);
break;
}
case 'unlink': {
events.onUnlink?.(item.value);
break;
}
case 'error': {
events.onError?.(new Error(item.value));
}
}
}
return () => close?.();
};
export const newStorageRepositoryMock = (reset = true): jest.Mocked<IStorageRepository> => { export const newStorageRepositoryMock = (reset = true): jest.Mocked<IStorageRepository> => {
if (reset) { if (reset) {
@ -21,7 +53,7 @@ export const newStorageRepositoryMock = (reset = true): jest.Mocked<IStorageRepo
crawl: jest.fn(), crawl: jest.fn(),
rename: jest.fn(), rename: jest.fn(),
copyFile: jest.fn(), copyFile: jest.fn(),
watch: jest.fn(),
utimes: jest.fn(), utimes: jest.fn(),
watch: jest.fn().mockImplementation(makeMockWatcher({})),
}; };
}; };