self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-25 10:43:13 +02:00
Code Issues Releases Activity

fix(server): queue library asset refresh in batches (#7914)

Browse Source 
* add debug logs

* scan assets in batches

* Cleanup

* don't normalize

* Removing extra log

* remove unneeded code

* change log levels
This commit is contained in:
Jonathan Jogenfors 2024-03-14 14:43:05 +01:00 committed by GitHub
parent 428b7b0c4e
commit ba38713fbc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 27 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
server/src/domain/library/library.service.ts
View File

@ -300,17 +300,26 @@ export class LibraryService extends EventEmitter {
} }
private async scanAssets(libraryId: string, assetPaths: string[], ownerId: string, force = false) { private async scanAssets(libraryId: string, assetPaths: string[], ownerId: string, force = false) {
await this.jobRepository.queueAll( this.logger.verbose(`Queuing refresh of ${assetPaths.length} asset(s)`);
assetPaths.map((assetPath) => ({
name: JobName.LIBRARY_SCAN_ASSET, // We perform this in batches to save on memory when performing large refreshes (greater than 1M assets)
data: { const batchSize = 5000;
id: libraryId, for (let i = 0; i < assetPaths.length; i += batchSize) {
assetPath: path.normalize(assetPath), const batch = assetPaths.slice(i, i + batchSize);
ownerId, await this.jobRepository.queueAll(
force, batch.map((assetPath) => ({
}, name: JobName.LIBRARY_SCAN_ASSET,
})), data: {
); id: libraryId,
assetPath: assetPath,
ownerId,
force,
},
})),
);
}
this.logger.debug('Asset refresh queue completed');
} }
private async validateImportPath(importPath: string): Promise<ValidateLibraryImportPathResponseDto> { private async validateImportPath(importPath: string): Promise<ValidateLibraryImportPathResponseDto> {
@ -611,14 +620,6 @@ export class LibraryService extends EventEmitter {
return true; return true;
} }
// Check if a given path is in a user's external path. Both arguments are assumed to be normalized
private isInExternalPath(filePath: string, externalPath: string | null): boolean {
if (externalPath === null) {
return false;
}
return filePath.startsWith(externalPath);
}
async handleQueueAssetRefresh(job: ILibraryRefreshJob): Promise<boolean> { async handleQueueAssetRefresh(job: ILibraryRefreshJob): Promise<boolean> {
const library = await this.repository.get(job.id); const library = await this.repository.get(job.id);
if (!library || library.type !== LibraryType.EXTERNAL) { if (!library || library.type !== LibraryType.EXTERNAL) {
@ -626,7 +627,7 @@ export class LibraryService extends EventEmitter {
return false; return false;
} }
this.logger.verbose(`Refreshing library: ${job.id}`); this.logger.log(`Refreshing library: ${job.id}`);
const crawledAssetPaths = await this.getPathTrie(library); const crawledAssetPaths = await this.getPathTrie(library);
this.logger.debug(`Found ${crawledAssetPaths.size} asset(s) when crawling import paths ${library.importPaths}`); this.logger.debug(`Found ${crawledAssetPaths.size} asset(s) when crawling import paths ${library.importPaths}`);
@ -637,16 +638,20 @@ export class LibraryService extends EventEmitter {
this.assetRepository.getLibraryAssetPaths(pagination, library.id), this.assetRepository.getLibraryAssetPaths(pagination, library.id),
); );
this.logger.verbose(`Crawled asset paths paginated`);
const shouldScanAll = job.refreshAllFiles || job.refreshModifiedFiles; const shouldScanAll = job.refreshAllFiles || job.refreshModifiedFiles;
for await (const page of pagination) { for await (const page of pagination) {
for (const asset of page) { for (const asset of page) {
const isOffline = !crawledAssetPaths.has(asset.originalPath); const isOffline = !crawledAssetPaths.has(asset.originalPath);
if (isOffline && !asset.isOffline) { if (isOffline && !asset.isOffline) {
assetIdsToMarkOffline.push(asset.id); assetIdsToMarkOffline.push(asset.id);
this.logger.verbose(`Added to mark-offline list: ${asset.originalPath}`);
} }
if (!isOffline && asset.isOffline) { if (!isOffline && asset.isOffline) {
assetIdsToMarkOnline.push(asset.id); assetIdsToMarkOnline.push(asset.id);
this.logger.verbose(`Added to mark-online list: ${asset.originalPath}`);
} }
if (!shouldScanAll) { if (!shouldScanAll) {
@ -655,6 +660,8 @@ export class LibraryService extends EventEmitter {
} }
} }
this.logger.verbose(`Crawled assets have been checked for online/offline status`);
if (assetIdsToMarkOffline.length > 0) { if (assetIdsToMarkOffline.length > 0) {
this.logger.debug(`Found ${assetIdsToMarkOffline.length} offline asset(s) previously marked as online`); this.logger.debug(`Found ${assetIdsToMarkOffline.length} offline asset(s) previously marked as online`);
await this.assetRepository.updateAll(assetIdsToMarkOffline, { isOffline: true }); await this.assetRepository.updateAll(assetIdsToMarkOffline, { isOffline: true });