mirror of
https://github.com/immich-app/immich.git
synced 2024-12-25 10:43:13 +02:00
perf(server): use queries to refresh library assets (#7685)
* use queries instead of js * missing await * add mock methods * fix test * update sql * linting
This commit is contained in:
parent
fcb990665c
commit
1ec5d612fa
@ -144,6 +144,7 @@ describe(LibraryService.name, () => {
|
||||
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
storageMock.crawl.mockResolvedValue(['/data/user1/photo.jpg']);
|
||||
assetMock.getPathsNotInLibrary.mockResolvedValue(['/data/user1/photo.jpg']);
|
||||
assetMock.getByLibraryId.mockResolvedValue([]);
|
||||
userMock.get.mockResolvedValue(userStub.admin);
|
||||
|
||||
|
@ -621,29 +621,18 @@ export class LibraryService extends EventEmitter {
|
||||
pathsToCrawl: validImportPaths,
|
||||
exclusionPatterns: library.exclusionPatterns,
|
||||
});
|
||||
|
||||
const crawledAssetPaths = rawPaths.map((filePath) => path.normalize(filePath));
|
||||
|
||||
this.logger.debug(`Found ${crawledAssetPaths.length} asset(s) when crawling import paths ${library.importPaths}`);
|
||||
const assetsInLibrary = await this.assetRepository.getByLibraryId([job.id]);
|
||||
const onlineFiles = new Set(crawledAssetPaths);
|
||||
const offlineAssetIds = assetsInLibrary
|
||||
.filter((asset) => !onlineFiles.has(asset.originalPath))
|
||||
.filter((asset) => !asset.isOffline)
|
||||
.map((asset) => asset.id);
|
||||
this.logger.debug(`Marking ${offlineAssetIds.length} assets as offline`);
|
||||
|
||||
await this.assetRepository.updateAll(offlineAssetIds, { isOffline: true });
|
||||
await this.assetRepository.updateOfflineLibraryAssets(library.id, crawledAssetPaths);
|
||||
|
||||
if (crawledAssetPaths.length > 0) {
|
||||
let filteredPaths: string[] = [];
|
||||
if (job.refreshAllFiles || job.refreshModifiedFiles) {
|
||||
filteredPaths = crawledAssetPaths;
|
||||
} else {
|
||||
const onlinePathsInLibrary = new Set(
|
||||
assetsInLibrary.filter((asset) => !asset.isOffline).map((asset) => asset.originalPath),
|
||||
);
|
||||
filteredPaths = crawledAssetPaths.filter((assetPath) => !onlinePathsInLibrary.has(assetPath));
|
||||
filteredPaths = await this.assetRepository.getPathsNotInLibrary(library.id, crawledAssetPaths);
|
||||
|
||||
this.logger.debug(`Will import ${filteredPaths.length} new asset(s)`);
|
||||
}
|
||||
|
@ -136,6 +136,8 @@ export interface IAssetRepository {
|
||||
getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
|
||||
getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]>;
|
||||
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
|
||||
getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise<string[]>;
|
||||
updateOfflineLibraryAssets(libraryId: string, originalPaths: string[]): Promise<void>;
|
||||
deleteAll(ownerId: string): Promise<void>;
|
||||
getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
|
||||
getAllByFileCreationDate(
|
||||
|
@ -199,6 +199,29 @@ export class AssetRepository implements IAssetRepository {
|
||||
});
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING]] })
|
||||
@ChunkedArray({ paramIndex: 1 })
|
||||
async getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise<string[]> {
|
||||
const result = await this.repository.query(
|
||||
`
|
||||
WITH paths AS (SELECT unnest($2::text[]) AS path)
|
||||
SELECT path FROM paths
|
||||
WHERE NOT EXISTS (SELECT 1 FROM assets WHERE "libraryId" = $1 AND "originalPath" = path);
|
||||
`,
|
||||
[libraryId, originalPaths],
|
||||
);
|
||||
return result.map((row: { path: string }) => row.path);
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING]] })
|
||||
@ChunkedArray({ paramIndex: 1 })
|
||||
async updateOfflineLibraryAssets(libraryId: string, originalPaths: string[]): Promise<void> {
|
||||
await this.repository.update(
|
||||
{ library: { id: libraryId }, originalPath: Not(In(originalPaths)), isOffline: false },
|
||||
{ isOffline: true },
|
||||
);
|
||||
}
|
||||
|
||||
getAll(pagination: PaginationOptions, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
|
||||
let builder = this.repository.createQueryBuilder('asset');
|
||||
builder = searchAssetBuilder(builder, options);
|
||||
|
@ -395,6 +395,39 @@ ORDER BY
|
||||
LIMIT
|
||||
1
|
||||
|
||||
-- AssetRepository.getPathsNotInLibrary
|
||||
WITH
|
||||
paths AS (
|
||||
SELECT
|
||||
unnest($2::text []) AS path
|
||||
)
|
||||
SELECT
|
||||
path
|
||||
FROM
|
||||
paths
|
||||
WHERE
|
||||
NOT EXISTS (
|
||||
SELECT
|
||||
1
|
||||
FROM
|
||||
assets
|
||||
WHERE
|
||||
"libraryId" = $1
|
||||
AND "originalPath" = path
|
||||
);
|
||||
|
||||
-- AssetRepository.updateOfflineLibraryAssets
|
||||
UPDATE "assets"
|
||||
SET
|
||||
"isOffline" = $1,
|
||||
"updatedAt" = CURRENT_TIMESTAMP
|
||||
WHERE
|
||||
(
|
||||
"libraryId" = $2
|
||||
AND NOT ("originalPath" IN ($3))
|
||||
AND "isOffline" = $4
|
||||
)
|
||||
|
||||
-- AssetRepository.getAllByFileCreationDate
|
||||
SELECT
|
||||
"asset"."id" AS "asset_id",
|
||||
|
@ -23,6 +23,8 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
|
||||
updateAll: jest.fn(),
|
||||
getByLibraryId: jest.fn(),
|
||||
getByLibraryIdAndOriginalPath: jest.fn(),
|
||||
updateOfflineLibraryAssets: jest.fn(),
|
||||
getPathsNotInLibrary: jest.fn(),
|
||||
deleteAll: jest.fn(),
|
||||
save: jest.fn(),
|
||||
remove: jest.fn(),
|
||||
|
Loading…
Reference in New Issue
Block a user