mirror of
https://github.com/immich-app/immich.git
synced 2024-12-22 01:47:08 +02:00
fix(server): external library sync not working for large libraries (#7759)
This commit is contained in:
Mert
GitHub
parent
49d9051879
commit
5bd597f14b
33
server/package-lock.json
generated
33
server/package-lock.json
generated
@ -33,6 +33,7 @@
|
||||
"cookie-parser": "^1.4.6",
|
||||
"exiftool-vendored": "~24.5.0",
|
||||
"exiftool-vendored.pl": "12.76",
|
||||
"fast-glob": "^3.3.2",
|
||||
"fluent-ffmpeg": "^2.1.2",
|
||||
"geo-tz": "^8.0.0",
|
||||
"glob": "^10.3.3",
|
||||
@ -2657,7 +2658,6 @@
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||
"integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@nodelib/fs.stat": "2.0.5",
|
||||
"run-parallel": "^1.1.9"
|
||||
@ -2670,7 +2670,6 @@
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
|
||||
"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
@ -2679,7 +2678,6 @@
|
||||
"version": "1.2.8",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
|
||||
"integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@nodelib/fs.scandir": "2.1.5",
|
||||
"fastq": "^1.6.0"
|
||||
@ -6345,7 +6343,6 @@
|
||||
"version": "3.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz",
|
||||
"integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@nodelib/fs.stat": "^2.0.2",
|
||||
"@nodelib/fs.walk": "^1.2.3",
|
||||
@ -6378,7 +6375,6 @@
|
||||
"version": "1.15.0",
|
||||
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz",
|
||||
"integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"reusify": "^1.0.4"
|
||||
}
|
||||
@ -8665,7 +8661,6 @@
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
|
||||
"integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
@ -8682,7 +8677,6 @@
|
||||
"version": "4.0.5",
|
||||
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
|
||||
"integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"braces": "^3.0.2",
|
||||
"picomatch": "^2.3.1"
|
||||
@ -8695,7 +8689,6 @@
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
|
||||
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=8.6"
|
||||
},
|
||||
@ -9965,7 +9958,6 @@
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
|
||||
"integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@ -10405,7 +10397,6 @@
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
|
||||
"integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"iojs": ">=1.0.0",
|
||||
"node": ">=0.10.0"
|
||||
@ -10441,7 +10432,6 @@
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
|
||||
"integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@ -14454,7 +14444,6 @@
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
|
||||
"integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@nodelib/fs.stat": "2.0.5",
|
||||
"run-parallel": "^1.1.9"
|
||||
@ -14463,14 +14452,12 @@
|
||||
"@nodelib/fs.stat": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
|
||||
"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
|
||||
"dev": true
|
||||
"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="
|
||||
},
|
||||
"@nodelib/fs.walk": {
|
||||
"version": "1.2.8",
|
||||
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
|
||||
"integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@nodelib/fs.scandir": "2.1.5",
|
||||
"fastq": "^1.6.0"
|
||||
@ -17321,7 +17308,6 @@
|
||||
"version": "3.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz",
|
||||
"integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@nodelib/fs.stat": "^2.0.2",
|
||||
"@nodelib/fs.walk": "^1.2.3",
|
||||
@ -17351,7 +17337,6 @@
|
||||
"version": "1.15.0",
|
||||
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz",
|
||||
"integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"reusify": "^1.0.4"
|
||||
}
|
||||
@ -19073,8 +19058,7 @@
|
||||
"merge2": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
|
||||
"integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
|
||||
"dev": true
|
||||
"integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="
|
||||
},
|
||||
"methods": {
|
||||
"version": "1.1.2",
|
||||
@ -19085,7 +19069,6 @@
|
||||
"version": "4.0.5",
|
||||
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
|
||||
"integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"braces": "^3.0.2",
|
||||
"picomatch": "^2.3.1"
|
||||
@ -19094,8 +19077,7 @@
|
||||
"picomatch": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
|
||||
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
|
||||
"dev": true
|
||||
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="
|
||||
}
|
||||
}
|
||||
},
|
||||
@ -20035,8 +20017,7 @@
|
||||
"queue-microtask": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
|
||||
"integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
|
||||
"dev": true
|
||||
"integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="
|
||||
},
|
||||
"queue-tick": {
|
||||
"version": "1.0.1",
|
||||
@ -20367,8 +20348,7 @@
|
||||
"reusify": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
|
||||
"integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
|
||||
"dev": true
|
||||
"integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw=="
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "5.0.5",
|
||||
@ -20388,7 +20368,6 @@
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
|
||||
"integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"queue-microtask": "^1.2.2"
|
||||
}
|
||||
|
||||
@ -57,6 +57,7 @@
|
||||
"cookie-parser": "^1.4.6",
|
||||
"exiftool-vendored": "~24.5.0",
|
||||
"exiftool-vendored.pl": "12.76",
|
||||
"fast-glob": "^3.3.2",
|
||||
"fluent-ffmpeg": "^2.1.2",
|
||||
"geo-tz": "^8.0.0",
|
||||
"glob": "^10.3.3",
|
||||
|
||||
@ -156,8 +156,7 @@ describe(LibraryService.name, () => {
|
||||
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
storageMock.crawl.mockResolvedValue(['/data/user1/photo.jpg']);
|
||||
assetMock.getPathsNotInLibrary.mockResolvedValue(['/data/user1/photo.jpg']);
|
||||
assetMock.getByLibraryId.mockResolvedValue([]);
|
||||
assetMock.getLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
|
||||
@ -183,7 +182,7 @@ describe(LibraryService.name, () => {
|
||||
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
storageMock.crawl.mockResolvedValue(['/data/user1/photo.jpg']);
|
||||
assetMock.getByLibraryId.mockResolvedValue([]);
|
||||
assetMock.getLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
|
||||
@ -233,7 +232,7 @@ describe(LibraryService.name, () => {
|
||||
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
|
||||
storageMock.crawl.mockResolvedValue([]);
|
||||
assetMock.getByLibraryId.mockResolvedValue([]);
|
||||
assetMock.getLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
|
||||
@ -242,6 +241,48 @@ describe(LibraryService.name, () => {
|
||||
exclusionPatterns: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should set missing assets offline', async () => {
|
||||
const mockLibraryJob: ILibraryRefreshJob = {
|
||||
id: libraryStub.externalLibrary1.id,
|
||||
refreshModifiedFiles: false,
|
||||
refreshAllFiles: false,
|
||||
};
|
||||
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
storageMock.crawl.mockResolvedValue([]);
|
||||
assetMock.getLibraryAssetPaths.mockResolvedValue({
|
||||
items: [assetStub.image],
|
||||
hasNextPage: false,
|
||||
});
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
|
||||
expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.image.id], { isOffline: true });
|
||||
expect(assetMock.updateAll).not.toHaveBeenCalledWith(expect.anything(), { isOffline: false });
|
||||
expect(jobMock.queueAll).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should set crawled assets that were previously offline back online', async () => {
|
||||
const mockLibraryJob: ILibraryRefreshJob = {
|
||||
id: libraryStub.externalLibrary1.id,
|
||||
refreshModifiedFiles: false,
|
||||
refreshAllFiles: false,
|
||||
};
|
||||
|
||||
libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
|
||||
storageMock.crawl.mockResolvedValue([assetStub.offline.originalPath]);
|
||||
assetMock.getLibraryAssetPaths.mockResolvedValue({
|
||||
items: [assetStub.offline],
|
||||
hasNextPage: false,
|
||||
});
|
||||
|
||||
await sut.handleQueueAssetRefresh(mockLibraryJob);
|
||||
|
||||
expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.offline.id], { isOffline: false });
|
||||
expect(assetMock.updateAll).not.toHaveBeenCalledWith(expect.anything(), { isOffline: true });
|
||||
expect(jobMock.queueAll).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleAssetRefresh', () => {
|
||||
|
||||
@ -640,27 +640,56 @@ export class LibraryService extends EventEmitter {
|
||||
.filter((validation) => validation.isValid)
|
||||
.map((validation) => validation.importPath);
|
||||
|
||||
const rawPaths = await this.storageRepository.crawl({
|
||||
let rawPaths = await this.storageRepository.crawl({
|
||||
pathsToCrawl: validImportPaths,
|
||||
exclusionPatterns: library.exclusionPatterns,
|
||||
});
|
||||
const crawledAssetPaths = rawPaths.map((filePath) => path.normalize(filePath));
|
||||
const crawledAssetPaths = new Set<string>(rawPaths);
|
||||
|
||||
this.logger.debug(`Found ${crawledAssetPaths.length} asset(s) when crawling import paths ${library.importPaths}`);
|
||||
const shouldScanAll = job.refreshAllFiles || job.refreshModifiedFiles;
|
||||
let pathsToScan: string[] = shouldScanAll ? rawPaths : [];
|
||||
rawPaths = [];
|
||||
|
||||
await this.assetRepository.updateOfflineLibraryAssets(library.id, crawledAssetPaths);
|
||||
this.logger.debug(`Found ${crawledAssetPaths.size} asset(s) when crawling import paths ${library.importPaths}`);
|
||||
|
||||
if (crawledAssetPaths.length > 0) {
|
||||
let filteredPaths: string[] = [];
|
||||
if (job.refreshAllFiles || job.refreshModifiedFiles) {
|
||||
filteredPaths = crawledAssetPaths;
|
||||
} else {
|
||||
filteredPaths = await this.assetRepository.getPathsNotInLibrary(library.id, crawledAssetPaths);
|
||||
const assetIdsToMarkOffline = [];
|
||||
const assetIdsToMarkOnline = [];
|
||||
const pagination = usePagination(5000, (pagination) =>
|
||||
this.assetRepository.getLibraryAssetPaths(pagination, library.id),
|
||||
);
|
||||
|
||||
this.logger.debug(`Will import ${filteredPaths.length} new asset(s)`);
|
||||
for await (const page of pagination) {
|
||||
for (const asset of page) {
|
||||
const isOffline = !crawledAssetPaths.has(asset.originalPath);
|
||||
if (isOffline && !asset.isOffline) {
|
||||
assetIdsToMarkOffline.push(asset.id);
|
||||
}
|
||||
|
||||
if (!isOffline && asset.isOffline) {
|
||||
assetIdsToMarkOnline.push(asset.id);
|
||||
}
|
||||
|
||||
crawledAssetPaths.delete(asset.originalPath);
|
||||
}
|
||||
}
|
||||
|
||||
await this.scanAssets(job.id, filteredPaths, library.ownerId, job.refreshAllFiles ?? false);
|
||||
if (assetIdsToMarkOffline.length > 0) {
|
||||
this.logger.debug(`Found ${assetIdsToMarkOffline.length} offline asset(s) previously marked as online`);
|
||||
await this.assetRepository.updateAll(assetIdsToMarkOffline, { isOffline: true });
|
||||
}
|
||||
|
||||
if (assetIdsToMarkOnline.length > 0) {
|
||||
this.logger.debug(`Found ${assetIdsToMarkOnline.length} online asset(s) previously marked as offline`);
|
||||
await this.assetRepository.updateAll(assetIdsToMarkOnline, { isOffline: false });
|
||||
}
|
||||
|
||||
if (!shouldScanAll) {
|
||||
pathsToScan = [...crawledAssetPaths];
|
||||
this.logger.debug(`Will import ${pathsToScan.length} new asset(s)`);
|
||||
}
|
||||
|
||||
if (pathsToScan.length > 0) {
|
||||
await this.scanAssets(job.id, pathsToScan, library.ownerId, job.refreshAllFiles ?? false);
|
||||
}
|
||||
|
||||
await this.repository.update({ id: job.id, refreshedAt: new Date() });
|
||||
|
||||
@ -109,6 +109,8 @@ export interface MetadataSearchOptions {
|
||||
numResults: number;
|
||||
}
|
||||
|
||||
export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>;
|
||||
|
||||
export const IAssetRepository = 'IAssetRepository';
|
||||
|
||||
export interface IAssetRepository {
|
||||
@ -129,10 +131,8 @@ export interface IAssetRepository {
|
||||
getRandom(userId: string, count: number): Promise<AssetEntity[]>;
|
||||
getFirstAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
|
||||
getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
|
||||
getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]>;
|
||||
getLibraryAssetPaths(pagination: PaginationOptions, libraryId: string): Paginated<AssetPathEntity>;
|
||||
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
|
||||
getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise<string[]>;
|
||||
updateOfflineLibraryAssets(libraryId: string, originalPaths: string[]): Promise<void>;
|
||||
deleteAll(ownerId: string): Promise<void>;
|
||||
getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
|
||||
getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
|
||||
|
||||
@ -2,6 +2,7 @@ import {
|
||||
AssetBuilderOptions,
|
||||
AssetCreate,
|
||||
AssetExploreFieldOptions,
|
||||
AssetPathEntity,
|
||||
AssetSearchOptions,
|
||||
AssetStats,
|
||||
AssetStatsOptions,
|
||||
@ -184,10 +185,10 @@ export class AssetRepository implements IAssetRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID]] })
|
||||
@ChunkedArray()
|
||||
getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]> {
|
||||
return this.repository.find({
|
||||
where: { library: { id: In(libraryIds) } },
|
||||
getLibraryAssetPaths(pagination: PaginationOptions, libraryId: string): Paginated<AssetPathEntity> {
|
||||
return paginate(this.repository, pagination, {
|
||||
select: { id: true, originalPath: true, isOffline: true },
|
||||
where: { library: { id: libraryId } },
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ import {
|
||||
import { ImmichLogger } from '@app/infra/logger';
|
||||
import archiver from 'archiver';
|
||||
import chokidar, { WatchOptions } from 'chokidar';
|
||||
import { glob } from 'glob';
|
||||
import { glob } from 'fast-glob';
|
||||
import { constants, createReadStream, existsSync, mkdirSync } from 'node:fs';
|
||||
import fs, { copyFile, readdir, rename, stat, utimes, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
@ -123,7 +123,7 @@ export class FilesystemProvider implements IStorageRepository {
|
||||
|
||||
crawl(crawlOptions: CrawlOptionsDto): Promise<string[]> {
|
||||
const { pathsToCrawl, exclusionPatterns, includeHidden } = crawlOptions;
|
||||
if (!pathsToCrawl) {
|
||||
if (pathsToCrawl.length === 0) {
|
||||
return Promise.resolve([]);
|
||||
}
|
||||
|
||||
@ -132,8 +132,8 @@ export class FilesystemProvider implements IStorageRepository {
|
||||
|
||||
return glob(`${base}/**/${extensions}`, {
|
||||
absolute: true,
|
||||
nocase: true,
|
||||
nodir: true,
|
||||
caseSensitiveMatch: false,
|
||||
onlyFiles: true,
|
||||
dot: includeHidden,
|
||||
ignore: exclusionPatterns,
|
||||
});
|
||||
|
||||
@ -293,53 +293,6 @@ DELETE FROM "assets"
|
||||
WHERE
|
||||
"ownerId" = $1
|
||||
|
||||
-- AssetRepository.getByLibraryId
|
||||
SELECT
|
||||
"AssetEntity"."id" AS "AssetEntity_id",
|
||||
"AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",
|
||||
"AssetEntity"."ownerId" AS "AssetEntity_ownerId",
|
||||
"AssetEntity"."libraryId" AS "AssetEntity_libraryId",
|
||||
"AssetEntity"."deviceId" AS "AssetEntity_deviceId",
|
||||
"AssetEntity"."type" AS "AssetEntity_type",
|
||||
"AssetEntity"."originalPath" AS "AssetEntity_originalPath",
|
||||
"AssetEntity"."resizePath" AS "AssetEntity_resizePath",
|
||||
"AssetEntity"."webpPath" AS "AssetEntity_webpPath",
|
||||
"AssetEntity"."thumbhash" AS "AssetEntity_thumbhash",
|
||||
"AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath",
|
||||
"AssetEntity"."createdAt" AS "AssetEntity_createdAt",
|
||||
"AssetEntity"."updatedAt" AS "AssetEntity_updatedAt",
|
||||
"AssetEntity"."deletedAt" AS "AssetEntity_deletedAt",
|
||||
"AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt",
|
||||
"AssetEntity"."localDateTime" AS "AssetEntity_localDateTime",
|
||||
"AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt",
|
||||
"AssetEntity"."isFavorite" AS "AssetEntity_isFavorite",
|
||||
"AssetEntity"."isArchived" AS "AssetEntity_isArchived",
|
||||
"AssetEntity"."isExternal" AS "AssetEntity_isExternal",
|
||||
"AssetEntity"."isReadOnly" AS "AssetEntity_isReadOnly",
|
||||
"AssetEntity"."isOffline" AS "AssetEntity_isOffline",
|
||||
"AssetEntity"."checksum" AS "AssetEntity_checksum",
|
||||
"AssetEntity"."duration" AS "AssetEntity_duration",
|
||||
"AssetEntity"."isVisible" AS "AssetEntity_isVisible",
|
||||
"AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
|
||||
"AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
|
||||
"AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
|
||||
"AssetEntity"."stackId" AS "AssetEntity_stackId"
|
||||
FROM
|
||||
"assets" "AssetEntity"
|
||||
LEFT JOIN "libraries" "AssetEntity__AssetEntity_library" ON "AssetEntity__AssetEntity_library"."id" = "AssetEntity"."libraryId"
|
||||
AND (
|
||||
"AssetEntity__AssetEntity_library"."deletedAt" IS NULL
|
||||
)
|
||||
WHERE
|
||||
(
|
||||
(
|
||||
(
|
||||
(("AssetEntity__AssetEntity_library"."id" IN ($1)))
|
||||
)
|
||||
)
|
||||
)
|
||||
AND ("AssetEntity"."deletedAt" IS NULL)
|
||||
|
||||
-- AssetRepository.getByLibraryIdAndOriginalPath
|
||||
SELECT DISTINCT
|
||||
"distinctAlias"."AssetEntity_id" AS "ids_AssetEntity_id"
|
||||
|
||||
@ -20,10 +20,8 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
|
||||
getAll: jest.fn().mockResolvedValue({ items: [], hasNextPage: false }),
|
||||
getAllByDeviceId: jest.fn(),
|
||||
updateAll: jest.fn(),
|
||||
getByLibraryId: jest.fn(),
|
||||
getLibraryAssetPaths: jest.fn(),
|
||||
getByLibraryIdAndOriginalPath: jest.fn(),
|
||||
updateOfflineLibraryAssets: jest.fn(),
|
||||
getPathsNotInLibrary: jest.fn(),
|
||||
deleteAll: jest.fn(),
|
||||
save: jest.fn(),
|
||||
remove: jest.fn(),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user