From 07e8f795630ebb542e2a55c1c18b9e3e7dbec6cd Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Fri, 15 Mar 2024 09:16:08 -0400 Subject: [PATCH] refactor: library e2e (#7969) --- e2e/src/api/specs/library.e2e-spec.ts | 147 +++++++++++++++++++++- e2e/src/utils.ts | 95 +++++++++++--- server/e2e/jobs/specs/library.e2e-spec.ts | 123 ------------------ 3 files changed, 223 insertions(+), 142 deletions(-) diff --git a/e2e/src/api/specs/library.e2e-spec.ts b/e2e/src/api/specs/library.e2e-spec.ts index e8f9a46bb2..d9ac1eddbe 100644 --- a/e2e/src/api/specs/library.e2e-spec.ts +++ b/e2e/src/api/specs/library.e2e-spec.ts @@ -1,20 +1,48 @@ -import { LibraryResponseDto, LibraryType, LoginResponseDto, getAllLibraries } from '@immich/sdk'; +import { + LibraryResponseDto, + LibraryType, + LoginResponseDto, + ScanLibraryDto, + getAllLibraries, + scanLibrary, +} from '@immich/sdk'; +import { existsSync, rmdirSync } from 'node:fs'; +import { Socket } from 'socket.io-client'; import { userDto, uuidDto } from 'src/fixtures'; import { errorDto } from 'src/responses'; -import { app, asBearerAuth, testAssetDirInternal, utils } from 'src/utils'; +import { app, asBearerAuth, testAssetDir, testAssetDirInternal, utils } from 'src/utils'; import request from 'supertest'; -import { beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; + +const scan = async (accessToken: string, id: string, dto: ScanLibraryDto = {}) => + scanLibrary({ id, scanLibraryDto: dto }, { headers: asBearerAuth(accessToken) }); describe('/library', () => { let admin: LoginResponseDto; let user: LoginResponseDto; let library: LibraryResponseDto; + let websocket: Socket; beforeAll(async () => { await utils.resetDatabase(); admin = await utils.adminSetup(); user = await utils.userSetup(admin.accessToken, userDto.user1); library = await utils.createLibrary(admin.accessToken, { type: LibraryType.External }); + websocket = await utils.connectWebsocket(admin.accessToken); + }); + + afterAll(() => { + utils.disconnectWebsocket(websocket); + }); + + beforeEach(() => { + utils.resetEvents(); + const tempDir = `${testAssetDir}/temp`; + if (existsSync(tempDir)) { + rmdirSync(tempDir, { recursive: true }); + } + utils.createImageFile(`${testAssetDir}/temp/directoryA/assetA.png`); + utils.createImageFile(`${testAssetDir}/temp/directoryB/assetB.png`); }); describe('GET /library', () => { @@ -376,6 +404,36 @@ describe('/library', () => { ]), ); }); + + it('should delete an external library with assets', async () => { + const library = await utils.createLibrary(admin.accessToken, { + type: LibraryType.External, + importPaths: [`${testAssetDirInternal}/temp`], + }); + + await scan(admin.accessToken, library.id); + await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 2 }); + + const { status, body } = await request(app) + .delete(`/library/${library.id}`) + .set('Authorization', `Bearer ${admin.accessToken}`); + + expect(status).toBe(204); + expect(body).toEqual({}); + + const libraries = await getAllLibraries({}, { headers: asBearerAuth(admin.accessToken) }); + expect(libraries).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: library.id, + }), + ]), + ); + + // ensure no files get deleted + expect(existsSync(`${testAssetDir}/temp/directoryA/assetA.png`)).toBe(true); + expect(existsSync(`${testAssetDir}/temp/directoryB/assetB.png`)).toBe(true); + }); }); describe('GET /library/:id/statistics', () => { @@ -394,6 +452,89 @@ describe('/library', () => { expect(status).toBe(401); expect(body).toEqual(errorDto.unauthorized); }); + + it('should not scan an upload library', async () => { + const library = await utils.createLibrary(admin.accessToken, { + type: LibraryType.Upload, + }); + + const { status, body } = await request(app) + .post(`/library/${library.id}/scan`) + .set('Authorization', `Bearer ${admin.accessToken}`); + + expect(status).toBe(400); + expect(body).toEqual(errorDto.badRequest('Can only refresh external libraries')); + }); + + it('should scan external library', async () => { + const library = await utils.createLibrary(admin.accessToken, { + type: LibraryType.External, + importPaths: [`${testAssetDirInternal}/temp/directoryA`], + }); + + await scan(admin.accessToken, library.id); + await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 1 }); + + const { assets } = await utils.metadataSearch(admin.accessToken, { + originalPath: `${testAssetDirInternal}/temp/directoryA/assetA.png`, + }); + expect(assets.count).toBe(1); + }); + + it('should scan external library with exclusion pattern', async () => { + const library = await utils.createLibrary(admin.accessToken, { + type: LibraryType.External, + importPaths: [`${testAssetDirInternal}/temp`], + exclusionPatterns: ['**/directoryA'], + }); + + await scan(admin.accessToken, library.id); + await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 1 }); + + const { assets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id }); + + expect(assets.count).toBe(1); + expect(assets.items[0].originalPath.includes('directoryB')); + }); + + it('should scan multiple import paths', async () => { + const library = await utils.createLibrary(admin.accessToken, { + type: LibraryType.External, + importPaths: [`${testAssetDirInternal}/temp/directoryA`, `${testAssetDirInternal}/temp/directoryB`], + }); + + await scan(admin.accessToken, library.id); + await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 2 }); + + const { assets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id }); + + expect(assets.count).toBe(2); + expect(assets.items.find((asset) => asset.originalPath.includes('directoryA'))).toBeDefined(); + expect(assets.items.find((asset) => asset.originalPath.includes('directoryB'))).toBeDefined(); + }); + + it('should pick up new files', async () => { + const library = await utils.createLibrary(admin.accessToken, { + type: LibraryType.External, + importPaths: [`${testAssetDirInternal}/temp`], + }); + + await scan(admin.accessToken, library.id); + await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 2 }); + + const { assets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id }); + + expect(assets.count).toBe(2); + + utils.createImageFile(`${testAssetDir}/temp/directoryA/assetB.png`); + + await scan(admin.accessToken, library.id); + await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 3 }); + + const { assets: newAssets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id }); + + expect(newAssets.count).toBe(3); + }); }); describe('POST /library/:id/removeOffline', () => { diff --git a/e2e/src/utils.ts b/e2e/src/utils.ts index dde9ed22ce..8ca7fba606 100644 --- a/e2e/src/utils.ts +++ b/e2e/src/utils.ts @@ -5,6 +5,7 @@ import { CreateAssetDto, CreateLibraryDto, CreateUserDto, + MetadataSearchDto, PersonCreateDto, SharedLinkCreateDto, ValidateLibraryDto, @@ -16,8 +17,10 @@ import { createUser, defaults, deleteAssets, + getAllAssets, getAssetInfo, login, + searchMetadata, setAdminOnboarding, signUpAdmin, validate, @@ -25,9 +28,9 @@ import { import { BrowserContext } from '@playwright/test'; import { exec, spawn } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { existsSync } from 'node:fs'; +import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import path from 'node:path'; +import path, { dirname } from 'node:path'; import { promisify } from 'node:util'; import pg from 'pg'; import { io, type Socket } from 'socket.io-client'; @@ -37,7 +40,7 @@ import request from 'supertest'; type CliResponse = { stdout: string; stderr: string; exitCode: number | null }; type EventType = 'assetUpload' | 'assetDelete' | 'userDelete'; -type WaitOptions = { event: EventType; id: string; timeout?: number }; +type WaitOptions = { event: EventType; id?: string; total?: number; timeout?: number }; type AdminSetupOptions = { onboarding?: boolean }; type AssetData = { bytes?: Buffer; filename: string }; @@ -83,16 +86,30 @@ const events: Record> = { userDelete: new Set(), }; -const callbacks: Record void> = {}; +const idCallbacks: Record void> = {}; +const countCallbacks: Record void }> = {}; const execPromise = promisify(exec); const onEvent = ({ event, id }: { event: EventType; id: string }) => { - events[event].add(id); - const callback = callbacks[id]; - if (callback) { - callback(); - delete callbacks[id]; + // console.log(`Received event: ${event} [id=${id}]`); + const set = events[event]; + set.add(id); + + const idCallback = idCallbacks[id]; + if (idCallback) { + idCallback(); + delete idCallbacks[id]; + } + + const item = countCallbacks[event]; + if (item) { + const { count, callback: countCallback } = item; + + if (set.size >= count) { + countCallback(); + delete countCallbacks[event]; + } } }; @@ -184,20 +201,43 @@ export const utils = { } }, - waitForWebsocketEvent: async ({ event, id, timeout: ms }: WaitOptions): Promise => { - console.log(`Waiting for ${event} [${id}]`); + resetEvents: () => { + for (const set of Object.values(events)) { + set.clear(); + } + }, + + waitForWebsocketEvent: async ({ event, id, total: count, timeout: ms }: WaitOptions): Promise => { + if (!id && !count) { + throw new Error('id or count must be provided for waitForWebsocketEvent'); + } + + const type = id ? `id=${id}` : `count=${count}`; + console.log(`Waiting for ${event} [${type}]`); const set = events[event]; - if (set.has(id)) { + if ((id && set.has(id)) || (count && set.size >= count)) { return; } return new Promise((resolve, reject) => { const timeout = setTimeout(() => reject(new Error(`Timed out waiting for ${event} event`)), ms || 10_000); - callbacks[id] = () => { - clearTimeout(timeout); - resolve(); - }; + if (id) { + idCallbacks[id] = () => { + clearTimeout(timeout); + resolve(); + }; + } + + if (count) { + countCallbacks[event] = { + count, + callback: () => { + clearTimeout(timeout); + resolve(); + }, + }; + } }); }, @@ -263,8 +303,31 @@ export const utils = { return body as AssetFileUploadResponseDto; }, + createImageFile: (path: string) => { + if (!existsSync(dirname(path))) { + mkdirSync(dirname(path), { recursive: true }); + } + if (!existsSync(path)) { + writeFileSync(path, makeRandomImage()); + } + }, + + removeImageFile: (path: string) => { + if (!existsSync(path)) { + return; + } + + rmSync(path); + }, + getAssetInfo: (accessToken: string, id: string) => getAssetInfo({ id }, { headers: asBearerAuth(accessToken) }), + getAllAssets: (accessToken: string) => getAllAssets({}, { headers: asBearerAuth(accessToken) }), + + metadataSearch: async (accessToken: string, dto: MetadataSearchDto) => { + return searchMetadata({ metadataSearchDto: dto }, { headers: asBearerAuth(accessToken) }); + }, + deleteAssets: (accessToken: string, ids: string[]) => deleteAssets({ assetBulkDeleteDto: { ids } }, { headers: asBearerAuth(accessToken) }), diff --git a/server/e2e/jobs/specs/library.e2e-spec.ts b/server/e2e/jobs/specs/library.e2e-spec.ts index 6dca783c42..4ebb00c4df 100644 --- a/server/e2e/jobs/specs/library.e2e-spec.ts +++ b/server/e2e/jobs/specs/library.e2e-spec.ts @@ -34,117 +34,7 @@ describe(`${LibraryController.name} (e2e)`, () => { await restoreTempFolder(); }); - describe('DELETE /library/:id', () => { - it('should delete an external library with assets', async () => { - const library = await api.libraryApi.create(server, admin.accessToken, { - type: LibraryType.EXTERNAL, - importPaths: [`${IMMICH_TEST_ASSET_PATH}/albums/nature`], - }); - await api.libraryApi.scanLibrary(server, admin.accessToken, library.id); - - const assets = await api.assetApi.getAllAssets(server, admin.accessToken); - expect(assets.length).toBeGreaterThan(2); - - const { status } = await request(server) - .delete(`/library/${library.id}`) - .set('Authorization', `Bearer ${admin.accessToken}`); - - expect(status).toBe(204); - - const libraries = await api.libraryApi.getAll(server, admin.accessToken); - expect(libraries).toHaveLength(1); - expect(libraries).not.toEqual( - expect.arrayContaining([ - expect.objectContaining({ - id: library.id, - }), - ]), - ); - }); - }); - describe('POST /library/:id/scan', () => { - it('should require authentication', async () => { - const { status, body } = await request(server).post(`/library/${uuidStub.notFound}/scan`).send({}); - - expect(status).toBe(401); - expect(body).toEqual(errorStub.unauthorized); - }); - - it('should scan external library with import paths', async () => { - const library = await api.libraryApi.create(server, admin.accessToken, { - type: LibraryType.EXTERNAL, - importPaths: [`${IMMICH_TEST_ASSET_PATH}/albums/nature`], - }); - await api.libraryApi.scanLibrary(server, admin.accessToken, library.id); - - const assets = await api.assetApi.getAllAssets(server, admin.accessToken); - - expect(assets).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - type: AssetType.IMAGE, - originalFileName: 'el_torcal_rocks', - libraryId: library.id, - resized: true, - thumbhash: expect.any(String), - exifInfo: expect.objectContaining({ - exifImageWidth: 512, - exifImageHeight: 341, - latitude: null, - longitude: null, - }), - }), - expect.objectContaining({ - type: AssetType.IMAGE, - originalFileName: 'silver_fir', - libraryId: library.id, - resized: true, - thumbhash: expect.any(String), - exifInfo: expect.objectContaining({ - exifImageWidth: 511, - exifImageHeight: 323, - latitude: null, - longitude: null, - }), - }), - ]), - ); - }); - - it('should scan external library with exclusion pattern', async () => { - const library = await api.libraryApi.create(server, admin.accessToken, { - type: LibraryType.EXTERNAL, - importPaths: [`${IMMICH_TEST_ASSET_PATH}/albums/nature`], - exclusionPatterns: ['**/el_corcal*'], - }); - - await api.libraryApi.scanLibrary(server, admin.accessToken, library.id); - - const assets = await api.assetApi.getAllAssets(server, admin.accessToken); - - expect(assets).toEqual( - expect.arrayContaining([ - expect.not.objectContaining({ - // Excluded by exclusion pattern - originalFileName: 'el_torcal_rocks', - }), - expect.objectContaining({ - type: AssetType.IMAGE, - originalFileName: 'silver_fir', - libraryId: library.id, - resized: true, - exifInfo: expect.objectContaining({ - exifImageWidth: 511, - exifImageHeight: 323, - latitude: null, - longitude: null, - }), - }), - ]), - ); - }); - it('should offline missing files', async () => { await fs.promises.cp(`${IMMICH_TEST_ASSET_PATH}/albums/nature`, `${IMMICH_TEST_ASSET_TEMP_PATH}/albums/nature`, { recursive: true, @@ -345,19 +235,6 @@ describe(`${LibraryController.name} (e2e)`, () => { ); }); }); - - it('should not scan an upload library', async () => { - const library = await api.libraryApi.create(server, admin.accessToken, { - type: LibraryType.UPLOAD, - }); - - const { status, body } = await request(server) - .post(`/library/${library.id}/scan`) - .set('Authorization', `Bearer ${admin.accessToken}`); - - expect(status).toBe(400); - expect(body).toEqual(errorStub.badRequest('Can only refresh external libraries')); - }); }); describe('POST /library/:id/removeOffline', () => {