diff --git a/docs/docs/features/libraries.md b/docs/docs/features/libraries.md index 6dcb9982bf..995d652492 100644 --- a/docs/docs/features/libraries.md +++ b/docs/docs/features/libraries.md @@ -42,23 +42,24 @@ Finally, files can be deleted from Immich via the `Remove Offline Files` job. Th ### Import Paths -External libraries use import paths to determine which files to scan. Each library can have multiple import paths so that files from different locations can be added to the same library. Import paths are scanned recursively, and if a file is in multiple import paths, it will only be added once. If the import paths are edited in a way that an external file is no longer in any import path, it will be removed from the library in the same way a deleted file would. If the file is moved back to an import path, it will be added again as if it was a new file. +External libraries use import paths to determine which files to scan. Each library can have multiple import paths so that files from different locations can be added to the same library. Import paths are scanned recursively, and if a file is in multiple import paths, it will only be added once. Each import file must be a readable directory that exists on the filesystem; the import path dialog will alert you of any paths that are not accessible. + +If the import paths are edited in a way that an external file is no longer in any import path, it will be removed from the library in the same way a deleted file would. If the file is moved back to an import path, it will be added again as if it was a new file. ### Troubleshooting -Sometimes, an external library will not scan correctly. This can happen if the immich_server or immich_microservices can't access the files. Here are some things to check: +Sometimes, an external library will not scan correctly. This can happen if immich_server or immich_microservices can't access the files. Here are some things to check: -- Is the external path set correctly? +- Is the external path set correctly? Each import path must be contained in the external path. +- Make sure the external path does not contain spaces - In the docker-compose file, are the volumes mounted correctly? - Are the volumes identical between the `server` and `microservices` container? - Are the import paths set correctly, and do they match the path set in docker-compose file? -- Are you using symbolic link in your import library? +- Make sure you don't use symlinks in your import libraries, and that you aren't linking across docker mounts. - Are the permissions set correctly? -- Are you using forward slashes everywhere? (`/`) -- Are you using symlink across docker mounts? -- Are you using [spaces in the internal path](/docs/features/libraries#:~:text=can%20be%20accessed.-,NOTE,-Spaces%20in%20the)? +- Make sure you are using forward slashes (`/`) and not backward slashes. -If all else fails, you can always start a shell inside the container and check if the path is accessible. For example, `docker exec -it immich_microservices /bin/bash` will start a bash shell. If your import path, for instance, is `/data/import/photos`, you can check if the files are accessible by running `ls /data/import/photos`. Also check the `immich_server` container in the same way. +To validate that Immich can reach your external library, start a shell inside the container. Run `docker exec -it immich_microservices /bin/bash` to a bash shell. If your import path is `/data/import/photos`, check it with `ls /data/import/photos`. Do the same check for the `immich_server` container. If you cannot access this directory in both the `microservices` and `server` containers, Immich won't be able to import files. ### Security Considerations diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index f42a9c82ca..0679a1749d 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -182,6 +182,9 @@ doc/UserAvatarColor.md doc/UserDto.md doc/UserResponseDto.md doc/ValidateAccessTokenResponseDto.md +doc/ValidateLibraryDto.md +doc/ValidateLibraryImportPathResponseDto.md +doc/ValidateLibraryResponseDto.md doc/VideoCodec.md git_push.sh lib/api.dart @@ -372,6 +375,9 @@ lib/model/user_avatar_color.dart lib/model/user_dto.dart lib/model/user_response_dto.dart lib/model/validate_access_token_response_dto.dart +lib/model/validate_library_dto.dart +lib/model/validate_library_import_path_response_dto.dart +lib/model/validate_library_response_dto.dart lib/model/video_codec.dart pubspec.yaml test/activity_api_test.dart @@ -553,4 +559,7 @@ test/user_avatar_color_test.dart test/user_dto_test.dart test/user_response_dto_test.dart test/validate_access_token_response_dto_test.dart +test/validate_library_dto_test.dart +test/validate_library_import_path_response_dto_test.dart +test/validate_library_response_dto_test.dart test/video_codec_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 9e241b0a44..8cb0919df1 100644 Binary files a/mobile/openapi/README.md and b/mobile/openapi/README.md differ diff --git a/mobile/openapi/doc/LibraryApi.md b/mobile/openapi/doc/LibraryApi.md index 5ecd46503a..ecaa73d86b 100644 Binary files a/mobile/openapi/doc/LibraryApi.md and b/mobile/openapi/doc/LibraryApi.md differ diff --git a/mobile/openapi/doc/ValidateLibraryDto.md b/mobile/openapi/doc/ValidateLibraryDto.md new file mode 100644 index 0000000000..13c74a6a66 Binary files /dev/null and b/mobile/openapi/doc/ValidateLibraryDto.md differ diff --git a/mobile/openapi/doc/ValidateLibraryImportPathResponseDto.md b/mobile/openapi/doc/ValidateLibraryImportPathResponseDto.md new file mode 100644 index 0000000000..4601d8d2f2 Binary files /dev/null and b/mobile/openapi/doc/ValidateLibraryImportPathResponseDto.md differ diff --git a/mobile/openapi/doc/ValidateLibraryResponseDto.md b/mobile/openapi/doc/ValidateLibraryResponseDto.md new file mode 100644 index 0000000000..7b22fdfe90 Binary files /dev/null and b/mobile/openapi/doc/ValidateLibraryResponseDto.md differ diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 3816a05b3b..72a6567648 100644 Binary files a/mobile/openapi/lib/api.dart and b/mobile/openapi/lib/api.dart differ diff --git a/mobile/openapi/lib/api/library_api.dart b/mobile/openapi/lib/api/library_api.dart index 23de593eac..21cec23eba 100644 Binary files a/mobile/openapi/lib/api/library_api.dart and b/mobile/openapi/lib/api/library_api.dart differ diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 3a0e535e73..2df5e67119 100644 Binary files a/mobile/openapi/lib/api_client.dart and b/mobile/openapi/lib/api_client.dart differ diff --git a/mobile/openapi/lib/model/validate_library_dto.dart b/mobile/openapi/lib/model/validate_library_dto.dart new file mode 100644 index 0000000000..a29a622079 Binary files /dev/null and b/mobile/openapi/lib/model/validate_library_dto.dart differ diff --git a/mobile/openapi/lib/model/validate_library_import_path_response_dto.dart b/mobile/openapi/lib/model/validate_library_import_path_response_dto.dart new file mode 100644 index 0000000000..1297c824c2 Binary files /dev/null and b/mobile/openapi/lib/model/validate_library_import_path_response_dto.dart differ diff --git a/mobile/openapi/lib/model/validate_library_response_dto.dart b/mobile/openapi/lib/model/validate_library_response_dto.dart new file mode 100644 index 0000000000..bb975da23a Binary files /dev/null and b/mobile/openapi/lib/model/validate_library_response_dto.dart differ diff --git a/mobile/openapi/test/library_api_test.dart b/mobile/openapi/test/library_api_test.dart index 85882edbd1..20abcffe70 100644 Binary files a/mobile/openapi/test/library_api_test.dart and b/mobile/openapi/test/library_api_test.dart differ diff --git a/mobile/openapi/test/validate_library_dto_test.dart b/mobile/openapi/test/validate_library_dto_test.dart new file mode 100644 index 0000000000..8d4922ee4a Binary files /dev/null and b/mobile/openapi/test/validate_library_dto_test.dart differ diff --git a/mobile/openapi/test/validate_library_import_path_response_dto_test.dart b/mobile/openapi/test/validate_library_import_path_response_dto_test.dart new file mode 100644 index 0000000000..a118698a13 Binary files /dev/null and b/mobile/openapi/test/validate_library_import_path_response_dto_test.dart differ diff --git a/mobile/openapi/test/validate_library_response_dto_test.dart b/mobile/openapi/test/validate_library_response_dto_test.dart new file mode 100644 index 0000000000..6750809066 Binary files /dev/null and b/mobile/openapi/test/validate_library_response_dto_test.dart differ diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 5dd394108e..e15073e139 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -3618,6 +3618,58 @@ ] } }, + "/library/{id}/validate": { + "post": { + "operationId": "validate", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateLibraryDto" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateLibraryResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Library" + ] + } + }, "/oauth/authorize": { "post": { "operationId": "startOAuth", @@ -10406,6 +10458,52 @@ ], "type": "object" }, + "ValidateLibraryDto": { + "properties": { + "exclusionPatterns": { + "items": { + "type": "string" + }, + "type": "array" + }, + "importPaths": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "ValidateLibraryImportPathResponseDto": { + "properties": { + "importPath": { + "type": "string" + }, + "isValid": { + "default": false, + "type": "boolean" + }, + "message": { + "type": "string" + } + }, + "required": [ + "importPath" + ], + "type": "object" + }, + "ValidateLibraryResponseDto": { + "properties": { + "importPaths": { + "items": { + "$ref": "#/components/schemas/ValidateLibraryImportPathResponseDto" + }, + "type": "array" + } + }, + "type": "object" + }, "VideoCodec": { "enum": [ "h264", diff --git a/open-api/typescript-sdk/axios-client/api.ts b/open-api/typescript-sdk/axios-client/api.ts index 5d37706904..49cf286f3b 100644 --- a/open-api/typescript-sdk/axios-client/api.ts +++ b/open-api/typescript-sdk/axios-client/api.ts @@ -5316,6 +5316,63 @@ export interface ValidateAccessTokenResponseDto { */ 'authStatus': boolean; } +/** + * + * @export + * @interface ValidateLibraryDto + */ +export interface ValidateLibraryDto { + /** + * + * @type {Array} + * @memberof ValidateLibraryDto + */ + 'exclusionPatterns'?: Array; + /** + * + * @type {Array} + * @memberof ValidateLibraryDto + */ + 'importPaths'?: Array; +} +/** + * + * @export + * @interface ValidateLibraryImportPathResponseDto + */ +export interface ValidateLibraryImportPathResponseDto { + /** + * + * @type {string} + * @memberof ValidateLibraryImportPathResponseDto + */ + 'importPath': string; + /** + * + * @type {boolean} + * @memberof ValidateLibraryImportPathResponseDto + */ + 'isValid'?: boolean; + /** + * + * @type {string} + * @memberof ValidateLibraryImportPathResponseDto + */ + 'message'?: string; +} +/** + * + * @export + * @interface ValidateLibraryResponseDto + */ +export interface ValidateLibraryResponseDto { + /** + * + * @type {Array} + * @memberof ValidateLibraryResponseDto + */ + 'importPaths'?: Array; +} /** * * @export @@ -12813,6 +12870,54 @@ export const LibraryApiAxiosParamCreator = function (configuration?: Configurati localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; localVarRequestOptions.data = serializeDataIfNeeded(updateLibraryDto, localVarRequestOptions, configuration) + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * + * @param {string} id + * @param {ValidateLibraryDto} validateLibraryDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + validate: async (id: string, validateLibraryDto: ValidateLibraryDto, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'id' is not null or undefined + assertParamExists('validate', 'id', id) + // verify required parameter 'validateLibraryDto' is not null or undefined + assertParamExists('validate', 'validateLibraryDto', validateLibraryDto) + const localVarPath = `/library/{id}/validate` + .replace(`{${"id"}}`, encodeURIComponent(String(id))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + localVarHeaderParameter['Content-Type'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(validateLibraryDto, localVarRequestOptions, configuration) + return { url: toPathString(localVarUrlObj), options: localVarRequestOptions, @@ -12925,6 +13030,19 @@ export const LibraryApiFp = function(configuration?: Configuration) { const operationBasePath = operationServerMap['LibraryApi.updateLibrary']?.[index]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, operationBasePath || basePath); }, + /** + * + * @param {string} id + * @param {ValidateLibraryDto} validateLibraryDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async validate(id: string, validateLibraryDto: ValidateLibraryDto, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.validate(id, validateLibraryDto, options); + const index = configuration?.serverIndex ?? 0; + const operationBasePath = operationServerMap['LibraryApi.validate']?.[index]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, operationBasePath || basePath); + }, } }; @@ -13006,6 +13124,15 @@ export const LibraryApiFactory = function (configuration?: Configuration, basePa updateLibrary(requestParameters: LibraryApiUpdateLibraryRequest, options?: RawAxiosRequestConfig): AxiosPromise { return localVarFp.updateLibrary(requestParameters.id, requestParameters.updateLibraryDto, options).then((request) => request(axios, basePath)); }, + /** + * + * @param {LibraryApiValidateRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + validate(requestParameters: LibraryApiValidateRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.validate(requestParameters.id, requestParameters.validateLibraryDto, options).then((request) => request(axios, basePath)); + }, }; }; @@ -13121,6 +13248,27 @@ export interface LibraryApiUpdateLibraryRequest { readonly updateLibraryDto: UpdateLibraryDto } +/** + * Request parameters for validate operation in LibraryApi. + * @export + * @interface LibraryApiValidateRequest + */ +export interface LibraryApiValidateRequest { + /** + * + * @type {string} + * @memberof LibraryApiValidate + */ + readonly id: string + + /** + * + * @type {ValidateLibraryDto} + * @memberof LibraryApiValidate + */ + readonly validateLibraryDto: ValidateLibraryDto +} + /** * LibraryApi - object-oriented interface * @export @@ -13214,6 +13362,17 @@ export class LibraryApi extends BaseAPI { public updateLibrary(requestParameters: LibraryApiUpdateLibraryRequest, options?: RawAxiosRequestConfig) { return LibraryApiFp(this.configuration).updateLibrary(requestParameters.id, requestParameters.updateLibraryDto, options).then((request) => request(this.axios, this.basePath)); } + + /** + * + * @param {LibraryApiValidateRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof LibraryApi + */ + public validate(requestParameters: LibraryApiValidateRequest, options?: RawAxiosRequestConfig) { + return LibraryApiFp(this.configuration).validate(requestParameters.id, requestParameters.validateLibraryDto, options).then((request) => request(this.axios, this.basePath)); + } } diff --git a/open-api/typescript-sdk/fetch-client.ts b/open-api/typescript-sdk/fetch-client.ts index 0f42ff9178..40600e56fa 100644 Binary files a/open-api/typescript-sdk/fetch-client.ts and b/open-api/typescript-sdk/fetch-client.ts differ diff --git a/server/e2e/api/setup.ts b/server/e2e/api/setup.ts index 8d44b07cbb..88f2f598bd 100644 --- a/server/e2e/api/setup.ts +++ b/server/e2e/api/setup.ts @@ -1,6 +1,16 @@ import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import path from 'node:path'; export default async () => { + let IMMICH_TEST_ASSET_PATH: string = ''; + + if (process.env.IMMICH_TEST_ASSET_PATH === undefined) { + IMMICH_TEST_ASSET_PATH = path.normalize(`${__dirname}/../../test/assets/`); + process.env.IMMICH_TEST_ASSET_PATH = IMMICH_TEST_ASSET_PATH; + } else { + IMMICH_TEST_ASSET_PATH = process.env.IMMICH_TEST_ASSET_PATH; + } + const pg = await new PostgreSqlContainer('tensorchord/pgvecto-rs:pg14-v0.2.0') .withDatabase('immich') .withUsername('postgres') @@ -11,6 +21,9 @@ export default async () => { process.env.DB_URL = pg.getConnectionUri(); process.env.NODE_ENV = 'development'; - process.env.LOG_LEVEL = 'fatal'; process.env.TZ = 'Z'; + + if (process.env.LOG_LEVEL === undefined) { + process.env.LOG_LEVEL = 'fatal'; + } }; diff --git a/server/e2e/api/specs/library.e2e-spec.ts b/server/e2e/api/specs/library.e2e-spec.ts index 75c973466e..5be9e3035b 100644 --- a/server/e2e/api/specs/library.e2e-spec.ts +++ b/server/e2e/api/specs/library.e2e-spec.ts @@ -2,6 +2,7 @@ import { LibraryResponseDto, LoginResponseDto } from '@app/domain'; import { LibraryController } from '@app/immich'; import { LibraryType } from '@app/infra/entities'; import { errorStub, userDto, uuidStub } from '@test/fixtures'; +import { IMMICH_TEST_ASSET_TEMP_PATH, restoreTempFolder } from 'src/test-utils/utils'; import request from 'supertest'; import { api } from '../../client'; import { testApp } from '../utils'; @@ -20,6 +21,7 @@ describe(`${LibraryController.name} (e2e)`, () => { }); beforeEach(async () => { + await restoreTempFolder(); await testApp.reset(); await api.authApi.adminSignUp(server); admin = await api.authApi.adminLogin(server); @@ -247,15 +249,16 @@ describe(`${LibraryController.name} (e2e)`, () => { }); it('should change the import paths', async () => { + await api.userApi.setExternalPath(server, admin.accessToken, admin.userId, IMMICH_TEST_ASSET_TEMP_PATH); const { status, body } = await request(server) .put(`/library/${library.id}`) .set('Authorization', `Bearer ${admin.accessToken}`) - .send({ importPaths: ['/path/to/import'] }); + .send({ importPaths: [IMMICH_TEST_ASSET_TEMP_PATH] }); expect(status).toBe(200); expect(body).toEqual( expect.objectContaining({ - importPaths: ['/path/to/import'], + importPaths: [IMMICH_TEST_ASSET_TEMP_PATH], }), ); }); @@ -435,4 +438,93 @@ describe(`${LibraryController.name} (e2e)`, () => { expect(body).toEqual(errorStub.unauthorized); }); }); + + describe('POST /library/:id/validate', () => { + it('should require authentication', async () => { + const { status, body } = await request(server).post(`/library/${uuidStub.notFound}/validate`).send({}); + + expect(status).toBe(401); + expect(body).toEqual(errorStub.unauthorized); + }); + + describe('Validate import path', () => { + let library: LibraryResponseDto; + + beforeEach(async () => { + // Create an external library with default settings + library = await api.libraryApi.create(server, admin.accessToken, { type: LibraryType.EXTERNAL }); + }); + + it('should fail with no external path set', async () => { + const { status, body } = await request(server) + .post(`/library/${library.id}/validate`) + .set('Authorization', `Bearer ${admin.accessToken}`) + + .send({ importPaths: [] }); + + expect(status).toBe(400); + expect(body).toEqual(errorStub.badRequest('User has no external path set')); + }); + + describe('With external path set', () => { + beforeEach(async () => { + await api.userApi.setExternalPath(server, admin.accessToken, admin.userId, IMMICH_TEST_ASSET_TEMP_PATH); + }); + + it('should pass with no import paths', async () => { + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { importPaths: [] }); + expect(response.importPaths).toEqual([]); + }); + + it('should not allow paths outside of the external path', async () => { + const pathToTest = `${IMMICH_TEST_ASSET_TEMP_PATH}/../`; + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { + importPaths: [pathToTest], + }); + expect(response.importPaths?.length).toEqual(1); + const pathResponse = response?.importPaths?.at(0); + + expect(pathResponse).toEqual({ + importPath: pathToTest, + isValid: false, + message: `Not contained in user's external path`, + }); + }); + + it('should fail if path does not exist', async () => { + const pathToTest = `${IMMICH_TEST_ASSET_TEMP_PATH}/does/not/exist`; + + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { + importPaths: [pathToTest], + }); + + expect(response.importPaths?.length).toEqual(1); + const pathResponse = response?.importPaths?.at(0); + + expect(pathResponse).toEqual({ + importPath: pathToTest, + isValid: false, + message: `Path does not exist (ENOENT)`, + }); + }); + + it('should fail if path is a file', async () => { + const pathToTest = `${IMMICH_TEST_ASSET_TEMP_PATH}/does/not/exist`; + + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { + importPaths: [pathToTest], + }); + + expect(response.importPaths?.length).toEqual(1); + const pathResponse = response?.importPaths?.at(0); + + expect(pathResponse).toEqual({ + importPath: pathToTest, + isValid: false, + message: `Path does not exist (ENOENT)`, + }); + }); + }); + }); + }); }); diff --git a/server/e2e/client/library-api.ts b/server/e2e/client/library-api.ts index c40f005457..9f2d0b77ef 100644 --- a/server/e2e/client/library-api.ts +++ b/server/e2e/client/library-api.ts @@ -4,6 +4,8 @@ import { LibraryStatsResponseDto, ScanLibraryDto, UpdateLibraryDto, + ValidateLibraryDto, + ValidateLibraryResponseDto, } from '@app/domain'; import request from 'supertest'; @@ -58,4 +60,12 @@ export const libraryApi = { expect(status).toBe(200); return body as LibraryResponseDto; }, + validate: async (server: any, accessToken: string, id: string, data: ValidateLibraryDto) => { + const { body, status } = await request(server) + .post(`/library/${id}/validate`) + .set('Authorization', `Bearer ${accessToken}`) + .send(data); + expect(status).toBe(200); + return body as ValidateLibraryResponseDto; + }, }; diff --git a/server/src/domain/library/library.dto.ts b/server/src/domain/library/library.dto.ts index 638841ec63..db6119d4de 100644 --- a/server/src/domain/library/library.dto.ts +++ b/server/src/domain/library/library.dto.ts @@ -1,6 +1,6 @@ import { LibraryEntity, LibraryType } from '@app/infra/entities'; import { ApiProperty } from '@nestjs/swagger'; -import { ArrayUnique, IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString } from 'class-validator'; +import { ArrayMaxSize, ArrayUnique, IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString } from 'class-validator'; import { ValidateUUID } from '../domain.util'; export class CreateLibraryDto { @@ -21,12 +21,14 @@ export class CreateLibraryDto { @IsString({ each: true }) @IsNotEmpty({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) importPaths?: string[]; @IsOptional() @IsString({ each: true }) @IsNotEmpty({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) exclusionPatterns?: string[]; @IsOptional() @@ -48,12 +50,14 @@ export class UpdateLibraryDto { @IsString({ each: true }) @IsNotEmpty({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) importPaths?: string[]; @IsOptional() @IsNotEmpty({ each: true }) @IsString({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) exclusionPatterns?: string[]; } @@ -63,6 +67,32 @@ export class CrawlOptionsDto { exclusionPatterns?: string[]; } +export class ValidateLibraryDto { + @IsOptional() + @IsString({ each: true }) + @IsNotEmpty({ each: true }) + @ArrayUnique() + @ArrayMaxSize(128) + importPaths?: string[]; + + @IsOptional() + @IsNotEmpty({ each: true }) + @IsString({ each: true }) + @ArrayUnique() + @ArrayMaxSize(128) + exclusionPatterns?: string[]; +} + +export class ValidateLibraryResponseDto { + importPaths?: ValidateLibraryImportPathResponseDto[]; +} + +export class ValidateLibraryImportPathResponseDto { + importPath!: string; + isValid?: boolean = false; + message?: string; +} + export class LibrarySearchDto { @ValidateUUID({ optional: true }) userId?: string; diff --git a/server/src/domain/library/library.service.spec.ts b/server/src/domain/library/library.service.spec.ts index c86aa4a4e7..cafe70b4d7 100644 --- a/server/src/domain/library/library.service.spec.ts +++ b/server/src/domain/library/library.service.spec.ts @@ -54,12 +54,6 @@ describe(LibraryService.name, () => { cryptoMock = newCryptoRepositoryMock(); storageMock = newStorageRepositoryMock(); - storageMock.stat.mockResolvedValue({ - size: 100, - mtime: new Date('2023-01-01'), - ctime: new Date('2023-01-01'), - } as Stats); - // Always validate owner access for library. accessMock.library.checkOwnerAccess.mockImplementation(async (_, libraryIds) => libraryIds); @@ -270,6 +264,39 @@ describe(LibraryService.name, () => { await expect(sut.handleQueueAssetRefresh(mockLibraryJob)).resolves.toBe(false); }); + + it('should ignore import paths that do not exist', async () => { + storageMock.stat.mockImplementation((path): Promise => { + if (path === libraryStub.externalLibraryWithImportPaths1.importPaths[0]) { + const error = { code: 'ENOENT' } as any; + throw error; + } + return Promise.resolve({ + isDirectory: () => true, + } as Stats); + }); + + storageMock.checkFileExists.mockResolvedValue(true); + + const mockLibraryJob: ILibraryRefreshJob = { + id: libraryStub.externalLibraryWithImportPaths1.id, + refreshModifiedFiles: false, + refreshAllFiles: false, + }; + + libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); + storageMock.crawl.mockResolvedValue([]); + assetMock.getByLibraryId.mockResolvedValue([]); + libraryMock.getOnlineAssetPaths.mockResolvedValue([]); + userMock.get.mockResolvedValue(userStub.externalPathRoot); + + await sut.handleQueueAssetRefresh(mockLibraryJob); + + expect(storageMock.crawl).toHaveBeenCalledWith({ + pathsToCrawl: [libraryStub.externalLibraryWithImportPaths1.importPaths[1]], + exclusionPatterns: [], + }); + }); }); describe('handleAssetRefresh', () => { @@ -278,6 +305,12 @@ describe(LibraryService.name, () => { beforeEach(() => { mockUser = userStub.externalPath1; userMock.get.mockResolvedValue(mockUser); + + storageMock.stat.mockResolvedValue({ + size: 100, + mtime: new Date('2023-01-01'), + ctime: new Date('2023-01-01'), + } as Stats); }); it('should reject an unknown file extension', async () => { @@ -1104,13 +1137,19 @@ describe(LibraryService.name, () => { libraryMock.update.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); - await expect(sut.update(authStub.admin, authStub.admin.user.id, { importPaths: ['/foo'] })).resolves.toEqual( - mapLibrary(libraryStub.externalLibraryWithImportPaths1), - ); + storageMock.stat.mockResolvedValue({ + isDirectory: () => true, + } as Stats); + + storageMock.checkFileExists.mockResolvedValue(true); + + await expect( + sut.update(authStub.external1, authStub.external1.user.id, { importPaths: ['/data/user1/foo'] }), + ).resolves.toEqual(mapLibrary(libraryStub.externalLibraryWithImportPaths1)); expect(libraryMock.update).toHaveBeenCalledWith( expect.objectContaining({ - id: authStub.admin.user.id, + id: authStub.external1.user.id, }), ); expect(storageMock.watch).toHaveBeenCalledWith( @@ -1142,7 +1181,7 @@ describe(LibraryService.name, () => { }); }); - describe('watchAll new', () => { + describe('watchAll', () => { describe('watching disabled', () => { beforeEach(async () => { configMock.load.mockResolvedValue(systemConfigStub.libraryWatchDisabled); @@ -1523,4 +1562,121 @@ describe(LibraryService.name, () => { ]); }); }); + + describe('validate', () => { + it('should validate directory', async () => { + storageMock.stat.mockResolvedValue({ + isDirectory: () => true, + } as Stats); + + storageMock.checkFileExists.mockResolvedValue(true); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: true, + message: undefined, + }, + ]); + }); + + it('should error when no external path is set', async () => { + await expect( + sut.validate(authStub.admin, libraryStub.externalLibrary1.id, { importPaths: ['/photos'] }), + ).rejects.toBeInstanceOf(BadRequestException); + }); + + it('should detect when path is outside external path', async () => { + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user2'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user2', + isValid: false, + message: "Not contained in user's external path", + }, + ]); + }); + + it('should detect when path does not exist', async () => { + storageMock.stat.mockImplementation(() => { + const error = { code: 'ENOENT' } as any; + throw error; + }); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: false, + message: 'Path does not exist (ENOENT)', + }, + ]); + }); + + it('should detect when path is not a directory', async () => { + storageMock.stat.mockResolvedValue({ + isDirectory: () => false, + } as Stats); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/file'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/file', + isValid: false, + message: 'Not a directory', + }, + ]); + }); + + it('should return an unknown exception from stat', async () => { + storageMock.stat.mockImplementation(() => { + throw new Error('Unknown error'); + }); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: false, + message: 'Error: Unknown error', + }, + ]); + }); + + it('should detect when access rights are missing', async () => { + storageMock.stat.mockResolvedValue({ + isDirectory: () => true, + } as Stats); + + storageMock.checkFileExists.mockResolvedValue(false); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: false, + message: 'Lacking read permission for folder', + }, + ]); + }); + }); }); diff --git a/server/src/domain/library/library.service.ts b/server/src/domain/library/library.service.ts index 3454f3547b..7c961963ec 100644 --- a/server/src/domain/library/library.service.ts +++ b/server/src/domain/library/library.service.ts @@ -30,6 +30,9 @@ import { LibraryStatsResponseDto, ScanLibraryDto, UpdateLibraryDto, + ValidateLibraryDto, + ValidateLibraryImportPathResponseDto, + ValidateLibraryResponseDto, mapLibrary, } from './library.dto'; @@ -270,10 +273,81 @@ export class LibraryService extends EventEmitter { ); } + private async validateImportPath(importPath: string): Promise { + const validation = new ValidateLibraryImportPathResponseDto(); + validation.importPath = importPath; + + try { + const stat = await this.storageRepository.stat(importPath); + + if (!stat.isDirectory()) { + validation.message = 'Not a directory'; + return validation; + } + } catch (error: any) { + if (error.code === 'ENOENT') { + validation.message = 'Path does not exist (ENOENT)'; + return validation; + } + validation.message = String(error); + return validation; + } + + const access = await this.storageRepository.checkFileExists(importPath, R_OK); + + if (!access) { + validation.message = 'Lacking read permission for folder'; + return validation; + } + + validation.isValid = true; + return validation; + } + + public async validate(auth: AuthDto, id: string, dto: ValidateLibraryDto): Promise { + await this.access.requirePermission(auth, Permission.LIBRARY_UPDATE, id); + + if (!auth.user.externalPath) { + throw new BadRequestException('User has no external path set'); + } + + const response = new ValidateLibraryResponseDto(); + + if (dto.importPaths) { + response.importPaths = await Promise.all( + dto.importPaths.map(async (importPath) => { + const normalizedPath = path.normalize(importPath); + + if (!this.isInExternalPath(normalizedPath, auth.user.externalPath)) { + const validation = new ValidateLibraryImportPathResponseDto(); + validation.importPath = importPath; + validation.message = `Not contained in user's external path`; + return validation; + } + + return await this.validateImportPath(importPath); + }), + ); + } + + return response; + } + async update(auth: AuthDto, id: string, dto: UpdateLibraryDto): Promise { await this.access.requirePermission(auth, Permission.LIBRARY_UPDATE, id); const library = await this.repository.update({ id, ...dto }); + if (dto.importPaths) { + const validation = await this.validate(auth, id, { importPaths: dto.importPaths }); + if (validation.importPaths) { + for (const path of validation.importPaths) { + if (!path.isValid) { + throw new BadRequestException(`Invalid import path: ${path.message}`); + } + } + } + } + if (dto.importPaths || dto.exclusionPatterns) { // Re-watch library to use new paths and/or exclusion patterns await this.watch(id); @@ -509,6 +583,14 @@ export class LibraryService extends EventEmitter { return true; } + // Check if a given path is in a user's external path. Both arguments are assumed to be normalized + private isInExternalPath(filePath: string, externalPath: string | null): boolean { + if (externalPath === null) { + return false; + } + return filePath.startsWith(externalPath); + } + async handleQueueAssetRefresh(job: ILibraryRefreshJob): Promise { const library = await this.repository.get(job.id); if (!library || library.type !== LibraryType.EXTERNAL) { @@ -523,17 +605,31 @@ export class LibraryService extends EventEmitter { } this.logger.verbose(`Refreshing library: ${job.id}`); + + const pathValidation = await Promise.all( + library.importPaths.map(async (importPath) => await this.validateImportPath(importPath)), + ); + + const validImportPaths = pathValidation + .map((validation) => { + if (!validation.isValid) { + this.logger.error(`Skipping invalid import path: ${validation.importPath}. Reason: ${validation.message}`); + } + return validation; + }) + .filter((validation) => validation.isValid) + .map((validation) => validation.importPath); + const rawPaths = await this.storageRepository.crawl({ - pathsToCrawl: library.importPaths, + pathsToCrawl: validImportPaths, exclusionPatterns: library.exclusionPatterns, }); const crawledAssetPaths = rawPaths + // Normalize file paths. This is important to prevent security issues like path traversal .map((filePath) => path.normalize(filePath)) - .filter((assetPath) => - // Filter out paths that are not within the user's external path - assetPath.match(new RegExp(`^${user.externalPath}`)), - ) as string[]; + // Filter out paths that are not within the user's external path + .filter((assetPath) => this.isInExternalPath(assetPath, user.externalPath)) as string[]; this.logger.debug(`Found ${crawledAssetPaths.length} asset(s) when crawling import paths ${library.importPaths}`); const assetsInLibrary = await this.assetRepository.getByLibraryId([job.id]); diff --git a/server/src/immich/controllers/library.controller.ts b/server/src/immich/controllers/library.controller.ts index 56dd5d8e76..173e632110 100644 --- a/server/src/immich/controllers/library.controller.ts +++ b/server/src/immich/controllers/library.controller.ts @@ -6,8 +6,10 @@ import { LibraryResponseDto as ResponseDto, ScanLibraryDto, UpdateLibraryDto as UpdateDto, + ValidateLibraryDto, + ValidateLibraryResponseDto, } from '@app/domain'; -import { Body, Controller, Delete, Get, Param, Post, Put } from '@nestjs/common'; +import { Body, Controller, Delete, Get, HttpCode, Param, Post, Put } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; import { Auth, Authenticated } from '../app.guard'; import { UseValidation } from '../app.utils'; @@ -40,6 +42,16 @@ export class LibraryController { return this.service.get(auth, id); } + @Post(':id/validate') + @HttpCode(200) + validate( + @Auth() auth: AuthDto, + @Param() { id }: UUIDParamDto, + @Body() dto: ValidateLibraryDto, + ): Promise { + return this.service.validate(auth, id, dto); + } + @Delete(':id') deleteLibrary(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto): Promise { return this.service.delete(auth, id); diff --git a/server/src/infra/infra.utils.ts b/server/src/infra/infra.utils.ts index d9eb823639..ab7d744317 100644 --- a/server/src/infra/infra.utils.ts +++ b/server/src/infra/infra.utils.ts @@ -164,7 +164,21 @@ export function searchAssetBuilder( builder.andWhere(_.omitBy(path, _.isUndefined)); const status = _.pick(options, ['isExternal', 'isFavorite', 'isOffline', 'isReadOnly', 'isVisible', 'type']); - const { isArchived, isEncoded, isMotion, withArchived } = options; + const { + isArchived, + isEncoded, + isMotion, + withArchived, + isNotInAlbum, + withFaces, + withPeople, + withSmartInfo, + personIds, + withExif, + withStacked, + trashedAfter, + trashedBefore, + } = options; builder.andWhere( _.omitBy( { @@ -177,38 +191,38 @@ export function searchAssetBuilder( ), ); - if (options.isNotInAlbum) { + if (isNotInAlbum) { builder .leftJoin(`${builder.alias}.albums`, 'albums') .andWhere('albums.id IS NULL') .andWhere(`${builder.alias}.isVisible = true`); } - if (options.withFaces || options.withPeople) { + if (withFaces || withPeople) { builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces'); } - if (options.withPeople) { + if (withPeople) { builder.leftJoinAndSelect(`${builder.alias}.person`, 'person'); } - if (options.withSmartInfo) { + if (withSmartInfo) { builder.leftJoinAndSelect(`${builder.alias}.smartInfo`, 'smartInfo'); } - if (options.personIds && options.personIds.length > 0) { + if (personIds && personIds.length > 0) { builder .leftJoin(`${builder.alias}.faces`, 'faces') - .andWhere('faces.personId IN (:...personIds)', { personIds: options.personIds }) + .andWhere('faces.personId IN (:...personIds)', { personIds: personIds }) .addGroupBy(`${builder.alias}.id`) - .having('COUNT(faces.id) = :personCount', { personCount: options.personIds.length }); + .having('COUNT(faces.id) = :personCount', { personCount: personIds.length }); - if (options.withExif) { + if (withExif) { builder.addGroupBy('exifInfo.assetId'); } } - if (options.withStacked) { + if (withStacked) { builder .leftJoinAndSelect(`${builder.alias}.stack`, 'stack') .leftJoinAndSelect('stack.assets', 'stackedAssets') @@ -217,8 +231,7 @@ export function searchAssetBuilder( ); } - const withDeleted = - options.withDeleted ?? (options.trashedAfter !== undefined || options.trashedBefore !== undefined); + const withDeleted = options.withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined); if (withDeleted) { builder.withDeleted(); } diff --git a/server/src/infra/repositories/filesystem.provider.ts b/server/src/infra/repositories/filesystem.provider.ts index ed009da76f..3ffcd8111f 100644 --- a/server/src/infra/repositories/filesystem.provider.ts +++ b/server/src/infra/repositories/filesystem.provider.ts @@ -12,12 +12,24 @@ import archiver from 'archiver'; import chokidar, { WatchOptions } from 'chokidar'; import { glob } from 'glob'; import { constants, createReadStream, existsSync, mkdirSync } from 'node:fs'; -import fs, { copyFile, readdir, rename, utimes, writeFile } from 'node:fs/promises'; +import fs, { copyFile, readdir, rename, stat, utimes, writeFile } from 'node:fs/promises'; import path from 'node:path'; export class FilesystemProvider implements IStorageRepository { private logger = new ImmichLogger(FilesystemProvider.name); + readdir = readdir; + + copyFile = copyFile; + + stat = stat; + + writeFile = writeFile; + + rename = rename; + + utimes = utimes; + createZipStream(): ImmichZipStream { const archive = archiver('zip', { store: true }); @@ -50,14 +62,6 @@ export class FilesystemProvider implements IStorageRepository { } } - writeFile = writeFile; - - rename = rename; - - copyFile = copyFile; - - utimes = utimes; - async checkFileExists(filepath: string, mode = constants.F_OK): Promise { try { await fs.access(filepath, mode); @@ -79,8 +83,6 @@ export class FilesystemProvider implements IStorageRepository { } } - stat = fs.stat; - async unlinkDir(folder: string, options: { recursive?: boolean; force?: boolean }) { await fs.rm(folder, options); } @@ -146,6 +148,4 @@ export class FilesystemProvider implements IStorageRepository { return () => watcher.close(); } - - readdir = readdir; } diff --git a/server/test/fixtures/user.stub.ts b/server/test/fixtures/user.stub.ts index 770e615306..1b8a3b3fed 100644 --- a/server/test/fixtures/user.stub.ts +++ b/server/test/fixtures/user.stub.ts @@ -140,6 +140,25 @@ export const userStub = { quotaSizeInBytes: null, quotaUsageInBytes: 0, }), + externalPathRoot: Object.freeze({ + ...authStub.user1.user, + password: 'immich_password', + name: 'immich_name', + storageLabel: 'label-1', + externalPath: '/', + oauthId: '', + shouldChangePassword: false, + profileImagePath: '', + createdAt: new Date('2021-01-01'), + deletedAt: null, + updatedAt: new Date('2021-01-01'), + tags: [], + assets: [], + memoriesEnabled: true, + avatarColor: UserAvatarColor.PRIMARY, + quotaSizeInBytes: null, + quotaUsageInBytes: 0, + }), profilePath: Object.freeze({ ...authStub.user1.user, password: 'immich_password', diff --git a/web/src/lib/components/forms/library-import-paths-form.svelte b/web/src/lib/components/forms/library-import-paths-form.svelte index fddb4b9867..17903526be 100644 --- a/web/src/lib/components/forms/library-import-paths-form.svelte +++ b/web/src/lib/components/forms/library-import-paths-form.svelte @@ -1,13 +1,15 @@