self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-25 10:43:13 +02:00
Code Issues Releases Activity

feat(web,server)!: configure machine learning via the UI (#3768)

Browse Source
This commit is contained in:
Jason Rasmussen 2023-08-25 00:15:03 -04:00 committed by GitHub
parent 2cccef174a
commit 8211afb726
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 575 additions and 406 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
cli/src/api/open-api/api.ts generated
View File

@ -2066,19 +2066,6 @@ export interface SearchAssetResponseDto {
*/
'total': number;
}
/**
*
* @export
* @interface SearchConfigResponseDto
*/
export interface SearchConfigResponseDto {
/**
*
* @type {boolean}
* @memberof SearchConfigResponseDto
*/
'enabled': boolean;
}
/**
*
* @export
@ -2185,7 +2172,13 @@ export interface ServerFeaturesDto {
* @type {boolean}
* @memberof ServerFeaturesDto
*/
'machineLearning': boolean;
'clipEncode': boolean;
/**
*
* @type {boolean}
* @memberof ServerFeaturesDto
*/
'facialRecognition': boolean;
/**
*
* @type {boolean}
@ -2210,6 +2203,18 @@ export interface ServerFeaturesDto {
* @memberof ServerFeaturesDto
*/
'search': boolean;
/**
*
* @type {boolean}
* @memberof ServerFeaturesDto
*/
'sidecar': boolean;
/**
*
* @type {boolean}
* @memberof ServerFeaturesDto
*/
'tagImage': boolean;
}
/**
*
@ -2611,6 +2616,12 @@ export interface SystemConfigDto {
* @memberof SystemConfigDto
*/
'job': SystemConfigJobDto;
/**
*
* @type {SystemConfigMachineLearningDto}
* @memberof SystemConfigDto
*/
'machineLearning': SystemConfigMachineLearningDto;
/**
*
* @type {SystemConfigOAuthDto}
@ -2778,6 +2789,43 @@ export interface SystemConfigJobDto {
*/
'videoConversion': JobSettingsDto;
}
/**
*
* @export
* @interface SystemConfigMachineLearningDto
*/
export interface SystemConfigMachineLearningDto {
/**
*
* @type {boolean}
* @memberof SystemConfigMachineLearningDto
*/
'clipEncodeEnabled': boolean;
/**
*
* @type {boolean}
* @memberof SystemConfigMachineLearningDto
*/
'enabled': boolean;
/**
*
* @type {boolean}
* @memberof SystemConfigMachineLearningDto
*/
'facialRecognitionEnabled': boolean;
/**
*
* @type {boolean}
* @memberof SystemConfigMachineLearningDto
*/
'tagImageEnabled': boolean;
/**
*
* @type {string}
* @memberof SystemConfigMachineLearningDto
*/
'url': string;
}
/**
*
* @export
@ -10106,44 +10154,6 @@ export const SearchApiAxiosParamCreator = function (configuration?: Configuratio
setSearchParams(localVarUrlObj, localVarQueryParameter);
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
return {
url: toPathString(localVarUrlObj),
options: localVarRequestOptions,
};
},
/**
*
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
getSearchConfig: async (options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
const localVarPath = `/search/config`;
// use dummy base URL string because the URL constructor only accepts absolute URLs.
const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
let baseOptions;
if (configuration) {
baseOptions = configuration.baseOptions;
}
const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options};
const localVarHeaderParameter = {} as any;
const localVarQueryParameter = {} as any;
// authentication cookie required
// authentication api_key required
await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration)
// authentication bearer required
// http bearer authentication required
await setBearerAuthToObject(localVarHeaderParameter, configuration)
setSearchParams(localVarUrlObj, localVarQueryParameter);
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
@ -10290,15 +10300,6 @@ export const SearchApiFp = function(configuration?: Configuration) {
const localVarAxiosArgs = await localVarAxiosParamCreator.getExploreData(options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
*
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async getSearchConfig(options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<SearchConfigResponseDto>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.getSearchConfig(options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
*
* @param {string} [q]
@ -10342,14 +10343,6 @@ export const SearchApiFactory = function (configuration?: Configuration, basePat
getExploreData(options?: AxiosRequestConfig): AxiosPromise<Array<SearchExploreResponseDto>> {
return localVarFp.getExploreData(options).then((request) => request(axios, basePath));
},
/**
*
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
getSearchConfig(options?: AxiosRequestConfig): AxiosPromise<SearchConfigResponseDto> {
return localVarFp.getSearchConfig(options).then((request) => request(axios, basePath));
},
/**
*
* @param {SearchApiSearchRequest} requestParameters Request parameters.
@ -10498,16 +10491,6 @@ export class SearchApi extends BaseAPI {
return SearchApiFp(this.configuration).getExploreData(options).then((request) => request(this.axios, this.basePath));
}
/**
*
* @param {*} [options] Override http request option.
* @throws {RequiredError}
* @memberof SearchApi
*/
public getSearchConfig(options?: AxiosRequestConfig) {
return SearchApiFp(this.configuration).getSearchConfig(options).then((request) => request(this.axios, this.basePath));
}
/**
*
* @param {SearchApiSearchRequest} requestParameters Request parameters.

4
docs/docs/FAQ.md
View File

@ -39,7 +39,7 @@ This often happens when using a reverse proxy or cloudflare tunnel in front of I
### Why is Immich slow on low-memory systems like the Raspberry Pi?
Immich uses optional machine-learning features to enhance search results. This feature, however, can be too heavy to run on a Raspberry Pi. To disable machine learning, comment out the `immich-machine-learning` section of your docker-compose.yml and set `IMMICH_MACHINE_LEARNING_URL=false` in your .env file.
Immich uses optional machine-learning features to enhance search results. This feature, however, can be too heavy to run on a Raspberry Pi. To disable machine learning, comment out the `immich-machine-learning` section of your docker-compose.yml and set `IMMICH_MACHINE_LEARNING_ENABLED=false` in your .env file.
### How to disable machine-learning and TypeSense?
@ -47,7 +47,7 @@ Immich uses optional machine-learning features to enhance search results. This f
Disabling both will result in poor search experience and typesense utilizes CLIP embeddings which are generated by machine-learning.
:::
These features can be disabled by commenting out `immich-typesense` and `immich-machine-learning` sections of the docker-compose.yml and setting `IMMICH_MACHINE_LEARNING_URL=false` & `TYPESENSE_ENABLED=false` in your .env file.
These features can be disabled by commenting out `immich-typesense` and `immich-machine-learning` sections of the docker-compose.yml and setting `IMMICH_MACHINE_LEARNING_ENABLED=false` & `TYPESENSE_ENABLED=false` in your .env file.
### What happens to existing files after I choose a new [Storage Template](/docs/administration/storage-template.mdx)?

1
docs/docs/install/docker-compose.md
View File

@ -132,7 +132,6 @@ PUBLIC_LOGIN_PAGE_MESSAGE="My Family Photos and Videos Backup Server"
IMMICH_WEB_URL=http://immich-web:3000
IMMICH_SERVER_URL=http://immich-server:3001
IMMICH_MACHINE_LEARNING_URL=http://immich-machine-learning:3003
####################################################################################
# Alternative API's External Address - Optional

5
docs/docs/install/environment-variables.md
View File

@ -51,10 +51,11 @@ These environment variables are used by the `docker-compose.yml` file and do **N
## URLs
| Variable | Description | Default | Services |
| :---------------------------- | :------------------------------------------------------- | :-----------------------------------: | :-------------------- |
| :-------------------------------- | :--------------------------- | :-----------------------------------: | :-------------------- |
| `IMMICH_WEB_URL` | Immich Web URL | `http://immich-web:3000` | proxy |
| `IMMICH_SERVER_URL` | Immich Server URL | `http://immich-server:3001` | web, proxy |
| `IMMICH_MACHINE_LEARNING_URL` | Immich Machine Learning URL, set `"false"` to disable ML | `http://immich-machine-learning:3003` | server, microservices |
| `IMMICH_MACHINE_LEARNING_ENABLED` | Enabled machine learning | `true` | server, microservices |
| `IMMICH_MACHINE_LEARNING_URL` | Immich Machine Learning URL, | `http://immich-machine-learning:3003` | server, microservices |
| `PUBLIC_IMMICH_SERVER_URL` | Public Immich URL | `http://immich-server:3001` | web |
| `IMMICH_API_URL_EXTERNAL` | Immich API URL External | `/api` | web |

6
mobile/openapi/.openapi-generator/FILES generated
View File

@ -84,7 +84,6 @@ doc/SearchAlbumResponseDto.md
doc/SearchApi.md
doc/SearchAssetDto.md
doc/SearchAssetResponseDto.md
doc/SearchConfigResponseDto.md
doc/SearchExploreItem.md
doc/SearchExploreResponseDto.md
doc/SearchFacetCountResponseDto.md
@ -108,6 +107,7 @@ doc/SystemConfigApi.md
doc/SystemConfigDto.md
doc/SystemConfigFFmpegDto.md
doc/SystemConfigJobDto.md
doc/SystemConfigMachineLearningDto.md
doc/SystemConfigOAuthDto.md
doc/SystemConfigPasswordLoginDto.md
doc/SystemConfigStorageTemplateDto.md
@ -228,7 +228,6 @@ lib/model/queue_status_dto.dart
lib/model/search_album_response_dto.dart
lib/model/search_asset_dto.dart
lib/model/search_asset_response_dto.dart
lib/model/search_config_response_dto.dart
lib/model/search_explore_item.dart
lib/model/search_explore_response_dto.dart
lib/model/search_facet_count_response_dto.dart
@ -249,6 +248,7 @@ lib/model/smart_info_response_dto.dart
lib/model/system_config_dto.dart
lib/model/system_config_f_fmpeg_dto.dart
lib/model/system_config_job_dto.dart
lib/model/system_config_machine_learning_dto.dart
lib/model/system_config_o_auth_dto.dart
lib/model/system_config_password_login_dto.dart
lib/model/system_config_storage_template_dto.dart
@ -353,7 +353,6 @@ test/search_album_response_dto_test.dart
test/search_api_test.dart
test/search_asset_dto_test.dart
test/search_asset_response_dto_test.dart
test/search_config_response_dto_test.dart
test/search_explore_item_test.dart
test/search_explore_response_dto_test.dart
test/search_facet_count_response_dto_test.dart
@ -377,6 +376,7 @@ test/system_config_api_test.dart
test/system_config_dto_test.dart
test/system_config_f_fmpeg_dto_test.dart
test/system_config_job_dto_test.dart
test/system_config_machine_learning_dto_test.dart
test/system_config_o_auth_dto_test.dart
test/system_config_password_login_dto_test.dart
test/system_config_storage_template_dto_test.dart

BIN
mobile/openapi/README.md generated
View File

Binary file not shown.

BIN
mobile/openapi/doc/SearchApi.md generated
View File

Binary file not shown.

BIN
mobile/openapi/doc/ServerFeaturesDto.md generated
View File

Binary file not shown.

BIN
mobile/openapi/doc/SystemConfigDto.md generated
View File

Binary file not shown.

BIN
mobile/openapi/doc/SearchConfigResponseDto.md → mobile/openapi/doc/SystemConfigMachineLearningDto.md generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/api.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/api/search_api.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/api_client.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/model/search_config_response_dto.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/model/server_features_dto.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/model/system_config_dto.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/model/system_config_machine_learning_dto.dart generated Normal file
View File

Binary file not shown.

BIN
mobile/openapi/test/search_api_test.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/test/search_config_response_dto_test.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/test/server_features_dto_test.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/test/system_config_dto_test.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/test/system_config_machine_learning_dto_test.dart generated Normal file
View File

Binary file not shown.

90
server/immich-openapi-specs.json
View File

@ -3243,38 +3243,6 @@
]
}
},
"/search/config": {
"get": {
"operationId": "getSearchConfig",
"parameters": [],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/SearchConfigResponseDto"
}
}
},
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Search"
]
}
},
"/search/explore": {
"get": {
"operationId": "getExploreData",
@ -6424,17 +6392,6 @@
],
"type": "object"
},
"SearchConfigResponseDto": {
"properties": {
"enabled": {
"type": "boolean"
}
},
"required": [
"enabled"
],
"type": "object"
},
"SearchExploreItem": {
"properties": {
"data": {
@ -6518,7 +6475,10 @@
},
"ServerFeaturesDto": {
"properties": {
"machineLearning": {
"clipEncode": {
"type": "boolean"
},
"facialRecognition": {
"type": "boolean"
},
"oauth": {
@ -6532,11 +6492,20 @@
},
"search": {
"type": "boolean"
},
"sidecar": {
"type": "boolean"
},
"tagImage": {
"type": "boolean"
}
},
"required": [
"machineLearning",
"clipEncode",
"facialRecognition",
"sidecar",
"search",
"tagImage",
"oauth",
"oauthAutoLaunch",
"passwordLogin"
@ -6868,6 +6837,9 @@
"job": {
"$ref": "#/components/schemas/SystemConfigJobDto"
},
"machineLearning": {
"$ref": "#/components/schemas/SystemConfigMachineLearningDto"
},
"oauth": {
"$ref": "#/components/schemas/SystemConfigOAuthDto"
},
@ -6883,6 +6855,7 @@
},
"required": [
"ffmpeg",
"machineLearning",
"oauth",
"passwordLogin",
"storageTemplate",
@ -6989,6 +6962,33 @@
],
"type": "object"
},
"SystemConfigMachineLearningDto": {
"properties": {
"clipEncodeEnabled": {
"type": "boolean"
},
"enabled": {
"type": "boolean"
},
"facialRecognitionEnabled": {
"type": "boolean"
},
"tagImageEnabled": {
"type": "boolean"
},
"url": {
"type": "string"
}
},
"required": [
"enabled",
"url",
"clipEncodeEnabled",
"facialRecognitionEnabled",
"tagImageEnabled"
],
"type": "object"
},
"SystemConfigOAuthDto": {
"properties": {
"autoLaunch": {

12
server/src/domain/domain.constant.ts
View File

@ -1,5 +1,4 @@
import { AssetType } from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common';
import { Duration } from 'luxon';
import { extname } from 'node:path';
import pkg from 'src/../../package.json';
@ -24,17 +23,6 @@ export const SERVER_VERSION = `${serverVersion.major}.${serverVersion.minor}.${s
export const APP_MEDIA_LOCATION = process.env.IMMICH_MEDIA_LOCATION || './upload';
export const SEARCH_ENABLED = process.env.TYPESENSE_ENABLED !== 'false';
export const MACHINE_LEARNING_URL = process.env.IMMICH_MACHINE_LEARNING_URL || 'http://immich-machine-learning:3003';
export const MACHINE_LEARNING_ENABLED = MACHINE_LEARNING_URL !== 'false';
export function assertMachineLearningEnabled() {
if (!MACHINE_LEARNING_ENABLED) {
throw new BadRequestException('Machine learning is not enabled.');
}
}
const image: Record<string, string[]> = {
'.3fr': ['image/3fr', 'image/x-hasselblad-3fr'],
'.ari': ['image/ari', 'image/x-arriflex-ari'],

7
server/src/domain/facial-recognition/facial-recognition.service.spec.ts
View File

@ -9,6 +9,7 @@ import {
newPersonRepositoryMock,
newSearchRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
personStub,
} from '@test';
import { IAssetRepository, WithoutProperty } from '../asset';
@ -18,6 +19,7 @@ import { IPersonRepository } from '../person';
import { ISearchRepository } from '../search';
import { IMachineLearningRepository } from '../smart-info';
import { IStorageRepository } from '../storage';
import { ISystemConfigRepository } from '../system-config';
import { IFaceRepository } from './face.repository';
import { FacialRecognitionService } from './facial-recognition.services';
@ -94,6 +96,7 @@ const faceSearch = {
describe(FacialRecognitionService.name, () => {
let sut: FacialRecognitionService;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let faceMock: jest.Mocked<IFaceRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let machineLearningMock: jest.Mocked<IMachineLearningRepository>;
@ -104,6 +107,7 @@ describe(FacialRecognitionService.name, () => {
beforeEach(async () => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
faceMock = newFaceRepositoryMock();
jobMock = newJobRepositoryMock();
machineLearningMock = newMachineLearningRepositoryMock();
@ -116,6 +120,7 @@ describe(FacialRecognitionService.name, () => {
sut = new FacialRecognitionService(
assetMock,
configMock,
faceMock,
jobMock,
machineLearningMock,
@ -174,7 +179,7 @@ describe(FacialRecognitionService.name, () => {
machineLearningMock.detectFaces.mockResolvedValue([]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
expect(machineLearningMock.detectFaces).toHaveBeenCalledWith({
expect(machineLearningMock.detectFaces).toHaveBeenCalledWith('http://immich-machine-learning:3003', {
imagePath: assetStub.image.resizePath,
});
expect(faceMock.create).not.toHaveBeenCalled();

27
server/src/domain/facial-recognition/facial-recognition.services.ts
View File

@ -1,7 +1,6 @@
import { Inject, Logger } from '@nestjs/common';
import { join } from 'path';
import { IAssetRepository, WithoutProperty } from '../asset';
import { MACHINE_LEARNING_ENABLED } from '../domain.constant';
import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, IFaceThumbnailJob, IJobRepository, JobName, JOBS_ASSET_PAGINATION_SIZE } from '../job';
import { CropOptions, FACE_THUMBNAIL_SIZE, IMediaRepository } from '../media';
@ -9,14 +8,17 @@ import { IPersonRepository } from '../person/person.repository';
import { ISearchRepository } from '../search/search.repository';
import { IMachineLearningRepository } from '../smart-info';
import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
import { ISystemConfigRepository, SystemConfigCore } from '../system-config';
import { AssetFaceId, IFaceRepository } from './face.repository';
export class FacialRecognitionService {
private logger = new Logger(FacialRecognitionService.name);
private storageCore = new StorageCore();
private configCore: SystemConfigCore;
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IFaceRepository) private faceRepository: IFaceRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository,
@ -24,9 +26,16 @@ export class FacialRecognitionService {
@Inject(IPersonRepository) private personRepository: IPersonRepository,
@Inject(ISearchRepository) private searchRepository: ISearchRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {}
) {
this.configCore = new SystemConfigCore(configRepository);
}
async handleQueueRecognizeFaces({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.facialRecognitionEnabled) {
return true;
}
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination, { order: 'DESC' })
@ -49,12 +58,17 @@ export class FacialRecognitionService {
}
async handleRecognizeFaces({ id }: IEntityJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.facialRecognitionEnabled) {
return true;
}
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset || !MACHINE_LEARNING_ENABLED || !asset.resizePath) {
if (!asset || !asset.resizePath) {
return false;
}
const faces = await this.machineLearning.detectFaces({ imagePath: asset.resizePath });
const faces = await this.machineLearning.detectFaces(machineLearning.url, { imagePath: asset.resizePath });
this.logger.debug(`${faces.length} faces detected in ${asset.resizePath}`);
this.logger.verbose(faces.map((face) => ({ ...face, embedding: `float[${face.embedding.length}]` })));
@ -100,6 +114,11 @@ export class FacialRecognitionService {
}
async handleGenerateFaceThumbnail(data: IFaceThumbnailJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.facialRecognitionEnabled) {
return true;
}
const { assetId, personId, boundingBox, imageWidth, imageHeight } = data;
const [asset] = await this.assetRepository.getByIds([assetId]);

9
server/src/domain/job/job.service.ts
View File

@ -2,8 +2,7 @@ import { AssetType } from '@app/infra/entities';
import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common';
import { IAssetRepository, mapAsset } from '../asset';
import { CommunicationEvent, ICommunicationRepository } from '../communication';
import { assertMachineLearningEnabled } from '../domain.constant';
import { ISystemConfigRepository } from '../system-config';
import { FeatureFlag, ISystemConfigRepository } from '../system-config';
import { SystemConfigCore } from '../system-config/system-config.core';
import { JobCommand, JobName, QueueName } from './job.constants';
import { AllJobStatusResponseDto, JobCommandDto, JobStatusDto } from './job.dto';
@ -78,23 +77,25 @@ export class JobService {
return this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
case QueueName.OBJECT_TAGGING:
assertMachineLearningEnabled();
await this.configCore.requireFeature(FeatureFlag.TAG_IMAGE);
return this.jobRepository.queue({ name: JobName.QUEUE_OBJECT_TAGGING, data: { force } });
case QueueName.CLIP_ENCODING:
assertMachineLearningEnabled();
await this.configCore.requireFeature(FeatureFlag.CLIP_ENCODE);
return this.jobRepository.queue({ name: JobName.QUEUE_ENCODE_CLIP, data: { force } });
case QueueName.METADATA_EXTRACTION:
return this.jobRepository.queue({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force } });
case QueueName.SIDECAR:
await this.configCore.requireFeature(FeatureFlag.SIDECAR);
return this.jobRepository.queue({ name: JobName.QUEUE_SIDECAR, data: { force } });
case QueueName.THUMBNAIL_GENERATION:
return this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force } });
case QueueName.RECOGNIZE_FACES:
await this.configCore.requireFeature(FeatureFlag.FACIAL_RECOGNITION);
return this.jobRepository.queue({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force } });
default:

1
server/src/domain/search/response-dto/index.ts
View File

@ -1,3 +1,2 @@
export * from './search-config-response.dto';
export * from './search-explore.response.dto';
export * from './search-response.dto';

3
server/src/domain/search/response-dto/search-config-response.dto.ts
View File

@ -1,3 +0,0 @@
export class SearchConfigResponseDto {
enabled!: boolean;
}

94
server/src/domain/search/search.service.spec.ts
View File

@ -1,5 +1,3 @@
import { BadRequestException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import {
albumStub,
assetStub,
@ -12,12 +10,14 @@ import {
newJobRepositoryMock,
newMachineLearningRepositoryMock,
newSearchRepositoryMock,
newSystemConfigRepositoryMock,
searchStub,
} from '@test';
import { plainToInstance } from 'class-transformer';
import { IAlbumRepository } from '../album/album.repository';
import { IAssetRepository } from '../asset/asset.repository';
import { IFaceRepository } from '../facial-recognition';
import { ISystemConfigRepository } from '../index';
import { JobName } from '../job';
import { IJobRepository } from '../job/job.repository';
import { IMachineLearningRepository } from '../smart-info';
@ -31,29 +31,26 @@ describe(SearchService.name, () => {
let sut: SearchService;
let albumMock: jest.Mocked<IAlbumRepository>;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let faceMock: jest.Mocked<IFaceRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let machineMock: jest.Mocked<IMachineLearningRepository>;
let searchMock: jest.Mocked<ISearchRepository>;
let configMock: jest.Mocked<ConfigService>;
const makeSut = (value?: string) => {
if (value) {
configMock.get.mockReturnValue(value);
}
return new SearchService(albumMock, assetMock, faceMock, jobMock, machineMock, searchMock, configMock);
};
beforeEach(() => {
beforeEach(async () => {
albumMock = newAlbumRepositoryMock();
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
faceMock = newFaceRepositoryMock();
jobMock = newJobRepositoryMock();
machineMock = newMachineLearningRepositoryMock();
searchMock = newSearchRepositoryMock();
configMock = { get: jest.fn() } as unknown as jest.Mocked<ConfigService>;
sut = makeSut();
sut = new SearchService(albumMock, assetMock, configMock, faceMock, jobMock, machineMock, searchMock);
searchMock.checkMigrationStatus.mockResolvedValue({ assets: false, albums: false, faces: false });
await sut.init();
});
afterEach(() => {
@ -86,45 +83,18 @@ describe(SearchService.name, () => {
});
});
describe('isEnabled', () => {
it('should be enabled by default', () => {
expect(sut.isEnabled()).toBe(true);
});
it('should be disabled via an env variable', () => {
const sut = makeSut('false');
expect(sut.isEnabled()).toBe(false);
});
});
describe('getConfig', () => {
it('should return the config', () => {
expect(sut.getConfig()).toEqual({ enabled: true });
});
it('should return the config when search is disabled', () => {
const sut = makeSut('false');
expect(sut.getConfig()).toEqual({ enabled: false });
});
});
describe(`init`, () => {
it('should skip when search is disabled', async () => {
const sut = makeSut('false');
// it('should skip when search is disabled', async () => {
// await sut.init();
await sut.init();
// expect(searchMock.setup).not.toHaveBeenCalled();
// expect(searchMock.checkMigrationStatus).not.toHaveBeenCalled();
// expect(jobMock.queue).not.toHaveBeenCalled();
expect(searchMock.setup).not.toHaveBeenCalled();
expect(searchMock.checkMigrationStatus).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
sut.teardown();
});
// sut.teardown();
// });
it('should skip schema migration if not needed', async () => {
searchMock.checkMigrationStatus.mockResolvedValue({ assets: false, albums: false, faces: false });
await sut.init();
expect(searchMock.setup).toHaveBeenCalled();
@ -145,14 +115,14 @@ describe(SearchService.name, () => {
});
describe('search', () => {
it('should throw an error is search is disabled', async () => {
const sut = makeSut('false');
// it('should throw an error is search is disabled', async () => {
// sut['enabled'] = false;
await expect(sut.search(authStub.admin, {})).rejects.toBeInstanceOf(BadRequestException);
// await expect(sut.search(authStub.admin, {})).rejects.toBeInstanceOf(BadRequestException);
expect(searchMock.searchAlbums).not.toHaveBeenCalled();
expect(searchMock.searchAssets).not.toHaveBeenCalled();
});
// expect(searchMock.searchAlbums).not.toHaveBeenCalled();
// expect(searchMock.searchAssets).not.toHaveBeenCalled();
// });
it('should search assets and albums', async () => {
searchMock.searchAssets.mockResolvedValue(searchStub.emptyResults);
@ -205,7 +175,7 @@ describe(SearchService.name, () => {
});
it('should skip if search is disabled', async () => {
const sut = makeSut('false');
sut['enabled'] = false;
await sut.handleIndexAssets();
@ -216,7 +186,7 @@ describe(SearchService.name, () => {
describe('handleIndexAsset', () => {
it('should skip if search is disabled', () => {
const sut = makeSut('false');
sut['enabled'] = false;
sut.handleIndexAsset({ ids: [assetStub.image.id] });
});
@ -227,7 +197,7 @@ describe(SearchService.name, () => {
describe('handleIndexAlbums', () => {
it('should skip if search is disabled', () => {
const sut = makeSut('false');
sut['enabled'] = false;
sut.handleIndexAlbums();
});
@ -242,7 +212,7 @@ describe(SearchService.name, () => {
describe('handleIndexAlbum', () => {
it('should skip if search is disabled', () => {
const sut = makeSut('false');
sut['enabled'] = false;
sut.handleIndexAlbum({ ids: [albumStub.empty.id] });
});
@ -253,7 +223,7 @@ describe(SearchService.name, () => {
describe('handleRemoveAlbum', () => {
it('should skip if search is disabled', () => {
const sut = makeSut('false');
sut['enabled'] = false;
sut.handleRemoveAlbum({ ids: ['album1'] });
});
@ -264,7 +234,7 @@ describe(SearchService.name, () => {
describe('handleRemoveAsset', () => {
it('should skip if search is disabled', () => {
const sut = makeSut('false');
sut['enabled'] = false;
sut.handleRemoveAsset({ ids: ['asset1'] });
});
@ -305,7 +275,7 @@ describe(SearchService.name, () => {
});
it('should skip if search is disabled', async () => {
const sut = makeSut('false');
sut['enabled'] = false;
await sut.handleIndexFaces();
@ -315,7 +285,7 @@ describe(SearchService.name, () => {
describe('handleIndexAsset', () => {
it('should skip if search is disabled', () => {
const sut = makeSut('false');
sut['enabled'] = false;
sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' });
expect(searchMock.importFaces).not.toHaveBeenCalled();
@ -333,7 +303,7 @@ describe(SearchService.name, () => {
describe('handleRemoveFace', () => {
it('should skip if search is disabled', () => {
const sut = makeSut('false');
sut['enabled'] = false;
sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' });
});

47
server/src/domain/search/search.service.ts
View File

@ -1,18 +1,17 @@
import { AlbumEntity, AssetEntity, AssetFaceEntity } from '@app/infra/entities';
import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { mapAlbumWithAssets } from '../album';
import { IAlbumRepository } from '../album/album.repository';
import { AssetResponseDto, mapAsset } from '../asset';
import { IAssetRepository } from '../asset/asset.repository';
import { AuthUserDto } from '../auth';
import { MACHINE_LEARNING_ENABLED } from '../domain.constant';
import { usePagination } from '../domain.util';
import { AssetFaceId, IFaceRepository } from '../facial-recognition';
import { IAssetFaceJob, IBulkEntityJob, IJobRepository, JobName, JOBS_ASSET_PAGINATION_SIZE } from '../job';
import { IMachineLearningRepository } from '../smart-info';
import { FeatureFlag, ISystemConfigRepository, SystemConfigCore } from '../system-config';
import { SearchDto } from './dto';
import { SearchConfigResponseDto, SearchResponseDto } from './response-dto';
import { SearchResponseDto } from './response-dto';
import {
ISearchRepository,
OwnedFaceEntity,
@ -30,8 +29,9 @@ interface SyncQueue {
@Injectable()
export class SearchService {
private logger = new Logger(SearchService.name);
private enabled: boolean;
private enabled = false;
private timer: NodeJS.Timer | null = null;
private configCore: SystemConfigCore;
private albumQueue: SyncQueue = {
upsert: new Set(),
@ -51,16 +51,13 @@ export class SearchService {
constructor(
@Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IFaceRepository) private faceRepository: IFaceRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository,
@Inject(ISearchRepository) private searchRepository: ISearchRepository,
configService: ConfigService,
) {
this.enabled = configService.get('TYPESENSE_ENABLED') !== 'false';
if (this.enabled) {
this.timer = setInterval(() => this.flush(), 5_000);
}
this.configCore = new SystemConfigCore(configRepository);
}
teardown() {
@ -70,17 +67,8 @@ export class SearchService {
}
}
isEnabled() {
return this.enabled;
}
getConfig(): SearchConfigResponseDto {
return {
enabled: this.enabled,
};
}
async init() {
this.enabled = await this.configCore.hasFeature(FeatureFlag.SEARCH);
if (!this.enabled) {
return;
}
@ -101,10 +89,13 @@ export class SearchService {
this.logger.debug('Queueing job to re-index all faces');
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACES });
}
this.timer = setInterval(() => this.flush(), 5_000);
}
async getExploreData(authUser: AuthUserDto): Promise<SearchExploreItem<AssetResponseDto>[]> {
this.assertEnabled();
await this.configCore.requireFeature(FeatureFlag.SEARCH);
const results = await this.searchRepository.explore(authUser.id);
const lookup = await this.getLookupMap(
results.reduce(
@ -126,16 +117,18 @@ export class SearchService {
}
async search(authUser: AuthUserDto, dto: SearchDto): Promise<SearchResponseDto> {
this.assertEnabled();
const { machineLearning } = await this.configCore.getConfig();
await this.configCore.requireFeature(FeatureFlag.SEARCH);
const query = dto.q || dto.query || '*';
const strategy = dto.clip && MACHINE_LEARNING_ENABLED ? SearchStrategy.CLIP : SearchStrategy.TEXT;
const hasClip = machineLearning.enabled && machineLearning.clipEncodeEnabled;
const strategy = dto.clip && hasClip ? SearchStrategy.CLIP : SearchStrategy.TEXT;
const filters = { userId: authUser.id, ...dto };
let assets: SearchResult<AssetEntity>;
switch (strategy) {
case SearchStrategy.CLIP:
const clip = await this.machineLearning.encodeText(query);
const clip = await this.machineLearning.encodeText(machineLearning.url, query);
assets = await this.searchRepository.vectorSearch(clip, filters);
break;
case SearchStrategy.TEXT:
@ -333,12 +326,6 @@ export class SearchService {
}
}
private assertEnabled() {
if (!this.enabled) {
throw new BadRequestException('Search is disabled');
}
}
private async idsToAlbums(ids: string[]): Promise<AlbumEntity[]> {
const entities = await this.albumRepository.getByIds(ids);
return this.patchAlbums(entities);

10
server/src/domain/server-info/server-info.dto.ts
View File

@ -1,4 +1,4 @@
import { IServerVersion } from '@app/domain';
import { FeatureFlags, IServerVersion } from '@app/domain';
import { ApiProperty, ApiResponseProperty } from '@nestjs/swagger';
export class ServerPingResponse {
@ -79,10 +79,14 @@ export class ServerMediaTypesResponseDto {
sidecar!: string[];
}
export class ServerFeaturesDto {
machineLearning!: boolean;
export class ServerFeaturesDto implements FeatureFlags {
clipEncode!: boolean;
facialRecognition!: boolean;
sidecar!: boolean;
search!: boolean;
tagImage!: boolean;
// TODO: use these instead of `POST oauth/config`
oauth!: boolean;
oauthAutoLaunch!: boolean;
passwordLogin!: boolean;

5
server/src/domain/server-info/server-info.service.spec.ts
View File

@ -147,11 +147,14 @@ describe(ServerInfoService.name, () => {
describe('getFeatures', () => {
it('should respond the server features', async () => {
await expect(sut.getFeatures()).resolves.toEqual({
machineLearning: true,
clipEncode: true,
facialRecognition: true,
oauth: false,
oauthAutoLaunch: false,
passwordLogin: true,
search: true,
sidecar: true,
tagImage: true,
});
expect(configMock.load).toHaveBeenCalled();
});

19
server/src/domain/server-info/server-info.service.ts
View File

@ -1,9 +1,8 @@
import { Inject, Injectable } from '@nestjs/common';
import { MACHINE_LEARNING_ENABLED, mimeTypes, SEARCH_ENABLED, serverVersion } from '../domain.constant';
import { mimeTypes, serverVersion } from '../domain.constant';
import { asHumanReadable } from '../domain.util';
import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
import { ISystemConfigRepository } from '../system-config';
import { SystemConfigCore } from '../system-config/system-config.core';
import { ISystemConfigRepository, SystemConfigCore } from '../system-config';
import { IUserRepository, UserStatsQueryResponse } from '../user';
import {
ServerFeaturesDto,
@ -52,18 +51,8 @@ export class ServerInfoService {
return serverVersion;
}
async getFeatures(): Promise<ServerFeaturesDto> {
const config = await this.configCore.getConfig();
return {
machineLearning: MACHINE_LEARNING_ENABLED,
search: SEARCH_ENABLED,
// TODO: use these instead of `POST oauth/config`
oauth: config.oauth.enabled,
oauthAutoLaunch: config.oauth.autoLaunch,
passwordLogin: config.passwordLogin.enabled,
};
getFeatures(): Promise<ServerFeaturesDto> {
return this.configCore.getFeatures();
}
async getStats(): Promise<ServerStatsResponseDto> {

8
server/src/domain/smart-info/machine-learning.interface.ts
View File

@ -20,8 +20,8 @@ export interface DetectFaceResult {
}
export interface IMachineLearningRepository {
classifyImage(input: MachineLearningInput): Promise<string[]>;
encodeImage(input: MachineLearningInput): Promise<number[]>;
encodeText(input: string): Promise<number[]>;
detectFaces(input: MachineLearningInput): Promise<DetectFaceResult[]>;
classifyImage(url: string, input: MachineLearningInput): Promise<string[]>;
encodeImage(url: string, input: MachineLearningInput): Promise<number[]>;
encodeText(url: string, input: string): Promise<number[]>;
detectFaces(url: string, input: MachineLearningInput): Promise<DetectFaceResult[]>;
}

14
server/src/domain/smart-info/smart-info.service.spec.ts
View File

@ -5,9 +5,11 @@ import {
newJobRepositoryMock,
newMachineLearningRepositoryMock,
newSmartInfoRepositoryMock,
newSystemConfigRepositoryMock,
} from '@test';
import { IAssetRepository, WithoutProperty } from '../asset';
import { IJobRepository, JobName } from '../job';
import { ISystemConfigRepository } from '../system-config';
import { IMachineLearningRepository } from './machine-learning.interface';
import { ISmartInfoRepository } from './smart-info.repository';
import { SmartInfoService } from './smart-info.service';
@ -20,16 +22,18 @@ const asset = {
describe(SmartInfoService.name, () => {
let sut: SmartInfoService;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let smartMock: jest.Mocked<ISmartInfoRepository>;
let machineMock: jest.Mocked<IMachineLearningRepository>;
beforeEach(async () => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
smartMock = newSmartInfoRepositoryMock();
jobMock = newJobRepositoryMock();
machineMock = newMachineLearningRepositoryMock();
sut = new SmartInfoService(assetMock, jobMock, smartMock, machineMock);
sut = new SmartInfoService(assetMock, configMock, jobMock, smartMock, machineMock);
assetMock.getByIds.mockResolvedValue([asset]);
});
@ -80,7 +84,9 @@ describe(SmartInfoService.name, () => {
await sut.handleClassifyImage({ id: asset.id });
expect(machineMock.classifyImage).toHaveBeenCalledWith({ imagePath: 'path/to/resize.ext' });
expect(machineMock.classifyImage).toHaveBeenCalledWith('http://immich-machine-learning:3003', {
imagePath: 'path/to/resize.ext',
});
expect(smartMock.upsert).toHaveBeenCalledWith({
assetId: 'asset-1',
tags: ['tag1', 'tag2', 'tag3'],
@ -139,7 +145,9 @@ describe(SmartInfoService.name, () => {
await sut.handleEncodeClip({ id: asset.id });
expect(machineMock.encodeImage).toHaveBeenCalledWith({ imagePath: 'path/to/resize.ext' });
expect(machineMock.encodeImage).toHaveBeenCalledWith('http://immich-machine-learning:3003', {
imagePath: 'path/to/resize.ext',
});
expect(smartMock.upsert).toHaveBeenCalledWith({
assetId: 'asset-1',
clipEmbedding: [0.01, 0.02, 0.03],

41
server/src/domain/smart-info/smart-info.service.ts
View File

@ -1,23 +1,31 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
import { IAssetRepository, WithoutProperty } from '../asset';
import { MACHINE_LEARNING_ENABLED } from '../domain.constant';
import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, IJobRepository, JobName, JOBS_ASSET_PAGINATION_SIZE } from '../job';
import { ISystemConfigRepository, SystemConfigCore } from '../system-config';
import { IMachineLearningRepository } from './machine-learning.interface';
import { ISmartInfoRepository } from './smart-info.repository';
@Injectable()
export class SmartInfoService {
private logger = new Logger(SmartInfoService.name);
private configCore: SystemConfigCore;
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISmartInfoRepository) private repository: ISmartInfoRepository,
@Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository,
) {}
) {
this.configCore = new SystemConfigCore(configRepository);
}
async handleQueueObjectTagging({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.tagImageEnabled) {
return true;
}
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
@ -34,19 +42,28 @@ export class SmartInfoService {
}
async handleClassifyImage({ id }: IEntityJob) {
const [asset] = await this.assetRepository.getByIds([id]);
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.tagImageEnabled) {
return true;
}
if (!MACHINE_LEARNING_ENABLED || !asset.resizePath) {
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset.resizePath) {
return false;
}
const tags = await this.machineLearning.classifyImage({ imagePath: asset.resizePath });
const tags = await this.machineLearning.classifyImage(machineLearning.url, { imagePath: asset.resizePath });
await this.repository.upsert({ assetId: asset.id, tags });
return true;
}
async handleQueueEncodeClip({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.clipEncodeEnabled) {
return true;
}
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
@ -63,13 +80,17 @@ export class SmartInfoService {
}
async handleEncodeClip({ id }: IEntityJob) {
const [asset] = await this.assetRepository.getByIds([id]);
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.clipEncodeEnabled) {
return true;
}
if (!MACHINE_LEARNING_ENABLED || !asset.resizePath) {
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset.resizePath) {
return false;
}
const clipEmbedding = await this.machineLearning.encodeImage({ imagePath: asset.resizePath });
const clipEmbedding = await this.machineLearning.encodeImage(machineLearning.url, { imagePath: asset.resizePath });
await this.repository.upsert({ assetId: asset.id, clipEmbedding: clipEmbedding });
return true;

19
server/src/domain/system-config/dto/system-config-machine-learning.dto.ts Normal file
View File

@ -0,0 +1,19 @@
import { IsBoolean, IsUrl, ValidateIf } from 'class-validator';
export class SystemConfigMachineLearningDto {
@IsBoolean()
enabled!: boolean;
@IsUrl({ require_tld: false })
@ValidateIf((dto) => dto.enabled)
url!: string;
@IsBoolean()
clipEncodeEnabled!: boolean;
@IsBoolean()
facialRecognitionEnabled!: boolean;
@IsBoolean()
tagImageEnabled!: boolean;
}

8
server/src/domain/system-config/dto/system-config.dto.ts
View File

@ -4,16 +4,22 @@ import { Type } from 'class-transformer';
import { IsObject, ValidateNested } from 'class-validator';
import { SystemConfigFFmpegDto } from './system-config-ffmpeg.dto';
import { SystemConfigJobDto } from './system-config-job.dto';
import { SystemConfigMachineLearningDto } from './system-config-machine-learning.dto';
import { SystemConfigOAuthDto } from './system-config-oauth.dto';
import { SystemConfigPasswordLoginDto } from './system-config-password-login.dto';
import { SystemConfigStorageTemplateDto } from './system-config-storage-template.dto';
export class SystemConfigDto {
export class SystemConfigDto implements SystemConfig {
@Type(() => SystemConfigFFmpegDto)
@ValidateNested()
@IsObject()
ffmpeg!: SystemConfigFFmpegDto;
@Type(() => SystemConfigMachineLearningDto)
@ValidateNested()
@IsObject()
machineLearning!: SystemConfigMachineLearningDto;
@Type(() => SystemConfigOAuthDto)
@ValidateNested()
@IsObject()

1
server/src/domain/system-config/index.ts
View File

@ -1,5 +1,6 @@
export * from './dto';
export * from './response-dto';
export * from './system-config.constants';
export * from './system-config.core';
export * from './system-config.repository';
export * from './system-config.service';

69
server/src/domain/system-config/system-config.core.ts
View File

@ -9,7 +9,7 @@ import {
TranscodePolicy,
VideoCodec,
} from '@app/infra/entities';
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import { BadRequestException, ForbiddenException, Injectable, Logger } from '@nestjs/common';
import * as _ from 'lodash';
import { Subject } from 'rxjs';
import { DeepPartial } from 'typeorm';
@ -44,6 +44,13 @@ export const defaults = Object.freeze<SystemConfig>({
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
},
machineLearning: {
enabled: process.env.IMMICH_MACHINE_LEARNING_ENABLED !== 'false',
url: process.env.IMMICH_MACHINE_LEARNING_URL || 'http://immich-machine-learning:3003',
facialRecognitionEnabled: true,
tagImageEnabled: true,
clipEncodeEnabled: true,
},
oauth: {
enabled: false,
issuerUrl: '',
@ -71,6 +78,19 @@ export const defaults = Object.freeze<SystemConfig>({
},
});
export enum FeatureFlag {
CLIP_ENCODE = 'clipEncode',
FACIAL_RECOGNITION = 'facialRecognition',
TAG_IMAGE = 'tagImage',
SIDECAR = 'sidecar',
SEARCH = 'search',
OAUTH = 'oauth',
OAUTH_AUTO_LAUNCH = 'oauthAutoLaunch',
PASSWORD_LOGIN = 'passwordLogin',
}
export type FeatureFlags = Record<FeatureFlag, boolean>;
const singleton = new Subject<SystemConfig>();
@Injectable()
@ -82,6 +102,53 @@ export class SystemConfigCore {
constructor(private repository: ISystemConfigRepository) {}
async requireFeature(feature: FeatureFlag) {
const hasFeature = await this.hasFeature(feature);
if (!hasFeature) {
switch (feature) {
case FeatureFlag.CLIP_ENCODE:
throw new BadRequestException('Clip encoding is not enabled');
case FeatureFlag.FACIAL_RECOGNITION:
throw new BadRequestException('Facial recognition is not enabled');
case FeatureFlag.TAG_IMAGE:
throw new BadRequestException('Image tagging is not enabled');
case FeatureFlag.SIDECAR:
throw new BadRequestException('Sidecar is not enabled');
case FeatureFlag.SEARCH:
throw new BadRequestException('Search is not enabled');
case FeatureFlag.OAUTH:
throw new BadRequestException('OAuth is not enabled');
case FeatureFlag.PASSWORD_LOGIN:
throw new BadRequestException('Password login is not enabled');
default:
throw new ForbiddenException(`Missing required feature: ${feature}`);
}
}
}
async hasFeature(feature: FeatureFlag) {
const features = await this.getFeatures();
return features[feature] ?? false;
}
async getFeatures(): Promise<FeatureFlags> {
const config = await this.getConfig();
const mlEnabled = config.machineLearning.enabled;
return {
[FeatureFlag.CLIP_ENCODE]: mlEnabled && config.machineLearning.clipEncodeEnabled,
[FeatureFlag.FACIAL_RECOGNITION]: mlEnabled && config.machineLearning.facialRecognitionEnabled,
[FeatureFlag.TAG_IMAGE]: mlEnabled && config.machineLearning.tagImageEnabled,
[FeatureFlag.SIDECAR]: true,
[FeatureFlag.SEARCH]: process.env.TYPESENSE_ENABLED !== 'false',
// TODO: use these instead of `POST oauth/config`
[FeatureFlag.OAUTH]: config.oauth.enabled,
[FeatureFlag.OAUTH_AUTO_LAUNCH]: config.oauth.autoLaunch,
[FeatureFlag.PASSWORD_LOGIN]: config.passwordLogin.enabled,
};
}
public getDefaults(): SystemConfig {
return defaults;
}

7
server/src/domain/system-config/system-config.service.spec.ts
View File

@ -46,6 +46,13 @@ const updatedConfig = Object.freeze<SystemConfig>({
accel: TranscodeHWAccel.DISABLED,
tonemap: ToneMapping.HABLE,
},
machineLearning: {
enabled: true,
url: 'http://immich-machine-learning:3003',
facialRecognitionEnabled: true,
tagImageEnabled: true,
clipEncodeEnabled: true,
},
oauth: {
autoLaunch: true,
autoRegister: true,

7
server/src/immich/app.service.ts
View File

@ -1,4 +1,4 @@
import { JobService, MACHINE_LEARNING_ENABLED, SearchService, StorageService } from '@app/domain';
import { JobService, SearchService, ServerInfoService, StorageService } from '@app/domain';
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
@ -10,6 +10,7 @@ export class AppService {
private jobService: JobService,
private searchService: SearchService,
private storageService: StorageService,
private serverService: ServerInfoService,
) {}
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
@ -20,8 +21,6 @@ export class AppService {
async init() {
this.storageService.init();
await this.searchService.init();
this.logger.log(`Machine learning is ${MACHINE_LEARNING_ENABLED ? 'enabled' : 'disabled'}`);
this.logger.log(`Search is ${this.searchService.isEnabled() ? 'enabled' : 'disabled'}`);
this.logger.log(`Feature Flags: ${JSON.stringify(await this.serverService.getFeatures(), null, 2)}`);
}
}

14
server/src/immich/controllers/search.controller.ts
View File

@ -1,11 +1,4 @@
import {
AuthUserDto,
SearchConfigResponseDto,
SearchDto,
SearchExploreResponseDto,
SearchResponseDto,
SearchService,
} from '@app/domain';
import { AuthUserDto, SearchDto, SearchExploreResponseDto, SearchResponseDto, SearchService } from '@app/domain';
import { Controller, Get, Query } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { Authenticated, AuthUser } from '../app.guard';
@ -23,11 +16,6 @@ export class SearchController {
return this.service.search(authUser, dto);
}
@Get('config')
getSearchConfig(): SearchConfigResponseDto {
return this.service.getConfig();
}
@Get('explore')
getExploreData(@AuthUser() authUser: AuthUserDto): Promise<SearchExploreResponseDto[]> {
return this.service.getExploreData(authUser) as Promise<SearchExploreResponseDto[]>;

13
server/src/infra/entities/system-config.entity.ts
View File

@ -37,6 +37,12 @@ export enum SystemConfigKey {
JOB_SEARCH_CONCURRENCY = 'job.search.concurrency',
JOB_SIDECAR_CONCURRENCY = 'job.sidecar.concurrency',
MACHINE_LEARNING_ENABLED = 'machineLearning.enabled',
MACHINE_LEARNING_URL = 'machineLearning.url',
MACHINE_LEARNING_FACIAL_RECOGNITION_ENABLED = 'machineLearning.facialRecognitionEnabled',
MACHINE_LEARNING_TAG_IMAGE_ENABLED = 'machineLearning.tagImageEnabled',
MACHINE_LEARNING_CLIP_ENCODE_ENABLED = 'machineLearning.clipEncodeEnabled',
OAUTH_ENABLED = 'oauth.enabled',
OAUTH_ISSUER_URL = 'oauth.issuerUrl',
OAUTH_CLIENT_ID = 'oauth.clientId',
@ -105,6 +111,13 @@ export interface SystemConfig {
tonemap: ToneMapping;
};
job: Record<QueueName, { concurrency: number }>;
machineLearning: {
enabled: boolean;
url: string;
clipEncodeEnabled: boolean;
facialRecognitionEnabled: boolean;
tagImageEnabled: boolean;
};
oauth: {
enabled: boolean;
issuerUrl: string;

20
server/src/infra/repositories/machine-learning.repository.ts
View File

@ -1,9 +1,9 @@
import { DetectFaceResult, IMachineLearningRepository, MachineLearningInput, MACHINE_LEARNING_URL } from '@app/domain';
import { DetectFaceResult, IMachineLearningRepository, MachineLearningInput } from '@app/domain';
import { Injectable } from '@nestjs/common';
import axios from 'axios';
import { createReadStream } from 'fs';
const client = axios.create({ baseURL: MACHINE_LEARNING_URL });
const client = axios.create();
@Injectable()
export class MachineLearningRepository implements IMachineLearningRepository {
@ -11,19 +11,19 @@ export class MachineLearningRepository implements IMachineLearningRepository {
return client.post<T>(endpoint, createReadStream(input.imagePath)).then((res) => res.data);
}
classifyImage(input: MachineLearningInput): Promise<string[]> {
return this.post<string[]>(input, '/image-classifier/tag-image');
classifyImage(url: string, input: MachineLearningInput): Promise<string[]> {
return this.post<string[]>(input, `${url}/image-classifier/tag-image`);
}
detectFaces(input: MachineLearningInput): Promise<DetectFaceResult[]> {
return this.post<DetectFaceResult[]>(input, '/facial-recognition/detect-faces');
detectFaces(url: string, input: MachineLearningInput): Promise<DetectFaceResult[]> {
return this.post<DetectFaceResult[]>(input, `${url}/facial-recognition/detect-faces`);
}
encodeImage(input: MachineLearningInput): Promise<number[]> {
return this.post<number[]>(input, '/sentence-transformer/encode-image');
encodeImage(url: string, input: MachineLearningInput): Promise<number[]> {
return this.post<number[]>(input, `${url}/sentence-transformer/encode-image`);
}
encodeText(input: string): Promise<number[]> {
return client.post<number[]>('/sentence-transformer/encode-text', { text: input }).then((res) => res.data);
encodeText(url: string, input: string): Promise<number[]> {
return client.post<number[]>(`${url}/sentence-transformer/encode-text`, { text: input }).then((res) => res.data);
}
}

141
web/src/api/open-api/api.ts generated
View File

@ -2066,19 +2066,6 @@ export interface SearchAssetResponseDto {
*/
'total': number;
}
/**
*
* @export
* @interface SearchConfigResponseDto
*/
export interface SearchConfigResponseDto {
/**
*
* @type {boolean}
* @memberof SearchConfigResponseDto
*/
'enabled': boolean;
}
/**
*
* @export
@ -2185,7 +2172,13 @@ export interface ServerFeaturesDto {
* @type {boolean}
* @memberof ServerFeaturesDto
*/
'machineLearning': boolean;
'clipEncode': boolean;
/**
*
* @type {boolean}
* @memberof ServerFeaturesDto
*/
'facialRecognition': boolean;
/**
*
* @type {boolean}
@ -2210,6 +2203,18 @@ export interface ServerFeaturesDto {
* @memberof ServerFeaturesDto
*/
'search': boolean;
/**
*
* @type {boolean}
* @memberof ServerFeaturesDto
*/
'sidecar': boolean;
/**
*
* @type {boolean}
* @memberof ServerFeaturesDto
*/
'tagImage': boolean;
}
/**
*
@ -2611,6 +2616,12 @@ export interface SystemConfigDto {
* @memberof SystemConfigDto
*/
'job': SystemConfigJobDto;
/**
*
* @type {SystemConfigMachineLearningDto}
* @memberof SystemConfigDto
*/
'machineLearning': SystemConfigMachineLearningDto;
/**
*
* @type {SystemConfigOAuthDto}
@ -2778,6 +2789,43 @@ export interface SystemConfigJobDto {
*/
'videoConversion': JobSettingsDto;
}
/**
*
* @export
* @interface SystemConfigMachineLearningDto
*/
export interface SystemConfigMachineLearningDto {
/**
*
* @type {boolean}
* @memberof SystemConfigMachineLearningDto
*/
'clipEncodeEnabled': boolean;
/**
*
* @type {boolean}
* @memberof SystemConfigMachineLearningDto
*/
'enabled': boolean;
/**
*
* @type {boolean}
* @memberof SystemConfigMachineLearningDto
*/
'facialRecognitionEnabled': boolean;
/**
*
* @type {boolean}
* @memberof SystemConfigMachineLearningDto
*/
'tagImageEnabled': boolean;
/**
*
* @type {string}
* @memberof SystemConfigMachineLearningDto
*/
'url': string;
}
/**
*
* @export
@ -10106,44 +10154,6 @@ export const SearchApiAxiosParamCreator = function (configuration?: Configuratio
setSearchParams(localVarUrlObj, localVarQueryParameter);
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
return {
url: toPathString(localVarUrlObj),
options: localVarRequestOptions,
};
},
/**
*
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
getSearchConfig: async (options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
const localVarPath = `/search/config`;
// use dummy base URL string because the URL constructor only accepts absolute URLs.
const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
let baseOptions;
if (configuration) {
baseOptions = configuration.baseOptions;
}
const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options};
const localVarHeaderParameter = {} as any;
const localVarQueryParameter = {} as any;
// authentication cookie required
// authentication api_key required
await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration)
// authentication bearer required
// http bearer authentication required
await setBearerAuthToObject(localVarHeaderParameter, configuration)
setSearchParams(localVarUrlObj, localVarQueryParameter);
let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
@ -10290,15 +10300,6 @@ export const SearchApiFp = function(configuration?: Configuration) {
const localVarAxiosArgs = await localVarAxiosParamCreator.getExploreData(options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
*
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async getSearchConfig(options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<SearchConfigResponseDto>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.getSearchConfig(options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
*
* @param {string} [q]
@ -10342,14 +10343,6 @@ export const SearchApiFactory = function (configuration?: Configuration, basePat
getExploreData(options?: AxiosRequestConfig): AxiosPromise<Array<SearchExploreResponseDto>> {
return localVarFp.getExploreData(options).then((request) => request(axios, basePath));
},
/**
*
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
getSearchConfig(options?: AxiosRequestConfig): AxiosPromise<SearchConfigResponseDto> {
return localVarFp.getSearchConfig(options).then((request) => request(axios, basePath));
},
/**
*
* @param {SearchApiSearchRequest} requestParameters Request parameters.
@ -10498,16 +10491,6 @@ export class SearchApi extends BaseAPI {
return SearchApiFp(this.configuration).getExploreData(options).then((request) => request(this.axios, this.basePath));
}
/**
*
* @param {*} [options] Override http request option.
* @throws {RequiredError}
* @memberof SearchApi
*/
public getSearchConfig(options?: AxiosRequestConfig) {
return SearchApiFp(this.configuration).getSearchConfig(options).then((request) => request(this.axios, this.basePath));
}
/**
*
* @param {SearchApiSearchRequest} requestParameters Request parameters.

7
web/src/lib/components/admin-page/jobs/jobs-panel.svelte
View File

@ -70,25 +70,26 @@
subtitle: 'Discover or synchronize sidecar metadata from the filesystem',
allText: 'SYNC',
missingText: 'DISCOVER',
disabled: !$featureFlags.sidecar,
},
[JobName.ObjectTagging]: {
icon: TagMultiple,
title: api.getJobName(JobName.ObjectTagging),
subtitle: 'Run machine learning to tag objects\nNote that some assets may not have any objects detected',
disabled: !$featureFlags.machineLearning,
disabled: !$featureFlags.tagImage,
},
[JobName.ClipEncoding]: {
icon: VectorCircle,
title: api.getJobName(JobName.ClipEncoding),
subtitle: 'Run machine learning to generate clip embeddings',
disabled: !$featureFlags.machineLearning,
disabled: !$featureFlags.clipEncode,
},
[JobName.RecognizeFaces]: {
icon: FaceRecognition,
title: api.getJobName(JobName.RecognizeFaces),
subtitle: 'Run machine learning to recognize faces',
handleCommand: handleFaceCommand,
disabled: !$featureFlags.machineLearning,
disabled: !$featureFlags.facialRecognition,
},
[JobName.VideoConversion]: {
icon: Video,

104
web/src/lib/components/admin-page/settings/machine-learning-settings/machine-learning-settings.svelte Normal file
View File

@ -0,0 +1,104 @@
<script lang="ts">
import {
notificationController,
NotificationType,
} from '$lib/components/shared-components/notification/notification';
import { handleError } from '$lib/utils/handle-error';
import { api, SystemConfigDto } from '@api';
import { isEqual } from 'lodash-es';
import { fade } from 'svelte/transition';
import SettingButtonsRow from '../setting-buttons-row.svelte';
import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
import SettingSwitch from '../setting-switch.svelte';
let config: SystemConfigDto;
let defaultConfig: SystemConfigDto;
async function refreshConfig() {
[config, defaultConfig] = await Promise.all([
api.systemConfigApi.getConfig().then((res) => res.data),
api.systemConfigApi.getDefaults().then((res) => res.data),
]);
}
async function reset() {
const { data: resetConfig } = await api.systemConfigApi.getConfig();
config = resetConfig;
notificationController.show({ message: 'Reset to the last saved settings', type: NotificationType.Info });
}
async function saveSetting() {
try {
const { data: current } = await api.systemConfigApi.getConfig();
await api.systemConfigApi.updateConfig({
systemConfigDto: { ...current, machineLearning: config.machineLearning },
});
await refreshConfig();
notificationController.show({ message: 'Settings saved', type: NotificationType.Info });
} catch (error) {
handleError(error, 'Unable to save settings');
}
}
async function resetToDefault() {
await refreshConfig();
const { data: defaults } = await api.systemConfigApi.getDefaults();
config = defaults;
notificationController.show({ message: 'Reset settings to defaults', type: NotificationType.Info });
}
</script>
<div class="mt-2">
{#await refreshConfig() then}
<div in:fade={{ duration: 500 }}>
<form autocomplete="off" on:submit|preventDefault class="mx-4 flex flex-col gap-4 py-4">
<SettingSwitch
title="Enabled"
subtitle="Use machine learning features"
bind:checked={config.machineLearning.enabled}
/>
<hr />
<SettingInputField
inputType={SettingInputFieldType.TEXT}
label="URL"
desc="URL of machine learning server"
bind:value={config.machineLearning.url}
required={true}
disabled={!config.machineLearning.enabled}
isEdited={!(config.machineLearning.url === config.machineLearning.url)}
/>
<SettingSwitch
title="SMART SEARCH"
subtitle="Extract CLIP embeddings for smart search"
bind:checked={config.machineLearning.clipEncodeEnabled}
disabled={!config.machineLearning.enabled}
/>
<SettingSwitch
title="FACIAL RECOGNITION"
subtitle="Recognize and group faces in photos"
disabled={!config.machineLearning.enabled}
bind:checked={config.machineLearning.facialRecognitionEnabled}
/>
<SettingSwitch
title="IMAGE TAGGING"
subtitle="Tag and classify images"
disabled={!config.machineLearning.enabled}
bind:checked={config.machineLearning.tagImageEnabled}
/>
<SettingButtonsRow
on:reset={reset}
on:save={saveSetting}
on:reset-to-default={resetToDefault}
showResetToDefault={!isEqual(config, defaultConfig)}
/>
</form>
</div>
{/await}
</div>

5
web/src/lib/components/admin-page/settings/setting-switch.svelte
View File

@ -32,9 +32,9 @@
<input class="disabled::cursor-not-allowed h-0 w-0 opacity-0" type="checkbox" bind:checked on:click {disabled} />
{#if disabled}
<span class="slider-disable" />
<span class="slider-disable cursor-not-allowed" />
{:else}
<span class="slider" />
<span class="slider cursor-pointer" />
{/if}
</label>
</div>
@ -43,7 +43,6 @@
.slider,
.slider-disable {
position: absolute;
cursor: pointer;
top: 0;
left: 0;
right: 0;

5
web/src/lib/stores/feature-flags.store.ts
View File

@ -4,7 +4,10 @@ import { writable } from 'svelte/store';
export type FeatureFlags = ServerFeaturesDto;
export const featureFlags = writable<FeatureFlags>({
machineLearning: true,
clipEncode: true,
facialRecognition: true,
sidecar: true,
tagImage: true,
search: true,
oauth: true,
oauthAutoLaunch: true,

7
web/src/routes/admin/system-settings/+page.svelte
View File

@ -2,11 +2,12 @@
import { page } from '$app/stores';
import FFmpegSettings from '$lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte';
import JobSettings from '$lib/components/admin-page/settings/job-settings/job-settings.svelte';
import ThumbnailSettings from '$lib/components/admin-page/settings/thumbnail/thumbnail-settings.svelte';
import MachineLearningSettings from '$lib/components/admin-page/settings/machine-learning-settings/machine-learning-settings.svelte';
import OAuthSettings from '$lib/components/admin-page/settings/oauth/oauth-settings.svelte';
import PasswordLoginSettings from '$lib/components/admin-page/settings/password-login/password-login-settings.svelte';
import SettingAccordion from '$lib/components/admin-page/settings/setting-accordion.svelte';
import StorageTemplateSettings from '$lib/components/admin-page/settings/storage-template/storage-template-settings.svelte';
import ThumbnailSettings from '$lib/components/admin-page/settings/thumbnail/thumbnail-settings.svelte';
import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte';
import { api } from '@api';
import type { PageData } from './$types';
@ -50,6 +51,10 @@
<OAuthSettings oauthConfig={configs.oauth} />
</SettingAccordion>
<SettingAccordion title="Machine Learning" subtitle="Manage machine learning settings">
<MachineLearningSettings />
</SettingAccordion>
<SettingAccordion
title="Storage Template"
subtitle="Manage the folder structure and file name of the upload asset"