refactor(server): telemetry env (#13564)

server/src/app.module.ts

@@ -23,7 +23,6 @@ import { repositories } from 'src/repositories';
 import { ConfigRepository } from 'src/repositories/config.repository';
 import { services } from 'src/services';
 import { DatabaseService } from 'src/services/database.service';
-import { otelConfig } from 'src/utils/instrumentation';
 
 const common = [...services, ...repositories];
 
@@ -37,14 +36,14 @@ const middleware = [
 ];
 
 const configRepository = new ConfigRepository();
-const { bull } = configRepository.getEnv();
+const { bull, otel } = configRepository.getEnv();
 
 const imports = [
   BullModule.forRoot(bull.config),
   BullModule.registerQueue(...bull.queues),
   ClsModule.forRoot(clsConfig),
   ConfigModule.forRoot(immichAppConfig),
-  OpenTelemetryModule.forRoot(otelConfig),
+  OpenTelemetryModule.forRoot(otel),
   TypeOrmModule.forRootAsync({
     inject: [ModuleRef],
     useFactory: (moduleRef: ModuleRef) => {

server/src/bin/sync-sql.ts

@@ -14,8 +14,8 @@ import { entities } from 'src/entities';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { repositories } from 'src/repositories';
 import { AccessRepository } from 'src/repositories/access.repository';
+import { ConfigRepository } from 'src/repositories/config.repository';
 import { AuthService } from 'src/services/auth.service';
-import { otelConfig } from 'src/utils/instrumentation';
 import { Logger } from 'typeorm';
 
 export class SqlLogger implements Logger {
@@ -74,6 +74,8 @@ class SqlGenerator {
     await rm(this.options.targetDir, { force: true, recursive: true });
     await mkdir(this.options.targetDir);
 
+    const { otel } = new ConfigRepository().getEnv();
+
     const moduleFixture = await Test.createTestingModule({
       imports: [
         TypeOrmModule.forRoot({
@@ -84,7 +86,7 @@ class SqlGenerator {
           logger: this.sqlLogger,
         }),
         TypeOrmModule.forFeature(entities),
-        OpenTelemetryModule.forRoot(otelConfig),
+        OpenTelemetryModule.forRoot(otel),
       ],
       providers: [...repositories, AuthService, SchedulerRegistry],
     }).compile();

server/src/interfaces/config.interface.ts

@@ -1,6 +1,7 @@
 import { RegisterQueueOptions } from '@nestjs/bullmq';
 import { QueueOptions } from 'bullmq';
 import { RedisOptions } from 'ioredis';
+import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
 import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';
 import { VectorExtension } from 'src/interfaces/database.interface';
 
@@ -54,6 +55,8 @@ export interface EnvData {
     trustedProxies: string[];
   };
 
+  otel: OpenTelemetryModuleOptions;
+
   resourcePaths: {
     lockFile: string;
     geodata: {
@@ -74,6 +77,11 @@ export interface EnvData {
   telemetry: {
     apiPort: number;
     microservicesPort: number;
+    enabled: boolean;
+    apiMetrics: boolean;
+    hostMetrics: boolean;
+    repoMetrics: boolean;
+    jobMetrics: boolean;
   };
 
   storage: {

server/src/repositories/config.repository.spec.ts

@@ -12,6 +12,11 @@ const resetEnv = () => {
     'IMMICH_TRUSTED_PROXIES',
     'IMMICH_API_METRICS_PORT',
     'IMMICH_MICROSERVICES_METRICS_PORT',
+    'IMMICH_METRICS',
+    'IMMICH_API_METRICS',
+    'IMMICH_HOST_METRICS',
+    'IMMICH_IO_METRICS',
+    'IMMICH_JOB_METRICS',
 
     'DB_URL',
     'DB_HOSTNAME',
@@ -200,11 +205,16 @@ describe('getEnv', () => {
   });
 
   describe('telemetry', () => {
-    it('should return default ports', () => {
+    it('should have default values', () => {
       const { telemetry } = getEnv();
       expect(telemetry).toEqual({
         apiPort: 8081,
         microservicesPort: 8082,
+        enabled: false,
+        apiMetrics: false,
+        hostMetrics: false,
+        jobMetrics: false,
+        repoMetrics: false,
       });
     });
 
@@ -212,10 +222,35 @@ describe('getEnv', () => {
       process.env.IMMICH_API_METRICS_PORT = '2001';
       process.env.IMMICH_MICROSERVICES_METRICS_PORT = '2002';
       const { telemetry } = getEnv();
-      expect(telemetry).toEqual({
+      expect(telemetry).toMatchObject({
         apiPort: 2001,
         microservicesPort: 2002,
       });
     });
+
+    it('should run with telemetry enabled', () => {
+      process.env.IMMICH_METRICS = 'true';
+      const { telemetry } = getEnv();
+      expect(telemetry).toMatchObject({
+        enabled: true,
+        apiMetrics: true,
+        hostMetrics: true,
+        jobMetrics: true,
+        repoMetrics: true,
+      });
+    });
+
+    it('should run with telemetry enabled and jobs disabled', () => {
+      process.env.IMMICH_METRICS = 'true';
+      process.env.IMMICH_JOB_METRICS = 'false';
+      const { telemetry } = getEnv();
+      expect(telemetry).toMatchObject({
+        enabled: true,
+        apiMetrics: true,
+        hostMetrics: true,
+        jobMetrics: false,
+        repoMetrics: true,
+      });
+    });
   });
 });

server/src/repositories/config.repository.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import { join } from 'node:path';
-import { citiesFile } from 'src/constants';
+import { citiesFile, excludePaths } from 'src/constants';
 import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';
 import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';
 import { DatabaseExtension } from 'src/interfaces/database.interface';
@@ -30,6 +30,8 @@ const asSet = (value: string | undefined, defaults: ImmichWorker[]) => {
   return new Set(values.length === 0 ? defaults : (values as ImmichWorker[]));
 };
 
+const parseBoolean = (value: string | undefined, defaultValue: boolean) => (value ? value === 'true' : defaultValue);
+
 const getEnv = (): EnvData => {
   const included = asSet(process.env.IMMICH_WORKERS_INCLUDE, [ImmichWorker.API, ImmichWorker.MICROSERVICES]);
   const excluded = asSet(process.env.IMMICH_WORKERS_EXCLUDE, []);
@@ -66,6 +68,16 @@ const getEnv = (): EnvData => {
     }
   }
 
+  const globalEnabled = parseBoolean(process.env.IMMICH_METRICS, false);
+  const hostMetrics = parseBoolean(process.env.IMMICH_HOST_METRICS, globalEnabled);
+  const apiMetrics = parseBoolean(process.env.IMMICH_API_METRICS, globalEnabled);
+  const repoMetrics = parseBoolean(process.env.IMMICH_IO_METRICS, globalEnabled);
+  const jobMetrics = parseBoolean(process.env.IMMICH_JOB_METRICS, globalEnabled);
+  const telemetryEnabled = globalEnabled || hostMetrics || apiMetrics || repoMetrics || jobMetrics;
+  if (!telemetryEnabled && process.env.OTEL_SDK_DISABLED === undefined) {
+    process.env.OTEL_SDK_DISABLED = 'true';
+  }
+
   return {
     host: process.env.IMMICH_HOST,
     port: Number(process.env.IMMICH_PORT) || 2283,
@@ -124,6 +136,16 @@ const getEnv = (): EnvData => {
         .filter(Boolean),
     },
 
+    otel: {
+      metrics: {
+        hostMetrics,
+        apiMetrics: {
+          enable: apiMetrics,
+          ignoreRoutes: excludePaths,
+        },
+      },
+    },
+
     redis: redisConfig,
 
     resourcePaths: {
@@ -148,6 +170,11 @@ const getEnv = (): EnvData => {
     telemetry: {
       apiPort: Number(process.env.IMMICH_API_METRICS_PORT || '') || 8081,
       microservicesPort: Number(process.env.IMMICH_MICROSERVICES_METRICS_PORT || '') || 8082,
+      enabled: telemetryEnabled,
+      hostMetrics,
+      apiMetrics,
+      repoMetrics,
+      jobMetrics,
     },
 
     workers,

server/src/repositories/metric.repository.ts

@@ -1,11 +1,12 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import { MetricOptions } from '@opentelemetry/api';
 import { MetricService } from 'nestjs-otel';
+import { IConfigRepository } from 'src/interfaces/config.interface';
 import { IMetricGroupRepository, IMetricRepository, MetricGroupOptions } from 'src/interfaces/metric.interface';
-import { apiMetrics, hostMetrics, jobMetrics, repoMetrics } from 'src/utils/instrumentation';
 
 class MetricGroupRepository implements IMetricGroupRepository {
   private enabled = false;
+
   constructor(private metricService: MetricService) {}
 
   addToCounter(name: string, value: number, options?: MetricOptions): void {
@@ -39,10 +40,11 @@ export class MetricRepository implements IMetricRepository {
   jobs: MetricGroupRepository;
   repo: MetricGroupRepository;
 
-  constructor(metricService: MetricService) {
+  constructor(metricService: MetricService, @Inject(IConfigRepository) configRepository: IConfigRepository) {
-    this.api = new MetricGroupRepository(metricService).configure({ enabled: apiMetrics });
+    const { telemetry } = configRepository.getEnv();
-    this.host = new MetricGroupRepository(metricService).configure({ enabled: hostMetrics });
+    this.api = new MetricGroupRepository(metricService).configure({ enabled: telemetry.apiMetrics });
-    this.jobs = new MetricGroupRepository(metricService).configure({ enabled: jobMetrics });
+    this.host = new MetricGroupRepository(metricService).configure({ enabled: telemetry.hostMetrics });
-    this.repo = new MetricGroupRepository(metricService).configure({ enabled: repoMetrics });
+    this.jobs = new MetricGroupRepository(metricService).configure({ enabled: telemetry.jobMetrics });
+    this.repo = new MetricGroupRepository(metricService).configure({ enabled: telemetry.repoMetrics });
   }
 }

server/src/utils/instrumentation.ts

@@ -7,32 +7,19 @@ import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
 import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node';
 import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
 import { snakeCase, startCase } from 'lodash';
-import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
 import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';
 import { performance } from 'node:perf_hooks';
-import { excludePaths, serverVersion } from 'src/constants';
+import { serverVersion } from 'src/constants';
 import { DecorateAll } from 'src/decorators';
-
+import { ConfigRepository } from 'src/repositories/config.repository';
-let metricsEnabled = process.env.IMMICH_METRICS === 'true';
-export const hostMetrics =
-  process.env.IMMICH_HOST_METRICS == null ? metricsEnabled : process.env.IMMICH_HOST_METRICS === 'true';
-export const apiMetrics =
-  process.env.IMMICH_API_METRICS == null ? metricsEnabled : process.env.IMMICH_API_METRICS === 'true';
-export const repoMetrics =
-  process.env.IMMICH_IO_METRICS == null ? metricsEnabled : process.env.IMMICH_IO_METRICS === 'true';
-export const jobMetrics =
-  process.env.IMMICH_JOB_METRICS == null ? metricsEnabled : process.env.IMMICH_JOB_METRICS === 'true';
-
-metricsEnabled ||= hostMetrics || apiMetrics || repoMetrics || jobMetrics;
-if (!metricsEnabled && process.env.OTEL_SDK_DISABLED === undefined) {
-  process.env.OTEL_SDK_DISABLED = 'true';
-}
 
 const aggregation = new metrics.ExplicitBucketHistogramAggregation(
   [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000],
   true,
 );
 
+const { telemetry } = new ConfigRepository().getEnv();
+
 let otelSingleton: NodeSDK | undefined;
 
 export const otelStart = (port: number) => {
@@ -64,23 +51,13 @@ export const otelShutdown = async () => {
   }
 };
 
-export const otelConfig: OpenTelemetryModuleOptions = {
-  metrics: {
-    hostMetrics,
-    apiMetrics: {
-      enable: apiMetrics,
-      ignoreRoutes: excludePaths,
-    },
-  },
-};
-
 function ExecutionTimeHistogram({
   description,
   unit = 'ms',
   valueType = contextBase.ValueType.DOUBLE,
 }: contextBase.MetricOptions = {}) {
   return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
-    if (!repoMetrics || process.env.OTEL_SDK_DISABLED) {
+    if (!telemetry.repoMetrics || process.env.OTEL_SDK_DISABLED) {
       return;
     }
 

server/test/repositories/config.repository.mock.ts

@@ -35,6 +35,16 @@ const envData: EnvData = {
     trustedProxies: [],
   },
 
+  otel: {
+    metrics: {
+      hostMetrics: false,
+      apiMetrics: {
+        enable: false,
+        ignoreRoutes: [],
+      },
+    },
+  },
+
   redis: {
     host: 'redis',
     port: 6379,
@@ -63,6 +73,11 @@ const envData: EnvData = {
   telemetry: {
     apiPort: 8081,
     microservicesPort: 8082,
+    enabled: false,
+    hostMetrics: false,
+    apiMetrics: false,
+    jobMetrics: false,
+    repoMetrics: false,
   },
 
   workers: [ImmichWorker.API, ImmichWorker.MICROSERVICES],