From 49f66be8afc2521122fc45b2e9428a7cf311dccc Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Fri, 31 Mar 2023 10:36:08 -0400 Subject: [PATCH] chore(server): use ioredis (#2116) --- server/apps/immich/src/main.ts | 6 +-- .../redis-io.adapter.middleware.ts | 37 ------------- server/apps/microservices/src/main.ts | 6 +-- server/libs/infra/src/index.ts | 1 + server/libs/infra/src/infra.config.ts | 19 ++++--- server/libs/infra/src/redis-io.adapter.ts | 15 ++++++ server/package-lock.json | 52 +++++++++++++++---- server/package.json | 2 +- 8 files changed, 74 insertions(+), 64 deletions(-) delete mode 100644 server/apps/immich/src/middlewares/redis-io.adapter.middleware.ts create mode 100644 server/libs/infra/src/redis-io.adapter.ts diff --git a/server/apps/immich/src/main.ts b/server/apps/immich/src/main.ts index df59aef457..7dae533d73 100644 --- a/server/apps/immich/src/main.ts +++ b/server/apps/immich/src/main.ts @@ -6,7 +6,7 @@ import cookieParser from 'cookie-parser'; import { writeFileSync } from 'fs'; import path from 'path'; import { AppModule } from './app.module'; -import { RedisIoAdapter } from './middlewares/redis-io.adapter.middleware'; +import { RedisIoAdapter } from '@app/infra'; import { json } from 'body-parser'; import { patchOpenAPI } from './utils/patch-open-api.util'; import { getLogLevels, MACHINE_LEARNING_ENABLED } from '@app/domain'; @@ -29,9 +29,7 @@ async function bootstrap() { const serverPort = Number(process.env.SERVER_PORT) || 3001; - const redisIoAdapter = new RedisIoAdapter(app); - await redisIoAdapter.connectToRedis(); - app.useWebSocketAdapter(redisIoAdapter); + app.useWebSocketAdapter(new RedisIoAdapter(app)); const config = new DocumentBuilder() .setTitle('Immich') diff --git a/server/apps/immich/src/middlewares/redis-io.adapter.middleware.ts b/server/apps/immich/src/middlewares/redis-io.adapter.middleware.ts deleted file mode 100644 index 9e385cb296..0000000000 --- a/server/apps/immich/src/middlewares/redis-io.adapter.middleware.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { IoAdapter } from '@nestjs/platform-socket.io'; -import { createClient } from 'redis'; -import { ServerOptions } from 'socket.io'; -import { createAdapter } from '@socket.io/redis-adapter'; - -const redisHost = process.env.REDIS_HOSTNAME || 'immich_redis'; -const redisPort = parseInt(process.env.REDIS_PORT || '6379'); -const redisDb = parseInt(process.env.REDIS_DBINDEX || '0'); -const redisPassword = process.env.REDIS_PASSWORD || undefined; -const redisSocket = process.env.REDIS_SOCKET || undefined; - -export class RedisIoAdapter extends IoAdapter { - private adapterConstructor: any; - - async connectToRedis(): Promise { - const pubClient = createClient({ - password: redisPassword, - database: redisDb, - socket: { - host: redisHost, - port: redisPort, - path: redisSocket, - }, - }); - const subClient = pubClient.duplicate(); - - await Promise.all([pubClient.connect(), subClient.connect()]); - - this.adapterConstructor = createAdapter(pubClient, subClient); - } - - createIOServer(port: number, options?: ServerOptions): any { - const server = super.createIOServer(port, options); - server.adapter(this.adapterConstructor); - return server; - } -} diff --git a/server/apps/microservices/src/main.ts b/server/apps/microservices/src/main.ts index bccf01115d..aa8c3efd42 100644 --- a/server/apps/microservices/src/main.ts +++ b/server/apps/microservices/src/main.ts @@ -2,7 +2,7 @@ import { Logger } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import { SERVER_VERSION } from '@app/domain'; import { getLogLevels } from '@app/domain'; -import { RedisIoAdapter } from '../../immich/src/middlewares/redis-io.adapter.middleware'; +import { RedisIoAdapter } from '@app/infra'; import { MicroservicesModule } from './microservices.module'; const logger = new Logger('ImmichMicroservice'); @@ -14,9 +14,7 @@ async function bootstrap() { const listeningPort = Number(process.env.MICROSERVICES_PORT) || 3002; - const redisIoAdapter = new RedisIoAdapter(app); - await redisIoAdapter.connectToRedis(); - app.useWebSocketAdapter(redisIoAdapter); + app.useWebSocketAdapter(new RedisIoAdapter(app)); await app.listen(listeningPort, () => { const envName = (process.env.NODE_ENV || 'development').toUpperCase(); diff --git a/server/libs/infra/src/index.ts b/server/libs/infra/src/index.ts index 27d0864535..158385ace5 100644 --- a/server/libs/infra/src/index.ts +++ b/server/libs/infra/src/index.ts @@ -1,3 +1,4 @@ export * from './database.config'; export * from './infra.config'; export * from './infra.module'; +export * from './redis-io.adapter'; diff --git a/server/libs/infra/src/infra.config.ts b/server/libs/infra/src/infra.config.ts index 0fa88760e5..c0f6f94279 100644 --- a/server/libs/infra/src/infra.config.ts +++ b/server/libs/infra/src/infra.config.ts @@ -1,16 +1,19 @@ +import { QueueName } from '@app/domain'; import { BullModuleOptions } from '@nestjs/bull'; +import { RedisOptions } from 'ioredis'; import { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration'; -import { QueueName } from '../../domain/src'; + +export const redisConfig: RedisOptions = { + host: process.env.REDIS_HOSTNAME || 'immich_redis', + port: parseInt(process.env.REDIS_PORT || '6379'), + db: parseInt(process.env.REDIS_DBINDEX || '0'), + password: process.env.REDIS_PASSWORD || undefined, + path: process.env.REDIS_SOCKET || undefined, +}; export const bullConfig: BullModuleOptions = { prefix: 'immich_bull', - redis: { - host: process.env.REDIS_HOSTNAME || 'immich_redis', - port: parseInt(process.env.REDIS_PORT || '6379'), - db: parseInt(process.env.REDIS_DBINDEX || '0'), - password: process.env.REDIS_PASSWORD || undefined, - path: process.env.REDIS_SOCKET || undefined, - }, + redis: redisConfig, defaultJobOptions: { attempts: 3, removeOnComplete: true, diff --git a/server/libs/infra/src/redis-io.adapter.ts b/server/libs/infra/src/redis-io.adapter.ts new file mode 100644 index 0000000000..9637d812a6 --- /dev/null +++ b/server/libs/infra/src/redis-io.adapter.ts @@ -0,0 +1,15 @@ +import { IoAdapter } from '@nestjs/platform-socket.io'; +import { createAdapter } from '@socket.io/redis-adapter'; +import Redis from 'ioredis'; +import { ServerOptions } from 'socket.io'; +import { redisConfig } from './infra.config'; + +export class RedisIoAdapter extends IoAdapter { + createIOServer(port: number, options?: ServerOptions): any { + const pubClient = new Redis(redisConfig); + const subClient = pubClient.duplicate(); + const server = super.createIOServer(port, options); + server.adapter(createAdapter(pubClient, subClient)); + return server; + } +} diff --git a/server/package-lock.json b/server/package-lock.json index 31f95d8bf0..d80fb49227 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -34,6 +34,7 @@ "fluent-ffmpeg": "^2.1.2", "handlebars": "^4.7.7", "i18n-iso-countries": "^7.5.0", + "ioredis": "^5.3.1", "joi": "^17.5.0", "local-reverse-geocoder": "0.12.5", "lodash": "^4.17.21", @@ -42,7 +43,6 @@ "nest-commander": "^3.3.0", "openid-client": "^5.2.1", "pg": "^8.8.0", - "redis": "^4.5.1", "reflect-metadata": "^0.1.13", "rxjs": "^7.2.0", "sanitize-filename": "^1.6.3", @@ -2286,6 +2286,8 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.1.0.tgz", "integrity": "sha512-9QovlxmpRtvxVbN0UBcv8WfdSMudNZZTFqCsnBszcQXqaZb/TVe30ScgGEO7u1EAIacTPAo7/oCYjYAxiHLanQ==", + "optional": true, + "peer": true, "peerDependencies": { "@redis/client": "^1.0.0" } @@ -2294,6 +2296,8 @@ "version": "1.4.2", "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.4.2.tgz", "integrity": "sha512-oUdEjE0I7JS5AyaAjkD3aOXn9NhO7XKyPyXEyrgFDu++VrVBHUPnV6dgEya9TcMuj5nIJRuCzCm8ZP+c9zCHPw==", + "optional": true, + "peer": true, "dependencies": { "cluster-key-slot": "1.1.1", "generic-pool": "3.9.0", @@ -2307,6 +2311,8 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz", "integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==", + "optional": true, + "peer": true, "peerDependencies": { "@redis/client": "^1.0.0" } @@ -2315,6 +2321,8 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz", "integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==", + "optional": true, + "peer": true, "peerDependencies": { "@redis/client": "^1.0.0" } @@ -2323,6 +2331,8 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.0.tgz", "integrity": "sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ==", + "optional": true, + "peer": true, "peerDependencies": { "@redis/client": "^1.0.0" } @@ -2331,6 +2341,8 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz", "integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==", + "optional": true, + "peer": true, "peerDependencies": { "@redis/client": "^1.0.0" } @@ -6198,6 +6210,8 @@ "version": "3.9.0", "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "optional": true, + "peer": true, "engines": { "node": ">= 4" } @@ -6730,14 +6744,14 @@ } }, "node_modules/ioredis": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", - "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.1.tgz", + "integrity": "sha512-C+IBcMysM6v52pTLItYMeV4Hz7uriGtoJdz7SSBDX6u+zwSYGirLdQh3L7t/OItWITcw3gTFMjJReYUwS4zihg==", "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -9461,6 +9475,8 @@ "version": "4.5.1", "resolved": "https://registry.npmjs.org/redis/-/redis-4.5.1.tgz", "integrity": "sha512-oxXSoIqMJCQVBTfxP6BNTCtDMyh9G6Vi5wjdPdV/sRKkufyZslDqCScSGcOr6XGR/reAWZefz7E4leM31RgdBA==", + "optional": true, + "peer": true, "dependencies": { "@redis/bloom": "1.1.0", "@redis/client": "1.4.2", @@ -13238,12 +13254,16 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.1.0.tgz", "integrity": "sha512-9QovlxmpRtvxVbN0UBcv8WfdSMudNZZTFqCsnBszcQXqaZb/TVe30ScgGEO7u1EAIacTPAo7/oCYjYAxiHLanQ==", + "optional": true, + "peer": true, "requires": {} }, "@redis/client": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.4.2.tgz", "integrity": "sha512-oUdEjE0I7JS5AyaAjkD3aOXn9NhO7XKyPyXEyrgFDu++VrVBHUPnV6dgEya9TcMuj5nIJRuCzCm8ZP+c9zCHPw==", + "optional": true, + "peer": true, "requires": { "cluster-key-slot": "1.1.1", "generic-pool": "3.9.0", @@ -13254,24 +13274,32 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz", "integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==", + "optional": true, + "peer": true, "requires": {} }, "@redis/json": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz", "integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==", + "optional": true, + "peer": true, "requires": {} }, "@redis/search": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.0.tgz", "integrity": "sha512-NyFZEVnxIJEybpy+YskjgOJRNsfTYqaPbK/Buv6W2kmFNaRk85JiqjJZA5QkRmWvGbyQYwoO5QfDi2wHskKrQQ==", + "optional": true, + "peer": true, "requires": {} }, "@redis/time-series": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz", "integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==", + "optional": true, + "peer": true, "requires": {} }, "@sideway/address": { @@ -16358,7 +16386,9 @@ "generic-pool": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", - "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==" + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "optional": true, + "peer": true }, "gensync": { "version": "1.0.0-beta.2", @@ -16735,14 +16765,14 @@ "dev": true }, "ioredis": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", - "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.1.tgz", + "integrity": "sha512-C+IBcMysM6v52pTLItYMeV4Hz7uriGtoJdz7SSBDX6u+zwSYGirLdQh3L7t/OItWITcw3gTFMjJReYUwS4zihg==", "requires": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -18833,6 +18863,8 @@ "version": "4.5.1", "resolved": "https://registry.npmjs.org/redis/-/redis-4.5.1.tgz", "integrity": "sha512-oxXSoIqMJCQVBTfxP6BNTCtDMyh9G6Vi5wjdPdV/sRKkufyZslDqCScSGcOr6XGR/reAWZefz7E4leM31RgdBA==", + "optional": true, + "peer": true, "requires": { "@redis/bloom": "1.1.0", "@redis/client": "1.4.2", diff --git a/server/package.json b/server/package.json index 7a6d609ada..64c263cd58 100644 --- a/server/package.json +++ b/server/package.json @@ -65,6 +65,7 @@ "fluent-ffmpeg": "^2.1.2", "handlebars": "^4.7.7", "i18n-iso-countries": "^7.5.0", + "ioredis": "^5.3.1", "joi": "^17.5.0", "local-reverse-geocoder": "0.12.5", "lodash": "^4.17.21", @@ -73,7 +74,6 @@ "nest-commander": "^3.3.0", "openid-client": "^5.2.1", "pg": "^8.8.0", - "redis": "^4.5.1", "reflect-metadata": "^0.1.13", "rxjs": "^7.2.0", "sanitize-filename": "^1.6.3",