1
0
mirror of https://github.com/immich-app/immich.git synced 2025-01-30 18:00:27 +02:00

refactor(server): version logic (#9615)

* refactor(server): version

* test: better version and log checks
This commit is contained in:
Jason Rasmussen 2024-05-20 20:31:36 -04:00 committed by GitHub
parent 5f25f28c42
commit 1df7be8436
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 404 additions and 509 deletions

View File

@ -57,6 +57,7 @@
"reflect-metadata": "^0.2.0", "reflect-metadata": "^0.2.0",
"rxjs": "^7.8.1", "rxjs": "^7.8.1",
"sanitize-filename": "^1.6.3", "sanitize-filename": "^1.6.3",
"semver": "^7.6.2",
"sharp": "^0.33.0", "sharp": "^0.33.0",
"sirv": "^2.0.4", "sirv": "^2.0.4",
"thumbhash": "^0.1.1", "thumbhash": "^0.1.1",

View File

@ -81,6 +81,7 @@
"reflect-metadata": "^0.2.0", "reflect-metadata": "^0.2.0",
"rxjs": "^7.8.1", "rxjs": "^7.8.1",
"sanitize-filename": "^1.6.3", "sanitize-filename": "^1.6.3",
"semver": "^7.6.2",
"sharp": "^0.33.0", "sharp": "^0.33.0",
"sirv": "^2.0.4", "sirv": "^2.0.4",
"thumbhash": "^0.1.1", "thumbhash": "^0.1.1",

View File

@ -1,7 +1,11 @@
import { Duration } from 'luxon'; import { Duration } from 'luxon';
import { readFileSync } from 'node:fs'; import { readFileSync } from 'node:fs';
import { join } from 'node:path'; import { join } from 'node:path';
import { Version } from 'src/utils/version'; import { SemVer } from 'semver';
export const POSTGRES_VERSION_RANGE = '>=14.0.0';
export const VECTORS_VERSION_RANGE = '0.2.x';
export const VECTOR_VERSION_RANGE = '>=0.5 <1';
export const NEXT_RELEASE = 'NEXT_RELEASE'; export const NEXT_RELEASE = 'NEXT_RELEASE';
export const LIFECYCLE_EXTENSION = 'x-immich-lifecycle'; export const LIFECYCLE_EXTENSION = 'x-immich-lifecycle';
@ -11,7 +15,7 @@ export const ADDED_IN_PREFIX = 'This property was added in ';
export const SALT_ROUNDS = 10; export const SALT_ROUNDS = 10;
const { version } = JSON.parse(readFileSync('./package.json', 'utf8')); const { version } = JSON.parse(readFileSync('./package.json', 'utf8'));
export const serverVersion = Version.fromString(version); export const serverVersion = new SemVer(version);
export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 }); export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 });
export const ONE_HOUR = Duration.fromObject({ hours: 1 }); export const ONE_HOUR = Duration.fromObject({ hours: 1 });

View File

@ -33,5 +33,5 @@ export const databaseConfig: PostgresConnectionOptions = {
*/ */
export const dataSource = new DataSource({ ...databaseConfig, host: 'localhost' }); export const dataSource = new DataSource({ ...databaseConfig, host: 'localhost' });
export const vectorExt = export const getVectorExtension = () =>
process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS; process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;

View File

@ -1,6 +1,6 @@
import { ApiProperty, ApiResponseProperty } from '@nestjs/swagger'; import { ApiProperty, ApiResponseProperty } from '@nestjs/swagger';
import { SemVer } from 'semver';
import { SystemConfigThemeDto } from 'src/dtos/system-config.dto'; import { SystemConfigThemeDto } from 'src/dtos/system-config.dto';
import { IVersion } from 'src/utils/version';
export class ServerPingResponse { export class ServerPingResponse {
@ApiResponseProperty({ type: String, example: 'pong' }) @ApiResponseProperty({ type: String, example: 'pong' })
@ -25,13 +25,17 @@ export class ServerInfoResponseDto {
diskUsagePercentage!: number; diskUsagePercentage!: number;
} }
export class ServerVersionResponseDto implements IVersion { export class ServerVersionResponseDto {
@ApiProperty({ type: 'integer' }) @ApiProperty({ type: 'integer' })
major!: number; major!: number;
@ApiProperty({ type: 'integer' }) @ApiProperty({ type: 'integer' })
minor!: number; minor!: number;
@ApiProperty({ type: 'integer' }) @ApiProperty({ type: 'integer' })
patch!: number; patch!: number;
static fromSemVer(value: SemVer) {
return { major: value.major, minor: value.minor, patch: value.patch };
}
} }
export class UsageByUserDto { export class UsageByUserDto {

View File

@ -1,5 +1,3 @@
import { Version } from 'src/utils/version';
export enum DatabaseExtension { export enum DatabaseExtension {
CUBE = 'cube', CUBE = 'cube',
EARTH_DISTANCE = 'earthdistance', EARTH_DISTANCE = 'earthdistance',
@ -23,7 +21,7 @@ export enum DatabaseLock {
GetSystemConfig = 69, GetSystemConfig = 69,
} }
export const extName: Record<DatabaseExtension, string> = { export const EXTENSION_NAMES: Record<DatabaseExtension, string> = {
cube: 'cube', cube: 'cube',
earthdistance: 'earthdistance', earthdistance: 'earthdistance',
vector: 'pgvector', vector: 'pgvector',
@ -37,13 +35,12 @@ export interface VectorUpdateResult {
export const IDatabaseRepository = 'IDatabaseRepository'; export const IDatabaseRepository = 'IDatabaseRepository';
export interface IDatabaseRepository { export interface IDatabaseRepository {
getExtensionVersion(extensionName: string): Promise<Version | null>; getExtensionVersion(extensionName: string): Promise<string | undefined>;
getAvailableExtensionVersion(extension: DatabaseExtension): Promise<Version | null>; getAvailableExtensionVersion(extension: DatabaseExtension): Promise<string | undefined>;
getPreferredVectorExtension(): VectorExtension; getPostgresVersion(): Promise<string>;
getPostgresVersion(): Promise<Version>;
createExtension(extension: DatabaseExtension): Promise<void>; createExtension(extension: DatabaseExtension): Promise<void>;
updateExtension(extension: DatabaseExtension, version?: Version): Promise<void>; updateExtension(extension: DatabaseExtension, version?: string): Promise<void>;
updateVectorExtension(extension: VectorExtension, version?: Version): Promise<VectorUpdateResult>; updateVectorExtension(extension: VectorExtension, version?: string): Promise<VectorUpdateResult>;
reindex(index: VectorIndex): Promise<void>; reindex(index: VectorIndex): Promise<void>;
shouldReindex(name: VectorIndex): Promise<boolean>; shouldReindex(name: VectorIndex): Promise<boolean>;
runMigrations(options?: { transaction?: 'all' | 'none' | 'each' }): Promise<void>; runMigrations(options?: { transaction?: 'all' | 'none' | 'each' }): Promise<void>;

View File

@ -1,4 +1,4 @@
import { vectorExt } from 'src/database.config'; import { getVectorExtension } from 'src/database.config';
import { getCLIPModelInfo } from 'src/utils/misc'; import { getCLIPModelInfo } from 'src/utils/misc';
import { MigrationInterface, QueryRunner } from 'typeorm'; import { MigrationInterface, QueryRunner } from 'typeorm';
@ -7,7 +7,7 @@ export class UsePgVectors1700713871511 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> { public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`SET search_path TO "$user", public, vectors`); await queryRunner.query(`SET search_path TO "$user", public, vectors`);
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${vectorExt}`); await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${getVectorExtension()}`);
const faceDimQuery = await queryRunner.query(` const faceDimQuery = await queryRunner.query(`
SELECT CARDINALITY(embedding::real[]) as dimsize SELECT CARDINALITY(embedding::real[]) as dimsize
FROM asset_faces FROM asset_faces

View File

@ -1,4 +1,4 @@
import { vectorExt } from 'src/database.config'; import { getVectorExtension } from 'src/database.config';
import { DatabaseExtension } from 'src/interfaces/database.interface'; import { DatabaseExtension } from 'src/interfaces/database.interface';
import { MigrationInterface, QueryRunner } from 'typeorm'; import { MigrationInterface, QueryRunner } from 'typeorm';
@ -6,7 +6,7 @@ export class AddCLIPEmbeddingIndex1700713994428 implements MigrationInterface {
name = 'AddCLIPEmbeddingIndex1700713994428'; name = 'AddCLIPEmbeddingIndex1700713994428';
public async up(queryRunner: QueryRunner): Promise<void> { public async up(queryRunner: QueryRunner): Promise<void> {
if (vectorExt === DatabaseExtension.VECTORS) { if (getVectorExtension() === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET vectors.pgvector_compatibility=on`); await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
} }
await queryRunner.query(`SET search_path TO "$user", public, vectors`); await queryRunner.query(`SET search_path TO "$user", public, vectors`);

View File

@ -1,4 +1,4 @@
import { vectorExt } from 'src/database.config'; import { getVectorExtension } from 'src/database.config';
import { DatabaseExtension } from 'src/interfaces/database.interface'; import { DatabaseExtension } from 'src/interfaces/database.interface';
import { MigrationInterface, QueryRunner } from 'typeorm'; import { MigrationInterface, QueryRunner } from 'typeorm';
@ -6,7 +6,7 @@ export class AddFaceEmbeddingIndex1700714033632 implements MigrationInterface {
name = 'AddFaceEmbeddingIndex1700714033632'; name = 'AddFaceEmbeddingIndex1700714033632';
public async up(queryRunner: QueryRunner): Promise<void> { public async up(queryRunner: QueryRunner): Promise<void> {
if (vectorExt === DatabaseExtension.VECTORS) { if (getVectorExtension() === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET vectors.pgvector_compatibility=on`); await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
} }
await queryRunner.query(`SET search_path TO "$user", public, vectors`); await queryRunner.query(`SET search_path TO "$user", public, vectors`);

View File

@ -1,19 +1,19 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm'; import { InjectDataSource } from '@nestjs/typeorm';
import AsyncLock from 'async-lock'; import AsyncLock from 'async-lock';
import { vectorExt } from 'src/database.config'; import semver from 'semver';
import { getVectorExtension } from 'src/database.config';
import { import {
DatabaseExtension, DatabaseExtension,
DatabaseLock, DatabaseLock,
EXTENSION_NAMES,
IDatabaseRepository, IDatabaseRepository,
VectorExtension, VectorExtension,
VectorIndex, VectorIndex,
VectorUpdateResult, VectorUpdateResult,
extName,
} from 'src/interfaces/database.interface'; } from 'src/interfaces/database.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { Instrumentation } from 'src/utils/instrumentation'; import { Instrumentation } from 'src/utils/instrumentation';
import { Version, VersionType } from 'src/utils/version';
import { isValidInteger } from 'src/validation'; import { isValidInteger } from 'src/validation';
import { DataSource, EntityManager, QueryRunner } from 'typeorm'; import { DataSource, EntityManager, QueryRunner } from 'typeorm';
@ -29,22 +29,12 @@ export class DatabaseRepository implements IDatabaseRepository {
this.logger.setContext(DatabaseRepository.name); this.logger.setContext(DatabaseRepository.name);
} }
async getExtensionVersion(extension: DatabaseExtension): Promise<Version | null> { async getExtensionVersion(extension: DatabaseExtension): Promise<string | undefined> {
const res = await this.dataSource.query(`SELECT extversion FROM pg_extension WHERE extname = $1`, [extension]); const res = await this.dataSource.query(`SELECT extversion FROM pg_extension WHERE extname = $1`, [extension]);
const extVersion = res[0]?.['extversion']; return res[0]?.['extversion'];
if (extVersion == null) {
return null;
}
const version = Version.fromString(extVersion);
if (version.isEqual(new Version(0, 1, 1))) {
return new Version(0, 1, 11);
}
return version;
} }
async getAvailableExtensionVersion(extension: DatabaseExtension): Promise<Version | null> { async getAvailableExtensionVersion(extension: DatabaseExtension): Promise<string | undefined> {
const res = await this.dataSource.query( const res = await this.dataSource.query(
` `
SELECT version FROM pg_available_extension_versions SELECT version FROM pg_available_extension_versions
@ -52,45 +42,41 @@ export class DatabaseRepository implements IDatabaseRepository {
ORDER BY version DESC`, ORDER BY version DESC`,
[extension], [extension],
); );
const version = res[0]?.['version']; return res[0]?.['version'];
return version == null ? null : Version.fromString(version);
} }
getPreferredVectorExtension(): VectorExtension { async getPostgresVersion(): Promise<string> {
return vectorExt; const [{ server_version: version }] = await this.dataSource.query(`SHOW server_version`);
} return version;
async getPostgresVersion(): Promise<Version> {
const res = await this.dataSource.query(`SHOW server_version`);
return Version.fromString(res[0]['server_version']);
} }
async createExtension(extension: DatabaseExtension): Promise<void> { async createExtension(extension: DatabaseExtension): Promise<void> {
await this.dataSource.query(`CREATE EXTENSION IF NOT EXISTS ${extension}`); await this.dataSource.query(`CREATE EXTENSION IF NOT EXISTS ${extension}`);
} }
async updateExtension(extension: DatabaseExtension, version?: Version): Promise<void> { async updateExtension(extension: DatabaseExtension, version?: string): Promise<void> {
await this.dataSource.query(`ALTER EXTENSION ${extension} UPDATE${version ? ` TO '${version}'` : ''}`); await this.dataSource.query(`ALTER EXTENSION ${extension} UPDATE${version ? ` TO '${version}'` : ''}`);
} }
async updateVectorExtension(extension: VectorExtension, version?: Version): Promise<VectorUpdateResult> { async updateVectorExtension(extension: VectorExtension, targetVersion?: string): Promise<VectorUpdateResult> {
const curVersion = await this.getExtensionVersion(extension); const currentVersion = await this.getExtensionVersion(extension);
if (!curVersion) { if (!currentVersion) {
throw new Error(`${extName[extension]} extension is not installed`); throw new Error(`${EXTENSION_NAMES[extension]} extension is not installed`);
} }
const minorOrMajor = version && curVersion.isOlderThan(version) >= VersionType.MINOR;
const isVectors = extension === DatabaseExtension.VECTORS; const isVectors = extension === DatabaseExtension.VECTORS;
let restartRequired = false; let restartRequired = false;
await this.dataSource.manager.transaction(async (manager) => { await this.dataSource.manager.transaction(async (manager) => {
await this.setSearchPath(manager); await this.setSearchPath(manager);
if (minorOrMajor && isVectors) {
await this.updateVectorsSchema(manager, curVersion); const isSchemaUpgrade = targetVersion && semver.satisfies(targetVersion, '0.1.1 || 0.1.11');
if (isSchemaUpgrade && isVectors) {
await this.updateVectorsSchema(manager, currentVersion);
} }
await manager.query(`ALTER EXTENSION ${extension} UPDATE${version ? ` TO '${version}'` : ''}`); await manager.query(`ALTER EXTENSION ${extension} UPDATE${targetVersion ? ` TO '${targetVersion}'` : ''}`);
if (!minorOrMajor) { if (!isSchemaUpgrade) {
return; return;
} }
@ -110,7 +96,7 @@ export class DatabaseRepository implements IDatabaseRepository {
try { try {
await this.dataSource.query(`REINDEX INDEX ${index}`); await this.dataSource.query(`REINDEX INDEX ${index}`);
} catch (error) { } catch (error) {
if (vectorExt === DatabaseExtension.VECTORS) { if (getVectorExtension() === DatabaseExtension.VECTORS) {
this.logger.warn(`Could not reindex index ${index}. Attempting to auto-fix.`); this.logger.warn(`Could not reindex index ${index}. Attempting to auto-fix.`);
const table = index === VectorIndex.CLIP ? 'smart_search' : 'asset_faces'; const table = index === VectorIndex.CLIP ? 'smart_search' : 'asset_faces';
const dimSize = await this.getDimSize(table); const dimSize = await this.getDimSize(table);
@ -132,7 +118,7 @@ export class DatabaseRepository implements IDatabaseRepository {
} }
async shouldReindex(name: VectorIndex): Promise<boolean> { async shouldReindex(name: VectorIndex): Promise<boolean> {
if (vectorExt !== DatabaseExtension.VECTORS) { if (getVectorExtension() !== DatabaseExtension.VECTORS) {
return false; return false;
} }
@ -160,10 +146,10 @@ export class DatabaseRepository implements IDatabaseRepository {
await manager.query(`SET search_path TO "$user", public, vectors`); await manager.query(`SET search_path TO "$user", public, vectors`);
} }
private async updateVectorsSchema(manager: EntityManager, curVersion: Version): Promise<void> { private async updateVectorsSchema(manager: EntityManager, currentVersion: string): Promise<void> {
await manager.query('CREATE SCHEMA IF NOT EXISTS vectors'); await manager.query('CREATE SCHEMA IF NOT EXISTS vectors');
await manager.query(`UPDATE pg_catalog.pg_extension SET extversion = $1 WHERE extname = $2`, [ await manager.query(`UPDATE pg_catalog.pg_extension SET extversion = $1 WHERE extname = $2`, [
curVersion.toString(), currentVersion,
DatabaseExtension.VECTORS, DatabaseExtension.VECTORS,
]); ]);
await manager.query('UPDATE pg_catalog.pg_extension SET extrelocatable = true WHERE extname = $1', [ await manager.query('UPDATE pg_catalog.pg_extension SET extrelocatable = true WHERE extname = $1', [

View File

@ -1,6 +1,6 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm'; import { InjectRepository } from '@nestjs/typeorm';
import { vectorExt } from 'src/database.config'; import { getVectorExtension } from 'src/database.config';
import { DummyValue, GenerateSql } from 'src/decorators'; import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity, AssetType } from 'src/entities/asset.entity'; import { AssetEntity, AssetType } from 'src/entities/asset.entity';
@ -336,7 +336,7 @@ export class SearchRepository implements ISearchRepository {
} }
private getRuntimeConfig(numResults?: number): string { private getRuntimeConfig(numResults?: number): string {
if (vectorExt === DatabaseExtension.VECTOR) { if (getVectorExtension() === DatabaseExtension.VECTOR) {
return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall
} }

View File

@ -1,7 +1,6 @@
import { DatabaseExtension, IDatabaseRepository, VectorIndex } from 'src/interfaces/database.interface'; import { DatabaseExtension, IDatabaseRepository } from 'src/interfaces/database.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { DatabaseService } from 'src/services/database.service'; import { DatabaseService } from 'src/services/database.service';
import { Version, VersionType } from 'src/utils/version';
import { newDatabaseRepositoryMock } from 'test/repositories/database.repository.mock'; import { newDatabaseRepositoryMock } from 'test/repositories/database.repository.mock';
import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock'; import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock';
import { Mocked } from 'vitest'; import { Mocked } from 'vitest';
@ -13,137 +12,174 @@ describe(DatabaseService.name, () => {
beforeEach(() => { beforeEach(() => {
delete process.env.DB_SKIP_MIGRATIONS; delete process.env.DB_SKIP_MIGRATIONS;
delete process.env.DB_VECTOR_EXTENSION;
databaseMock = newDatabaseRepositoryMock(); databaseMock = newDatabaseRepositoryMock();
loggerMock = newLoggerRepositoryMock(); loggerMock = newLoggerRepositoryMock();
sut = new DatabaseService(databaseMock, loggerMock); sut = new DatabaseService(databaseMock, loggerMock);
databaseMock.getExtensionVersion.mockResolvedValue('0.2.0');
}); });
it('should work', () => { it('should work', () => {
expect(sut).toBeDefined(); expect(sut).toBeDefined();
}); });
describe.each([ it('should throw an error if PostgreSQL version is below minimum supported version', async () => {
[{ vectorExt: DatabaseExtension.VECTORS, extName: 'pgvecto.rs', minVersion: new Version(0, 1, 1) }], databaseMock.getPostgresVersion.mockResolvedValueOnce('13.10.0');
[{ vectorExt: DatabaseExtension.VECTOR, extName: 'pgvector', minVersion: new Version(0, 5, 0) }],
] as const)('init', ({ vectorExt, extName, minVersion }) => {
beforeEach(() => {
databaseMock.getPreferredVectorExtension.mockReturnValue(vectorExt);
databaseMock.getExtensionVersion.mockResolvedValue(minVersion);
sut = new DatabaseService(databaseMock, loggerMock); await expect(sut.init()).rejects.toThrow('Invalid PostgreSQL version. Found 13.10.0');
sut.minVectorVersion = minVersion; expect(databaseMock.getPostgresVersion).toHaveBeenCalledTimes(1);
sut.minVectorsVersion = minVersion; });
sut.vectorVersionPin = VersionType.MINOR;
sut.vectorsVersionPin = VersionType.MINOR;
});
it(`should resolve successfully if minimum supported PostgreSQL and ${extName} version are installed`, async () => { it(`should start up successfully with pgvectors`, async () => {
databaseMock.getPostgresVersion.mockResolvedValueOnce(new Version(14, 0, 0)); databaseMock.getPostgresVersion.mockResolvedValue('14.0.0');
await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.getPostgresVersion).toHaveBeenCalled();
expect(databaseMock.createExtension).toHaveBeenCalledWith(DatabaseExtension.VECTORS);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.getExtensionVersion).toHaveBeenCalled();
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled();
});
it(`should start up successfully with pgvector`, async () => {
process.env.DB_VECTOR_EXTENSION = 'pgvector';
databaseMock.getPostgresVersion.mockResolvedValue('14.0.0');
databaseMock.getExtensionVersion.mockResolvedValue('0.5.0');
await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.createExtension).toHaveBeenCalledWith(DatabaseExtension.VECTOR);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled();
});
it(`should throw an error if the pgvecto.rs extension is not installed`, async () => {
databaseMock.getExtensionVersion.mockResolvedValue('');
await expect(sut.init()).rejects.toThrow(`Unexpected: The pgvecto.rs extension is not installed.`);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it(`should throw an error if the pgvector extension is not installed`, async () => {
process.env.DB_VECTOR_EXTENSION = 'pgvector';
databaseMock.getExtensionVersion.mockResolvedValue('');
await expect(sut.init()).rejects.toThrow(`Unexpected: The pgvector extension is not installed.`);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it(`should throw an error if the pgvecto.rs extension version is below minimum supported version`, async () => {
databaseMock.getExtensionVersion.mockResolvedValue('0.1.0');
await expect(sut.init()).rejects.toThrow(
'The pgvecto.rs extension version is 0.1.0, but Immich only supports 0.2.x.',
);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it(`should throw an error if the pgvector extension version is below minimum supported version`, async () => {
process.env.DB_VECTOR_EXTENSION = 'pgvector';
databaseMock.getExtensionVersion.mockResolvedValue('0.1.0');
await expect(sut.init()).rejects.toThrow(
'The pgvector extension version is 0.1.0, but Immich only supports >=0.5 <1',
);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it(`should throw an error if pgvecto.rs extension version is a nightly`, async () => {
databaseMock.getExtensionVersion.mockResolvedValue('0.0.0');
await expect(sut.init()).rejects.toThrow(
'The pgvecto.rs extension version is 0.0.0, which means it is a nightly release.',
);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it(`should throw an error if pgvector extension version is a nightly`, async () => {
process.env.DB_VECTOR_EXTENSION = 'pgvector';
databaseMock.getExtensionVersion.mockResolvedValue('0.0.0');
await expect(sut.init()).rejects.toThrow(
'The pgvector extension version is 0.0.0, which means it is a nightly release.',
);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it(`should throw error if pgvecto.rs extension could not be created`, async () => {
databaseMock.createExtension.mockRejectedValue(new Error('Failed to create extension'));
await expect(sut.init()).rejects.toThrow('Failed to create extension');
expect(loggerMock.fatal).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal.mock.calls[0][0]).toContain(
'Alternatively, if your Postgres instance has pgvector, you may use this instead',
);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it(`should throw error if pgvector extension could not be created`, async () => {
process.env.DB_VECTOR_EXTENSION = 'pgvector';
databaseMock.getExtensionVersion.mockResolvedValue('0.0.0');
databaseMock.createExtension.mockRejectedValue(new Error('Failed to create extension'));
await expect(sut.init()).rejects.toThrow('Failed to create extension');
expect(loggerMock.fatal).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal.mock.calls[0][0]).toContain(
'Alternatively, if your Postgres instance has pgvecto.rs, you may use this instead',
);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
for (const version of ['0.2.1', '0.2.0', '0.2.9']) {
it(`should update the pgvecto.rs extension to ${version}`, async () => {
databaseMock.getAvailableExtensionVersion.mockResolvedValue(version);
databaseMock.getExtensionVersion.mockResolvedValue(version);
await expect(sut.init()).resolves.toBeUndefined(); await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.getPostgresVersion).toHaveBeenCalled(); expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vectors', version);
expect(databaseMock.createExtension).toHaveBeenCalledWith(vectorExt); expect(databaseMock.updateVectorExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.getExtensionVersion).toHaveBeenCalled(); expect(databaseMock.getExtensionVersion).toHaveBeenCalled();
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1); expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled(); expect(loggerMock.fatal).not.toHaveBeenCalled();
}); });
}
it('should throw an error if PostgreSQL version is below minimum supported version', async () => { for (const version of ['0.5.1', '0.6.0', '0.7.10']) {
databaseMock.getPostgresVersion.mockResolvedValueOnce(new Version(13, 0, 0)); it(`should update the pgvectors extension to ${version}`, async () => {
process.env.DB_VECTOR_EXTENSION = 'pgvector';
await expect(sut.init()).rejects.toThrow('PostgreSQL version is 13');
expect(databaseMock.getPostgresVersion).toHaveBeenCalledTimes(1);
});
it(`should resolve successfully if minimum supported ${extName} version is installed`, async () => {
await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.createExtension).toHaveBeenCalledWith(vectorExt);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled();
});
it(`should throw an error if ${extName} version is not installed even after createVectorExtension`, async () => {
databaseMock.getExtensionVersion.mockResolvedValue(null);
await expect(sut.init()).rejects.toThrow(`Unexpected: ${extName} extension is not installed.`);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it(`should throw an error if ${extName} version is below minimum supported version`, async () => {
databaseMock.getExtensionVersion.mockResolvedValue(
new Version(minVersion.major, minVersion.minor - 1, minVersion.patch),
);
await expect(sut.init()).rejects.toThrow(extName);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it.each([
{ type: VersionType.EQUAL, max: 'no', actual: 'patch' },
{ type: VersionType.PATCH, max: 'patch', actual: 'minor' },
{ type: VersionType.MINOR, max: 'minor', actual: 'major' },
] as const)(
`should throw an error if $max upgrade from min version is allowed and ${extName} version is $actual`,
async ({ type, actual }) => {
const version = new Version(minVersion.major, minVersion.minor, minVersion.patch);
version[actual] = minVersion[actual] + 1;
databaseMock.getExtensionVersion.mockResolvedValue(version);
if (vectorExt === DatabaseExtension.VECTOR) {
sut.minVectorVersion = minVersion;
sut.vectorVersionPin = type;
} else {
sut.minVectorsVersion = minVersion;
sut.vectorsVersionPin = type;
}
await expect(sut.init()).rejects.toThrow(extName);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
},
);
it(`should throw an error if ${extName} version is a nightly`, async () => {
databaseMock.getExtensionVersion.mockResolvedValue(new Version(0, 0, 0));
await expect(sut.init()).rejects.toThrow(extName);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it(`should throw error if ${extName} extension could not be created`, async () => {
databaseMock.createExtension.mockRejectedValue(new Error('Failed to create extension'));
await expect(sut.init()).rejects.toThrow('Failed to create extension');
expect(loggerMock.fatal).toHaveBeenCalledTimes(1);
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
});
it(`should update ${extName} if a newer version is available`, async () => {
const version = new Version(minVersion.major, minVersion.minor + 1, minVersion.patch);
databaseMock.getAvailableExtensionVersion.mockResolvedValue(version); databaseMock.getAvailableExtensionVersion.mockResolvedValue(version);
databaseMock.getExtensionVersion.mockResolvedValue(version);
await expect(sut.init()).resolves.toBeUndefined(); await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith(vectorExt, version); expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vector', version);
expect(databaseMock.updateVectorExtension).toHaveBeenCalledTimes(1); expect(databaseMock.updateVectorExtension).toHaveBeenCalledTimes(1);
expect(databaseMock.getExtensionVersion).toHaveBeenCalled();
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1); expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled(); expect(loggerMock.fatal).not.toHaveBeenCalled();
}); });
}
it(`should not update ${extName} if a newer version is higher than the maximum`, async () => { for (const version of ['0.1.0', '0.3.0', '1.0.0']) {
const version = new Version(minVersion.major + 1, minVersion.minor, minVersion.patch); it(`should not upgrade pgvecto.rs to ${version}`, async () => {
databaseMock.getAvailableExtensionVersion.mockResolvedValue(version); databaseMock.getAvailableExtensionVersion.mockResolvedValue(version);
await expect(sut.init()).resolves.toBeUndefined(); await expect(sut.init()).resolves.toBeUndefined();
@ -152,72 +188,106 @@ describe(DatabaseService.name, () => {
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1); expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled(); expect(loggerMock.fatal).not.toHaveBeenCalled();
}); });
}
it(`should warn if attempted to update ${extName} and failed`, async () => { for (const version of ['0.4.0', '1.0.0']) {
const version = new Version(minVersion.major, minVersion.minor, minVersion.patch + 1); it(`should not upgrade pgvector to ${version}`, async () => {
process.env.DB_VECTOR_EXTENSION = 'pgvector';
databaseMock.getExtensionVersion.mockResolvedValue('0.5.0');
databaseMock.getAvailableExtensionVersion.mockResolvedValue(version); databaseMock.getAvailableExtensionVersion.mockResolvedValue(version);
databaseMock.updateVectorExtension.mockRejectedValue(new Error('Failed to update extension'));
await expect(sut.init()).resolves.toBeUndefined(); await expect(sut.init()).resolves.toBeUndefined();
expect(loggerMock.warn).toHaveBeenCalledTimes(1); expect(databaseMock.updateVectorExtension).not.toHaveBeenCalled();
expect(loggerMock.warn.mock.calls[0][0]).toContain(extName);
expect(loggerMock.error).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled();
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith(vectorExt, version);
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
});
it(`should warn if ${extName} update requires restart`, async () => {
const version = new Version(minVersion.major, minVersion.minor, minVersion.patch + 1);
databaseMock.getAvailableExtensionVersion.mockResolvedValue(version);
databaseMock.updateVectorExtension.mockResolvedValue({ restartRequired: true });
await expect(sut.init()).resolves.toBeUndefined();
expect(loggerMock.warn).toHaveBeenCalledTimes(1);
expect(loggerMock.warn.mock.calls[0][0]).toContain(extName);
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith(vectorExt, version);
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1); expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled(); expect(loggerMock.fatal).not.toHaveBeenCalled();
}); });
}
it.each([{ index: VectorIndex.CLIP }, { index: VectorIndex.FACE }])( it(`should warn if the pgvecto.rs extension upgrade failed`, async () => {
`should reindex $index if necessary`, process.env.DB_VECTOR_EXTENSION = 'pgvector';
async ({ index }) => { databaseMock.getExtensionVersion.mockResolvedValue('0.5.0');
databaseMock.shouldReindex.mockImplementation((indexArg) => Promise.resolve(indexArg === index)); databaseMock.getAvailableExtensionVersion.mockResolvedValue('0.5.2');
databaseMock.updateVectorExtension.mockRejectedValue(new Error('Failed to update extension'));
await expect(sut.init()).resolves.toBeUndefined(); await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.shouldReindex).toHaveBeenCalledWith(index); expect(loggerMock.warn.mock.calls[0][0]).toContain('The pgvector extension can be updated to 0.5.2.');
expect(databaseMock.shouldReindex).toHaveBeenCalledTimes(2); expect(loggerMock.error).toHaveBeenCalledTimes(1);
expect(databaseMock.reindex).toHaveBeenCalledWith(index); expect(loggerMock.fatal).not.toHaveBeenCalled();
expect(databaseMock.reindex).toHaveBeenCalledTimes(1); expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vector', '0.5.2');
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1); expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled(); });
},
);
it.each([{ index: VectorIndex.CLIP }, { index: VectorIndex.FACE }])( it(`should warn if the pgvector extension upgrade failed`, async () => {
`should not reindex $index if not necessary`, databaseMock.getAvailableExtensionVersion.mockResolvedValue('0.2.1');
async () => { databaseMock.updateVectorExtension.mockRejectedValue(new Error('Failed to update extension'));
databaseMock.shouldReindex.mockResolvedValue(false);
await expect(sut.init()).resolves.toBeUndefined(); await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.shouldReindex).toHaveBeenCalledTimes(2); expect(loggerMock.warn.mock.calls[0][0]).toContain('The pgvecto.rs extension can be updated to 0.2.1.');
expect(databaseMock.reindex).not.toHaveBeenCalled(); expect(loggerMock.error).toHaveBeenCalledTimes(1);
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1); expect(loggerMock.fatal).not.toHaveBeenCalled();
expect(loggerMock.fatal).not.toHaveBeenCalled(); expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vectors', '0.2.1');
}, expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
); });
it('should skip migrations if DB_SKIP_MIGRATIONS=true', async () => { it(`should warn if the pgvecto.rs extension update requires restart`, async () => {
process.env.DB_SKIP_MIGRATIONS = 'true'; databaseMock.getAvailableExtensionVersion.mockResolvedValue('0.2.1');
databaseMock.updateVectorExtension.mockResolvedValue({ restartRequired: true });
await expect(sut.init()).resolves.toBeUndefined(); await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.runMigrations).not.toHaveBeenCalled(); expect(loggerMock.warn).toHaveBeenCalledTimes(1);
}); expect(loggerMock.warn.mock.calls[0][0]).toContain('pgvecto.rs');
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vectors', '0.2.1');
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled();
});
it(`should warn if the pgvector extension update requires restart`, async () => {
process.env.DB_VECTOR_EXTENSION = 'pgvector';
databaseMock.getExtensionVersion.mockResolvedValue('0.5.0');
databaseMock.getAvailableExtensionVersion.mockResolvedValue('0.5.1');
databaseMock.updateVectorExtension.mockResolvedValue({ restartRequired: true });
await expect(sut.init()).resolves.toBeUndefined();
expect(loggerMock.warn).toHaveBeenCalledTimes(1);
expect(loggerMock.warn.mock.calls[0][0]).toContain('pgvector');
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vector', '0.5.1');
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled();
});
it('should reindex if needed', async () => {
databaseMock.shouldReindex.mockResolvedValue(true);
await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.shouldReindex).toHaveBeenCalledTimes(2);
expect(databaseMock.reindex).toHaveBeenCalledTimes(2);
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled();
});
it('should not reindex if not needed', async () => {
databaseMock.shouldReindex.mockResolvedValue(false);
await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.shouldReindex).toHaveBeenCalledTimes(2);
expect(databaseMock.reindex).toHaveBeenCalledTimes(0);
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
expect(loggerMock.fatal).not.toHaveBeenCalled();
});
it('should skip migrations if DB_SKIP_MIGRATIONS=true', async () => {
process.env.DB_SKIP_MIGRATIONS = 'true';
databaseMock.getExtensionVersion.mockResolvedValue('0.2.0');
await expect(sut.init()).resolves.toBeUndefined();
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
}); });
}); });

View File

@ -1,38 +1,126 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import semver from 'semver';
import { POSTGRES_VERSION_RANGE, VECTORS_VERSION_RANGE, VECTOR_VERSION_RANGE } from 'src/constants';
import { getVectorExtension } from 'src/database.config';
import { import {
DatabaseExtension, DatabaseExtension,
DatabaseLock, DatabaseLock,
EXTENSION_NAMES,
IDatabaseRepository, IDatabaseRepository,
VectorExtension,
VectorIndex, VectorIndex,
extName,
} from 'src/interfaces/database.interface'; } from 'src/interfaces/database.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { Version, VersionType } from 'src/utils/version';
type CreateFailedArgs = { name: string; extension: string; otherName: string };
type UpdateFailedArgs = { name: string; extension: string; availableVersion: string };
type RestartRequiredArgs = { name: string; availableVersion: string };
type NightlyVersionArgs = { name: string; extension: string; version: string };
type OutOfRangeArgs = { name: string; extension: string; version: string; range: string };
const EXTENSION_RANGES = {
[DatabaseExtension.VECTOR]: VECTOR_VERSION_RANGE,
[DatabaseExtension.VECTORS]: VECTORS_VERSION_RANGE,
};
const messages = {
notInstalled: (name: string) => `Unexpected: The ${name} extension is not installed.`,
nightlyVersion: ({ name, extension, version }: NightlyVersionArgs) => `
The ${name} extension version is ${version}, which means it is a nightly release.
Please run 'DROP EXTENSION IF EXISTS ${extension}' and switch to a release version.
See https://immich.app/docs/guides/database-queries for how to query the database.`,
outOfRange: ({ name, extension, version, range }: OutOfRangeArgs) => `
The ${name} extension version is ${version}, but Immich only supports ${range}.
If the Postgres instance already has a compatible version installed, Immich may not have the necessary permissions to activate it.
In this case, please run 'ALTER EXTENSION UPDATE ${extension}' manually as a superuser.
See https://immich.app/docs/guides/database-queries for how to query the database.
Otherwise, please update the version of ${name} in the Postgres instance to a compatible version.`,
createFailed: ({ name, extension, otherName }: CreateFailedArgs) => `
Failed to activate ${name} extension.
Please ensure the Postgres instance has ${name} installed.
If the Postgres instance already has ${name} installed, Immich may not have the necessary permissions to activate it.
In this case, please run 'CREATE EXTENSION IF NOT EXISTS ${extension}' manually as a superuser.
See https://immich.app/docs/guides/database-queries for how to query the database.
Alternatively, if your Postgres instance has ${otherName}, you may use this instead by setting the environment variable 'DB_VECTOR_EXTENSION=${otherName}'.
Note that switching between the two extensions after a successful startup is not supported.
The exception is if your version of Immich prior to upgrading was 1.90.2 or earlier.
In this case, you may set either extension now, but you will not be able to switch to the other extension following a successful startup.
`,
updateFailed: ({ name, extension, availableVersion }: UpdateFailedArgs) => `
The ${name} extension can be updated to ${availableVersion}.
Immich attempted to update the extension, but failed to do so.
This may be because Immich does not have the necessary permissions to update the extension.
Please run 'ALTER EXTENSION ${extension} UPDATE' manually as a superuser.
See https://immich.app/docs/guides/database-queries for how to query the database.`,
restartRequired: ({ name, availableVersion }: RestartRequiredArgs) => `
The ${name} extension has been updated to ${availableVersion}.
Please restart the Postgres instance to complete the update.`,
};
@Injectable() @Injectable()
export class DatabaseService { export class DatabaseService {
private vectorExt: VectorExtension;
minPostgresVersion = 14;
minVectorsVersion = new Version(0, 2, 0);
vectorsVersionPin = VersionType.MINOR;
minVectorVersion = new Version(0, 5, 0);
vectorVersionPin = VersionType.MAJOR;
constructor( constructor(
@Inject(IDatabaseRepository) private databaseRepository: IDatabaseRepository, @Inject(IDatabaseRepository) private databaseRepository: IDatabaseRepository,
@Inject(ILoggerRepository) private logger: ILoggerRepository, @Inject(ILoggerRepository) private logger: ILoggerRepository,
) { ) {
this.logger.setContext(DatabaseService.name); this.logger.setContext(DatabaseService.name);
this.vectorExt = this.databaseRepository.getPreferredVectorExtension();
} }
async init() { async init() {
await this.assertPostgresql(); const version = await this.databaseRepository.getPostgresVersion();
const current = semver.coerce(version);
if (!current || !semver.satisfies(current, POSTGRES_VERSION_RANGE)) {
throw new Error(
`Invalid PostgreSQL version. Found ${version}, but needed ${POSTGRES_VERSION_RANGE}. Please use a supported version.`,
);
}
await this.databaseRepository.withLock(DatabaseLock.Migrations, async () => { await this.databaseRepository.withLock(DatabaseLock.Migrations, async () => {
await this.createVectorExtension(); const extension = getVectorExtension();
await this.updateVectorExtension(); const otherExtension =
await this.assertVectorExtension(); extension === DatabaseExtension.VECTORS ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;
const otherName = EXTENSION_NAMES[otherExtension];
const name = EXTENSION_NAMES[extension];
const extensionRange = EXTENSION_RANGES[extension];
try {
await this.databaseRepository.createExtension(extension);
} catch (error) {
this.logger.fatal(messages.createFailed({ name, extension, otherName }));
throw error;
}
const availableVersion = await this.databaseRepository.getAvailableExtensionVersion(extension);
if (availableVersion && semver.satisfies(availableVersion, extensionRange)) {
try {
this.logger.log(`Updating ${name} extension to ${availableVersion}`);
const { restartRequired } = await this.databaseRepository.updateVectorExtension(extension, availableVersion);
if (restartRequired) {
this.logger.warn(messages.restartRequired({ name, availableVersion }));
}
} catch (error) {
this.logger.warn(messages.updateFailed({ name, extension, availableVersion }));
this.logger.error(error);
}
}
const version = await this.databaseRepository.getExtensionVersion(extension);
if (!version) {
throw new Error(messages.notInstalled(name));
}
if (semver.eq(version, '0.0.0')) {
throw new Error(messages.nightlyVersion({ name, extension, version }));
}
if (!semver.satisfies(version, extensionRange)) {
throw new Error(messages.outOfRange({ name, extension, version, range: extensionRange }));
}
try { try {
if (await this.databaseRepository.shouldReindex(VectorIndex.CLIP)) { if (await this.databaseRepository.shouldReindex(VectorIndex.CLIP)) {
@ -54,110 +142,4 @@ export class DatabaseService {
} }
}); });
} }
private async assertPostgresql() {
const { major } = await this.databaseRepository.getPostgresVersion();
if (major < this.minPostgresVersion) {
throw new Error(`
The PostgreSQL version is ${major}, which is older than the minimum supported version ${this.minPostgresVersion}.
Please upgrade to this version or later.`);
}
}
private async createVectorExtension() {
try {
await this.databaseRepository.createExtension(this.vectorExt);
} catch (error) {
const otherExt =
this.vectorExt === DatabaseExtension.VECTORS ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;
this.logger.fatal(`
Failed to activate ${extName[this.vectorExt]} extension.
Please ensure the Postgres instance has ${extName[this.vectorExt]} installed.
If the Postgres instance already has ${extName[this.vectorExt]} installed, Immich may not have the necessary permissions to activate it.
In this case, please run 'CREATE EXTENSION IF NOT EXISTS ${this.vectorExt}' manually as a superuser.
See https://immich.app/docs/guides/database-queries for how to query the database.
Alternatively, if your Postgres instance has ${extName[otherExt]}, you may use this instead by setting the environment variable 'DB_VECTOR_EXTENSION=${extName[otherExt]}'.
Note that switching between the two extensions after a successful startup is not supported.
The exception is if your version of Immich prior to upgrading was 1.90.2 or earlier.
In this case, you may set either extension now, but you will not be able to switch to the other extension following a successful startup.
`);
throw error;
}
}
private async updateVectorExtension() {
const [version, availableVersion] = await Promise.all([
this.databaseRepository.getExtensionVersion(this.vectorExt),
this.databaseRepository.getAvailableExtensionVersion(this.vectorExt),
]);
if (version == null) {
throw new Error(`Unexpected: ${extName[this.vectorExt]} extension is not installed.`);
}
if (availableVersion == null) {
return;
}
const maxVersion = this.vectorExt === DatabaseExtension.VECTOR ? this.vectorVersionPin : this.vectorsVersionPin;
const isNewer = availableVersion.isNewerThan(version);
if (isNewer == null || isNewer > maxVersion) {
return;
}
try {
this.logger.log(`Updating ${extName[this.vectorExt]} extension to ${availableVersion}`);
const { restartRequired } = await this.databaseRepository.updateVectorExtension(this.vectorExt, availableVersion);
if (restartRequired) {
this.logger.warn(`
The ${extName[this.vectorExt]} extension has been updated to ${availableVersion}.
Please restart the Postgres instance to complete the update.`);
}
} catch (error) {
this.logger.warn(`
The ${extName[this.vectorExt]} extension version is ${version}, but ${availableVersion} is available.
Immich attempted to update the extension, but failed to do so.
This may be because Immich does not have the necessary permissions to update the extension.
Please run 'ALTER EXTENSION ${this.vectorExt} UPDATE' manually as a superuser.
See https://immich.app/docs/guides/database-queries for how to query the database.`);
this.logger.error(error);
}
}
private async assertVectorExtension() {
const version = await this.databaseRepository.getExtensionVersion(this.vectorExt);
if (version == null) {
throw new Error(`Unexpected: The ${extName[this.vectorExt]} extension is not installed.`);
}
if (version.isEqual(new Version(0, 0, 0))) {
throw new Error(`
The ${extName[this.vectorExt]} extension version is ${version}, which means it is a nightly release.
Please run 'DROP EXTENSION IF EXISTS ${this.vectorExt}' and switch to a release version.
See https://immich.app/docs/guides/database-queries for how to query the database.`);
}
const minVersion = this.vectorExt === DatabaseExtension.VECTOR ? this.minVectorVersion : this.minVectorsVersion;
const maxVersion = this.vectorExt === DatabaseExtension.VECTOR ? this.vectorVersionPin : this.vectorsVersionPin;
if (version.isOlderThan(minVersion) || version.isNewerThan(minVersion) > maxVersion) {
const allowedReleaseType = maxVersion === VersionType.MAJOR ? '' : ` ${VersionType[maxVersion].toLowerCase()}`;
const releases =
maxVersion === VersionType.EQUAL
? minVersion.toString()
: `${minVersion} and later${allowedReleaseType} releases`;
throw new Error(`
The ${extName[this.vectorExt]} extension version is ${version}, but Immich only supports ${releases}.
If the Postgres instance already has a compatible version installed, Immich may not have the necessary permissions to activate it.
In this case, please run 'ALTER EXTENSION UPDATE ${this.vectorExt}' manually as a superuser.
See https://immich.app/docs/guides/database-queries for how to query the database.
Otherwise, please update the version of ${extName[this.vectorExt]} in the Postgres instance to a compatible version.`);
}
}
} }

View File

@ -14,10 +14,10 @@ import { newServerInfoRepositoryMock } from 'test/repositories/system-info.repos
import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock'; import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock';
import { Mocked } from 'vitest'; import { Mocked } from 'vitest';
const mockRelease = (version = '100.0.0') => ({ const mockRelease = (version: string) => ({
id: 1, id: 1,
url: 'https://api.github.com/repos/owner/repo/releases/1', url: 'https://api.github.com/repos/owner/repo/releases/1',
tag_name: 'v' + version, tag_name: version,
name: 'Release 1000', name: 'Release 1000',
created_at: DateTime.utc().toISO(), created_at: DateTime.utc().toISO(),
published_at: DateTime.utc().toISO(), published_at: DateTime.utc().toISO(),
@ -48,7 +48,11 @@ describe(VersionService.name, () => {
describe('getVersion', () => { describe('getVersion', () => {
it('should respond the server version', () => { it('should respond the server version', () => {
expect(sut.getVersion()).toEqual(serverVersion); expect(sut.getVersion()).toEqual({
major: serverVersion.major,
minor: serverVersion.minor,
patch: serverVersion.patch,
});
}); });
}); });
@ -78,7 +82,7 @@ describe(VersionService.name, () => {
}); });
it('should run if it has been > 60 minutes', async () => { it('should run if it has been > 60 minutes', async () => {
serverMock.getGitHubRelease.mockResolvedValue(mockRelease()); serverMock.getGitHubRelease.mockResolvedValue(mockRelease('v100.0.0'));
systemMock.get.mockResolvedValue({ systemMock.get.mockResolvedValue({
checkedAt: DateTime.utc().minus({ minutes: 65 }).toISO(), checkedAt: DateTime.utc().minus({ minutes: 65 }).toISO(),
releaseVersion: '1.0.0', releaseVersion: '1.0.0',

View File

@ -1,24 +1,23 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { DateTime } from 'luxon'; import { DateTime } from 'luxon';
import semver, { SemVer } from 'semver';
import { isDev, serverVersion } from 'src/constants'; import { isDev, serverVersion } from 'src/constants';
import { SystemConfigCore } from 'src/cores/system-config.core'; import { SystemConfigCore } from 'src/cores/system-config.core';
import { OnServerEvent } from 'src/decorators'; import { OnServerEvent } from 'src/decorators';
import { ReleaseNotification } from 'src/dtos/server-info.dto'; import { ReleaseNotification, ServerVersionResponseDto } from 'src/dtos/server-info.dto';
import { SystemMetadataKey, VersionCheckMetadata } from 'src/entities/system-metadata.entity'; import { SystemMetadataKey, VersionCheckMetadata } from 'src/entities/system-metadata.entity';
import { ClientEvent, IEventRepository, ServerEvent, ServerEventMap } from 'src/interfaces/event.interface'; import { ClientEvent, IEventRepository, ServerEvent, ServerEventMap } from 'src/interfaces/event.interface';
import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface'; import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IServerInfoRepository } from 'src/interfaces/server-info.interface'; import { IServerInfoRepository } from 'src/interfaces/server-info.interface';
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface'; import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
import { Version } from 'src/utils/version';
const asNotification = ({ releaseVersion, checkedAt }: VersionCheckMetadata): ReleaseNotification => { const asNotification = ({ checkedAt, releaseVersion }: VersionCheckMetadata): ReleaseNotification => {
const version = Version.fromString(releaseVersion);
return { return {
isAvailable: version.isNewerThan(serverVersion) !== 0, isAvailable: semver.gt(releaseVersion, serverVersion),
checkedAt, checkedAt,
serverVersion, serverVersion: ServerVersionResponseDto.fromSemVer(serverVersion),
releaseVersion: version, releaseVersion: ServerVersionResponseDto.fromSemVer(new SemVer(releaseVersion)),
}; };
}; };
@ -42,7 +41,7 @@ export class VersionService {
} }
getVersion() { getVersion() {
return serverVersion; return ServerVersionResponseDto.fromSemVer(serverVersion);
} }
async handleQueueVersionCheck() { async handleQueueVersionCheck() {
@ -72,18 +71,13 @@ export class VersionService {
} }
} }
const githubRelease = await this.repository.getGitHubRelease(); const { tag_name: releaseVersion, published_at: publishedAt } = await this.repository.getGitHubRelease();
const githubVersion = Version.fromString(githubRelease.tag_name); const metadata: VersionCheckMetadata = { checkedAt: DateTime.utc().toISO(), releaseVersion };
const metadata: VersionCheckMetadata = {
checkedAt: DateTime.utc().toISO(),
releaseVersion: githubVersion.toString(),
};
await this.systemMetadataRepository.set(SystemMetadataKey.VERSION_CHECK_STATE, metadata); await this.systemMetadataRepository.set(SystemMetadataKey.VERSION_CHECK_STATE, metadata);
if (githubVersion.isNewerThan(serverVersion)) { if (semver.gt(releaseVersion, serverVersion)) {
const publishedAt = new Date(githubRelease.published_at); this.logger.log(`Found ${releaseVersion}, released at ${new Date(publishedAt).toLocaleString()}`);
this.logger.log(`Found ${githubVersion.toString()}, released at ${publishedAt.toLocaleString()}`);
this.eventRepository.clientBroadcast(ClientEvent.NEW_RELEASE, asNotification(metadata)); this.eventRepository.clientBroadcast(ClientEvent.NEW_RELEASE, asNotification(metadata));
} }
} catch (error: Error | any) { } catch (error: Error | any) {

View File

@ -3,8 +3,8 @@ import { OpenAPIObject } from '@nestjs/swagger';
import { SchemaObject } from '@nestjs/swagger/dist/interfaces/open-api-spec.interface'; import { SchemaObject } from '@nestjs/swagger/dist/interfaces/open-api-spec.interface';
import { readFileSync } from 'node:fs'; import { readFileSync } from 'node:fs';
import { resolve } from 'node:path'; import { resolve } from 'node:path';
import { SemVer } from 'semver';
import { ADDED_IN_PREFIX, DEPRECATED_IN_PREFIX, LIFECYCLE_EXTENSION, NEXT_RELEASE } from 'src/constants'; import { ADDED_IN_PREFIX, DEPRECATED_IN_PREFIX, LIFECYCLE_EXTENSION, NEXT_RELEASE } from 'src/constants';
import { Version } from 'src/utils/version';
const outputPath = resolve(process.cwd(), '../open-api/immich-openapi-specs.json'); const outputPath = resolve(process.cwd(), '../open-api/immich-openapi-specs.json');
const spec = JSON.parse(readFileSync(outputPath).toString()) as OpenAPIObject; const spec = JSON.parse(readFileSync(outputPath).toString()) as OpenAPIObject;
@ -69,9 +69,7 @@ const sortedVersions = Object.keys(metadata).sort((a, b) => {
return 1; return 1;
} }
const versionA = Version.fromString(a); return new SemVer(b).compare(new SemVer(a));
const versionB = Version.fromString(b);
return versionB.compareTo(versionA);
}); });
for (const version of sortedVersions) { for (const version of sortedVersions) {

View File

@ -1,72 +0,0 @@
import { Version, VersionType } from 'src/utils/version';
describe('Version', () => {
const tests = [
{ this: new Version(0, 0, 1), other: new Version(0, 0, 0), compare: 1, type: VersionType.PATCH },
{ this: new Version(0, 1, 0), other: new Version(0, 0, 0), compare: 1, type: VersionType.MINOR },
{ this: new Version(1, 0, 0), other: new Version(0, 0, 0), compare: 1, type: VersionType.MAJOR },
{ this: new Version(0, 0, 0), other: new Version(0, 0, 1), compare: -1, type: VersionType.PATCH },
{ this: new Version(0, 0, 0), other: new Version(0, 1, 0), compare: -1, type: VersionType.MINOR },
{ this: new Version(0, 0, 0), other: new Version(1, 0, 0), compare: -1, type: VersionType.MAJOR },
{ this: new Version(0, 0, 0), other: new Version(0, 0, 0), compare: 0, type: VersionType.EQUAL },
{ this: new Version(0, 0, 1), other: new Version(0, 0, 1), compare: 0, type: VersionType.EQUAL },
{ this: new Version(0, 1, 0), other: new Version(0, 1, 0), compare: 0, type: VersionType.EQUAL },
{ this: new Version(1, 0, 0), other: new Version(1, 0, 0), compare: 0, type: VersionType.EQUAL },
{ this: new Version(1, 0), other: new Version(1, 0, 0), compare: 0, type: VersionType.EQUAL },
{ this: new Version(1, 0), other: new Version(1, 0, 1), compare: -1, type: VersionType.PATCH },
{ this: new Version(1, 1), other: new Version(1, 0, 1), compare: 1, type: VersionType.MINOR },
{ this: new Version(1), other: new Version(1, 0, 0), compare: 0, type: VersionType.EQUAL },
{ this: new Version(1), other: new Version(1, 0, 1), compare: -1, type: VersionType.PATCH },
];
describe('isOlderThan', () => {
for (const { this: thisVersion, other: otherVersion, compare, type } of tests) {
const expected = compare < 0 ? type : VersionType.EQUAL;
it(`should return '${expected}' when comparing ${thisVersion} to ${otherVersion}`, () => {
expect(thisVersion.isOlderThan(otherVersion)).toEqual(expected);
});
}
});
describe('isEqual', () => {
for (const { this: thisVersion, other: otherVersion, compare } of tests) {
const bool = compare === 0;
it(`should return ${bool} when comparing ${thisVersion} to ${otherVersion}`, () => {
expect(thisVersion.isEqual(otherVersion)).toEqual(bool);
});
}
});
describe('isNewerThan', () => {
for (const { this: thisVersion, other: otherVersion, compare, type } of tests) {
const expected = compare > 0 ? type : VersionType.EQUAL;
it(`should return ${expected} when comparing ${thisVersion} to ${otherVersion}`, () => {
expect(thisVersion.isNewerThan(otherVersion)).toEqual(expected);
});
}
});
describe('fromString', () => {
const tests = [
{ scenario: 'leading v', value: 'v1.72.2', expected: new Version(1, 72, 2) },
{ scenario: 'uppercase v', value: 'V1.72.2', expected: new Version(1, 72, 2) },
{ scenario: 'missing v', value: '1.72.2', expected: new Version(1, 72, 2) },
{ scenario: 'large patch', value: '1.72.123', expected: new Version(1, 72, 123) },
{ scenario: 'large minor', value: '1.123.0', expected: new Version(1, 123, 0) },
{ scenario: 'large major', value: '123.0.0', expected: new Version(123, 0, 0) },
{ scenario: 'major bump', value: 'v2.0.0', expected: new Version(2, 0, 0) },
{ scenario: 'has dash', value: '14.10-1', expected: new Version(14, 10, 1) },
{ scenario: 'missing patch', value: '14.10', expected: new Version(14, 10, 0) },
{ scenario: 'only major', value: '14', expected: new Version(14, 0, 0) },
];
for (const { scenario, value, expected } of tests) {
it(`should correctly parse ${scenario}`, () => {
const actual = Version.fromString(value);
expect(actual.major).toEqual(expected.major);
expect(actual.minor).toEqual(expected.minor);
expect(actual.patch).toEqual(expected.patch);
});
}
});
});

View File

@ -1,72 +0,0 @@
export type IVersion = { major: number; minor: number; patch: number };
export enum VersionType {
EQUAL = 0,
PATCH = 1,
MINOR = 2,
MAJOR = 3,
}
export class Version implements IVersion {
public readonly types = ['major', 'minor', 'patch'] as const;
constructor(
public major: number,
public minor: number = 0,
public patch: number = 0,
) {}
toString() {
return `${this.major}.${this.minor}.${this.patch}`;
}
toJSON() {
const { major, minor, patch } = this;
return { major, minor, patch };
}
static fromString(version: string): Version {
const regex = /v?(?<major>\d+)(?:\.(?<minor>\d+))?(?:[.-](?<patch>\d+))?/i;
const matchResult = version.match(regex);
if (matchResult) {
const { major, minor = '0', patch = '0' } = matchResult.groups as { [K in keyof IVersion]: string };
return new Version(Number(major), Number(minor), Number(patch));
} else {
throw new Error(`Invalid version format: ${version}`);
}
}
private compare(version: Version): [number, VersionType] {
for (const [i, key] of this.types.entries()) {
const diff = this[key] - version[key];
if (diff !== 0) {
return [diff > 0 ? 1 : -1, (VersionType.MAJOR - i) as VersionType];
}
}
return [0, VersionType.EQUAL];
}
isOlderThan(version: Version): VersionType {
const [bool, type] = this.compare(version);
return bool < 0 ? type : VersionType.EQUAL;
}
isEqual(version: Version): boolean {
const [bool] = this.compare(version);
return bool === 0;
}
isNewerThan(version: Version): VersionType {
const [bool, type] = this.compare(version);
return bool > 0 ? type : VersionType.EQUAL;
}
compareTo(other: Version) {
if (this.isEqual(other)) {
return 0;
}
return this.isNewerThan(other) ? 1 : -1;
}
}

View File

@ -1,14 +1,12 @@
import { IDatabaseRepository } from 'src/interfaces/database.interface'; import { IDatabaseRepository } from 'src/interfaces/database.interface';
import { Version } from 'src/utils/version';
import { Mocked, vitest } from 'vitest'; import { Mocked, vitest } from 'vitest';
export const newDatabaseRepositoryMock = (): Mocked<IDatabaseRepository> => { export const newDatabaseRepositoryMock = (): Mocked<IDatabaseRepository> => {
return { return {
getExtensionVersion: vitest.fn(), getExtensionVersion: vitest.fn(),
getAvailableExtensionVersion: vitest.fn(), getAvailableExtensionVersion: vitest.fn(),
getPreferredVectorExtension: vitest.fn(), getPostgresVersion: vitest.fn().mockResolvedValue('14.10 (Debian 14.10-1.pgdg120+1)'),
getPostgresVersion: vitest.fn().mockResolvedValue(new Version(14, 0, 0)), createExtension: vitest.fn().mockResolvedValue(void 0),
createExtension: vitest.fn().mockImplementation(() => Promise.resolve()),
updateExtension: vitest.fn(), updateExtension: vitest.fn(),
updateVectorExtension: vitest.fn(), updateVectorExtension: vitest.fn(),
reindex: vitest.fn(), reindex: vitest.fn(),