mirror of
https://github.com/immich-app/immich.git
synced 2025-01-29 17:43:42 +02:00
refactor(server): version logic (#9615)
* refactor(server): version * test: better version and log checks
This commit is contained in:
Jason Rasmussen
GitHub
parent
5f25f28c42
commit
1df7be8436
1
server/package-lock.json
generated
1
server/package-lock.json
generated
@ -57,6 +57,7 @@
|
||||
"reflect-metadata": "^0.2.0",
|
||||
"rxjs": "^7.8.1",
|
||||
"sanitize-filename": "^1.6.3",
|
||||
"semver": "^7.6.2",
|
||||
"sharp": "^0.33.0",
|
||||
"sirv": "^2.0.4",
|
||||
"thumbhash": "^0.1.1",
|
||||
|
||||
@ -81,6 +81,7 @@
|
||||
"reflect-metadata": "^0.2.0",
|
||||
"rxjs": "^7.8.1",
|
||||
"sanitize-filename": "^1.6.3",
|
||||
"semver": "^7.6.2",
|
||||
"sharp": "^0.33.0",
|
||||
"sirv": "^2.0.4",
|
||||
"thumbhash": "^0.1.1",
|
||||
|
||||
@ -1,7 +1,11 @@
|
||||
import { Duration } from 'luxon';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { Version } from 'src/utils/version';
|
||||
import { SemVer } from 'semver';
|
||||
|
||||
export const POSTGRES_VERSION_RANGE = '>=14.0.0';
|
||||
export const VECTORS_VERSION_RANGE = '0.2.x';
|
||||
export const VECTOR_VERSION_RANGE = '>=0.5 <1';
|
||||
|
||||
export const NEXT_RELEASE = 'NEXT_RELEASE';
|
||||
export const LIFECYCLE_EXTENSION = 'x-immich-lifecycle';
|
||||
@ -11,7 +15,7 @@ export const ADDED_IN_PREFIX = 'This property was added in ';
|
||||
export const SALT_ROUNDS = 10;
|
||||
|
||||
const { version } = JSON.parse(readFileSync('./package.json', 'utf8'));
|
||||
export const serverVersion = Version.fromString(version);
|
||||
export const serverVersion = new SemVer(version);
|
||||
|
||||
export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 });
|
||||
export const ONE_HOUR = Duration.fromObject({ hours: 1 });
|
||||
|
||||
@ -33,5 +33,5 @@ export const databaseConfig: PostgresConnectionOptions = {
|
||||
*/
|
||||
export const dataSource = new DataSource({ ...databaseConfig, host: 'localhost' });
|
||||
|
||||
export const vectorExt =
|
||||
export const getVectorExtension = () =>
|
||||
process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { ApiProperty, ApiResponseProperty } from '@nestjs/swagger';
|
||||
import { SemVer } from 'semver';
|
||||
import { SystemConfigThemeDto } from 'src/dtos/system-config.dto';
|
||||
import { IVersion } from 'src/utils/version';
|
||||
|
||||
export class ServerPingResponse {
|
||||
@ApiResponseProperty({ type: String, example: 'pong' })
|
||||
@ -25,13 +25,17 @@ export class ServerInfoResponseDto {
|
||||
diskUsagePercentage!: number;
|
||||
}
|
||||
|
||||
export class ServerVersionResponseDto implements IVersion {
|
||||
export class ServerVersionResponseDto {
|
||||
@ApiProperty({ type: 'integer' })
|
||||
major!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
minor!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
patch!: number;
|
||||
|
||||
static fromSemVer(value: SemVer) {
|
||||
return { major: value.major, minor: value.minor, patch: value.patch };
|
||||
}
|
||||
}
|
||||
|
||||
export class UsageByUserDto {
|
||||
|
||||
@ -1,5 +1,3 @@
|
||||
import { Version } from 'src/utils/version';
|
||||
|
||||
export enum DatabaseExtension {
|
||||
CUBE = 'cube',
|
||||
EARTH_DISTANCE = 'earthdistance',
|
||||
@ -23,7 +21,7 @@ export enum DatabaseLock {
|
||||
GetSystemConfig = 69,
|
||||
}
|
||||
|
||||
export const extName: Record<DatabaseExtension, string> = {
|
||||
export const EXTENSION_NAMES: Record<DatabaseExtension, string> = {
|
||||
cube: 'cube',
|
||||
earthdistance: 'earthdistance',
|
||||
vector: 'pgvector',
|
||||
@ -37,13 +35,12 @@ export interface VectorUpdateResult {
|
||||
export const IDatabaseRepository = 'IDatabaseRepository';
|
||||
|
||||
export interface IDatabaseRepository {
|
||||
getExtensionVersion(extensionName: string): Promise<Version | null>;
|
||||
getAvailableExtensionVersion(extension: DatabaseExtension): Promise<Version | null>;
|
||||
getPreferredVectorExtension(): VectorExtension;
|
||||
getPostgresVersion(): Promise<Version>;
|
||||
getExtensionVersion(extensionName: string): Promise<string | undefined>;
|
||||
getAvailableExtensionVersion(extension: DatabaseExtension): Promise<string | undefined>;
|
||||
getPostgresVersion(): Promise<string>;
|
||||
createExtension(extension: DatabaseExtension): Promise<void>;
|
||||
updateExtension(extension: DatabaseExtension, version?: Version): Promise<void>;
|
||||
updateVectorExtension(extension: VectorExtension, version?: Version): Promise<VectorUpdateResult>;
|
||||
updateExtension(extension: DatabaseExtension, version?: string): Promise<void>;
|
||||
updateVectorExtension(extension: VectorExtension, version?: string): Promise<VectorUpdateResult>;
|
||||
reindex(index: VectorIndex): Promise<void>;
|
||||
shouldReindex(name: VectorIndex): Promise<boolean>;
|
||||
runMigrations(options?: { transaction?: 'all' | 'none' | 'each' }): Promise<void>;
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { vectorExt } from 'src/database.config';
|
||||
import { getVectorExtension } from 'src/database.config';
|
||||
import { getCLIPModelInfo } from 'src/utils/misc';
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
@ -7,7 +7,7 @@ export class UsePgVectors1700713871511 implements MigrationInterface {
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
|
||||
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${vectorExt}`);
|
||||
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${getVectorExtension()}`);
|
||||
const faceDimQuery = await queryRunner.query(`
|
||||
SELECT CARDINALITY(embedding::real[]) as dimsize
|
||||
FROM asset_faces
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { vectorExt } from 'src/database.config';
|
||||
import { getVectorExtension } from 'src/database.config';
|
||||
import { DatabaseExtension } from 'src/interfaces/database.interface';
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
@ -6,7 +6,7 @@ export class AddCLIPEmbeddingIndex1700713994428 implements MigrationInterface {
|
||||
name = 'AddCLIPEmbeddingIndex1700713994428';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
if (vectorExt === DatabaseExtension.VECTORS) {
|
||||
if (getVectorExtension() === DatabaseExtension.VECTORS) {
|
||||
await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
|
||||
}
|
||||
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { vectorExt } from 'src/database.config';
|
||||
import { getVectorExtension } from 'src/database.config';
|
||||
import { DatabaseExtension } from 'src/interfaces/database.interface';
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
@ -6,7 +6,7 @@ export class AddFaceEmbeddingIndex1700714033632 implements MigrationInterface {
|
||||
name = 'AddFaceEmbeddingIndex1700714033632';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
if (vectorExt === DatabaseExtension.VECTORS) {
|
||||
if (getVectorExtension() === DatabaseExtension.VECTORS) {
|
||||
await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
|
||||
}
|
||||
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
|
||||
|
||||
@ -1,19 +1,19 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { InjectDataSource } from '@nestjs/typeorm';
|
||||
import AsyncLock from 'async-lock';
|
||||
import { vectorExt } from 'src/database.config';
|
||||
import semver from 'semver';
|
||||
import { getVectorExtension } from 'src/database.config';
|
||||
import {
|
||||
DatabaseExtension,
|
||||
DatabaseLock,
|
||||
EXTENSION_NAMES,
|
||||
IDatabaseRepository,
|
||||
VectorExtension,
|
||||
VectorIndex,
|
||||
VectorUpdateResult,
|
||||
extName,
|
||||
} from 'src/interfaces/database.interface';
|
||||
import { ILoggerRepository } from 'src/interfaces/logger.interface';
|
||||
import { Instrumentation } from 'src/utils/instrumentation';
|
||||
import { Version, VersionType } from 'src/utils/version';
|
||||
import { isValidInteger } from 'src/validation';
|
||||
import { DataSource, EntityManager, QueryRunner } from 'typeorm';
|
||||
|
||||
@ -29,22 +29,12 @@ export class DatabaseRepository implements IDatabaseRepository {
|
||||
this.logger.setContext(DatabaseRepository.name);
|
||||
}
|
||||
|
||||
async getExtensionVersion(extension: DatabaseExtension): Promise<Version | null> {
|
||||
async getExtensionVersion(extension: DatabaseExtension): Promise<string | undefined> {
|
||||
const res = await this.dataSource.query(`SELECT extversion FROM pg_extension WHERE extname = $1`, [extension]);
|
||||
const extVersion = res[0]?.['extversion'];
|
||||
if (extVersion == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const version = Version.fromString(extVersion);
|
||||
if (version.isEqual(new Version(0, 1, 1))) {
|
||||
return new Version(0, 1, 11);
|
||||
}
|
||||
|
||||
return version;
|
||||
return res[0]?.['extversion'];
|
||||
}
|
||||
|
||||
async getAvailableExtensionVersion(extension: DatabaseExtension): Promise<Version | null> {
|
||||
async getAvailableExtensionVersion(extension: DatabaseExtension): Promise<string | undefined> {
|
||||
const res = await this.dataSource.query(
|
||||
`
|
||||
SELECT version FROM pg_available_extension_versions
|
||||
@ -52,45 +42,41 @@ export class DatabaseRepository implements IDatabaseRepository {
|
||||
ORDER BY version DESC`,
|
||||
[extension],
|
||||
);
|
||||
const version = res[0]?.['version'];
|
||||
return version == null ? null : Version.fromString(version);
|
||||
return res[0]?.['version'];
|
||||
}
|
||||
|
||||
getPreferredVectorExtension(): VectorExtension {
|
||||
return vectorExt;
|
||||
}
|
||||
|
||||
async getPostgresVersion(): Promise<Version> {
|
||||
const res = await this.dataSource.query(`SHOW server_version`);
|
||||
return Version.fromString(res[0]['server_version']);
|
||||
async getPostgresVersion(): Promise<string> {
|
||||
const [{ server_version: version }] = await this.dataSource.query(`SHOW server_version`);
|
||||
return version;
|
||||
}
|
||||
|
||||
async createExtension(extension: DatabaseExtension): Promise<void> {
|
||||
await this.dataSource.query(`CREATE EXTENSION IF NOT EXISTS ${extension}`);
|
||||
}
|
||||
|
||||
async updateExtension(extension: DatabaseExtension, version?: Version): Promise<void> {
|
||||
async updateExtension(extension: DatabaseExtension, version?: string): Promise<void> {
|
||||
await this.dataSource.query(`ALTER EXTENSION ${extension} UPDATE${version ? ` TO '${version}'` : ''}`);
|
||||
}
|
||||
|
||||
async updateVectorExtension(extension: VectorExtension, version?: Version): Promise<VectorUpdateResult> {
|
||||
const curVersion = await this.getExtensionVersion(extension);
|
||||
if (!curVersion) {
|
||||
throw new Error(`${extName[extension]} extension is not installed`);
|
||||
async updateVectorExtension(extension: VectorExtension, targetVersion?: string): Promise<VectorUpdateResult> {
|
||||
const currentVersion = await this.getExtensionVersion(extension);
|
||||
if (!currentVersion) {
|
||||
throw new Error(`${EXTENSION_NAMES[extension]} extension is not installed`);
|
||||
}
|
||||
|
||||
const minorOrMajor = version && curVersion.isOlderThan(version) >= VersionType.MINOR;
|
||||
const isVectors = extension === DatabaseExtension.VECTORS;
|
||||
let restartRequired = false;
|
||||
await this.dataSource.manager.transaction(async (manager) => {
|
||||
await this.setSearchPath(manager);
|
||||
if (minorOrMajor && isVectors) {
|
||||
await this.updateVectorsSchema(manager, curVersion);
|
||||
|
||||
const isSchemaUpgrade = targetVersion && semver.satisfies(targetVersion, '0.1.1 || 0.1.11');
|
||||
if (isSchemaUpgrade && isVectors) {
|
||||
await this.updateVectorsSchema(manager, currentVersion);
|
||||
}
|
||||
|
||||
await manager.query(`ALTER EXTENSION ${extension} UPDATE${version ? ` TO '${version}'` : ''}`);
|
||||
await manager.query(`ALTER EXTENSION ${extension} UPDATE${targetVersion ? ` TO '${targetVersion}'` : ''}`);
|
||||
|
||||
if (!minorOrMajor) {
|
||||
if (!isSchemaUpgrade) {
|
||||
return;
|
||||
}
|
||||
|
||||
@ -110,7 +96,7 @@ export class DatabaseRepository implements IDatabaseRepository {
|
||||
try {
|
||||
await this.dataSource.query(`REINDEX INDEX ${index}`);
|
||||
} catch (error) {
|
||||
if (vectorExt === DatabaseExtension.VECTORS) {
|
||||
if (getVectorExtension() === DatabaseExtension.VECTORS) {
|
||||
this.logger.warn(`Could not reindex index ${index}. Attempting to auto-fix.`);
|
||||
const table = index === VectorIndex.CLIP ? 'smart_search' : 'asset_faces';
|
||||
const dimSize = await this.getDimSize(table);
|
||||
@ -132,7 +118,7 @@ export class DatabaseRepository implements IDatabaseRepository {
|
||||
}
|
||||
|
||||
async shouldReindex(name: VectorIndex): Promise<boolean> {
|
||||
if (vectorExt !== DatabaseExtension.VECTORS) {
|
||||
if (getVectorExtension() !== DatabaseExtension.VECTORS) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -160,10 +146,10 @@ export class DatabaseRepository implements IDatabaseRepository {
|
||||
await manager.query(`SET search_path TO "$user", public, vectors`);
|
||||
}
|
||||
|
||||
private async updateVectorsSchema(manager: EntityManager, curVersion: Version): Promise<void> {
|
||||
private async updateVectorsSchema(manager: EntityManager, currentVersion: string): Promise<void> {
|
||||
await manager.query('CREATE SCHEMA IF NOT EXISTS vectors');
|
||||
await manager.query(`UPDATE pg_catalog.pg_extension SET extversion = $1 WHERE extname = $2`, [
|
||||
curVersion.toString(),
|
||||
currentVersion,
|
||||
DatabaseExtension.VECTORS,
|
||||
]);
|
||||
await manager.query('UPDATE pg_catalog.pg_extension SET extrelocatable = true WHERE extname = $1', [
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { vectorExt } from 'src/database.config';
|
||||
import { getVectorExtension } from 'src/database.config';
|
||||
import { DummyValue, GenerateSql } from 'src/decorators';
|
||||
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
|
||||
import { AssetEntity, AssetType } from 'src/entities/asset.entity';
|
||||
@ -336,7 +336,7 @@ export class SearchRepository implements ISearchRepository {
|
||||
}
|
||||
|
||||
private getRuntimeConfig(numResults?: number): string {
|
||||
if (vectorExt === DatabaseExtension.VECTOR) {
|
||||
if (getVectorExtension() === DatabaseExtension.VECTOR) {
|
||||
return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall
|
||||
}
|
||||
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import { DatabaseExtension, IDatabaseRepository, VectorIndex } from 'src/interfaces/database.interface';
|
||||
import { DatabaseExtension, IDatabaseRepository } from 'src/interfaces/database.interface';
|
||||
import { ILoggerRepository } from 'src/interfaces/logger.interface';
|
||||
import { DatabaseService } from 'src/services/database.service';
|
||||
import { Version, VersionType } from 'src/utils/version';
|
||||
import { newDatabaseRepositoryMock } from 'test/repositories/database.repository.mock';
|
||||
import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock';
|
||||
import { Mocked } from 'vitest';
|
||||
@ -13,137 +12,174 @@ describe(DatabaseService.name, () => {
|
||||
|
||||
beforeEach(() => {
|
||||
delete process.env.DB_SKIP_MIGRATIONS;
|
||||
delete process.env.DB_VECTOR_EXTENSION;
|
||||
databaseMock = newDatabaseRepositoryMock();
|
||||
loggerMock = newLoggerRepositoryMock();
|
||||
sut = new DatabaseService(databaseMock, loggerMock);
|
||||
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.2.0');
|
||||
});
|
||||
|
||||
it('should work', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe.each([
|
||||
[{ vectorExt: DatabaseExtension.VECTORS, extName: 'pgvecto.rs', minVersion: new Version(0, 1, 1) }],
|
||||
[{ vectorExt: DatabaseExtension.VECTOR, extName: 'pgvector', minVersion: new Version(0, 5, 0) }],
|
||||
] as const)('init', ({ vectorExt, extName, minVersion }) => {
|
||||
beforeEach(() => {
|
||||
databaseMock.getPreferredVectorExtension.mockReturnValue(vectorExt);
|
||||
databaseMock.getExtensionVersion.mockResolvedValue(minVersion);
|
||||
it('should throw an error if PostgreSQL version is below minimum supported version', async () => {
|
||||
databaseMock.getPostgresVersion.mockResolvedValueOnce('13.10.0');
|
||||
|
||||
sut = new DatabaseService(databaseMock, loggerMock);
|
||||
await expect(sut.init()).rejects.toThrow('Invalid PostgreSQL version. Found 13.10.0');
|
||||
|
||||
sut.minVectorVersion = minVersion;
|
||||
sut.minVectorsVersion = minVersion;
|
||||
sut.vectorVersionPin = VersionType.MINOR;
|
||||
sut.vectorsVersionPin = VersionType.MINOR;
|
||||
});
|
||||
expect(databaseMock.getPostgresVersion).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it(`should resolve successfully if minimum supported PostgreSQL and ${extName} version are installed`, async () => {
|
||||
databaseMock.getPostgresVersion.mockResolvedValueOnce(new Version(14, 0, 0));
|
||||
it(`should start up successfully with pgvectors`, async () => {
|
||||
databaseMock.getPostgresVersion.mockResolvedValue('14.0.0');
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.getPostgresVersion).toHaveBeenCalled();
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledWith(DatabaseExtension.VECTORS);
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.getExtensionVersion).toHaveBeenCalled();
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should start up successfully with pgvector`, async () => {
|
||||
process.env.DB_VECTOR_EXTENSION = 'pgvector';
|
||||
databaseMock.getPostgresVersion.mockResolvedValue('14.0.0');
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.5.0');
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledWith(DatabaseExtension.VECTOR);
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw an error if the pgvecto.rs extension is not installed`, async () => {
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('');
|
||||
await expect(sut.init()).rejects.toThrow(`Unexpected: The pgvecto.rs extension is not installed.`);
|
||||
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw an error if the pgvector extension is not installed`, async () => {
|
||||
process.env.DB_VECTOR_EXTENSION = 'pgvector';
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('');
|
||||
await expect(sut.init()).rejects.toThrow(`Unexpected: The pgvector extension is not installed.`);
|
||||
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw an error if the pgvecto.rs extension version is below minimum supported version`, async () => {
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.1.0');
|
||||
|
||||
await expect(sut.init()).rejects.toThrow(
|
||||
'The pgvecto.rs extension version is 0.1.0, but Immich only supports 0.2.x.',
|
||||
);
|
||||
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw an error if the pgvector extension version is below minimum supported version`, async () => {
|
||||
process.env.DB_VECTOR_EXTENSION = 'pgvector';
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.1.0');
|
||||
|
||||
await expect(sut.init()).rejects.toThrow(
|
||||
'The pgvector extension version is 0.1.0, but Immich only supports >=0.5 <1',
|
||||
);
|
||||
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw an error if pgvecto.rs extension version is a nightly`, async () => {
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.0.0');
|
||||
|
||||
await expect(sut.init()).rejects.toThrow(
|
||||
'The pgvecto.rs extension version is 0.0.0, which means it is a nightly release.',
|
||||
);
|
||||
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw an error if pgvector extension version is a nightly`, async () => {
|
||||
process.env.DB_VECTOR_EXTENSION = 'pgvector';
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.0.0');
|
||||
|
||||
await expect(sut.init()).rejects.toThrow(
|
||||
'The pgvector extension version is 0.0.0, which means it is a nightly release.',
|
||||
);
|
||||
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw error if pgvecto.rs extension could not be created`, async () => {
|
||||
databaseMock.createExtension.mockRejectedValue(new Error('Failed to create extension'));
|
||||
|
||||
await expect(sut.init()).rejects.toThrow('Failed to create extension');
|
||||
|
||||
expect(loggerMock.fatal).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal.mock.calls[0][0]).toContain(
|
||||
'Alternatively, if your Postgres instance has pgvector, you may use this instead',
|
||||
);
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw error if pgvector extension could not be created`, async () => {
|
||||
process.env.DB_VECTOR_EXTENSION = 'pgvector';
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.0.0');
|
||||
databaseMock.createExtension.mockRejectedValue(new Error('Failed to create extension'));
|
||||
|
||||
await expect(sut.init()).rejects.toThrow('Failed to create extension');
|
||||
|
||||
expect(loggerMock.fatal).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal.mock.calls[0][0]).toContain(
|
||||
'Alternatively, if your Postgres instance has pgvecto.rs, you may use this instead',
|
||||
);
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
for (const version of ['0.2.1', '0.2.0', '0.2.9']) {
|
||||
it(`should update the pgvecto.rs extension to ${version}`, async () => {
|
||||
databaseMock.getAvailableExtensionVersion.mockResolvedValue(version);
|
||||
databaseMock.getExtensionVersion.mockResolvedValue(version);
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.getPostgresVersion).toHaveBeenCalled();
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledWith(vectorExt);
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vectors', version);
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.getExtensionVersion).toHaveBeenCalled();
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
}
|
||||
|
||||
it('should throw an error if PostgreSQL version is below minimum supported version', async () => {
|
||||
databaseMock.getPostgresVersion.mockResolvedValueOnce(new Version(13, 0, 0));
|
||||
|
||||
await expect(sut.init()).rejects.toThrow('PostgreSQL version is 13');
|
||||
|
||||
expect(databaseMock.getPostgresVersion).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it(`should resolve successfully if minimum supported ${extName} version is installed`, async () => {
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledWith(vectorExt);
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw an error if ${extName} version is not installed even after createVectorExtension`, async () => {
|
||||
databaseMock.getExtensionVersion.mockResolvedValue(null);
|
||||
|
||||
await expect(sut.init()).rejects.toThrow(`Unexpected: ${extName} extension is not installed.`);
|
||||
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw an error if ${extName} version is below minimum supported version`, async () => {
|
||||
databaseMock.getExtensionVersion.mockResolvedValue(
|
||||
new Version(minVersion.major, minVersion.minor - 1, minVersion.patch),
|
||||
);
|
||||
|
||||
await expect(sut.init()).rejects.toThrow(extName);
|
||||
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ type: VersionType.EQUAL, max: 'no', actual: 'patch' },
|
||||
{ type: VersionType.PATCH, max: 'patch', actual: 'minor' },
|
||||
{ type: VersionType.MINOR, max: 'minor', actual: 'major' },
|
||||
] as const)(
|
||||
`should throw an error if $max upgrade from min version is allowed and ${extName} version is $actual`,
|
||||
async ({ type, actual }) => {
|
||||
const version = new Version(minVersion.major, minVersion.minor, minVersion.patch);
|
||||
version[actual] = minVersion[actual] + 1;
|
||||
databaseMock.getExtensionVersion.mockResolvedValue(version);
|
||||
if (vectorExt === DatabaseExtension.VECTOR) {
|
||||
sut.minVectorVersion = minVersion;
|
||||
sut.vectorVersionPin = type;
|
||||
} else {
|
||||
sut.minVectorsVersion = minVersion;
|
||||
sut.vectorsVersionPin = type;
|
||||
}
|
||||
|
||||
await expect(sut.init()).rejects.toThrow(extName);
|
||||
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it(`should throw an error if ${extName} version is a nightly`, async () => {
|
||||
databaseMock.getExtensionVersion.mockResolvedValue(new Version(0, 0, 0));
|
||||
|
||||
await expect(sut.init()).rejects.toThrow(extName);
|
||||
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should throw error if ${extName} extension could not be created`, async () => {
|
||||
databaseMock.createExtension.mockRejectedValue(new Error('Failed to create extension'));
|
||||
|
||||
await expect(sut.init()).rejects.toThrow('Failed to create extension');
|
||||
|
||||
expect(loggerMock.fatal).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.createExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should update ${extName} if a newer version is available`, async () => {
|
||||
const version = new Version(minVersion.major, minVersion.minor + 1, minVersion.patch);
|
||||
for (const version of ['0.5.1', '0.6.0', '0.7.10']) {
|
||||
it(`should update the pgvectors extension to ${version}`, async () => {
|
||||
process.env.DB_VECTOR_EXTENSION = 'pgvector';
|
||||
databaseMock.getAvailableExtensionVersion.mockResolvedValue(version);
|
||||
databaseMock.getExtensionVersion.mockResolvedValue(version);
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith(vectorExt, version);
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vector', version);
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.getExtensionVersion).toHaveBeenCalled();
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
}
|
||||
|
||||
it(`should not update ${extName} if a newer version is higher than the maximum`, async () => {
|
||||
const version = new Version(minVersion.major + 1, minVersion.minor, minVersion.patch);
|
||||
for (const version of ['0.1.0', '0.3.0', '1.0.0']) {
|
||||
it(`should not upgrade pgvecto.rs to ${version}`, async () => {
|
||||
databaseMock.getAvailableExtensionVersion.mockResolvedValue(version);
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
@ -152,72 +188,106 @@ describe(DatabaseService.name, () => {
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
}
|
||||
|
||||
it(`should warn if attempted to update ${extName} and failed`, async () => {
|
||||
const version = new Version(minVersion.major, minVersion.minor, minVersion.patch + 1);
|
||||
for (const version of ['0.4.0', '1.0.0']) {
|
||||
it(`should not upgrade pgvector to ${version}`, async () => {
|
||||
process.env.DB_VECTOR_EXTENSION = 'pgvector';
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.5.0');
|
||||
databaseMock.getAvailableExtensionVersion.mockResolvedValue(version);
|
||||
databaseMock.updateVectorExtension.mockRejectedValue(new Error('Failed to update extension'));
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(loggerMock.warn).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.warn.mock.calls[0][0]).toContain(extName);
|
||||
expect(loggerMock.error).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith(vectorExt, version);
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it(`should warn if ${extName} update requires restart`, async () => {
|
||||
const version = new Version(minVersion.major, minVersion.minor, minVersion.patch + 1);
|
||||
databaseMock.getAvailableExtensionVersion.mockResolvedValue(version);
|
||||
databaseMock.updateVectorExtension.mockResolvedValue({ restartRequired: true });
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(loggerMock.warn).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.warn.mock.calls[0][0]).toContain(extName);
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith(vectorExt, version);
|
||||
expect(databaseMock.updateVectorExtension).not.toHaveBeenCalled();
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
}
|
||||
|
||||
it.each([{ index: VectorIndex.CLIP }, { index: VectorIndex.FACE }])(
|
||||
`should reindex $index if necessary`,
|
||||
async ({ index }) => {
|
||||
databaseMock.shouldReindex.mockImplementation((indexArg) => Promise.resolve(indexArg === index));
|
||||
it(`should warn if the pgvecto.rs extension upgrade failed`, async () => {
|
||||
process.env.DB_VECTOR_EXTENSION = 'pgvector';
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.5.0');
|
||||
databaseMock.getAvailableExtensionVersion.mockResolvedValue('0.5.2');
|
||||
databaseMock.updateVectorExtension.mockRejectedValue(new Error('Failed to update extension'));
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.shouldReindex).toHaveBeenCalledWith(index);
|
||||
expect(databaseMock.shouldReindex).toHaveBeenCalledTimes(2);
|
||||
expect(databaseMock.reindex).toHaveBeenCalledWith(index);
|
||||
expect(databaseMock.reindex).toHaveBeenCalledTimes(1);
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
expect(loggerMock.warn.mock.calls[0][0]).toContain('The pgvector extension can be updated to 0.5.2.');
|
||||
expect(loggerMock.error).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vector', '0.5.2');
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([{ index: VectorIndex.CLIP }, { index: VectorIndex.FACE }])(
|
||||
`should not reindex $index if not necessary`,
|
||||
async () => {
|
||||
databaseMock.shouldReindex.mockResolvedValue(false);
|
||||
it(`should warn if the pgvector extension upgrade failed`, async () => {
|
||||
databaseMock.getAvailableExtensionVersion.mockResolvedValue('0.2.1');
|
||||
databaseMock.updateVectorExtension.mockRejectedValue(new Error('Failed to update extension'));
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.shouldReindex).toHaveBeenCalledTimes(2);
|
||||
expect(databaseMock.reindex).not.toHaveBeenCalled();
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
expect(loggerMock.warn.mock.calls[0][0]).toContain('The pgvecto.rs extension can be updated to 0.2.1.');
|
||||
expect(loggerMock.error).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vectors', '0.2.1');
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should skip migrations if DB_SKIP_MIGRATIONS=true', async () => {
|
||||
process.env.DB_SKIP_MIGRATIONS = 'true';
|
||||
it(`should warn if the pgvecto.rs extension update requires restart`, async () => {
|
||||
databaseMock.getAvailableExtensionVersion.mockResolvedValue('0.2.1');
|
||||
databaseMock.updateVectorExtension.mockResolvedValue({ restartRequired: true });
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
expect(loggerMock.warn).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.warn.mock.calls[0][0]).toContain('pgvecto.rs');
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vectors', '0.2.1');
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`should warn if the pgvector extension update requires restart`, async () => {
|
||||
process.env.DB_VECTOR_EXTENSION = 'pgvector';
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.5.0');
|
||||
databaseMock.getAvailableExtensionVersion.mockResolvedValue('0.5.1');
|
||||
databaseMock.updateVectorExtension.mockResolvedValue({ restartRequired: true });
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(loggerMock.warn).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.warn.mock.calls[0][0]).toContain('pgvector');
|
||||
expect(databaseMock.updateVectorExtension).toHaveBeenCalledWith('vector', '0.5.1');
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should reindex if needed', async () => {
|
||||
databaseMock.shouldReindex.mockResolvedValue(true);
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.shouldReindex).toHaveBeenCalledTimes(2);
|
||||
expect(databaseMock.reindex).toHaveBeenCalledTimes(2);
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should not reindex if not needed', async () => {
|
||||
databaseMock.shouldReindex.mockResolvedValue(false);
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.shouldReindex).toHaveBeenCalledTimes(2);
|
||||
expect(databaseMock.reindex).toHaveBeenCalledTimes(0);
|
||||
expect(databaseMock.runMigrations).toHaveBeenCalledTimes(1);
|
||||
expect(loggerMock.fatal).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should skip migrations if DB_SKIP_MIGRATIONS=true', async () => {
|
||||
process.env.DB_SKIP_MIGRATIONS = 'true';
|
||||
databaseMock.getExtensionVersion.mockResolvedValue('0.2.0');
|
||||
|
||||
await expect(sut.init()).resolves.toBeUndefined();
|
||||
|
||||
expect(databaseMock.runMigrations).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,38 +1,126 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import semver from 'semver';
|
||||
import { POSTGRES_VERSION_RANGE, VECTORS_VERSION_RANGE, VECTOR_VERSION_RANGE } from 'src/constants';
|
||||
import { getVectorExtension } from 'src/database.config';
|
||||
import {
|
||||
DatabaseExtension,
|
||||
DatabaseLock,
|
||||
EXTENSION_NAMES,
|
||||
IDatabaseRepository,
|
||||
VectorExtension,
|
||||
VectorIndex,
|
||||
extName,
|
||||
} from 'src/interfaces/database.interface';
|
||||
import { ILoggerRepository } from 'src/interfaces/logger.interface';
|
||||
import { Version, VersionType } from 'src/utils/version';
|
||||
|
||||
type CreateFailedArgs = { name: string; extension: string; otherName: string };
|
||||
type UpdateFailedArgs = { name: string; extension: string; availableVersion: string };
|
||||
type RestartRequiredArgs = { name: string; availableVersion: string };
|
||||
type NightlyVersionArgs = { name: string; extension: string; version: string };
|
||||
type OutOfRangeArgs = { name: string; extension: string; version: string; range: string };
|
||||
|
||||
const EXTENSION_RANGES = {
|
||||
[DatabaseExtension.VECTOR]: VECTOR_VERSION_RANGE,
|
||||
[DatabaseExtension.VECTORS]: VECTORS_VERSION_RANGE,
|
||||
};
|
||||
|
||||
const messages = {
|
||||
notInstalled: (name: string) => `Unexpected: The ${name} extension is not installed.`,
|
||||
nightlyVersion: ({ name, extension, version }: NightlyVersionArgs) => `
|
||||
The ${name} extension version is ${version}, which means it is a nightly release.
|
||||
|
||||
Please run 'DROP EXTENSION IF EXISTS ${extension}' and switch to a release version.
|
||||
See https://immich.app/docs/guides/database-queries for how to query the database.`,
|
||||
outOfRange: ({ name, extension, version, range }: OutOfRangeArgs) => `
|
||||
The ${name} extension version is ${version}, but Immich only supports ${range}.
|
||||
|
||||
If the Postgres instance already has a compatible version installed, Immich may not have the necessary permissions to activate it.
|
||||
In this case, please run 'ALTER EXTENSION UPDATE ${extension}' manually as a superuser.
|
||||
See https://immich.app/docs/guides/database-queries for how to query the database.
|
||||
|
||||
Otherwise, please update the version of ${name} in the Postgres instance to a compatible version.`,
|
||||
createFailed: ({ name, extension, otherName }: CreateFailedArgs) => `
|
||||
Failed to activate ${name} extension.
|
||||
Please ensure the Postgres instance has ${name} installed.
|
||||
|
||||
If the Postgres instance already has ${name} installed, Immich may not have the necessary permissions to activate it.
|
||||
In this case, please run 'CREATE EXTENSION IF NOT EXISTS ${extension}' manually as a superuser.
|
||||
See https://immich.app/docs/guides/database-queries for how to query the database.
|
||||
|
||||
Alternatively, if your Postgres instance has ${otherName}, you may use this instead by setting the environment variable 'DB_VECTOR_EXTENSION=${otherName}'.
|
||||
Note that switching between the two extensions after a successful startup is not supported.
|
||||
The exception is if your version of Immich prior to upgrading was 1.90.2 or earlier.
|
||||
In this case, you may set either extension now, but you will not be able to switch to the other extension following a successful startup.
|
||||
`,
|
||||
updateFailed: ({ name, extension, availableVersion }: UpdateFailedArgs) => `
|
||||
The ${name} extension can be updated to ${availableVersion}.
|
||||
Immich attempted to update the extension, but failed to do so.
|
||||
This may be because Immich does not have the necessary permissions to update the extension.
|
||||
|
||||
Please run 'ALTER EXTENSION ${extension} UPDATE' manually as a superuser.
|
||||
See https://immich.app/docs/guides/database-queries for how to query the database.`,
|
||||
restartRequired: ({ name, availableVersion }: RestartRequiredArgs) => `
|
||||
The ${name} extension has been updated to ${availableVersion}.
|
||||
Please restart the Postgres instance to complete the update.`,
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class DatabaseService {
|
||||
private vectorExt: VectorExtension;
|
||||
minPostgresVersion = 14;
|
||||
minVectorsVersion = new Version(0, 2, 0);
|
||||
vectorsVersionPin = VersionType.MINOR;
|
||||
minVectorVersion = new Version(0, 5, 0);
|
||||
vectorVersionPin = VersionType.MAJOR;
|
||||
|
||||
constructor(
|
||||
@Inject(IDatabaseRepository) private databaseRepository: IDatabaseRepository,
|
||||
@Inject(ILoggerRepository) private logger: ILoggerRepository,
|
||||
) {
|
||||
this.logger.setContext(DatabaseService.name);
|
||||
this.vectorExt = this.databaseRepository.getPreferredVectorExtension();
|
||||
}
|
||||
|
||||
async init() {
|
||||
await this.assertPostgresql();
|
||||
const version = await this.databaseRepository.getPostgresVersion();
|
||||
const current = semver.coerce(version);
|
||||
if (!current || !semver.satisfies(current, POSTGRES_VERSION_RANGE)) {
|
||||
throw new Error(
|
||||
`Invalid PostgreSQL version. Found ${version}, but needed ${POSTGRES_VERSION_RANGE}. Please use a supported version.`,
|
||||
);
|
||||
}
|
||||
|
||||
await this.databaseRepository.withLock(DatabaseLock.Migrations, async () => {
|
||||
await this.createVectorExtension();
|
||||
await this.updateVectorExtension();
|
||||
await this.assertVectorExtension();
|
||||
const extension = getVectorExtension();
|
||||
const otherExtension =
|
||||
extension === DatabaseExtension.VECTORS ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;
|
||||
const otherName = EXTENSION_NAMES[otherExtension];
|
||||
const name = EXTENSION_NAMES[extension];
|
||||
const extensionRange = EXTENSION_RANGES[extension];
|
||||
|
||||
try {
|
||||
await this.databaseRepository.createExtension(extension);
|
||||
} catch (error) {
|
||||
this.logger.fatal(messages.createFailed({ name, extension, otherName }));
|
||||
throw error;
|
||||
}
|
||||
|
||||
const availableVersion = await this.databaseRepository.getAvailableExtensionVersion(extension);
|
||||
if (availableVersion && semver.satisfies(availableVersion, extensionRange)) {
|
||||
try {
|
||||
this.logger.log(`Updating ${name} extension to ${availableVersion}`);
|
||||
const { restartRequired } = await this.databaseRepository.updateVectorExtension(extension, availableVersion);
|
||||
if (restartRequired) {
|
||||
this.logger.warn(messages.restartRequired({ name, availableVersion }));
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(messages.updateFailed({ name, extension, availableVersion }));
|
||||
this.logger.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
const version = await this.databaseRepository.getExtensionVersion(extension);
|
||||
if (!version) {
|
||||
throw new Error(messages.notInstalled(name));
|
||||
}
|
||||
|
||||
if (semver.eq(version, '0.0.0')) {
|
||||
throw new Error(messages.nightlyVersion({ name, extension, version }));
|
||||
}
|
||||
|
||||
if (!semver.satisfies(version, extensionRange)) {
|
||||
throw new Error(messages.outOfRange({ name, extension, version, range: extensionRange }));
|
||||
}
|
||||
|
||||
try {
|
||||
if (await this.databaseRepository.shouldReindex(VectorIndex.CLIP)) {
|
||||
@ -54,110 +142,4 @@ export class DatabaseService {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async assertPostgresql() {
|
||||
const { major } = await this.databaseRepository.getPostgresVersion();
|
||||
if (major < this.minPostgresVersion) {
|
||||
throw new Error(`
|
||||
The PostgreSQL version is ${major}, which is older than the minimum supported version ${this.minPostgresVersion}.
|
||||
Please upgrade to this version or later.`);
|
||||
}
|
||||
}
|
||||
|
||||
private async createVectorExtension() {
|
||||
try {
|
||||
await this.databaseRepository.createExtension(this.vectorExt);
|
||||
} catch (error) {
|
||||
const otherExt =
|
||||
this.vectorExt === DatabaseExtension.VECTORS ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;
|
||||
this.logger.fatal(`
|
||||
Failed to activate ${extName[this.vectorExt]} extension.
|
||||
Please ensure the Postgres instance has ${extName[this.vectorExt]} installed.
|
||||
|
||||
If the Postgres instance already has ${extName[this.vectorExt]} installed, Immich may not have the necessary permissions to activate it.
|
||||
In this case, please run 'CREATE EXTENSION IF NOT EXISTS ${this.vectorExt}' manually as a superuser.
|
||||
See https://immich.app/docs/guides/database-queries for how to query the database.
|
||||
|
||||
Alternatively, if your Postgres instance has ${extName[otherExt]}, you may use this instead by setting the environment variable 'DB_VECTOR_EXTENSION=${extName[otherExt]}'.
|
||||
Note that switching between the two extensions after a successful startup is not supported.
|
||||
The exception is if your version of Immich prior to upgrading was 1.90.2 or earlier.
|
||||
In this case, you may set either extension now, but you will not be able to switch to the other extension following a successful startup.
|
||||
`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async updateVectorExtension() {
|
||||
const [version, availableVersion] = await Promise.all([
|
||||
this.databaseRepository.getExtensionVersion(this.vectorExt),
|
||||
this.databaseRepository.getAvailableExtensionVersion(this.vectorExt),
|
||||
]);
|
||||
if (version == null) {
|
||||
throw new Error(`Unexpected: ${extName[this.vectorExt]} extension is not installed.`);
|
||||
}
|
||||
|
||||
if (availableVersion == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const maxVersion = this.vectorExt === DatabaseExtension.VECTOR ? this.vectorVersionPin : this.vectorsVersionPin;
|
||||
const isNewer = availableVersion.isNewerThan(version);
|
||||
if (isNewer == null || isNewer > maxVersion) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
this.logger.log(`Updating ${extName[this.vectorExt]} extension to ${availableVersion}`);
|
||||
const { restartRequired } = await this.databaseRepository.updateVectorExtension(this.vectorExt, availableVersion);
|
||||
if (restartRequired) {
|
||||
this.logger.warn(`
|
||||
The ${extName[this.vectorExt]} extension has been updated to ${availableVersion}.
|
||||
Please restart the Postgres instance to complete the update.`);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.warn(`
|
||||
The ${extName[this.vectorExt]} extension version is ${version}, but ${availableVersion} is available.
|
||||
Immich attempted to update the extension, but failed to do so.
|
||||
This may be because Immich does not have the necessary permissions to update the extension.
|
||||
|
||||
Please run 'ALTER EXTENSION ${this.vectorExt} UPDATE' manually as a superuser.
|
||||
See https://immich.app/docs/guides/database-queries for how to query the database.`);
|
||||
this.logger.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
private async assertVectorExtension() {
|
||||
const version = await this.databaseRepository.getExtensionVersion(this.vectorExt);
|
||||
if (version == null) {
|
||||
throw new Error(`Unexpected: The ${extName[this.vectorExt]} extension is not installed.`);
|
||||
}
|
||||
|
||||
if (version.isEqual(new Version(0, 0, 0))) {
|
||||
throw new Error(`
|
||||
The ${extName[this.vectorExt]} extension version is ${version}, which means it is a nightly release.
|
||||
|
||||
Please run 'DROP EXTENSION IF EXISTS ${this.vectorExt}' and switch to a release version.
|
||||
See https://immich.app/docs/guides/database-queries for how to query the database.`);
|
||||
}
|
||||
|
||||
const minVersion = this.vectorExt === DatabaseExtension.VECTOR ? this.minVectorVersion : this.minVectorsVersion;
|
||||
const maxVersion = this.vectorExt === DatabaseExtension.VECTOR ? this.vectorVersionPin : this.vectorsVersionPin;
|
||||
|
||||
if (version.isOlderThan(minVersion) || version.isNewerThan(minVersion) > maxVersion) {
|
||||
const allowedReleaseType = maxVersion === VersionType.MAJOR ? '' : ` ${VersionType[maxVersion].toLowerCase()}`;
|
||||
const releases =
|
||||
maxVersion === VersionType.EQUAL
|
||||
? minVersion.toString()
|
||||
: `${minVersion} and later${allowedReleaseType} releases`;
|
||||
|
||||
throw new Error(`
|
||||
The ${extName[this.vectorExt]} extension version is ${version}, but Immich only supports ${releases}.
|
||||
|
||||
If the Postgres instance already has a compatible version installed, Immich may not have the necessary permissions to activate it.
|
||||
In this case, please run 'ALTER EXTENSION UPDATE ${this.vectorExt}' manually as a superuser.
|
||||
See https://immich.app/docs/guides/database-queries for how to query the database.
|
||||
|
||||
Otherwise, please update the version of ${extName[this.vectorExt]} in the Postgres instance to a compatible version.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -14,10 +14,10 @@ import { newServerInfoRepositoryMock } from 'test/repositories/system-info.repos
|
||||
import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock';
|
||||
import { Mocked } from 'vitest';
|
||||
|
||||
const mockRelease = (version = '100.0.0') => ({
|
||||
const mockRelease = (version: string) => ({
|
||||
id: 1,
|
||||
url: 'https://api.github.com/repos/owner/repo/releases/1',
|
||||
tag_name: 'v' + version,
|
||||
tag_name: version,
|
||||
name: 'Release 1000',
|
||||
created_at: DateTime.utc().toISO(),
|
||||
published_at: DateTime.utc().toISO(),
|
||||
@ -48,7 +48,11 @@ describe(VersionService.name, () => {
|
||||
|
||||
describe('getVersion', () => {
|
||||
it('should respond the server version', () => {
|
||||
expect(sut.getVersion()).toEqual(serverVersion);
|
||||
expect(sut.getVersion()).toEqual({
|
||||
major: serverVersion.major,
|
||||
minor: serverVersion.minor,
|
||||
patch: serverVersion.patch,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -78,7 +82,7 @@ describe(VersionService.name, () => {
|
||||
});
|
||||
|
||||
it('should run if it has been > 60 minutes', async () => {
|
||||
serverMock.getGitHubRelease.mockResolvedValue(mockRelease());
|
||||
serverMock.getGitHubRelease.mockResolvedValue(mockRelease('v100.0.0'));
|
||||
systemMock.get.mockResolvedValue({
|
||||
checkedAt: DateTime.utc().minus({ minutes: 65 }).toISO(),
|
||||
releaseVersion: '1.0.0',
|
||||
|
||||
@ -1,24 +1,23 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { DateTime } from 'luxon';
|
||||
import semver, { SemVer } from 'semver';
|
||||
import { isDev, serverVersion } from 'src/constants';
|
||||
import { SystemConfigCore } from 'src/cores/system-config.core';
|
||||
import { OnServerEvent } from 'src/decorators';
|
||||
import { ReleaseNotification } from 'src/dtos/server-info.dto';
|
||||
import { ReleaseNotification, ServerVersionResponseDto } from 'src/dtos/server-info.dto';
|
||||
import { SystemMetadataKey, VersionCheckMetadata } from 'src/entities/system-metadata.entity';
|
||||
import { ClientEvent, IEventRepository, ServerEvent, ServerEventMap } from 'src/interfaces/event.interface';
|
||||
import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface';
|
||||
import { ILoggerRepository } from 'src/interfaces/logger.interface';
|
||||
import { IServerInfoRepository } from 'src/interfaces/server-info.interface';
|
||||
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
|
||||
import { Version } from 'src/utils/version';
|
||||
|
||||
const asNotification = ({ releaseVersion, checkedAt }: VersionCheckMetadata): ReleaseNotification => {
|
||||
const version = Version.fromString(releaseVersion);
|
||||
const asNotification = ({ checkedAt, releaseVersion }: VersionCheckMetadata): ReleaseNotification => {
|
||||
return {
|
||||
isAvailable: version.isNewerThan(serverVersion) !== 0,
|
||||
isAvailable: semver.gt(releaseVersion, serverVersion),
|
||||
checkedAt,
|
||||
serverVersion,
|
||||
releaseVersion: version,
|
||||
serverVersion: ServerVersionResponseDto.fromSemVer(serverVersion),
|
||||
releaseVersion: ServerVersionResponseDto.fromSemVer(new SemVer(releaseVersion)),
|
||||
};
|
||||
};
|
||||
|
||||
@ -42,7 +41,7 @@ export class VersionService {
|
||||
}
|
||||
|
||||
getVersion() {
|
||||
return serverVersion;
|
||||
return ServerVersionResponseDto.fromSemVer(serverVersion);
|
||||
}
|
||||
|
||||
async handleQueueVersionCheck() {
|
||||
@ -72,18 +71,13 @@ export class VersionService {
|
||||
}
|
||||
}
|
||||
|
||||
const githubRelease = await this.repository.getGitHubRelease();
|
||||
const githubVersion = Version.fromString(githubRelease.tag_name);
|
||||
const metadata: VersionCheckMetadata = {
|
||||
checkedAt: DateTime.utc().toISO(),
|
||||
releaseVersion: githubVersion.toString(),
|
||||
};
|
||||
const { tag_name: releaseVersion, published_at: publishedAt } = await this.repository.getGitHubRelease();
|
||||
const metadata: VersionCheckMetadata = { checkedAt: DateTime.utc().toISO(), releaseVersion };
|
||||
|
||||
await this.systemMetadataRepository.set(SystemMetadataKey.VERSION_CHECK_STATE, metadata);
|
||||
|
||||
if (githubVersion.isNewerThan(serverVersion)) {
|
||||
const publishedAt = new Date(githubRelease.published_at);
|
||||
this.logger.log(`Found ${githubVersion.toString()}, released at ${publishedAt.toLocaleString()}`);
|
||||
if (semver.gt(releaseVersion, serverVersion)) {
|
||||
this.logger.log(`Found ${releaseVersion}, released at ${new Date(publishedAt).toLocaleString()}`);
|
||||
this.eventRepository.clientBroadcast(ClientEvent.NEW_RELEASE, asNotification(metadata));
|
||||
}
|
||||
} catch (error: Error | any) {
|
||||
|
||||
@ -3,8 +3,8 @@ import { OpenAPIObject } from '@nestjs/swagger';
|
||||
import { SchemaObject } from '@nestjs/swagger/dist/interfaces/open-api-spec.interface';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { SemVer } from 'semver';
|
||||
import { ADDED_IN_PREFIX, DEPRECATED_IN_PREFIX, LIFECYCLE_EXTENSION, NEXT_RELEASE } from 'src/constants';
|
||||
import { Version } from 'src/utils/version';
|
||||
|
||||
const outputPath = resolve(process.cwd(), '../open-api/immich-openapi-specs.json');
|
||||
const spec = JSON.parse(readFileSync(outputPath).toString()) as OpenAPIObject;
|
||||
@ -69,9 +69,7 @@ const sortedVersions = Object.keys(metadata).sort((a, b) => {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const versionA = Version.fromString(a);
|
||||
const versionB = Version.fromString(b);
|
||||
return versionB.compareTo(versionA);
|
||||
return new SemVer(b).compare(new SemVer(a));
|
||||
});
|
||||
|
||||
for (const version of sortedVersions) {
|
||||
|
||||
@ -1,72 +0,0 @@
|
||||
import { Version, VersionType } from 'src/utils/version';
|
||||
|
||||
describe('Version', () => {
|
||||
const tests = [
|
||||
{ this: new Version(0, 0, 1), other: new Version(0, 0, 0), compare: 1, type: VersionType.PATCH },
|
||||
{ this: new Version(0, 1, 0), other: new Version(0, 0, 0), compare: 1, type: VersionType.MINOR },
|
||||
{ this: new Version(1, 0, 0), other: new Version(0, 0, 0), compare: 1, type: VersionType.MAJOR },
|
||||
{ this: new Version(0, 0, 0), other: new Version(0, 0, 1), compare: -1, type: VersionType.PATCH },
|
||||
{ this: new Version(0, 0, 0), other: new Version(0, 1, 0), compare: -1, type: VersionType.MINOR },
|
||||
{ this: new Version(0, 0, 0), other: new Version(1, 0, 0), compare: -1, type: VersionType.MAJOR },
|
||||
{ this: new Version(0, 0, 0), other: new Version(0, 0, 0), compare: 0, type: VersionType.EQUAL },
|
||||
{ this: new Version(0, 0, 1), other: new Version(0, 0, 1), compare: 0, type: VersionType.EQUAL },
|
||||
{ this: new Version(0, 1, 0), other: new Version(0, 1, 0), compare: 0, type: VersionType.EQUAL },
|
||||
{ this: new Version(1, 0, 0), other: new Version(1, 0, 0), compare: 0, type: VersionType.EQUAL },
|
||||
{ this: new Version(1, 0), other: new Version(1, 0, 0), compare: 0, type: VersionType.EQUAL },
|
||||
{ this: new Version(1, 0), other: new Version(1, 0, 1), compare: -1, type: VersionType.PATCH },
|
||||
{ this: new Version(1, 1), other: new Version(1, 0, 1), compare: 1, type: VersionType.MINOR },
|
||||
{ this: new Version(1), other: new Version(1, 0, 0), compare: 0, type: VersionType.EQUAL },
|
||||
{ this: new Version(1), other: new Version(1, 0, 1), compare: -1, type: VersionType.PATCH },
|
||||
];
|
||||
|
||||
describe('isOlderThan', () => {
|
||||
for (const { this: thisVersion, other: otherVersion, compare, type } of tests) {
|
||||
const expected = compare < 0 ? type : VersionType.EQUAL;
|
||||
it(`should return '${expected}' when comparing ${thisVersion} to ${otherVersion}`, () => {
|
||||
expect(thisVersion.isOlderThan(otherVersion)).toEqual(expected);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('isEqual', () => {
|
||||
for (const { this: thisVersion, other: otherVersion, compare } of tests) {
|
||||
const bool = compare === 0;
|
||||
it(`should return ${bool} when comparing ${thisVersion} to ${otherVersion}`, () => {
|
||||
expect(thisVersion.isEqual(otherVersion)).toEqual(bool);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('isNewerThan', () => {
|
||||
for (const { this: thisVersion, other: otherVersion, compare, type } of tests) {
|
||||
const expected = compare > 0 ? type : VersionType.EQUAL;
|
||||
it(`should return ${expected} when comparing ${thisVersion} to ${otherVersion}`, () => {
|
||||
expect(thisVersion.isNewerThan(otherVersion)).toEqual(expected);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('fromString', () => {
|
||||
const tests = [
|
||||
{ scenario: 'leading v', value: 'v1.72.2', expected: new Version(1, 72, 2) },
|
||||
{ scenario: 'uppercase v', value: 'V1.72.2', expected: new Version(1, 72, 2) },
|
||||
{ scenario: 'missing v', value: '1.72.2', expected: new Version(1, 72, 2) },
|
||||
{ scenario: 'large patch', value: '1.72.123', expected: new Version(1, 72, 123) },
|
||||
{ scenario: 'large minor', value: '1.123.0', expected: new Version(1, 123, 0) },
|
||||
{ scenario: 'large major', value: '123.0.0', expected: new Version(123, 0, 0) },
|
||||
{ scenario: 'major bump', value: 'v2.0.0', expected: new Version(2, 0, 0) },
|
||||
{ scenario: 'has dash', value: '14.10-1', expected: new Version(14, 10, 1) },
|
||||
{ scenario: 'missing patch', value: '14.10', expected: new Version(14, 10, 0) },
|
||||
{ scenario: 'only major', value: '14', expected: new Version(14, 0, 0) },
|
||||
];
|
||||
|
||||
for (const { scenario, value, expected } of tests) {
|
||||
it(`should correctly parse ${scenario}`, () => {
|
||||
const actual = Version.fromString(value);
|
||||
expect(actual.major).toEqual(expected.major);
|
||||
expect(actual.minor).toEqual(expected.minor);
|
||||
expect(actual.patch).toEqual(expected.patch);
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
@ -1,72 +0,0 @@
|
||||
export type IVersion = { major: number; minor: number; patch: number };
|
||||
|
||||
export enum VersionType {
|
||||
EQUAL = 0,
|
||||
PATCH = 1,
|
||||
MINOR = 2,
|
||||
MAJOR = 3,
|
||||
}
|
||||
|
||||
export class Version implements IVersion {
|
||||
public readonly types = ['major', 'minor', 'patch'] as const;
|
||||
|
||||
constructor(
|
||||
public major: number,
|
||||
public minor: number = 0,
|
||||
public patch: number = 0,
|
||||
) {}
|
||||
|
||||
toString() {
|
||||
return `${this.major}.${this.minor}.${this.patch}`;
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
const { major, minor, patch } = this;
|
||||
return { major, minor, patch };
|
||||
}
|
||||
|
||||
static fromString(version: string): Version {
|
||||
const regex = /v?(?<major>\d+)(?:\.(?<minor>\d+))?(?:[.-](?<patch>\d+))?/i;
|
||||
const matchResult = version.match(regex);
|
||||
if (matchResult) {
|
||||
const { major, minor = '0', patch = '0' } = matchResult.groups as { [K in keyof IVersion]: string };
|
||||
return new Version(Number(major), Number(minor), Number(patch));
|
||||
} else {
|
||||
throw new Error(`Invalid version format: ${version}`);
|
||||
}
|
||||
}
|
||||
|
||||
private compare(version: Version): [number, VersionType] {
|
||||
for (const [i, key] of this.types.entries()) {
|
||||
const diff = this[key] - version[key];
|
||||
if (diff !== 0) {
|
||||
return [diff > 0 ? 1 : -1, (VersionType.MAJOR - i) as VersionType];
|
||||
}
|
||||
}
|
||||
|
||||
return [0, VersionType.EQUAL];
|
||||
}
|
||||
|
||||
isOlderThan(version: Version): VersionType {
|
||||
const [bool, type] = this.compare(version);
|
||||
return bool < 0 ? type : VersionType.EQUAL;
|
||||
}
|
||||
|
||||
isEqual(version: Version): boolean {
|
||||
const [bool] = this.compare(version);
|
||||
return bool === 0;
|
||||
}
|
||||
|
||||
isNewerThan(version: Version): VersionType {
|
||||
const [bool, type] = this.compare(version);
|
||||
return bool > 0 ? type : VersionType.EQUAL;
|
||||
}
|
||||
|
||||
compareTo(other: Version) {
|
||||
if (this.isEqual(other)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return this.isNewerThan(other) ? 1 : -1;
|
||||
}
|
||||
}
|
||||
@ -1,14 +1,12 @@
|
||||
import { IDatabaseRepository } from 'src/interfaces/database.interface';
|
||||
import { Version } from 'src/utils/version';
|
||||
import { Mocked, vitest } from 'vitest';
|
||||
|
||||
export const newDatabaseRepositoryMock = (): Mocked<IDatabaseRepository> => {
|
||||
return {
|
||||
getExtensionVersion: vitest.fn(),
|
||||
getAvailableExtensionVersion: vitest.fn(),
|
||||
getPreferredVectorExtension: vitest.fn(),
|
||||
getPostgresVersion: vitest.fn().mockResolvedValue(new Version(14, 0, 0)),
|
||||
createExtension: vitest.fn().mockImplementation(() => Promise.resolve()),
|
||||
getPostgresVersion: vitest.fn().mockResolvedValue('14.10 (Debian 14.10-1.pgdg120+1)'),
|
||||
createExtension: vitest.fn().mockResolvedValue(void 0),
|
||||
updateExtension: vitest.fn(),
|
||||
updateVectorExtension: vitest.fn(),
|
||||
reindex: vitest.fn(),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user