self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-05 01:16:04 +02:00
Code Activity

fix(server): Split database queries based on PostgreSQL bound params limit (#6034)

Browse Source 
* fix(server): Split database queries based on PostgreSQL bound params limit

PostgreSQL uses a 16-bit integer to indicate the number of bound
parameters.

This means that the maximum number of parameters for any query is 65535.
Any query that tries to bind more than that (e.g. searching by a list of
IDs) requires splitting the query into multiple chunks.

This change includes refactoring every Repository that runs queries
using a list of ids, and either flattening or merging results.

Fixes #5788, #5997.

Also, potentially a fix for #4648 (at least based on
[this comment](https://github.com/immich-app/immich/issues/4648#issuecomment-1826134027)).

References:

* https://github.com/typeorm/typeorm/issues/7565
* [PostgreSQL message format - Bind](https://www.postgresql.org/docs/15/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BIND)

* misc: Create Chunked decorator to simplify implementation

* feat: Add ChunkedArray/ChunkedSet decorators
This commit is contained in:
Michael Manganiello
2024-01-06 20:36:12 -05:00
committed by GitHub
parent 6835d4519a
commit e262298090
11 changed files with 392 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
server/src/infra/infra.utils.ts
View File

@@ -1,5 +1,8 @@
import { Paginated, PaginationOptions } from '@app/domain';
import _ from 'lodash';
import { Between, FindOneOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm';
import { chunks, setUnion } from '../domain/domain.util';
import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util';
/**
* Allows optional values unlike the regular Between and uses MoreThanOrEqual
@@ -40,3 +43,42 @@ export const isValidInteger = (value: number, options: { min?: number; max?: num
const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
return Number.isInteger(value) && value >= min && value <= max;
};
/**
* Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection,
* to overcome the maximum number of parameters allowed by the database driver.
*
* @param options.paramIndex The index of the function parameter to chunk. Defaults to 0.
* @param options.flatten Whether to flatten the results. Defaults to false.
*/
export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator {
return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
const originalMethod = descriptor.value;
const paramIndex = options.paramIndex ?? 0;
descriptor.value = async function (...args: any[]) {
const arg = args[paramIndex];
// Early return if argument length is less than or equal to the chunk size.
if (
(arg instanceof Array && arg.length <= DATABASE_PARAMETER_CHUNK_SIZE) ||
(arg instanceof Set && arg.size <= DATABASE_PARAMETER_CHUNK_SIZE)
) {
return await originalMethod.apply(this, args);
}
return Promise.all(
chunks(arg, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => {
await originalMethod.apply(this, [...args.slice(0, paramIndex), chunk, ...args.slice(paramIndex + 1)]);
}),
).then((results) => (options.mergeFn ? options.mergeFn(results) : results));
};
};
}
export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator {
return Chunked({ ...options, mergeFn: _.flatten });
}
export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
return Chunked({ ...options, mergeFn: setUnion });
}