mirror of
https://github.com/immich-app/immich.git
synced 2024-12-25 10:43:13 +02:00
fix(server): Split database queries based on PostgreSQL bound params limit (#6034)
* fix(server): Split database queries based on PostgreSQL bound params limit PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the maximum number of parameters for any query is 65535. Any query that tries to bind more than that (e.g. searching by a list of IDs) requires splitting the query into multiple chunks. This change includes refactoring every Repository that runs queries using a list of ids, and either flattening or merging results. Fixes #5788, #5997. Also, potentially a fix for #4648 (at least based on [this comment](https://github.com/immich-app/immich/issues/4648#issuecomment-1826134027)). References: * https://github.com/typeorm/typeorm/issues/7565 * [PostgreSQL message format - Bind](https://www.postgresql.org/docs/15/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BIND) * misc: Create Chunked decorator to simplify implementation * feat: Add ChunkedArray/ChunkedSet decorators
This commit is contained in:
parent
6835d4519a
commit
e262298090
@ -679,7 +679,7 @@ describe(AlbumService.name, () => {
|
||||
]);
|
||||
|
||||
expect(albumMock.update).toHaveBeenCalledWith({ id: 'album-123', updatedAt: expect.any(Date) });
|
||||
expect(albumMock.removeAssets).toHaveBeenCalledWith({ assetIds: ['asset-id'], albumId: 'album-123' });
|
||||
expect(albumMock.removeAssets).toHaveBeenCalledWith('album-123', ['asset-id']);
|
||||
});
|
||||
|
||||
it('should skip assets not in the album', async () => {
|
||||
|
@ -231,7 +231,7 @@ export class AlbumService {
|
||||
|
||||
const removedIds = results.filter(({ success }) => success).map(({ id }) => id);
|
||||
if (removedIds.length > 0) {
|
||||
await this.albumRepository.removeAssets({ albumId: id, assetIds: removedIds });
|
||||
await this.albumRepository.removeAssets(id, removedIds);
|
||||
await this.albumRepository.update({ id, updatedAt: new Date() });
|
||||
if (album.albumThumbnailAssetId && removedIds.includes(album.albumThumbnailAssetId)) {
|
||||
await this.albumRepository.updateThumbnails();
|
||||
|
@ -13,6 +13,7 @@ import {
|
||||
ValidationOptions,
|
||||
} from 'class-validator';
|
||||
import { CronJob } from 'cron';
|
||||
import _ from 'lodash';
|
||||
import { basename, extname } from 'node:path';
|
||||
import sanitize from 'sanitize-filename';
|
||||
|
||||
@ -175,6 +176,32 @@ export function Optional({ nullable, ...validationOptions }: OptionalOptions = {
|
||||
return ValidateIf((obj: any, v: any) => v !== undefined, validationOptions);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunks an array or set into smaller arrays of the specified size.
|
||||
*
|
||||
* @param collection The collection to chunk.
|
||||
* @param size The size of each chunk.
|
||||
*/
|
||||
export function chunks<T>(collection: Array<T> | Set<T>, size: number): T[][] {
|
||||
if (collection instanceof Set) {
|
||||
const result = [];
|
||||
let chunk = [];
|
||||
for (const elem of collection) {
|
||||
chunk.push(elem);
|
||||
if (chunk.length === size) {
|
||||
result.push(chunk);
|
||||
chunk = [];
|
||||
}
|
||||
}
|
||||
if (chunk.length > 0) {
|
||||
result.push(chunk);
|
||||
}
|
||||
return result;
|
||||
} else {
|
||||
return _.chunk(collection, size);
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: The following Set utils have been added here, to easily determine where they are used.
|
||||
// They should be replaced with native Set operations, when they are added to the language.
|
||||
// Proposal reference: https://github.com/tc39/proposal-set-methods
|
||||
|
@ -31,7 +31,7 @@ export interface IAlbumRepository {
|
||||
getAssetIds(albumId: string, assetIds?: string[]): Promise<Set<string>>;
|
||||
hasAsset(asset: AlbumAsset): Promise<boolean>;
|
||||
removeAsset(assetId: string): Promise<void>;
|
||||
removeAssets(assets: AlbumAssets): Promise<void>;
|
||||
removeAssets(albumId: string, assetIds: string[]): Promise<void>;
|
||||
getMetadataForIds(ids: string[]): Promise<AlbumAssetCount[]>;
|
||||
getInvalidThumbnail(): Promise<string[]>;
|
||||
getOwned(ownerId: string): Promise<AlbumEntity[]>;
|
||||
|
@ -19,3 +19,9 @@ export const DummyValue = {
|
||||
DATE: new Date(),
|
||||
TIME_BUCKET: '2024-01-01T00:00:00.000Z',
|
||||
};
|
||||
|
||||
// PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the
|
||||
// maximum number of parameters is 65535. Any query that tries to bind more than that (e.g. searching
|
||||
// by a list of IDs) requires splitting the query into multiple chunks.
|
||||
// We are rounding down this limit, as queries commonly include other filters and parameters.
|
||||
export const DATABASE_PARAMETER_CHUNK_SIZE = 65500;
|
||||
|
@ -1,5 +1,8 @@
|
||||
import { Paginated, PaginationOptions } from '@app/domain';
|
||||
import _ from 'lodash';
|
||||
import { Between, FindOneOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm';
|
||||
import { chunks, setUnion } from '../domain/domain.util';
|
||||
import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util';
|
||||
|
||||
/**
|
||||
* Allows optional values unlike the regular Between and uses MoreThanOrEqual
|
||||
@ -40,3 +43,42 @@ export const isValidInteger = (value: number, options: { min?: number; max?: num
|
||||
const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
|
||||
return Number.isInteger(value) && value >= min && value <= max;
|
||||
};
|
||||
|
||||
/**
|
||||
* Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection,
|
||||
* to overcome the maximum number of parameters allowed by the database driver.
|
||||
*
|
||||
* @param options.paramIndex The index of the function parameter to chunk. Defaults to 0.
|
||||
* @param options.flatten Whether to flatten the results. Defaults to false.
|
||||
*/
|
||||
export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator {
|
||||
return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
|
||||
const originalMethod = descriptor.value;
|
||||
const paramIndex = options.paramIndex ?? 0;
|
||||
descriptor.value = async function (...args: any[]) {
|
||||
const arg = args[paramIndex];
|
||||
|
||||
// Early return if argument length is less than or equal to the chunk size.
|
||||
if (
|
||||
(arg instanceof Array && arg.length <= DATABASE_PARAMETER_CHUNK_SIZE) ||
|
||||
(arg instanceof Set && arg.size <= DATABASE_PARAMETER_CHUNK_SIZE)
|
||||
) {
|
||||
return await originalMethod.apply(this, args);
|
||||
}
|
||||
|
||||
return Promise.all(
|
||||
chunks(arg, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => {
|
||||
await originalMethod.apply(this, [...args.slice(0, paramIndex), chunk, ...args.slice(paramIndex + 1)]);
|
||||
}),
|
||||
).then((results) => (options.mergeFn ? options.mergeFn(results) : results));
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator {
|
||||
return Chunked({ ...options, mergeFn: _.flatten });
|
||||
}
|
||||
|
||||
export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
|
||||
return Chunked({ ...options, mergeFn: setUnion });
|
||||
}
|
||||
|
@ -1,6 +1,7 @@
|
||||
import { IAccessRepository } from '@app/domain';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Brackets, In, Repository } from 'typeorm';
|
||||
import { chunks, setUnion } from '../../domain/domain.util';
|
||||
import {
|
||||
ActivityEntity,
|
||||
AlbumEntity,
|
||||
@ -12,6 +13,7 @@ import {
|
||||
SharedLinkEntity,
|
||||
UserTokenEntity,
|
||||
} from '../entities';
|
||||
import { DATABASE_PARAMETER_CHUNK_SIZE } from '../infra.util';
|
||||
|
||||
export class AccessRepository implements IAccessRepository {
|
||||
constructor(
|
||||
@ -32,15 +34,19 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.activityRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In([...activityIds]),
|
||||
userId,
|
||||
},
|
||||
})
|
||||
.then((activities) => new Set(activities.map((activity) => activity.id)));
|
||||
return Promise.all(
|
||||
chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.activityRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In(idChunk),
|
||||
userId,
|
||||
},
|
||||
})
|
||||
.then((activities) => new Set(activities.map((activity) => activity.id))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
|
||||
checkAlbumOwnerAccess: async (userId: string, activityIds: Set<string>): Promise<Set<string>> => {
|
||||
@ -48,17 +54,21 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.activityRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In([...activityIds]),
|
||||
album: {
|
||||
ownerId: userId,
|
||||
},
|
||||
},
|
||||
})
|
||||
.then((activities) => new Set(activities.map((activity) => activity.id)));
|
||||
return Promise.all(
|
||||
chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.activityRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In(idChunk),
|
||||
album: {
|
||||
ownerId: userId,
|
||||
},
|
||||
},
|
||||
})
|
||||
.then((activities) => new Set(activities.map((activity) => activity.id))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
|
||||
checkCreateAccess: async (userId: string, albumIds: Set<string>): Promise<Set<string>> => {
|
||||
@ -66,19 +76,23 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.albumRepository
|
||||
.createQueryBuilder('album')
|
||||
.select('album.id')
|
||||
.leftJoin('album.sharedUsers', 'sharedUsers')
|
||||
.where('album.id IN (:...albumIds)', { albumIds: [...albumIds] })
|
||||
.andWhere('album.isActivityEnabled = true')
|
||||
.andWhere(
|
||||
new Brackets((qb) => {
|
||||
qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
|
||||
}),
|
||||
)
|
||||
.getMany()
|
||||
.then((albums) => new Set(albums.map((album) => album.id)));
|
||||
return Promise.all(
|
||||
chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.albumRepository
|
||||
.createQueryBuilder('album')
|
||||
.select('album.id')
|
||||
.leftJoin('album.sharedUsers', 'sharedUsers')
|
||||
.where('album.id IN (:...albumIds)', { albumIds: idChunk })
|
||||
.andWhere('album.isActivityEnabled = true')
|
||||
.andWhere(
|
||||
new Brackets((qb) => {
|
||||
qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
|
||||
}),
|
||||
)
|
||||
.getMany()
|
||||
.then((albums) => new Set(albums.map((album) => album.id))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
};
|
||||
|
||||
@ -88,15 +102,19 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.libraryRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In([...libraryIds]),
|
||||
ownerId: userId,
|
||||
},
|
||||
})
|
||||
.then((libraries) => new Set(libraries.map((library) => library.id)));
|
||||
return Promise.all(
|
||||
chunks(libraryIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.libraryRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In(idChunk),
|
||||
ownerId: userId,
|
||||
},
|
||||
})
|
||||
.then((libraries) => new Set(libraries.map((library) => library.id))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
|
||||
checkPartnerAccess: async (userId: string, partnerIds: Set<string>): Promise<Set<string>> => {
|
||||
@ -104,13 +122,17 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.partnerRepository
|
||||
.createQueryBuilder('partner')
|
||||
.select('partner.sharedById')
|
||||
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] })
|
||||
.andWhere('partner.sharedWithId = :userId', { userId })
|
||||
.getMany()
|
||||
.then((partners) => new Set(partners.map((partner) => partner.sharedById)));
|
||||
return Promise.all(
|
||||
chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.partnerRepository
|
||||
.createQueryBuilder('partner')
|
||||
.select('partner.sharedById')
|
||||
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk })
|
||||
.andWhere('partner.sharedWithId = :userId', { userId })
|
||||
.getMany()
|
||||
.then((partners) => new Set(partners.map((partner) => partner.sharedById))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
};
|
||||
|
||||
@ -120,13 +142,17 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.partnerRepository
|
||||
.createQueryBuilder('partner')
|
||||
.select('partner.sharedById')
|
||||
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] })
|
||||
.andWhere('partner.sharedWithId = :userId', { userId })
|
||||
.getMany()
|
||||
.then((partners) => new Set(partners.map((partner) => partner.sharedById)));
|
||||
return Promise.all(
|
||||
chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.partnerRepository
|
||||
.createQueryBuilder('partner')
|
||||
.select('partner.sharedById')
|
||||
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk })
|
||||
.andWhere('partner.sharedWithId = :userId', { userId })
|
||||
.getMany()
|
||||
.then((partners) => new Set(partners.map((partner) => partner.sharedById))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
};
|
||||
|
||||
@ -136,33 +162,37 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.albumRepository
|
||||
.createQueryBuilder('album')
|
||||
.innerJoin('album.assets', 'asset')
|
||||
.leftJoin('album.sharedUsers', 'sharedUsers')
|
||||
.select('asset.id', 'assetId')
|
||||
.addSelect('asset.livePhotoVideoId', 'livePhotoVideoId')
|
||||
.where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', {
|
||||
assetIds: [...assetIds],
|
||||
})
|
||||
.andWhere(
|
||||
new Brackets((qb) => {
|
||||
qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
|
||||
}),
|
||||
)
|
||||
.getRawMany()
|
||||
.then((rows) => {
|
||||
const allowedIds = new Set<string>();
|
||||
for (const row of rows) {
|
||||
if (row.assetId && assetIds.has(row.assetId)) {
|
||||
allowedIds.add(row.assetId);
|
||||
}
|
||||
if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) {
|
||||
allowedIds.add(row.livePhotoVideoId);
|
||||
}
|
||||
}
|
||||
return allowedIds;
|
||||
});
|
||||
return Promise.all(
|
||||
chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.albumRepository
|
||||
.createQueryBuilder('album')
|
||||
.innerJoin('album.assets', 'asset')
|
||||
.leftJoin('album.sharedUsers', 'sharedUsers')
|
||||
.select('asset.id', 'assetId')
|
||||
.addSelect('asset.livePhotoVideoId', 'livePhotoVideoId')
|
||||
.where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', {
|
||||
assetIds: idChunk,
|
||||
})
|
||||
.andWhere(
|
||||
new Brackets((qb) => {
|
||||
qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId });
|
||||
}),
|
||||
)
|
||||
.getRawMany()
|
||||
.then((rows) => {
|
||||
const allowedIds = new Set<string>();
|
||||
for (const row of rows) {
|
||||
if (row.assetId && assetIds.has(row.assetId)) {
|
||||
allowedIds.add(row.assetId);
|
||||
}
|
||||
if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) {
|
||||
allowedIds.add(row.livePhotoVideoId);
|
||||
}
|
||||
}
|
||||
return allowedIds;
|
||||
}),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
|
||||
checkOwnerAccess: async (userId: string, assetIds: Set<string>): Promise<Set<string>> => {
|
||||
@ -170,16 +200,20 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.assetRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In([...assetIds]),
|
||||
ownerId: userId,
|
||||
},
|
||||
withDeleted: true,
|
||||
})
|
||||
.then((assets) => new Set(assets.map((asset) => asset.id)));
|
||||
return Promise.all(
|
||||
chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.assetRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In(idChunk),
|
||||
ownerId: userId,
|
||||
},
|
||||
withDeleted: true,
|
||||
})
|
||||
.then((assets) => new Set(assets.map((asset) => asset.id))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
|
||||
checkPartnerAccess: async (userId: string, assetIds: Set<string>): Promise<Set<string>> => {
|
||||
@ -187,15 +221,19 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.partnerRepository
|
||||
.createQueryBuilder('partner')
|
||||
.innerJoin('partner.sharedBy', 'sharedBy')
|
||||
.innerJoin('sharedBy.assets', 'asset')
|
||||
.select('asset.id', 'assetId')
|
||||
.where('partner.sharedWithId = :userId', { userId })
|
||||
.andWhere('asset.id IN (:...assetIds)', { assetIds: [...assetIds] })
|
||||
.getRawMany()
|
||||
.then((rows) => new Set(rows.map((row) => row.assetId)));
|
||||
return Promise.all(
|
||||
chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.partnerRepository
|
||||
.createQueryBuilder('partner')
|
||||
.innerJoin('partner.sharedBy', 'sharedBy')
|
||||
.innerJoin('sharedBy.assets', 'asset')
|
||||
.select('asset.id', 'assetId')
|
||||
.where('partner.sharedWithId = :userId', { userId })
|
||||
.andWhere('asset.id IN (:...assetIds)', { assetIds: idChunk })
|
||||
.getRawMany()
|
||||
.then((rows) => new Set(rows.map((row) => row.assetId))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
|
||||
checkSharedLinkAccess: async (sharedLinkId: string, assetIds: Set<string>): Promise<Set<string>> => {
|
||||
@ -203,41 +241,45 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.sharedLinkRepository
|
||||
.createQueryBuilder('sharedLink')
|
||||
.leftJoin('sharedLink.album', 'album')
|
||||
.leftJoin('sharedLink.assets', 'assets')
|
||||
.leftJoin('album.assets', 'albumAssets')
|
||||
.select('assets.id', 'assetId')
|
||||
.addSelect('albumAssets.id', 'albumAssetId')
|
||||
.addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId')
|
||||
.addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId')
|
||||
.where('sharedLink.id = :sharedLinkId', { sharedLinkId })
|
||||
.andWhere(
|
||||
'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]',
|
||||
{
|
||||
assetIds: [...assetIds],
|
||||
},
|
||||
)
|
||||
.getRawMany()
|
||||
.then((rows) => {
|
||||
const allowedIds = new Set<string>();
|
||||
for (const row of rows) {
|
||||
if (row.assetId && assetIds.has(row.assetId)) {
|
||||
allowedIds.add(row.assetId);
|
||||
}
|
||||
if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) {
|
||||
allowedIds.add(row.assetLivePhotoVideoId);
|
||||
}
|
||||
if (row.albumAssetId && assetIds.has(row.albumAssetId)) {
|
||||
allowedIds.add(row.albumAssetId);
|
||||
}
|
||||
if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) {
|
||||
allowedIds.add(row.albumAssetLivePhotoVideoId);
|
||||
}
|
||||
}
|
||||
return allowedIds;
|
||||
});
|
||||
return Promise.all(
|
||||
chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.sharedLinkRepository
|
||||
.createQueryBuilder('sharedLink')
|
||||
.leftJoin('sharedLink.album', 'album')
|
||||
.leftJoin('sharedLink.assets', 'assets')
|
||||
.leftJoin('album.assets', 'albumAssets')
|
||||
.select('assets.id', 'assetId')
|
||||
.addSelect('albumAssets.id', 'albumAssetId')
|
||||
.addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId')
|
||||
.addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId')
|
||||
.where('sharedLink.id = :sharedLinkId', { sharedLinkId })
|
||||
.andWhere(
|
||||
'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]',
|
||||
{
|
||||
assetIds: idChunk,
|
||||
},
|
||||
)
|
||||
.getRawMany()
|
||||
.then((rows) => {
|
||||
const allowedIds = new Set<string>();
|
||||
for (const row of rows) {
|
||||
if (row.assetId && assetIds.has(row.assetId)) {
|
||||
allowedIds.add(row.assetId);
|
||||
}
|
||||
if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) {
|
||||
allowedIds.add(row.assetLivePhotoVideoId);
|
||||
}
|
||||
if (row.albumAssetId && assetIds.has(row.albumAssetId)) {
|
||||
allowedIds.add(row.albumAssetId);
|
||||
}
|
||||
if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) {
|
||||
allowedIds.add(row.albumAssetLivePhotoVideoId);
|
||||
}
|
||||
}
|
||||
return allowedIds;
|
||||
}),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
};
|
||||
|
||||
@ -247,15 +289,19 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.tokenRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
userId,
|
||||
id: In([...deviceIds]),
|
||||
},
|
||||
})
|
||||
.then((tokens) => new Set(tokens.map((token) => token.id)));
|
||||
return Promise.all(
|
||||
chunks(deviceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.tokenRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
userId,
|
||||
id: In(idChunk),
|
||||
},
|
||||
})
|
||||
.then((tokens) => new Set(tokens.map((token) => token.id))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
};
|
||||
|
||||
@ -265,15 +311,19 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.albumRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In([...albumIds]),
|
||||
ownerId: userId,
|
||||
},
|
||||
})
|
||||
.then((albums) => new Set(albums.map((album) => album.id)));
|
||||
return Promise.all(
|
||||
chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.albumRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In(idChunk),
|
||||
ownerId: userId,
|
||||
},
|
||||
})
|
||||
.then((albums) => new Set(albums.map((album) => album.id))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
|
||||
checkSharedAlbumAccess: async (userId: string, albumIds: Set<string>): Promise<Set<string>> => {
|
||||
@ -281,17 +331,21 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.albumRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In([...albumIds]),
|
||||
sharedUsers: {
|
||||
id: userId,
|
||||
},
|
||||
},
|
||||
})
|
||||
.then((albums) => new Set(albums.map((album) => album.id)));
|
||||
return Promise.all(
|
||||
chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.albumRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In(idChunk),
|
||||
sharedUsers: {
|
||||
id: userId,
|
||||
},
|
||||
},
|
||||
})
|
||||
.then((albums) => new Set(albums.map((album) => album.id))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
|
||||
checkSharedLinkAccess: async (sharedLinkId: string, albumIds: Set<string>): Promise<Set<string>> => {
|
||||
@ -299,18 +353,22 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.sharedLinkRepository
|
||||
.find({
|
||||
select: { albumId: true },
|
||||
where: {
|
||||
id: sharedLinkId,
|
||||
albumId: In([...albumIds]),
|
||||
},
|
||||
})
|
||||
.then(
|
||||
(sharedLinks) =>
|
||||
new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))),
|
||||
);
|
||||
return Promise.all(
|
||||
chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.sharedLinkRepository
|
||||
.find({
|
||||
select: { albumId: true },
|
||||
where: {
|
||||
id: sharedLinkId,
|
||||
albumId: In(idChunk),
|
||||
},
|
||||
})
|
||||
.then(
|
||||
(sharedLinks) =>
|
||||
new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))),
|
||||
),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
};
|
||||
|
||||
@ -320,32 +378,41 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.personRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In([...personIds]),
|
||||
ownerId: userId,
|
||||
},
|
||||
})
|
||||
.then((persons) => new Set(persons.map((person) => person.id)));
|
||||
return Promise.all(
|
||||
chunks(personIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.personRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In(idChunk),
|
||||
ownerId: userId,
|
||||
},
|
||||
})
|
||||
.then((persons) => new Set(persons.map((person) => person.id))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
|
||||
checkFaceOwnerAccess: async (userId: string, assetFaceIds: Set<string>): Promise<Set<string>> => {
|
||||
if (assetFaceIds.size === 0) {
|
||||
return new Set();
|
||||
}
|
||||
return this.assetFaceRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In([...assetFaceIds]),
|
||||
asset: {
|
||||
ownerId: userId,
|
||||
},
|
||||
},
|
||||
})
|
||||
.then((faces) => new Set(faces.map((face) => face.id)));
|
||||
|
||||
return Promise.all(
|
||||
chunks(assetFaceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.assetFaceRepository
|
||||
.find({
|
||||
select: { id: true },
|
||||
where: {
|
||||
id: In(idChunk),
|
||||
asset: {
|
||||
ownerId: userId,
|
||||
},
|
||||
},
|
||||
})
|
||||
.then((faces) => new Set(faces.map((face) => face.id))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
};
|
||||
|
||||
@ -355,13 +422,17 @@ export class AccessRepository implements IAccessRepository {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return this.partnerRepository
|
||||
.createQueryBuilder('partner')
|
||||
.select('partner.sharedById')
|
||||
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] })
|
||||
.andWhere('partner.sharedWithId = :userId', { userId })
|
||||
.getMany()
|
||||
.then((partners) => new Set(partners.map((partner) => partner.sharedById)));
|
||||
return Promise.all(
|
||||
chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
this.partnerRepository
|
||||
.createQueryBuilder('partner')
|
||||
.select('partner.sharedById')
|
||||
.where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk })
|
||||
.andWhere('partner.sharedWithId = :userId', { userId })
|
||||
.getMany()
|
||||
.then((partners) => new Set(partners.map((partner) => partner.sharedById))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
@ -1,10 +1,13 @@
|
||||
import { AlbumAsset, AlbumAssetCount, AlbumAssets, AlbumInfoOptions, IAlbumRepository } from '@app/domain';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
|
||||
import _ from 'lodash';
|
||||
import { DataSource, FindOptionsOrder, FindOptionsRelations, In, IsNull, Not, Repository } from 'typeorm';
|
||||
import { setUnion } from '../../domain/domain.util';
|
||||
import { dataSource } from '../database.config';
|
||||
import { AlbumEntity, AssetEntity } from '../entities';
|
||||
import { DummyValue, GenerateSql } from '../infra.util';
|
||||
import { DATABASE_PARAMETER_CHUNK_SIZE, DummyValue, GenerateSql } from '../infra.util';
|
||||
import { Chunked, ChunkedArray } from '../infra.utils';
|
||||
|
||||
@Injectable()
|
||||
export class AlbumRepository implements IAlbumRepository {
|
||||
@ -39,6 +42,7 @@ export class AlbumRepository implements IAlbumRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID]] })
|
||||
@ChunkedArray()
|
||||
getByIds(ids: string[]): Promise<AlbumEntity[]> {
|
||||
return this.repository.find({
|
||||
where: {
|
||||
@ -64,6 +68,7 @@ export class AlbumRepository implements IAlbumRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID]] })
|
||||
@ChunkedArray()
|
||||
async getMetadataForIds(ids: string[]): Promise<AlbumAssetCount[]> {
|
||||
// Guard against running invalid query when ids list is empty.
|
||||
if (!ids.length) {
|
||||
@ -188,15 +193,16 @@ export class AlbumRepository implements IAlbumRepository {
|
||||
.execute();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [{ albumId: DummyValue.UUID, assetIds: [DummyValue.UUID] }] })
|
||||
async removeAssets(asset: AlbumAssets): Promise<void> {
|
||||
@GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] })
|
||||
@Chunked({ paramIndex: 1 })
|
||||
async removeAssets(albumId: string, assetIds: string[]): Promise<void> {
|
||||
await this.dataSource
|
||||
.createQueryBuilder()
|
||||
.delete()
|
||||
.from('albums_assets_assets')
|
||||
.where({
|
||||
albumsId: asset.albumId,
|
||||
assetsId: In(asset.assetIds),
|
||||
albumsId: albumId,
|
||||
assetsId: In(assetIds),
|
||||
})
|
||||
.execute();
|
||||
}
|
||||
@ -216,12 +222,19 @@ export class AlbumRepository implements IAlbumRepository {
|
||||
.from('albums_assets_assets', 'albums_assets')
|
||||
.where('"albums_assets"."albumsId" = :albumId', { albumId });
|
||||
|
||||
if (assetIds?.length) {
|
||||
query.andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds });
|
||||
if (!assetIds?.length) {
|
||||
const result = await query.getRawMany();
|
||||
return new Set(result.map((row) => row['assetId']));
|
||||
}
|
||||
|
||||
const result = await query.getRawMany();
|
||||
return new Set(result.map((row) => row['assetId']));
|
||||
return Promise.all(
|
||||
_.chunk(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) =>
|
||||
query
|
||||
.andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds: idChunk })
|
||||
.getRawMany()
|
||||
.then((result) => new Set(result.map((row) => row['assetId']))),
|
||||
),
|
||||
).then((results) => setUnion(...results));
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [{ albumId: DummyValue.UUID, assetId: DummyValue.UUID }] })
|
||||
|
@ -27,7 +27,7 @@ import { DateTime } from 'luxon';
|
||||
import { And, FindOptionsRelations, FindOptionsWhere, In, IsNull, LessThan, Not, Repository } from 'typeorm';
|
||||
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities';
|
||||
import { DummyValue, GenerateSql } from '../infra.util';
|
||||
import { OptionalBetween, paginate } from '../infra.utils';
|
||||
import { Chunked, ChunkedArray, OptionalBetween, paginate } from '../infra.utils';
|
||||
|
||||
const DEFAULT_SEARCH_SIZE = 250;
|
||||
|
||||
@ -248,6 +248,7 @@ export class AssetRepository implements IAssetRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID]] })
|
||||
@ChunkedArray()
|
||||
getByIds(ids: string[], relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity[]> {
|
||||
if (!relations) {
|
||||
relations = {
|
||||
@ -301,6 +302,7 @@ export class AssetRepository implements IAssetRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID]] })
|
||||
@ChunkedArray()
|
||||
getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]> {
|
||||
return this.repository.find({
|
||||
where: { library: { id: In(libraryIds) } },
|
||||
@ -380,14 +382,17 @@ export class AssetRepository implements IAssetRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] })
|
||||
@Chunked()
|
||||
async updateAll(ids: string[], options: Partial<AssetEntity>): Promise<void> {
|
||||
await this.repository.update({ id: In(ids) }, options);
|
||||
}
|
||||
|
||||
@Chunked()
|
||||
async softDeleteAll(ids: string[]): Promise<void> {
|
||||
await this.repository.softDelete({ id: In(ids), isExternal: false });
|
||||
}
|
||||
|
||||
@Chunked()
|
||||
async restoreAll(ids: string[]): Promise<void> {
|
||||
await this.repository.restore({ id: In(ids) });
|
||||
}
|
||||
|
@ -10,7 +10,7 @@ import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { In, Repository } from 'typeorm';
|
||||
import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities';
|
||||
import { DummyValue, GenerateSql } from '../infra.util';
|
||||
import { asVector } from '../infra.utils';
|
||||
import { Chunked, ChunkedArray, asVector } from '../infra.utils';
|
||||
|
||||
export class PersonRepository implements IPersonRepository {
|
||||
constructor(
|
||||
@ -32,12 +32,16 @@ export class PersonRepository implements IPersonRepository {
|
||||
.getRawMany();
|
||||
|
||||
const assetIds = results.map(({ assetId }) => assetId);
|
||||
|
||||
await this.assetFaceRepository.delete({ personId: oldPersonId, assetId: In(assetIds) });
|
||||
await this.deletePersonFromAssets(oldPersonId, assetIds);
|
||||
|
||||
return assetIds;
|
||||
}
|
||||
|
||||
@Chunked({ paramIndex: 1 })
|
||||
async deletePersonFromAssets(personId: string, assetIds: string[]): Promise<void> {
|
||||
await this.assetFaceRepository.delete({ personId: personId, assetId: In(assetIds) });
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [{ oldPersonId: DummyValue.UUID, newPersonId: DummyValue.UUID }] })
|
||||
async reassignFaces({ oldPersonId, newPersonId }: UpdateFacesData): Promise<number> {
|
||||
const result = await this.assetFaceRepository
|
||||
@ -234,6 +238,7 @@ export class PersonRepository implements IPersonRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[{ assetId: DummyValue.UUID, personId: DummyValue.UUID }]] })
|
||||
@ChunkedArray()
|
||||
async getFacesByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]> {
|
||||
return this.assetFaceRepository.find({ where: ids, relations: { asset: true }, withDeleted: true });
|
||||
}
|
||||
|
@ -5,6 +5,7 @@ import { readFile } from 'fs/promises';
|
||||
import { In, Repository } from 'typeorm';
|
||||
import { SystemConfigEntity } from '../entities';
|
||||
import { DummyValue, GenerateSql } from '../infra.util';
|
||||
import { Chunked } from '../infra.utils';
|
||||
|
||||
export class SystemConfigRepository implements ISystemConfigRepository {
|
||||
constructor(
|
||||
@ -29,6 +30,7 @@ export class SystemConfigRepository implements ISystemConfigRepository {
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.STRING] })
|
||||
@Chunked()
|
||||
async deleteKeys(keys: string[]): Promise<void> {
|
||||
await this.repository.delete({ key: In(keys) });
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user