mirror of
https://github.com/immich-app/immich.git
synced 2024-12-25 10:43:13 +02:00
feat(server): optimize get asset query (#7176)
* faster query * add index * remove legacy code * update mock * remove unused imports * add relations * add stack * formatting * remove stack relation * remove unused import * increase chunk size * generate sql * linting * fix typing * formatting
This commit is contained in:
parent
8e1c85fe4f
commit
857ec0451d
@ -63,7 +63,7 @@ class AssetService {
|
||||
|
||||
/// Returns `null` if the server state did not change, else list of assets
|
||||
Future<List<Asset>?> _getRemoteAssets(User user) async {
|
||||
const int chunkSize = 5000;
|
||||
const int chunkSize = 10000;
|
||||
try {
|
||||
final DateTime now = DateTime.now().toUtc();
|
||||
final List<Asset> allAssets = [];
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { AssetSearchOptions, SearchExploreItem } from '@app/domain';
|
||||
import { AssetSearchOneToOneRelationOptions, AssetSearchOptions, SearchExploreItem } from '@app/domain';
|
||||
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity } from '@app/infra/entities';
|
||||
import { FindOptionsRelations, FindOptionsSelect } from 'typeorm';
|
||||
import { Paginated, PaginationOptions } from '../domain.util';
|
||||
@ -133,6 +133,10 @@ export interface IAssetRepository {
|
||||
getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
|
||||
deleteAll(ownerId: string): Promise<void>;
|
||||
getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
|
||||
getAllByFileCreationDate(
|
||||
pagination: PaginationOptions,
|
||||
options?: AssetSearchOneToOneRelationOptions,
|
||||
): Paginated<AssetEntity>;
|
||||
getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
|
||||
updateAll(ids: string[], options: Partial<AssetEntity>): Promise<void>;
|
||||
save(asset: Pick<AssetEntity, 'id'> & Partial<AssetEntity>): Promise<AssetEntity>;
|
||||
|
@ -69,7 +69,6 @@ export interface SearchAssetIDOptions {
|
||||
export interface SearchUserIdOptions {
|
||||
deviceId?: string;
|
||||
libraryId?: string;
|
||||
ownerId?: string;
|
||||
userIds?: string[];
|
||||
}
|
||||
|
||||
@ -147,16 +146,19 @@ export interface SearchPaginationOptions {
|
||||
size: number;
|
||||
}
|
||||
|
||||
export type AssetSearchOptions = SearchDateOptions &
|
||||
type BaseAssetSearchOptions = SearchDateOptions &
|
||||
SearchIdOptions &
|
||||
SearchExifOptions &
|
||||
SearchOrderOptions &
|
||||
SearchPathOptions &
|
||||
SearchRelationOptions &
|
||||
SearchStatusOptions &
|
||||
SearchUserIdOptions &
|
||||
SearchPeopleOptions;
|
||||
|
||||
export type AssetSearchOptions = BaseAssetSearchOptions & SearchRelationOptions;
|
||||
|
||||
export type AssetSearchOneToOneRelationOptions = BaseAssetSearchOptions & SearchOneToOneRelationOptions;
|
||||
|
||||
export type AssetSearchBuilderOptions = Omit<AssetSearchOptions, 'orderDirection'>;
|
||||
|
||||
export type SmartSearchOptions = SearchDateOptions &
|
||||
|
@ -1,10 +1,8 @@
|
||||
import { AssetEntity, ExifEntity } from '@app/infra/entities';
|
||||
import { OptionalBetween } from '@app/infra/infra.utils';
|
||||
import { AssetEntity } from '@app/infra/entities';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { In } from 'typeorm/find-options/operator/In.js';
|
||||
import { Repository } from 'typeorm/repository/Repository.js';
|
||||
import { AssetSearchDto } from './dto/asset-search.dto';
|
||||
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
|
||||
import { SearchPropertiesDto } from './dto/search-properties.dto';
|
||||
import { CuratedLocationsResponseDto } from './response-dto/curated-locations-response.dto';
|
||||
@ -21,7 +19,6 @@ export interface AssetOwnerCheck extends AssetCheck {
|
||||
|
||||
export interface IAssetRepositoryV1 {
|
||||
get(id: string): Promise<AssetEntity | null>;
|
||||
getAllByUserId(userId: string, dto: AssetSearchDto): Promise<AssetEntity[]>;
|
||||
getLocationsByUserId(userId: string): Promise<CuratedLocationsResponseDto[]>;
|
||||
getDetectedObjectsByUserId(userId: string): Promise<CuratedObjectsResponseDto[]>;
|
||||
getSearchPropertiesByUserId(userId: string): Promise<SearchPropertiesDto[]>;
|
||||
@ -34,10 +31,7 @@ export const IAssetRepositoryV1 = 'IAssetRepositoryV1';
|
||||
|
||||
@Injectable()
|
||||
export class AssetRepositoryV1 implements IAssetRepositoryV1 {
|
||||
constructor(
|
||||
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
|
||||
@InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
|
||||
) {}
|
||||
constructor(@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>) {}
|
||||
|
||||
getSearchPropertiesByUserId(userId: string): Promise<SearchPropertiesDto[]> {
|
||||
return this.assetRepository
|
||||
@ -89,33 +83,6 @@ export class AssetRepositoryV1 implements IAssetRepositoryV1 {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all assets belong to the user on the database
|
||||
* @param ownerId
|
||||
*/
|
||||
getAllByUserId(ownerId: string, dto: AssetSearchDto): Promise<AssetEntity[]> {
|
||||
return this.assetRepository.find({
|
||||
where: {
|
||||
ownerId,
|
||||
isVisible: true,
|
||||
isFavorite: dto.isFavorite,
|
||||
isArchived: dto.isArchived,
|
||||
updatedAt: OptionalBetween(dto.updatedAfter, dto.updatedBefore),
|
||||
},
|
||||
relations: {
|
||||
exifInfo: true,
|
||||
tags: true,
|
||||
stack: { assets: true },
|
||||
},
|
||||
skip: dto.skip || 0,
|
||||
take: dto.take,
|
||||
order: {
|
||||
fileCreatedAt: 'DESC',
|
||||
},
|
||||
withDeleted: true,
|
||||
});
|
||||
}
|
||||
|
||||
get(id: string): Promise<AssetEntity | null> {
|
||||
return this.assetRepository.findOne({
|
||||
where: { id },
|
||||
|
@ -77,7 +77,6 @@ describe('AssetService', () => {
|
||||
beforeEach(() => {
|
||||
assetRepositoryMockV1 = {
|
||||
get: jest.fn(),
|
||||
getAllByUserId: jest.fn(),
|
||||
getDetectedObjectsByUserId: jest.fn(),
|
||||
getLocationsByUserId: jest.fn(),
|
||||
getSearchPropertiesByUserId: jest.fn(),
|
||||
|
@ -114,8 +114,11 @@ export class AssetService {
|
||||
public async getAllAssets(auth: AuthDto, dto: AssetSearchDto): Promise<AssetResponseDto[]> {
|
||||
const userId = dto.userId || auth.user.id;
|
||||
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
|
||||
const assets = await this.assetRepositoryV1.getAllByUserId(userId, dto);
|
||||
return assets.map((asset) => mapAsset(asset, { withStack: true }));
|
||||
const assets = await this.assetRepository.getAllByFileCreationDate(
|
||||
{ take: dto.take ?? 1000, skip: dto.skip },
|
||||
{ ...dto, userIds: [userId], withDeleted: true, orderDirection: 'DESC', withExif: true },
|
||||
);
|
||||
return assets.items.map((asset) => mapAsset(asset));
|
||||
}
|
||||
|
||||
async serveThumbnail(auth: AuthDto, assetId: string, dto: GetAssetThumbnailDto): Promise<ImmichFileResponse> {
|
||||
|
@ -85,6 +85,7 @@ export class AssetEntity {
|
||||
@DeleteDateColumn({ type: 'timestamptz', nullable: true })
|
||||
deletedAt!: Date | null;
|
||||
|
||||
@Index('idx_asset_file_created_at')
|
||||
@Column({ type: 'timestamptz' })
|
||||
fileCreatedAt!: Date;
|
||||
|
||||
|
@ -0,0 +1,12 @@
|
||||
import { MigrationInterface, QueryRunner } from "typeorm";
|
||||
|
||||
export class AddFileCreatedAtIndex1708227417898 implements MigrationInterface {
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`CREATE INDEX idx_asset_file_created_at ON assets ("fileCreatedAt")`);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`DROP INDEX idx_asset_file_created_at`);
|
||||
}
|
||||
}
|
@ -2,6 +2,7 @@ import {
|
||||
AssetBuilderOptions,
|
||||
AssetCreate,
|
||||
AssetExploreFieldOptions,
|
||||
AssetSearchOneToOneRelationOptions,
|
||||
AssetSearchOptions,
|
||||
AssetStats,
|
||||
AssetStatsOptions,
|
||||
@ -175,8 +176,12 @@ export class AssetRepository implements IAssetRepository {
|
||||
});
|
||||
}
|
||||
|
||||
getByUserId(pagination: PaginationOptions, userId: string, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
|
||||
return this.getAll(pagination, { ...options, ownerId: userId });
|
||||
getByUserId(
|
||||
pagination: PaginationOptions,
|
||||
userId: string,
|
||||
options: Omit<AssetSearchOptions, 'userIds'> = {},
|
||||
): Paginated<AssetEntity> {
|
||||
return this.getAll(pagination, { ...options, userIds: [userId] });
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID]] })
|
||||
@ -205,6 +210,29 @@ export class AssetRepository implements IAssetRepository {
|
||||
});
|
||||
}
|
||||
|
||||
@GenerateSql({
|
||||
params: [
|
||||
{ skip: 20_000, take: 10_000 },
|
||||
{
|
||||
takenBefore: DummyValue.DATE,
|
||||
userIds: [DummyValue.UUID],
|
||||
},
|
||||
],
|
||||
})
|
||||
getAllByFileCreationDate(
|
||||
pagination: PaginationOptions,
|
||||
options: AssetSearchOneToOneRelationOptions = {},
|
||||
): Paginated<AssetEntity> {
|
||||
let builder = this.repository.createQueryBuilder('asset');
|
||||
builder = searchAssetBuilder(builder, options);
|
||||
builder.orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC');
|
||||
return paginatedBuilder<AssetEntity>(builder, {
|
||||
mode: PaginationMode.LIMIT_OFFSET,
|
||||
skip: pagination.skip,
|
||||
take: pagination.take,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get assets by device's Id on the database
|
||||
* @param ownerId
|
||||
|
@ -395,6 +395,55 @@ ORDER BY
|
||||
LIMIT
|
||||
1
|
||||
|
||||
-- AssetRepository.getAllByFileCreationDate
|
||||
SELECT
|
||||
"asset"."id" AS "asset_id",
|
||||
"asset"."deviceAssetId" AS "asset_deviceAssetId",
|
||||
"asset"."ownerId" AS "asset_ownerId",
|
||||
"asset"."libraryId" AS "asset_libraryId",
|
||||
"asset"."deviceId" AS "asset_deviceId",
|
||||
"asset"."type" AS "asset_type",
|
||||
"asset"."originalPath" AS "asset_originalPath",
|
||||
"asset"."resizePath" AS "asset_resizePath",
|
||||
"asset"."webpPath" AS "asset_webpPath",
|
||||
"asset"."thumbhash" AS "asset_thumbhash",
|
||||
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
|
||||
"asset"."createdAt" AS "asset_createdAt",
|
||||
"asset"."updatedAt" AS "asset_updatedAt",
|
||||
"asset"."deletedAt" AS "asset_deletedAt",
|
||||
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
|
||||
"asset"."localDateTime" AS "asset_localDateTime",
|
||||
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
|
||||
"asset"."isFavorite" AS "asset_isFavorite",
|
||||
"asset"."isArchived" AS "asset_isArchived",
|
||||
"asset"."isExternal" AS "asset_isExternal",
|
||||
"asset"."isReadOnly" AS "asset_isReadOnly",
|
||||
"asset"."isOffline" AS "asset_isOffline",
|
||||
"asset"."checksum" AS "asset_checksum",
|
||||
"asset"."duration" AS "asset_duration",
|
||||
"asset"."isVisible" AS "asset_isVisible",
|
||||
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
|
||||
"asset"."originalFileName" AS "asset_originalFileName",
|
||||
"asset"."sidecarPath" AS "asset_sidecarPath",
|
||||
"asset"."stackId" AS "asset_stackId"
|
||||
FROM
|
||||
"assets" "asset"
|
||||
WHERE
|
||||
(
|
||||
"asset"."fileCreatedAt" <= $1
|
||||
AND 1 = 1
|
||||
AND "asset"."ownerId" IN ($2)
|
||||
AND 1 = 1
|
||||
AND 1 = 1
|
||||
)
|
||||
AND ("asset"."deletedAt" IS NULL)
|
||||
ORDER BY
|
||||
"asset"."fileCreatedAt" DESC
|
||||
LIMIT
|
||||
10001
|
||||
OFFSET
|
||||
20000
|
||||
|
||||
-- AssetRepository.getAllByDeviceId
|
||||
SELECT
|
||||
"AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",
|
||||
|
@ -18,6 +18,7 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
|
||||
getFirstAssetForAlbumId: jest.fn(),
|
||||
getLastUpdatedAssetForAlbumId: jest.fn(),
|
||||
getAll: jest.fn().mockResolvedValue({ items: [], hasNextPage: false }),
|
||||
getAllByFileCreationDate: jest.fn(),
|
||||
getAllByDeviceId: jest.fn(),
|
||||
updateAll: jest.fn(),
|
||||
getByLibraryId: jest.fn(),
|
||||
|
Loading…
Reference in New Issue
Block a user