self-hosted/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-25 10:43:13 +02:00
Code Issues Releases Activity

feat(mobile)!: batched full/initial sync (#4840)

Browse Source 
* feat(mobile): batched full/initial sync

* use OptionalBetween

* skip/take as integer

---------

Co-authored-by: Fynn Petersen-Frey <zoodyy@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Fynn Petersen-Frey 2023-11-06 18:40:43 +01:00 committed by GitHub
parent 26fd9d7e5f
commit 21f2d3058a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 156 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
cli/src/api/open-api/api.ts generated
View File

@ -6695,16 +6695,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
},
/**
* Get all AssetEntity belong to the user
* @param {number} [skip]
* @param {number} [take]
* @param {string} [userId]
* @param {boolean} [isFavorite]
* @param {boolean} [isArchived]
* @param {number} [skip]
* @param {string} [updatedAfter]
* @param {string} [updatedBefore]
* @param {string} [ifNoneMatch] ETag of data already cached on the client
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
getAllAssets: async (userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
getAllAssets: async (skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
const localVarPath = `/asset`;
// use dummy base URL string because the URL constructor only accepts absolute URLs.
const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
@ -6726,6 +6728,14 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
// http bearer authentication required
await setBearerAuthToObject(localVarHeaderParameter, configuration)
if (skip !== undefined) {
localVarQueryParameter['skip'] = skip;
}
if (take !== undefined) {
localVarQueryParameter['take'] = take;
}
if (userId !== undefined) {
localVarQueryParameter['userId'] = userId;
}
@ -6738,16 +6748,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
localVarQueryParameter['isArchived'] = isArchived;
}
if (skip !== undefined) {
localVarQueryParameter['skip'] = skip;
}
if (updatedAfter !== undefined) {
localVarQueryParameter['updatedAfter'] = (updatedAfter as any instanceof Date) ?
(updatedAfter as any).toISOString() :
updatedAfter;
}
if (updatedBefore !== undefined) {
localVarQueryParameter['updatedBefore'] = (updatedBefore as any instanceof Date) ?
(updatedBefore as any).toISOString() :
updatedBefore;
}
if (ifNoneMatch != null) {
localVarHeaderParameter['if-none-match'] = String(ifNoneMatch);
}
@ -8066,17 +8078,19 @@ export const AssetApiFp = function(configuration?: Configuration) {
},
/**
* Get all AssetEntity belong to the user
* @param {number} [skip]
* @param {number} [take]
* @param {string} [userId]
* @param {boolean} [isFavorite]
* @param {boolean} [isArchived]
* @param {number} [skip]
* @param {string} [updatedAfter]
* @param {string} [updatedBefore]
* @param {string} [ifNoneMatch] ETag of data already cached on the client
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async getAllAssets(userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<Array<AssetResponseDto>>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(userId, isFavorite, isArchived, skip, updatedAfter, ifNoneMatch, options);
async getAllAssets(skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<Array<AssetResponseDto>>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(skip, take, userId, isFavorite, isArchived, updatedAfter, updatedBefore, ifNoneMatch, options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
@ -8421,7 +8435,7 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath
* @throws {RequiredError}
*/
getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig): AxiosPromise<Array<AssetResponseDto>> {
return localVarFp.getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath));
return localVarFp.getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath));
},
/**
* Get a single asset\'s information
@ -8719,6 +8733,20 @@ export interface AssetApiDownloadFileRequest {
* @interface AssetApiGetAllAssetsRequest
*/
export interface AssetApiGetAllAssetsRequest {
/**
*
* @type {number}
* @memberof AssetApiGetAllAssets
*/
readonly skip?: number
/**
*
* @type {number}
* @memberof AssetApiGetAllAssets
*/
readonly take?: number
/**
*
* @type {string}
@ -8742,17 +8770,17 @@ export interface AssetApiGetAllAssetsRequest {
/**
*
* @type {number}
* @type {string}
* @memberof AssetApiGetAllAssets
*/
readonly skip?: number
readonly updatedAfter?: string
/**
*
* @type {string}
* @memberof AssetApiGetAllAssets
*/
readonly updatedAfter?: string
readonly updatedBefore?: string
/**
* ETag of data already cached on the client
@ -9430,7 +9458,7 @@ export class AssetApi extends BaseAPI {
* @memberof AssetApi
*/
public getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig) {
return AssetApiFp(this.configuration).getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath));
return AssetApiFp(this.configuration).getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath));
}
/**

35
mobile/lib/shared/services/asset.service.dart
View File

@ -62,20 +62,31 @@ class AssetService {
/// Returns `null` if the server state did not change, else list of assets
Future<List<Asset>?> _getRemoteAssets(User user) async {
const int chunkSize = 5000;
try {
final List<AssetResponseDto>? assets =
await _apiService.assetApi.getAllAssets(
userId: user.id,
);
if (assets == null) {
return null;
} else if (assets.isNotEmpty && assets.first.ownerId != user.id) {
log.warning("Make sure that server and app versions match!"
" The server returned assets for user ${assets.first.ownerId}"
" while requesting assets of user ${user.id}");
return null;
final DateTime now = DateTime.now().toUtc();
final List<Asset> allAssets = [];
for (int i = 0;; i += chunkSize) {
final List<AssetResponseDto>? assets =
await _apiService.assetApi.getAllAssets(
userId: user.id,
// updatedBefore is important! without it we could
// a) get the same Asset multiple times in different versions (when
// the asset is modified while the chunks are loaded from the server)
// b) miss assets when new assets are inserted in between the calls
updatedBefore: now,
skip: i,
take: chunkSize,
);
if (assets == null) {
return null;
}
allAssets.addAll(assets.map(Asset.remote));
if (assets.length < chunkSize) {
break;
}
}
return assets.map(Asset.remote).toList();
return allAssets;
} catch (error, stack) {
log.severe(
'Error while getting remote assets: ${error.toString()}',

12
mobile/lib/shared/services/sync.service.dart
View File

@ -197,7 +197,7 @@ class SyncService {
User user,
FutureOr<List<Asset>?> Function(User user) loadAssets,
) async {
final DateTime now = DateTime.now();
final DateTime now = DateTime.now().toUtc();
final List<Asset>? remote = await loadAssets(user);
if (remote == null) {
return false;
@ -210,6 +210,10 @@ class SyncService {
assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!");
remote.sort(Asset.compareByChecksum);
// filter our duplicates that might be introduced by the chunked retrieval
remote.uniqueConsecutive(compare: Asset.compareByChecksum);
final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true);
if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) {
await _updateUserAssetsETag(user, now);
@ -759,6 +763,12 @@ class SyncService {
final List<Asset> toAdd = [];
final List<Asset> toUpdate = [];
final List<Asset> toRemove = [];
if (assets.isEmpty || inDb.isEmpty) {
// fast path for trivial cases: halfes memory usage during initial sync
return assets.isEmpty
? (toAdd, toUpdate, inDb) // remove all from DB
: (assets, toUpdate, toRemove); // add all assets
}
diffSortedListsSync(
inDb,
assets,

BIN
mobile/openapi/doc/AssetApi.md generated
View File

Binary file not shown.

BIN
mobile/openapi/lib/api/asset_api.dart generated
View File

Binary file not shown.

BIN
mobile/openapi/test/asset_api_test.dart generated
View File

Binary file not shown.

23
server/immich-openapi-specs.json
View File

@ -914,6 +914,22 @@
"description": "Get all AssetEntity belong to the user",
"operationId": "getAllAssets",
"parameters": [
{
"name": "skip",
"required": false,
"in": "query",
"schema": {
"type": "integer"
}
},
{
"name": "take",
"required": false,
"in": "query",
"schema": {
"type": "integer"
}
},
{
"name": "userId",
"required": false,
@ -940,15 +956,16 @@
}
},
{
"name": "skip",
"name": "updatedAfter",
"required": false,
"in": "query",
"schema": {
"type": "number"
"format": "date-time",
"type": "string"
}
},
{
"name": "updatedAfter",
"name": "updatedBefore",
"required": false,
"in": "query",
"schema": {

5
server/src/immich/api-v1/asset/asset-repository.ts
View File

@ -1,8 +1,8 @@
import { AssetCreate } from '@app/domain';
import { AssetEntity } from '@app/infra/entities';
import OptionalBetween from '@app/infra/utils/optional-between.util';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { MoreThan } from 'typeorm';
import { In } from 'typeorm/find-options/operator/In';
import { Repository } from 'typeorm/repository/Repository';
import { AssetSearchDto } from './dto/asset-search.dto';
@ -129,7 +129,7 @@ export class AssetRepository implements IAssetRepository {
isVisible: true,
isFavorite: dto.isFavorite,
isArchived: dto.isArchived,
updatedAt: dto.updatedAfter ? MoreThan(dto.updatedAfter) : undefined,
updatedAt: OptionalBetween(dto.updatedAfter, dto.updatedBefore),
},
relations: {
exifInfo: true,
@ -137,6 +137,7 @@ export class AssetRepository implements IAssetRepository {
stack: true,
},
skip: dto.skip || 0,
take: dto.take,
order: {
fileCreatedAt: 'DESC',
},

17
server/src/immich/api-v1/asset/dto/asset-search.dto.ts
View File

@ -1,7 +1,7 @@
import { Optional, toBoolean } from '@app/domain';
import { ApiProperty } from '@nestjs/swagger';
import { Transform, Type } from 'class-transformer';
import { IsBoolean, IsDate, IsNotEmpty, IsNumber, IsUUID } from 'class-validator';
import { IsBoolean, IsDate, IsInt, IsNotEmpty, IsUUID } from 'class-validator';
export class AssetSearchDto {
@Optional()
@ -17,9 +17,17 @@ export class AssetSearchDto {
isArchived?: boolean;
@Optional()
@IsNumber()
@IsInt()
@Type(() => Number)
@ApiProperty({ type: 'integer' })
skip?: number;
@Optional()
@IsInt()
@Type(() => Number)
@ApiProperty({ type: 'integer' })
take?: number;
@Optional()
@IsUUID('4')
@ApiProperty({ format: 'uuid' })
@ -29,4 +37,9 @@ export class AssetSearchDto {
@IsDate()
@Type(() => Date)
updatedAfter?: Date;
@Optional()
@IsDate()
@Type(() => Date)
updatedBefore?: Date;
}

56
web/src/api/open-api/api.ts generated
View File

@ -6695,16 +6695,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
},
/**
* Get all AssetEntity belong to the user
* @param {number} [skip]
* @param {number} [take]
* @param {string} [userId]
* @param {boolean} [isFavorite]
* @param {boolean} [isArchived]
* @param {number} [skip]
* @param {string} [updatedAfter]
* @param {string} [updatedBefore]
* @param {string} [ifNoneMatch] ETag of data already cached on the client
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
getAllAssets: async (userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
getAllAssets: async (skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
const localVarPath = `/asset`;
// use dummy base URL string because the URL constructor only accepts absolute URLs.
const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
@ -6726,6 +6728,14 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
// http bearer authentication required
await setBearerAuthToObject(localVarHeaderParameter, configuration)
if (skip !== undefined) {
localVarQueryParameter['skip'] = skip;
}
if (take !== undefined) {
localVarQueryParameter['take'] = take;
}
if (userId !== undefined) {
localVarQueryParameter['userId'] = userId;
}
@ -6738,16 +6748,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
localVarQueryParameter['isArchived'] = isArchived;
}
if (skip !== undefined) {
localVarQueryParameter['skip'] = skip;
}
if (updatedAfter !== undefined) {
localVarQueryParameter['updatedAfter'] = (updatedAfter as any instanceof Date) ?
(updatedAfter as any).toISOString() :
updatedAfter;
}
if (updatedBefore !== undefined) {
localVarQueryParameter['updatedBefore'] = (updatedBefore as any instanceof Date) ?
(updatedBefore as any).toISOString() :
updatedBefore;
}
if (ifNoneMatch != null) {
localVarHeaderParameter['if-none-match'] = String(ifNoneMatch);
}
@ -8066,17 +8078,19 @@ export const AssetApiFp = function(configuration?: Configuration) {
},
/**
* Get all AssetEntity belong to the user
* @param {number} [skip]
* @param {number} [take]
* @param {string} [userId]
* @param {boolean} [isFavorite]
* @param {boolean} [isArchived]
* @param {number} [skip]
* @param {string} [updatedAfter]
* @param {string} [updatedBefore]
* @param {string} [ifNoneMatch] ETag of data already cached on the client
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async getAllAssets(userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<Array<AssetResponseDto>>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(userId, isFavorite, isArchived, skip, updatedAfter, ifNoneMatch, options);
async getAllAssets(skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<Array<AssetResponseDto>>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(skip, take, userId, isFavorite, isArchived, updatedAfter, updatedBefore, ifNoneMatch, options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
/**
@ -8421,7 +8435,7 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath
* @throws {RequiredError}
*/
getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig): AxiosPromise<Array<AssetResponseDto>> {
return localVarFp.getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath));
return localVarFp.getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath));
},
/**
* Get a single asset\'s information
@ -8719,6 +8733,20 @@ export interface AssetApiDownloadFileRequest {
* @interface AssetApiGetAllAssetsRequest
*/
export interface AssetApiGetAllAssetsRequest {
/**
*
* @type {number}
* @memberof AssetApiGetAllAssets
*/
readonly skip?: number
/**
*
* @type {number}
* @memberof AssetApiGetAllAssets
*/
readonly take?: number
/**
*
* @type {string}
@ -8742,17 +8770,17 @@ export interface AssetApiGetAllAssetsRequest {
/**
*
* @type {number}
* @type {string}
* @memberof AssetApiGetAllAssets
*/
readonly skip?: number
readonly updatedAfter?: string
/**
*
* @type {string}
* @memberof AssetApiGetAllAssets
*/
readonly updatedAfter?: string
readonly updatedBefore?: string
/**
* ETag of data already cached on the client
@ -9430,7 +9458,7 @@ export class AssetApi extends BaseAPI {
* @memberof AssetApi
*/
public getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig) {
return AssetApiFp(this.configuration).getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath));
return AssetApiFp(this.configuration).getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath));
}
/**