From 118b55513e8a01179f14139c79ae4341b6a50496 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Sun, 17 Dec 2023 10:09:23 +0000 Subject: [PATCH] update --- packages/app-desktop/runForTesting.sh | 47 ++++++++- packages/lib/Synchronizer.ts | 10 +- packages/lib/file-api-driver-joplinServer.ts | 5 + packages/lib/file-api.ts | 9 ++ packages/server/src/env.ts | 3 + packages/server/src/models/ChangeModel.ts | 105 +++++-------------- 6 files changed, 94 insertions(+), 85 deletions(-) diff --git a/packages/app-desktop/runForTesting.sh b/packages/app-desktop/runForTesting.sh index f9eabc233..549b143dc 100755 --- a/packages/app-desktop/runForTesting.sh +++ b/packages/app-desktop/runForTesting.sh @@ -2,6 +2,8 @@ # Setup the sync parameters for user X and create a few folders and notes to # allow sharing. Also calls the API to create the test users and clear the data. +# This script was setup for the desktop app but can now also be used to test the +# CLI using the `clix` users. # ---------------------------------------------------------------------------------- # For example, to setup a user for sharing, and another as recipient with E2EE @@ -37,6 +39,13 @@ # ./runForTesting.sh 1 createTeams,createData,resetTeam,sync && ./runForTesting.sh 2 resetTeam,sync && ./runForTesting.sh 1 +# ---------------------------------------------------------------------------------- +# Testing the CLI app with commands: +# ---------------------------------------------------------------------------------- + +# ./runForTesting.sh cli1 createUsers,createData,reset,sync +# ./runForTesting.sh cli1 -- import /path/to/file.jex + set -e SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" @@ -49,6 +58,7 @@ fi USER_NUM=$1 USER_PROFILE_NUM=$USER_NUM +IS_DESKTOP=1 if [ "$USER_NUM" = "1a" ]; then USER_NUM=1 @@ -70,6 +80,18 @@ if [ "$USER_NUM" = "2b" ]; then USER_PROFILE_NUM=2b fi +if [ "$USER_NUM" = "cli1" ]; then + USER_NUM=1 + USER_PROFILE_NUM=1 + IS_DESKTOP=0 +fi + +if [ "$USER_NUM" = "cli1a" ]; then + USER_NUM=1 + USER_PROFILE_NUM=1a + IS_DESKTOP=0 +fi + COMMANDS=($(echo $2 | tr "," "\n")) PROFILE_DIR=~/.config/joplindev-desktop-$USER_PROFILE_NUM SYNC_TARGET=10 @@ -135,6 +157,10 @@ do echo "sync --use-lock 0" >> "$CMD_FILE" + elif [[ $CMD == "--" ]]; then + + break + else echo "Unknown command: $CMD" @@ -146,9 +172,22 @@ done cd "$ROOT_DIR/packages/app-cli" yarn start --profile "$PROFILE_DIR" batch "$CMD_FILE" -if [[ $COMMANDS != "" ]]; then - exit 0 +if [[ $CMD != "--" ]]; then + if [[ $COMMANDS != "" ]]; then + exit 0 + fi fi -cd "$ROOT_DIR/packages/app-desktop" -yarn start --profile "$PROFILE_DIR" +if [ "$IS_DESKTOP" = "1" ]; then + cd "$ROOT_DIR/packages/app-desktop" + yarn start --profile "$PROFILE_DIR" +else + cd "$ROOT_DIR/packages/app-cli" + if [[ $CMD == "--" ]]; then + shift + shift + yarn start --profile "$PROFILE_DIR" "$@" + else + yarn start --profile "$PROFILE_DIR" + fi +fi \ No newline at end of file diff --git a/packages/lib/Synchronizer.ts b/packages/lib/Synchronizer.ts index 6551c6842..fb992d2d5 100644 --- a/packages/lib/Synchronizer.ts +++ b/packages/lib/Synchronizer.ts @@ -401,7 +401,7 @@ export default class Synchronizer { this.dispatch({ type: 'SYNC_STARTED' }); eventManager.emit(EventName.SyncStart); - this.logSyncOperation('starting', null, null, `Starting synchronisation to target ${syncTargetId}... supportsAccurateTimestamp = ${this.api().supportsAccurateTimestamp}; supportsMultiPut = ${this.api().supportsMultiPut} [${synchronizationId}]`); + this.logSyncOperation('starting', null, null, `Starting synchronisation to target ${syncTargetId}... supportsAccurateTimestamp = ${this.api().supportsAccurateTimestamp}; supportsMultiPut = ${this.api().supportsMultiPut}; supportsDeltaWithItems = ${this.api().supportsDeltaWithItems} [${synchronizationId}]`); const handleCannotSyncItem = async (ItemClass: any, syncTargetId: any, item: any, cannotSyncReason: string, itemLocation: any = null) => { await ItemClass.saveSyncDisabled(syncTargetId, item, cannotSyncReason, itemLocation); @@ -843,6 +843,10 @@ export default class Synchronizer { if (local && local.updated_time === remote.jop_updated_time) needsToDownload = false; } + if (this.api().supportsDeltaWithItems) { + needsToDownload = false; + } + if (needsToDownload) { this.downloadQueue_.push(remote.path, async () => { return this.apiCall('get', remote.path); @@ -862,6 +866,10 @@ export default class Synchronizer { if (!BaseItem.isSystemPath(remote.path)) continue; // The delta API might return things like the .sync, .resource or the root folder const loadContent = async () => { + if (this.api().supportsDeltaWithItems) { + return remote.jopItem; + } + const task = await this.downloadQueue_.waitForResult(path); if (task.error) throw task.error; if (!task.result) return null; diff --git a/packages/lib/file-api-driver-joplinServer.ts b/packages/lib/file-api-driver-joplinServer.ts index 3ab3f1ef8..269d6065f 100644 --- a/packages/lib/file-api-driver-joplinServer.ts +++ b/packages/lib/file-api-driver-joplinServer.ts @@ -45,6 +45,10 @@ export default class FileApiDriverJoplinServer { return true; } + public get supportsDeltaWithItems() { + return true; + } + public requestRepeatCount() { return 3; } @@ -56,6 +60,7 @@ export default class FileApiDriverJoplinServer { jop_updated_time: md.jop_updated_time, isDir: false, isDeleted: isDeleted, + jopItem: md.jopItem, }; return output; diff --git a/packages/lib/file-api.ts b/packages/lib/file-api.ts index e695b5692..e7919cdd5 100644 --- a/packages/lib/file-api.ts +++ b/packages/lib/file-api.ts @@ -33,6 +33,8 @@ export interface RemoteItem { // value will always be ahead. However for synchronising we need to know the // exact Joplin item updated_time value. jop_updated_time?: number; + + jopItem?: any; } export interface PaginatedList { @@ -137,6 +139,13 @@ class FileApi { return !!this.driver().supportsLocks; } + // Tells whether the delta call is going to include the items themselves or + // just the metadata (which is the default). If the items are included it + // means less http request and faster processing. + public get supportsDeltaWithItems(): boolean { + return !!this.driver().supportsDeltaWithItems; + } + private async fetchRemoteDateOffset_() { const tempFile = `${this.tempDirName()}/timeCheck${Math.round(Math.random() * 1000000)}.txt`; const startTime = Date.now(); diff --git a/packages/server/src/env.ts b/packages/server/src/env.ts index a7c061cf6..e37861890 100644 --- a/packages/server/src/env.ts +++ b/packages/server/src/env.ts @@ -39,6 +39,8 @@ const defaultEnvValues: EnvVariables = { MAX_TIME_DRIFT: 2000, NTP_SERVER: 'pool.ntp.org:123', + DELTA_INCLUDES_ITEMS: true, + // ================================================== // URL config // ================================================== @@ -141,6 +143,7 @@ export interface EnvVariables { RUNNING_IN_DOCKER: boolean; MAX_TIME_DRIFT: number; NTP_SERVER: string; + DELTA_INCLUDES_ITEMS: boolean; IS_ADMIN_INSTANCE: boolean; INSTANCE_NAME: string; diff --git a/packages/server/src/models/ChangeModel.ts b/packages/server/src/models/ChangeModel.ts index d83a7d87d..b03345136 100644 --- a/packages/server/src/models/ChangeModel.ts +++ b/packages/server/src/models/ChangeModel.ts @@ -1,12 +1,14 @@ import { Knex } from 'knex'; import Logger from '@joplin/utils/Logger'; -import { SqliteMaxVariableNum, isPostgres } from '../db'; +import { DbConnection, SqliteMaxVariableNum, isPostgres } from '../db'; import { Change, ChangeType, Item, Uuid } from '../services/database/types'; import { md5 } from '../utils/crypto'; import { ErrorResyncRequired } from '../utils/errors'; import { Day, formatDateTime } from '../utils/time'; import BaseModel, { SaveOptions } from './BaseModel'; import { PaginatedResults } from './utils/pagination'; +import { NewModelFactoryHandler } from './factory'; +import { Config } from '../utils/types'; const logger = Logger.create('ChangeModel'); @@ -51,6 +53,13 @@ export function requestDeltaPagination(query: any): ChangePagination { export default class ChangeModel extends BaseModel { + private deltaIncludesItems_: boolean; + + public constructor(db: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) { + super(db, modelFactory, config); + this.deltaIncludesItems_ = config.DELTA_INCLUDES_ITEMS; + } + public get tableName(): string { return 'changes'; } @@ -89,43 +98,6 @@ export default class ChangeModel extends BaseModel { }; } - // private changesForUserQuery(userId: Uuid, count: boolean): Knex.QueryBuilder { - // // When need to get: - // // - // // - All the CREATE and DELETE changes associated with the user - // // - All the UPDATE changes that applies to items associated with the - // // user. - // // - // // UPDATE changes do not have the user_id set because they are specific - // // to the item, not to a particular user. - - // const query = this - // .db('changes') - // .where(function() { - // void this.whereRaw('((type = ? OR type = ?) AND user_id = ?)', [ChangeType.Create, ChangeType.Delete, userId]) - // // Need to use a RAW query here because Knex has a "not a - // // bug" bug that makes it go into infinite loop in some - // // contexts, possibly only when running inside Jest (didn't - // // test outside). - // // https://github.com/knex/knex/issues/1851 - // .orWhereRaw('type = ? AND item_id IN (SELECT item_id FROM user_items WHERE user_id = ?)', [ChangeType.Update, userId]); - // }); - - // if (count) { - // void query.countDistinct('id', { as: 'total' }); - // } else { - // void query.select([ - // 'id', - // 'item_id', - // 'item_name', - // 'type', - // 'updated_time', - // ]); - // } - - // return query; - // } - public async changesForUserQuery(userId: Uuid, fromCounter: number, limit: number, doCountQuery: boolean): Promise { // When need to get: // @@ -252,35 +224,6 @@ export default class ChangeModel extends BaseModel { return output; } - // public async allByUser(userId: Uuid, pagination: Pagination = null): Promise { - // pagination = { - // page: 1, - // limit: 100, - // order: [{ by: 'counter', dir: PaginationOrderDir.ASC }], - // ...pagination, - // }; - - // const query = this.changesForUserQuery(userId, false); - // const countQuery = this.changesForUserQuery(userId, true); - // const itemCount = (await countQuery.first()).total; - - // void query - // .orderBy(pagination.order[0].by, pagination.order[0].dir) - // .offset((pagination.page - 1) * pagination.limit) - // .limit(pagination.limit) as any[]; - - // const changes = await query; - - // return { - // items: changes, - // // If we have changes, we return the ID of the latest changes from which delta sync can resume. - // // If there's no change, we return the previous cursor. - // cursor: changes.length ? changes[changes.length - 1].id : pagination.cursor, - // has_more: changes.length >= pagination.limit, - // page_count: itemCount !== null ? Math.ceil(itemCount / pagination.limit) : undefined, - // }; - // } - public async delta(userId: Uuid, pagination: ChangePagination = null): Promise { pagination = { ...defaultDeltaPagination(), @@ -306,18 +249,20 @@ export default class ChangeModel extends BaseModel { let processedChanges = this.compressChanges(changes); processedChanges = await this.removeDeletedItems(processedChanges, items); - items = await this.models().item().loadWithContentMulti(processedChanges.map(c => c.item_id), { - fields: [ - 'content', - 'id', - 'jop_encryption_applied', - 'jop_id', - 'jop_parent_id', - 'jop_share_id', - 'jop_type', - 'jop_updated_time', - ], - }); + if (this.deltaIncludesItems_) { + items = await this.models().item().loadWithContentMulti(processedChanges.map(c => c.item_id), { + fields: [ + 'content', + 'id', + 'jop_encryption_applied', + 'jop_id', + 'jop_parent_id', + 'jop_share_id', + 'jop_type', + 'jop_updated_time', + ], + }); + } const finalChanges = processedChanges.map(change => { const item = items.find(item => item.id === change.item_id); @@ -325,7 +270,7 @@ export default class ChangeModel extends BaseModel { const deltaChange: DeltaChange = { ...change, jop_updated_time: item.jop_updated_time, - jopItem: item.jop_type ? this.models().item().itemToJoplinItem(item) : null, + jopItem: this.deltaIncludesItems_ && item.jop_type ? this.models().item().itemToJoplinItem(item) : null, }; return deltaChange; });