self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
Code Issues Releases Activity

update

Browse Source
This commit is contained in:
Laurent Cozic 2023-12-17 10:09:23 +00:00
parent c582c44436
commit 118b55513e
6 changed files with 94 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
packages/app-desktop/runForTesting.sh
View File

@ -2,6 +2,8 @@
# Setup the sync parameters for user X and create a few folders and notes to
# allow sharing. Also calls the API to create the test users and clear the data.
# This script was setup for the desktop app but can now also be used to test the
# CLI using the `clix` users.
# ----------------------------------------------------------------------------------
# For example, to setup a user for sharing, and another as recipient with E2EE
@ -37,6 +39,13 @@
# ./runForTesting.sh 1 createTeams,createData,resetTeam,sync && ./runForTesting.sh 2 resetTeam,sync && ./runForTesting.sh 1
# ----------------------------------------------------------------------------------
# Testing the CLI app with commands:
# ----------------------------------------------------------------------------------
# ./runForTesting.sh cli1 createUsers,createData,reset,sync
# ./runForTesting.sh cli1 -- import /path/to/file.jex
set -e
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
@ -49,6 +58,7 @@ fi
USER_NUM=$1
USER_PROFILE_NUM=$USER_NUM
IS_DESKTOP=1
if [ "$USER_NUM" = "1a" ]; then
USER_NUM=1
@ -70,6 +80,18 @@ if [ "$USER_NUM" = "2b" ]; then
USER_PROFILE_NUM=2b
fi
if [ "$USER_NUM" = "cli1" ]; then
USER_NUM=1
USER_PROFILE_NUM=1
IS_DESKTOP=0
fi
if [ "$USER_NUM" = "cli1a" ]; then
USER_NUM=1
USER_PROFILE_NUM=1a
IS_DESKTOP=0
fi
COMMANDS=($(echo $2 | tr "," "\n"))
PROFILE_DIR=~/.config/joplindev-desktop-$USER_PROFILE_NUM
SYNC_TARGET=10
@ -135,6 +157,10 @@ do
echo "sync --use-lock 0" >> "$CMD_FILE"
elif [[ $CMD == "--" ]]; then
break
else
echo "Unknown command: $CMD"
@ -146,9 +172,22 @@ done
cd "$ROOT_DIR/packages/app-cli"
yarn start --profile "$PROFILE_DIR" batch "$CMD_FILE"
if [[ $COMMANDS != "" ]]; then
exit 0
if [[ $CMD != "--" ]]; then
if [[ $COMMANDS != "" ]]; then
exit 0
fi
fi
cd "$ROOT_DIR/packages/app-desktop"
yarn start --profile "$PROFILE_DIR"
if [ "$IS_DESKTOP" = "1" ]; then
cd "$ROOT_DIR/packages/app-desktop"
yarn start --profile "$PROFILE_DIR"
else
cd "$ROOT_DIR/packages/app-cli"
if [[ $CMD == "--" ]]; then
shift
shift
yarn start --profile "$PROFILE_DIR" "$@"
else
yarn start --profile "$PROFILE_DIR"
fi
fi

10
packages/lib/Synchronizer.ts
View File

@ -401,7 +401,7 @@ export default class Synchronizer {
this.dispatch({ type: 'SYNC_STARTED' });
eventManager.emit(EventName.SyncStart);
this.logSyncOperation('starting', null, null, `Starting synchronisation to target ${syncTargetId}... supportsAccurateTimestamp = ${this.api().supportsAccurateTimestamp}; supportsMultiPut = ${this.api().supportsMultiPut} [${synchronizationId}]`);
this.logSyncOperation('starting', null, null, `Starting synchronisation to target ${syncTargetId}... supportsAccurateTimestamp = ${this.api().supportsAccurateTimestamp}; supportsMultiPut = ${this.api().supportsMultiPut}; supportsDeltaWithItems = ${this.api().supportsDeltaWithItems} [${synchronizationId}]`);
const handleCannotSyncItem = async (ItemClass: any, syncTargetId: any, item: any, cannotSyncReason: string, itemLocation: any = null) => {
await ItemClass.saveSyncDisabled(syncTargetId, item, cannotSyncReason, itemLocation);
@ -843,6 +843,10 @@ export default class Synchronizer {
if (local && local.updated_time === remote.jop_updated_time) needsToDownload = false;
}
if (this.api().supportsDeltaWithItems) {
needsToDownload = false;
}
if (needsToDownload) {
this.downloadQueue_.push(remote.path, async () => {
return this.apiCall('get', remote.path);
@ -862,6 +866,10 @@ export default class Synchronizer {
if (!BaseItem.isSystemPath(remote.path)) continue; // The delta API might return things like the .sync, .resource or the root folder
const loadContent = async () => {
if (this.api().supportsDeltaWithItems) {
return remote.jopItem;
}
const task = await this.downloadQueue_.waitForResult(path);
if (task.error) throw task.error;
if (!task.result) return null;

5
packages/lib/file-api-driver-joplinServer.ts
View File

@ -45,6 +45,10 @@ export default class FileApiDriverJoplinServer {
return true;
}
public get supportsDeltaWithItems() {
return true;
}
public requestRepeatCount() {
return 3;
}
@ -56,6 +60,7 @@ export default class FileApiDriverJoplinServer {
jop_updated_time: md.jop_updated_time,
isDir: false,
isDeleted: isDeleted,
jopItem: md.jopItem,
};
return output;

9
packages/lib/file-api.ts
View File

@ -33,6 +33,8 @@ export interface RemoteItem {
// value will always be ahead. However for synchronising we need to know the
// exact Joplin item updated_time value.
jop_updated_time?: number;
jopItem?: any;
}
export interface PaginatedList {
@ -137,6 +139,13 @@ class FileApi {
return !!this.driver().supportsLocks;
}
// Tells whether the delta call is going to include the items themselves or
// just the metadata (which is the default). If the items are included it
// means less http request and faster processing.
public get supportsDeltaWithItems(): boolean {
return !!this.driver().supportsDeltaWithItems;
}
private async fetchRemoteDateOffset_() {
const tempFile = `${this.tempDirName()}/timeCheck${Math.round(Math.random() * 1000000)}.txt`;
const startTime = Date.now();

3
packages/server/src/env.ts
View File

@ -39,6 +39,8 @@ const defaultEnvValues: EnvVariables = {
MAX_TIME_DRIFT: 2000,
NTP_SERVER: 'pool.ntp.org:123',
DELTA_INCLUDES_ITEMS: true,
// ==================================================
// URL config
// ==================================================
@ -141,6 +143,7 @@ export interface EnvVariables {
RUNNING_IN_DOCKER: boolean;
MAX_TIME_DRIFT: number;
NTP_SERVER: string;
DELTA_INCLUDES_ITEMS: boolean;
IS_ADMIN_INSTANCE: boolean;
INSTANCE_NAME: string;

105
packages/server/src/models/ChangeModel.ts
View File

@ -1,12 +1,14 @@
import { Knex } from 'knex';
import Logger from '@joplin/utils/Logger';
import { SqliteMaxVariableNum, isPostgres } from '../db';
import { DbConnection, SqliteMaxVariableNum, isPostgres } from '../db';
import { Change, ChangeType, Item, Uuid } from '../services/database/types';
import { md5 } from '../utils/crypto';
import { ErrorResyncRequired } from '../utils/errors';
import { Day, formatDateTime } from '../utils/time';
import BaseModel, { SaveOptions } from './BaseModel';
import { PaginatedResults } from './utils/pagination';
import { NewModelFactoryHandler } from './factory';
import { Config } from '../utils/types';
const logger = Logger.create('ChangeModel');
@ -51,6 +53,13 @@ export function requestDeltaPagination(query: any): ChangePagination {
export default class ChangeModel extends BaseModel<Change> {
private deltaIncludesItems_: boolean;
public constructor(db: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
super(db, modelFactory, config);
this.deltaIncludesItems_ = config.DELTA_INCLUDES_ITEMS;
}
public get tableName(): string {
return 'changes';
}
@ -89,43 +98,6 @@ export default class ChangeModel extends BaseModel<Change> {
};
}
// private changesForUserQuery(userId: Uuid, count: boolean): Knex.QueryBuilder {
// // When need to get:
// //
// // - All the CREATE and DELETE changes associated with the user
// // - All the UPDATE changes that applies to items associated with the
// // user.
// //
// // UPDATE changes do not have the user_id set because they are specific
// // to the item, not to a particular user.
// const query = this
// .db('changes')
// .where(function() {
// void this.whereRaw('((type = ? OR type = ?) AND user_id = ?)', [ChangeType.Create, ChangeType.Delete, userId])
// // Need to use a RAW query here because Knex has a "not a
// // bug" bug that makes it go into infinite loop in some
// // contexts, possibly only when running inside Jest (didn't
// // test outside).
// // https://github.com/knex/knex/issues/1851
// .orWhereRaw('type = ? AND item_id IN (SELECT item_id FROM user_items WHERE user_id = ?)', [ChangeType.Update, userId]);
// });
// if (count) {
// void query.countDistinct('id', { as: 'total' });
// } else {
// void query.select([
// 'id',
// 'item_id',
// 'item_name',
// 'type',
// 'updated_time',
// ]);
// }
// return query;
// }
public async changesForUserQuery(userId: Uuid, fromCounter: number, limit: number, doCountQuery: boolean): Promise<Change[]> {
// When need to get:
//
@ -252,35 +224,6 @@ export default class ChangeModel extends BaseModel<Change> {
return output;
}
// public async allByUser(userId: Uuid, pagination: Pagination = null): Promise<PaginatedDeltaChanges> {
// pagination = {
// page: 1,
// limit: 100,
// order: [{ by: 'counter', dir: PaginationOrderDir.ASC }],
// ...pagination,
// };
// const query = this.changesForUserQuery(userId, false);
// const countQuery = this.changesForUserQuery(userId, true);
// const itemCount = (await countQuery.first()).total;
// void query
// .orderBy(pagination.order[0].by, pagination.order[0].dir)
// .offset((pagination.page - 1) * pagination.limit)
// .limit(pagination.limit) as any[];
// const changes = await query;
// return {
// items: changes,
// // If we have changes, we return the ID of the latest changes from which delta sync can resume.
// // If there's no change, we return the previous cursor.
// cursor: changes.length ? changes[changes.length - 1].id : pagination.cursor,
// has_more: changes.length >= pagination.limit,
// page_count: itemCount !== null ? Math.ceil(itemCount / pagination.limit) : undefined,
// };
// }
public async delta(userId: Uuid, pagination: ChangePagination = null): Promise<PaginatedDeltaChanges> {
pagination = {
...defaultDeltaPagination(),
@ -306,18 +249,20 @@ export default class ChangeModel extends BaseModel<Change> {
let processedChanges = this.compressChanges(changes);
processedChanges = await this.removeDeletedItems(processedChanges, items);
items = await this.models().item().loadWithContentMulti(processedChanges.map(c => c.item_id), {
fields: [
'content',
'id',
'jop_encryption_applied',
'jop_id',
'jop_parent_id',
'jop_share_id',
'jop_type',
'jop_updated_time',
],
});
if (this.deltaIncludesItems_) {
items = await this.models().item().loadWithContentMulti(processedChanges.map(c => c.item_id), {
fields: [
'content',
'id',
'jop_encryption_applied',
'jop_id',
'jop_parent_id',
'jop_share_id',
'jop_type',
'jop_updated_time',
],
});
}
const finalChanges = processedChanges.map(change => {
const item = items.find(item => item.id === change.item_id);
@ -325,7 +270,7 @@ export default class ChangeModel extends BaseModel<Change> {
const deltaChange: DeltaChange = {
...change,
jop_updated_time: item.jop_updated_time,
jopItem: item.jop_type ? this.models().item().itemToJoplinItem(item) : null,
jopItem: this.deltaIncludesItems_ && item.jop_type ? this.models().item().itemToJoplinItem(item) : null,
};
return deltaChange;
});