1
0
mirror of https://github.com/laurent22/joplin.git synced 2025-12-23 23:33:01 +02:00

Compare commits

..

8 Commits

Author SHA1 Message Date
Laurent Cozic
ae696ce8f5 log 2021-11-02 10:53:27 +00:00
Laurent Cozic
11dee3cc52 validare 2021-11-02 10:21:38 +00:00
Laurent Cozic
6cb413deba v2.6 2021-11-01 19:45:11 +00:00
Laurent Cozic
d5f1075de4 Merge branch 'dev' into server_native_lock_2 2021-11-01 19:21:31 +00:00
Laurent Cozic
370441333f Server: Improved logging and rendering of low level middleware errors 2021-11-01 19:20:36 +00:00
Laurent Cozic
b4e9aeb6f8 locks 2021-11-01 18:57:31 +00:00
Laurent Cozic
b6ffc31dfc tests 2021-11-01 16:59:29 +00:00
Laurent Cozic
d735d14a01 reboot 2021-11-01 16:39:58 +00:00
22 changed files with 289 additions and 301 deletions

View File

@@ -142,7 +142,7 @@ export default class JoplinServerApi {
}
if (sessionId) headers['X-API-AUTH'] = sessionId;
headers['X-API-MIN-VERSION'] = '2.1.4';
headers['X-API-MIN-VERSION'] = '2.6.0'; // Need server 2.6 for new lock support
const fetchOptions: any = {};
fetchOptions.headers = headers;

View File

@@ -1,6 +1,6 @@
import Logger from './Logger';
import LockHandler, { hasActiveLock, LockType } from './services/synchronizer/LockHandler';
import Setting from './models/Setting';
import LockHandler, { hasActiveLock, LockClientType, LockType } from './services/synchronizer/LockHandler';
import Setting, { AppType } from './models/Setting';
import shim from './shim';
import MigrationHandler from './services/synchronizer/MigrationHandler';
import eventManager from './eventManager';
@@ -66,6 +66,7 @@ export default class Synchronizer {
private resourceService_: ResourceService = null;
private syncTargetIsLocked_: boolean = false;
private shareService_: ShareService = null;
private lockClientType_: LockClientType = null;
// Debug flags are used to test certain hard-to-test conditions
// such as cancelling in the middle of a loop.
@@ -120,9 +121,21 @@ export default class Synchronizer {
return this.lockHandler_;
}
private lockClientType(): LockClientType {
if (this.lockClientType_) return this.lockClientType_;
if (this.appType_ === AppType.Desktop) this.lockClientType_ = LockClientType.Desktop;
if (this.appType_ === AppType.Mobile) this.lockClientType_ = LockClientType.Mobile;
if (this.appType_ === AppType.Cli) this.lockClientType_ = LockClientType.Cli;
if (!this.lockClientType_) throw new Error(`Invalid client type: ${this.appType_}`);
return this.lockClientType_;
}
migrationHandler() {
if (this.migrationHandler_) return this.migrationHandler_;
this.migrationHandler_ = new MigrationHandler(this.api(), this.db(), this.lockHandler(), this.appType_, this.clientId_);
this.migrationHandler_ = new MigrationHandler(this.api(), this.db(), this.lockHandler(), this.lockClientType(), this.clientId_);
return this.migrationHandler_;
}
@@ -164,6 +177,12 @@ export default class Synchronizer {
return !!report && !!report.errors && !!report.errors.length;
}
private static completionTime(report: any): string {
const duration = report.completedTime - report.startTime;
if (duration > 1000) return `${Math.round(duration / 1000)}s`;
return `${duration}ms`;
}
static reportToLines(report: any) {
const lines = [];
if (report.createLocal) lines.push(_('Created local items: %d.', report.createLocal));
@@ -174,7 +193,7 @@ export default class Synchronizer {
if (report.deleteRemote) lines.push(_('Deleted remote items: %d.', report.deleteRemote));
if (report.fetchingTotal && report.fetchingProcessed) lines.push(_('Fetched items: %d/%d.', report.fetchingProcessed, report.fetchingTotal));
if (report.cancelling && !report.completedTime) lines.push(_('Cancelling...'));
if (report.completedTime) lines.push(_('Completed: %s (%s)', time.formatMsToLocal(report.completedTime), `${Math.round((report.completedTime - report.startTime) / 1000)}s`));
if (report.completedTime) lines.push(_('Completed: %s (%s)', time.formatMsToLocal(report.completedTime), this.completionTime(report)));
if (this.reportHasErrors(report)) lines.push(_('Last error: %s', report.errors[report.errors.length - 1].toString().substr(0, 500)));
return lines;
@@ -304,7 +323,7 @@ export default class Synchronizer {
const hasActiveExclusiveLock = await hasActiveLock(locks, currentDate, this.lockHandler().lockTtl, LockType.Exclusive);
if (hasActiveExclusiveLock) return 'hasExclusiveLock';
const hasActiveSyncLock = await hasActiveLock(locks, currentDate, this.lockHandler().lockTtl, LockType.Sync, this.appType_, this.clientId_);
const hasActiveSyncLock = await hasActiveLock(locks, currentDate, this.lockHandler().lockTtl, LockType.Sync, this.lockClientType(), this.clientId_);
if (!hasActiveSyncLock) return 'syncLockGone';
return '';
@@ -449,10 +468,10 @@ export default class Synchronizer {
const previousE2EE = localInfo.e2ee;
logger.info('Sync target info differs between local and remote - merging infos: ', newInfo.toObject());
await this.lockHandler().acquireLock(LockType.Exclusive, this.appType_, this.clientId_, { clearExistingSyncLocksFromTheSameClient: true });
await this.lockHandler().acquireLock(LockType.Exclusive, this.lockClientType(), this.clientId_, { clearExistingSyncLocksFromTheSameClient: true });
await uploadSyncInfo(this.api(), newInfo);
await saveLocalSyncInfo(newInfo);
await this.lockHandler().releaseLock(LockType.Exclusive, this.appType_, this.clientId_);
await this.lockHandler().releaseLock(LockType.Exclusive, this.lockClientType(), this.clientId_);
// console.info('NEW', newInfo);
@@ -476,7 +495,7 @@ export default class Synchronizer {
throw error;
}
syncLock = await this.lockHandler().acquireLock(LockType.Sync, this.appType_, this.clientId_);
syncLock = await this.lockHandler().acquireLock(LockType.Sync, this.lockClientType(), this.clientId_);
this.lockHandler().startAutoLockRefresh(syncLock, (error: any) => {
logger.warn('Could not refresh lock - cancelling sync. Error was:', error);
@@ -1087,7 +1106,7 @@ export default class Synchronizer {
if (syncLock) {
this.lockHandler().stopAutoLockRefresh(syncLock);
await this.lockHandler().releaseLock(LockType.Sync, this.appType_, this.clientId_);
await this.lockHandler().releaseLock(LockType.Sync, this.lockClientType(), this.clientId_);
}
this.syncTargetIsLocked_ = false;

View File

@@ -2,7 +2,7 @@ import { MultiPutItem } from './file-api';
import JoplinError from './JoplinError';
import JoplinServerApi from './JoplinServerApi';
import { trimSlashes } from './path-utils';
import { Lock, LockType } from './services/synchronizer/LockHandler';
import { Lock, LockClientType, LockType } from './services/synchronizer/LockHandler';
// All input paths should be in the format: "path/to/file". This is converted to
// "root:/path/to/file:" when doing the API call.
@@ -201,15 +201,43 @@ export default class FileApiDriverJoplinServer {
throw new Error('Not supported');
}
public async acquireLock(type: LockType, clientType: string, clientId: string): Promise<Lock> {
// private lockClientTypeToId(clientType:AppType):number {
// if (clientType === AppType.Desktop) return 1;
// if (clientType === AppType.Mobile) return 2;
// if (clientType === AppType.Cli) return 3;
// throw new Error('Invalid client type: ' + clientType);
// }
// private lockTypeToId(lockType:LockType):number {
// if (lockType === LockType.None) return 0; // probably not possible?
// if (lockType === LockType.Sync) return 1;
// if (lockType === LockType.Exclusive) return 2;
// throw new Error('Invalid lock type: ' + lockType);
// }
// private lockClientIdTypeToType(clientType:number):AppType {
// if (clientType === 1) return AppType.Desktop;
// if (clientType === 2) return AppType.Mobile;
// if (clientType === 3) return AppType.Cli;
// throw new Error('Invalid client type: ' + clientType);
// }
// private lockIdToType(lockType:number):LockType {
// if (lockType === 0) return LockType.None; // probably not possible?
// if (lockType === 1) return LockType.Sync;
// if (lockType === 2) return LockType.Exclusive;
// throw new Error('Invalid lock type: ' + lockType);
// }
public async acquireLock(type: LockType, clientType: LockClientType, clientId: string): Promise<Lock> {
return this.api().exec('POST', 'api/locks', null, {
type,
clientType: clientType,
clientType,
clientId: clientId,
});
}
public async releaseLock(type: LockType, clientType: string, clientId: string) {
public async releaseLock(type: LockType, clientType: LockClientType, clientId: string) {
await this.api().exec('DELETE', `api/locks/${type}_${clientType}_${clientId}`);
}

View File

@@ -5,7 +5,7 @@ import time from './time';
const { isHidden } = require('./path-utils');
import JoplinError from './JoplinError';
import { Lock, LockType } from './services/synchronizer/LockHandler';
import { Lock, LockClientType, LockType } from './services/synchronizer/LockHandler';
const ArrayUtils = require('./ArrayUtils');
const { sprintf } = require('sprintf-js');
const Mutex = require('async-mutex').Mutex;
@@ -355,12 +355,12 @@ class FileApi {
return tryAndRepeat(() => this.driver_.delta(this.fullPath(path), options), this.requestRepeatCount());
}
public async acquireLock(type: LockType, clientType: string, clientId: string): Promise<Lock> {
public async acquireLock(type: LockType, clientType: LockClientType, clientId: string): Promise<Lock> {
if (!this.supportsLocks) throw new Error('Sync target does not support built-in locks');
return tryAndRepeat(() => this.driver_.acquireLock(type, clientType, clientId), this.requestRepeatCount());
}
public async releaseLock(type: LockType, clientType: string, clientId: string) {
public async releaseLock(type: LockType, clientType: LockClientType, clientId: string) {
if (!this.supportsLocks) throw new Error('Sync target does not support built-in locks');
return tryAndRepeat(() => this.driver_.releaseLock(type, clientType, clientId), this.requestRepeatCount());
}

View File

@@ -1,21 +1,26 @@
import { Dirnames } from './utils/types';
import shim from '../../shim';
import JoplinError from '../../JoplinError';
import time from '../../time';
import { FileApi } from '../../file-api';
const { fileExtension, filename } = require('../../path-utils');
export enum LockType {
None = '',
Sync = 'sync',
Exclusive = 'exclusive',
None = 0,
Sync = 1,
Exclusive = 2,
}
export enum LockClientType {
Desktop = 1,
Mobile = 2,
Cli = 3,
}
export interface Lock {
id?: string;
type: LockType;
clientType: string;
clientType: LockClientType;
clientId: string;
updatedTime?: number;
}
@@ -27,15 +32,21 @@ function lockIsActive(lock: Lock, currentDate: Date, lockTtl: number): boolean {
export function lockNameToObject(name: string, updatedTime: number = null): Lock {
const p = name.split('_');
return {
type: p[0] as LockType,
clientType: p[1],
const lock: Lock = {
id: null,
type: Number(p[0]) as LockType,
clientType: Number(p[1]) as LockClientType,
clientId: p[2],
updatedTime: updatedTime,
updatedTime,
};
if (isNaN(lock.clientType)) throw new Error(`Invalid lock client type: ${name}`);
if (isNaN(lock.type)) throw new Error(`Invalid lock type: ${name}`);
return lock;
}
export function hasActiveLock(locks: Lock[], currentDate: Date, lockTtl: number, lockType: LockType, clientType: string = null, clientId: string = null) {
export function hasActiveLock(locks: Lock[], currentDate: Date, lockTtl: number, lockType: LockType, clientType: LockClientType = null, clientId: string = null) {
const lock = activeLock(locks, currentDate, lockTtl, lockType, clientType, clientId);
return !!lock;
}
@@ -43,7 +54,7 @@ export function hasActiveLock(locks: Lock[], currentDate: Date, lockTtl: number,
// Finds if there's an active lock for this clientType and clientId and returns it.
// If clientType and clientId are not specified, returns the first active lock
// of that type instead.
export function activeLock(locks: Lock[], currentDate: Date, lockTtl: number, lockType: LockType, clientType: string = null, clientId: string = null) {
export function activeLock(locks: Lock[], currentDate: Date, lockTtl: number, lockType: LockType, clientType: LockClientType = null, clientId: string = null) {
if (lockType === LockType.Exclusive) {
const activeLocks = locks
.slice()
@@ -112,14 +123,14 @@ export interface LockHandlerOptions {
lockTtl?: number;
}
export const lockDefaultTtl = 1000 * 60 * 3;
export const defaultLockTtl = 1000 * 60 * 3;
export default class LockHandler {
private api_: FileApi = null;
private refreshTimers_: RefreshTimers = {};
private autoRefreshInterval_: number = 1000 * 60;
private lockTtl_: number = lockDefaultTtl;
private lockTtl_: number = defaultLockTtl;
public constructor(api: FileApi, options: LockHandlerOptions = null) {
if (!options) options = {};
@@ -150,8 +161,8 @@ export default class LockHandler {
private lockTypeFromFilename(name: string): LockType {
const ext = fileExtension(name);
if (ext !== 'json') return LockType.None;
if (name.indexOf(LockType.Sync) === 0) return LockType.Sync;
if (name.indexOf(LockType.Exclusive) === 0) return LockType.Exclusive;
if (name.indexOf(LockType.Sync.toString()) === 0) return LockType.Sync;
if (name.indexOf(LockType.Exclusive.toString()) === 0) return LockType.Exclusive;
return LockType.None;
}
@@ -193,7 +204,7 @@ export default class LockHandler {
return this.saveLock(lock);
}
private async acquireSyncLock(clientType: string, clientId: string): Promise<Lock> {
private async acquireSyncLock(clientType: LockClientType, clientId: string): Promise<Lock> {
if (this.useBuiltInLocks) return this.api_.acquireLock(LockType.Sync, clientType, clientId);
try {
@@ -242,7 +253,7 @@ export default class LockHandler {
return `(${lock.clientType} #${lock.clientId})`;
}
private async acquireExclusiveLock(clientType: string, clientId: string, options: AcquireLockOptions = null): Promise<Lock> {
private async acquireExclusiveLock(clientType: LockClientType, clientId: string, options: AcquireLockOptions = null): Promise<Lock> {
if (this.useBuiltInLocks) return this.api_.acquireLock(LockType.Exclusive, clientType, clientId);
// The logic to acquire an exclusive lock, while avoiding race conditions is as follow:
@@ -399,7 +410,7 @@ export default class LockHandler {
delete this.refreshTimers_[handle];
}
public async acquireLock(lockType: LockType, clientType: string, clientId: string, options: AcquireLockOptions = null): Promise<Lock> {
public async acquireLock(lockType: LockType, clientType: LockClientType, clientId: string, options: AcquireLockOptions = null): Promise<Lock> {
options = {
...defaultAcquireLockOptions(),
...options,
@@ -414,7 +425,7 @@ export default class LockHandler {
}
}
public async releaseLock(lockType: LockType, clientType: string, clientId: string) {
public async releaseLock(lockType: LockType, clientType: LockClientType, clientId: string) {
if (this.useBuiltInLocks) {
await this.api_.releaseLock(lockType, clientType, clientId);
return;

View File

@@ -1,4 +1,4 @@
import LockHandler, { LockType } from './LockHandler';
import LockHandler, { LockClientType, LockType } from './LockHandler';
import { Dirnames } from './utils/types';
import BaseService from '../BaseService';
import migration1 from './migrations/1';
@@ -33,11 +33,11 @@ export default class MigrationHandler extends BaseService {
private api_: FileApi = null;
private lockHandler_: LockHandler = null;
private clientType_: string;
private clientType_: LockClientType;
private clientId_: string;
private db_: JoplinDatabase;
public constructor(api: FileApi, db: JoplinDatabase, lockHandler: LockHandler, clientType: string, clientId: string) {
public constructor(api: FileApi, db: JoplinDatabase, lockHandler: LockHandler, clientType: LockClientType, clientId: string) {
super();
this.api_ = api;
this.db_ = db;

View File

@@ -1,4 +1,4 @@
import LockHandler, { LockType, LockHandlerOptions, Lock, activeLock } from '../../services/synchronizer/LockHandler';
import LockHandler, { LockType, LockHandlerOptions, Lock, activeLock, LockClientType } from '../../services/synchronizer/LockHandler';
import { isNetworkSyncTarget, fileApi, setupDatabaseAndSynchronizer, synchronizer, switchClient, msleep, expectThrow, expectNotThrow } from '../../testing/test-utils';
// For tests with memory of file system we can use low intervals to make the tests faster.
@@ -34,14 +34,14 @@ describe('synchronizer_LockHandler', function() {
});
it('should acquire and release a sync lock', (async () => {
await lockHandler().acquireLock(LockType.Sync, 'mobile', '123456');
await lockHandler().acquireLock(LockType.Sync, LockClientType.Mobile, '123456');
const locks = await lockHandler().locks(LockType.Sync);
expect(locks.length).toBe(1);
expect(locks[0].type).toBe(LockType.Sync);
expect(locks[0].clientId).toBe('123456');
expect(locks[0].clientType).toBe('mobile');
expect(locks[0].clientType).toBe(LockClientType.Mobile);
await lockHandler().releaseLock(LockType.Sync, 'mobile', '123456');
await lockHandler().releaseLock(LockType.Sync, LockClientType.Mobile, '123456');
expect((await lockHandler().locks(LockType.Sync)).length).toBe(0);
}));
@@ -51,23 +51,29 @@ describe('synchronizer_LockHandler', function() {
await fileApi().put('locks/desktop.ini', 'a');
await fileApi().put('locks/exclusive.json', 'a');
await fileApi().put('locks/garbage.json', 'a');
await fileApi().put('locks/sync_mobile_72c4d1b7253a4475bfb2f977117d26ed.json', 'a');
await fileApi().put('locks/1_2_72c4d1b7253a4475bfb2f977117d26ed.json', 'a');
// Check that it doesn't cause an error if it fetches an old style lock
await fileApi().put('locks/sync_desktop_82c4d1b7253a4475bfb2f977117d26ed.json', 'a');
const locks = await lockHandler().locks(LockType.Sync);
expect(locks.length).toBe(1);
expect(locks[0].type).toBe(LockType.Sync);
expect(locks[0].clientType).toBe(LockClientType.Mobile);
expect(locks[0].clientId).toBe('72c4d1b7253a4475bfb2f977117d26ed');
}));
it('should allow multiple sync locks', (async () => {
await lockHandler().acquireLock(LockType.Sync, 'mobile', '111');
await lockHandler().acquireLock(LockType.Sync, LockClientType.Mobile, '111');
await switchClient(2);
await lockHandler().acquireLock(LockType.Sync, 'mobile', '222');
await lockHandler().acquireLock(LockType.Sync, LockClientType.Mobile, '222');
expect((await lockHandler().locks(LockType.Sync)).length).toBe(2);
{
await lockHandler().releaseLock(LockType.Sync, 'mobile', '222');
await lockHandler().releaseLock(LockType.Sync, LockClientType.Mobile, '222');
const locks = await lockHandler().locks(LockType.Sync);
expect(locks.length).toBe(1);
expect(locks[0].clientId).toBe('111');
@@ -76,11 +82,11 @@ describe('synchronizer_LockHandler', function() {
it('should auto-refresh a lock', (async () => {
const handler = newLockHandler({ autoRefreshInterval: 100 * timeoutMultipler });
const lock = await handler.acquireLock(LockType.Sync, 'desktop', '111');
const lockBefore = activeLock(await handler.locks(), new Date(), handler.lockTtl, LockType.Sync, 'desktop', '111');
const lock = await handler.acquireLock(LockType.Sync, LockClientType.Desktop, '111');
const lockBefore = activeLock(await handler.locks(), new Date(), handler.lockTtl, LockType.Sync, LockClientType.Desktop, '111');
handler.startAutoLockRefresh(lock, () => {});
await msleep(500 * timeoutMultipler);
const lockAfter = activeLock(await handler.locks(), new Date(), handler.lockTtl, LockType.Sync, 'desktop', '111');
const lockAfter = activeLock(await handler.locks(), new Date(), handler.lockTtl, LockType.Sync, LockClientType.Desktop, '111');
expect(lockAfter.updatedTime).toBeGreaterThan(lockBefore.updatedTime);
handler.stopAutoLockRefresh(lock);
}));
@@ -91,7 +97,7 @@ describe('synchronizer_LockHandler', function() {
autoRefreshInterval: 200 * timeoutMultipler,
});
const lock = await handler.acquireLock(LockType.Sync, 'desktop', '111');
const lock = await handler.acquireLock(LockType.Sync, LockClientType.Desktop, '111');
let autoLockError: any = null;
handler.startAutoLockRefresh(lock, (error: any) => {
autoLockError = error;
@@ -108,10 +114,10 @@ describe('synchronizer_LockHandler', function() {
}));
it('should not allow sync locks if there is an exclusive lock', (async () => {
await lockHandler().acquireLock(LockType.Exclusive, 'desktop', '111');
await lockHandler().acquireLock(LockType.Exclusive, LockClientType.Desktop, '111');
await expectThrow(async () => {
await lockHandler().acquireLock(LockType.Sync, 'mobile', '222');
await lockHandler().acquireLock(LockType.Sync, LockClientType.Mobile, '222');
}, 'hasExclusiveLock');
}));
@@ -119,11 +125,11 @@ describe('synchronizer_LockHandler', function() {
const lockHandler = newLockHandler({ lockTtl: 1000 * 60 * 60 });
if (lockHandler.useBuiltInLocks) return; // Tested server side
await lockHandler.acquireLock(LockType.Sync, 'mobile', '111');
await lockHandler.acquireLock(LockType.Sync, 'mobile', '222');
await lockHandler.acquireLock(LockType.Sync, LockClientType.Mobile, '111');
await lockHandler.acquireLock(LockType.Sync, LockClientType.Mobile, '222');
await expectThrow(async () => {
await lockHandler.acquireLock(LockType.Exclusive, 'desktop', '333');
await lockHandler.acquireLock(LockType.Exclusive, LockClientType.Desktop, '333');
}, 'hasSyncLock');
}));
@@ -131,13 +137,13 @@ describe('synchronizer_LockHandler', function() {
const lockHandler = newLockHandler({ lockTtl: 500 * timeoutMultipler });
if (lockHandler.useBuiltInLocks) return; // Tested server side
await lockHandler.acquireLock(LockType.Sync, 'mobile', '111');
await lockHandler.acquireLock(LockType.Sync, 'mobile', '222');
await lockHandler.acquireLock(LockType.Sync, LockClientType.Mobile, '111');
await lockHandler.acquireLock(LockType.Sync, LockClientType.Mobile, '222');
await msleep(600 * timeoutMultipler);
await expectNotThrow(async () => {
await lockHandler.acquireLock(LockType.Exclusive, 'desktop', '333');
await lockHandler.acquireLock(LockType.Exclusive, LockClientType.Desktop, '333');
});
}));
@@ -149,7 +155,7 @@ describe('synchronizer_LockHandler', function() {
{
type: LockType.Exclusive,
clientId: '1',
clientType: 'd',
clientType: LockClientType.Desktop,
updatedTime: Date.now(),
},
];
@@ -159,7 +165,7 @@ describe('synchronizer_LockHandler', function() {
locks.push({
type: LockType.Exclusive,
clientId: '2',
clientType: 'd',
clientType: LockClientType.Desktop,
updatedTime: Date.now(),
});
@@ -171,14 +177,14 @@ describe('synchronizer_LockHandler', function() {
// it('should ignore locks by same client when trying to acquire exclusive lock', (async () => {
// const lockHandler = newLockHandler();
// await lockHandler.acquireLock(LockType.Sync, 'desktop', '111');
// await lockHandler.acquireLock(LockType.Sync, LockClientType.Desktop, '111');
// await expectThrow(async () => {
// await lockHandler.acquireLock(LockType.Exclusive, 'desktop', '111', { clearExistingSyncLocksFromTheSameClient: false });
// await lockHandler.acquireLock(LockType.Exclusive, LockClientType.Desktop, '111', { clearExistingSyncLocksFromTheSameClient: false });
// }, 'hasSyncLock');
// await expectNotThrow(async () => {
// await lockHandler.acquireLock(LockType.Exclusive, 'desktop', '111', { clearExistingSyncLocksFromTheSameClient: true });
// await lockHandler.acquireLock(LockType.Exclusive, LockClientType.Desktop, '111', { clearExistingSyncLocksFromTheSameClient: true });
// });
// const lock = activeLock(await lockHandler.locks(), new Date(), lockHandler.lockTtl, LockType.Exclusive);

View File

@@ -5,7 +5,7 @@
//
// These tests work by a taking a sync target snapshot at a version n and upgrading it to n+1.
import LockHandler from './LockHandler';
import LockHandler, { LockClientType } from './LockHandler';
import MigrationHandler from './MigrationHandler';
import { Dirnames } from './utils/types';
import { setSyncTargetName, fileApi, synchronizer, decryptionWorker, encryptionService, setupDatabaseAndSynchronizer, switchClient, expectThrow, expectNotThrow, db } from '../../testing/test-utils';
@@ -28,7 +28,7 @@ function lockHandler(): LockHandler {
function migrationHandler(clientId: string = 'abcd'): MigrationHandler {
if (migrationHandler_) return migrationHandler_;
migrationHandler_ = new MigrationHandler(fileApi(), db(), lockHandler(), 'desktop', clientId);
migrationHandler_ = new MigrationHandler(fileApi(), db(), lockHandler(), LockClientType.Desktop, clientId);
return migrationHandler_;
}

View File

@@ -1,6 +1,6 @@
{
"name": "@joplin/server",
"version": "2.5.9",
"version": "2.6.0",
"private": true,
"scripts": {
"start-dev": "npm run build && nodemon --config nodemon.json --ext ts,js,mustache,css,tsx dist/app.js --env dev",

Binary file not shown.

View File

@@ -19,6 +19,7 @@ import apiVersionHandler from './middleware/apiVersionHandler';
import clickJackingHandler from './middleware/clickJackingHandler';
import newModelFactory from './models/factory';
import setupCommands from './utils/setupCommands';
import { RouteResponseFormat, routeResponseFormat } from './utils/routeUtils';
interface Argv {
env?: Env;
@@ -139,17 +140,28 @@ async function main() {
} catch (error) {
ctx.status = error.httpCode || 500;
// Since this is a low level error, rendering a view might fail too,
// so catch this and default to rendering JSON.
try {
ctx.body = await ctx.joplin.services.mustache.renderView({
name: 'error',
title: 'Error',
path: 'index/error',
content: { error },
});
} catch (anotherError) {
ctx.body = { error: anotherError.message };
appLogger().error(`Middleware error on ${ctx.path}:`, error);
const responseFormat = routeResponseFormat(ctx);
if (responseFormat === RouteResponseFormat.Html) {
// Since this is a low level error, rendering a view might fail too,
// so catch this and default to rendering JSON.
try {
ctx.response.set('Content-Type', 'text/html');
ctx.body = await ctx.joplin.services.mustache.renderView({
name: 'error',
title: 'Error',
path: 'index/error',
content: { error },
});
} catch (anotherError) {
ctx.response.set('Content-Type', 'application/json');
ctx.body = JSON.stringify({ error: error.message });
}
} else {
ctx.response.set('Content-Type', 'application/json');
ctx.body = JSON.stringify({ error: error.message });
}
}
});

View File

@@ -22,7 +22,6 @@ export interface EnvVariables {
ERROR_STACK_TRACES?: string;
COOKIES_SECURE?: string;
RUNNING_IN_DOCKER?: string;
BUILTIN_LOCKS_ENABLED?: string;
// ==================================================
// URL config
@@ -212,7 +211,6 @@ export async function initConfig(envType: Env, env: EnvVariables, overrides: any
supportName: env.SUPPORT_NAME || appName,
businessEmail: env.BUSINESS_EMAIL || supportEmail,
cookieSecure: env.COOKIES_SECURE === '1',
buildInLocksEnabled: envReadBool(env.BUILTIN_LOCKS_ENABLED, false),
...overrides,
};
}

View File

@@ -1,23 +0,0 @@
import { Knex } from 'knex';
import { DbConnection } from '../db';
export async function up(db: DbConnection): Promise<any> {
await db.schema.createTable('locks', (table: Knex.CreateTableBuilder) => {
table.uuid('id').unique().primary().notNullable();
table.string('user_id', 32).notNullable();
table.integer('type', 2).notNullable();
table.string('client_type', 32).notNullable();
table.string('client_id', 32).notNullable();
table.bigInteger('updated_time').notNullable();
table.bigInteger('created_time').notNullable();
});
await db.schema.alterTable('locks', (table: Knex.CreateTableBuilder) => {
table.index('user_id');
table.index('created_time');
});
}
export async function down(db: DbConnection): Promise<any> {
await db.schema.dropTable('locks');
}

View File

@@ -95,10 +95,6 @@ export default abstract class BaseModel<T> {
return this.db_;
}
protected get dbRead(): DbConnection {
return this.db;
}
protected get defaultFields(): string[] {
if (!this.defaultFields_.length) {
this.defaultFields_ = Object.keys(databaseSchema[this.tableName]);

View File

@@ -0,0 +1,94 @@
// Note that a lot of the testing logic is done from
// synchronizer_LockHandler.test so to fully test that it works, Joplin Server
// should be setup as a sync target for the test units.
import { ErrorConflict, ErrorUnprocessableEntity } from '../utils/errors';
import { beforeAllDb, afterAllTests, beforeEachDb, models, createUserAndSession, expectHttpError } from '../utils/testing/testUtils';
import { LockType, LockClientType, defaultLockTtl } from '@joplin/lib/services/synchronizer/LockHandler';
describe('LockModel', function() {
beforeAll(async () => {
await beforeAllDb('LockModel');
});
afterAll(async () => {
await afterAllTests();
});
beforeEach(async () => {
await beforeEachDb();
});
test('should allow exclusive lock if the sync locks have expired', async function() {
jest.useFakeTimers('modern');
const { user } = await createUserAndSession(1);
const t1 = new Date('2020-01-01').getTime();
jest.setSystemTime(t1);
await models().lock().acquireLock(user.id, LockType.Sync, LockClientType.Desktop, '1111');
await models().lock().acquireLock(user.id, LockType.Sync, LockClientType.Desktop, '2222');
// First confirm that it's not possible to acquire an exclusive lock if
// there are sync locks.
await expectHttpError(async () => models().lock().acquireLock(user.id, LockType.Exclusive, LockClientType.Desktop, '3333'), ErrorConflict.httpCode);
jest.setSystemTime(t1 + defaultLockTtl + 1);
// Now that the sync locks have expired check that it's possible to
// acquire a sync lock.
const exclusiveLock = await models().lock().acquireLock(user.id, LockType.Exclusive, LockClientType.Desktop, '3333');
expect(exclusiveLock).toBeTruthy();
jest.useRealTimers();
});
test('should keep user locks separated', async function() {
const { user: user1 } = await createUserAndSession(1);
const { user: user2 } = await createUserAndSession(2);
await models().lock().acquireLock(user1.id, LockType.Sync, LockClientType.Desktop, '1111');
// If user 1 tries to get an exclusive lock, it should fail
await expectHttpError(async () => models().lock().acquireLock(user1.id, LockType.Exclusive, LockClientType.Desktop, '3333'), ErrorConflict.httpCode);
// But it should work for user 2
const exclusiveLock = await models().lock().acquireLock(user2.id, LockType.Exclusive, LockClientType.Desktop, '3333');
expect(exclusiveLock).toBeTruthy();
});
test('should validate locks', async function() {
const { user: user1 } = await createUserAndSession(1);
await expectHttpError(async () => models().lock().acquireLock(user1.id, 'wrongtype' as any, LockClientType.Desktop, '1111'), ErrorUnprocessableEntity.httpCode);
await expectHttpError(async () => models().lock().acquireLock(user1.id, LockType.Exclusive, 'wrongclienttype' as any, '1111'), ErrorUnprocessableEntity.httpCode);
await expectHttpError(async () => models().lock().acquireLock(user1.id, LockType.Exclusive, LockClientType.Desktop, 'veryverylongclientidveryverylongclientidveryverylongclientidveryverylongclientid'), ErrorUnprocessableEntity.httpCode);
});
test('should expire locks', async function() {
const { user } = await createUserAndSession(1);
jest.useFakeTimers('modern');
const t1 = new Date('2020-01-01').getTime();
jest.setSystemTime(t1);
await models().lock().acquireLock(user.id, LockType.Sync, LockClientType.Desktop, '1111');
const lock1 = (await models().lock().allLocks(user.id))[0];
jest.setSystemTime(t1 + models().lock().lockTtl + 1);
// If we call this again, at the same time it should expire old timers.
await models().lock().acquireLock(user.id, LockType.Sync, LockClientType.Desktop, '2222');
expect((await models().lock().allLocks(user.id)).length).toBe(1);
const lock2 = (await models().lock().allLocks(user.id))[0];
expect(lock1.id).not.toBe(lock2.id);
jest.useRealTimers();
});
});

View File

@@ -1,13 +1,13 @@
import BaseModel, { UuidType } from './BaseModel';
import { Uuid } from '../services/database/types';
import { Lock, LockType, lockDefaultTtl, activeLock } from '@joplin/lib/services/synchronizer/LockHandler';
import { LockType, Lock, LockClientType, defaultLockTtl, activeLock } from '@joplin/lib/services/synchronizer/LockHandler';
import { Value } from './KeyValueModel';
import { ErrorConflict } from '../utils/errors';
import { ErrorConflict, ErrorUnprocessableEntity } from '../utils/errors';
import uuidgen from '../utils/uuidgen';
export default class LockModel extends BaseModel<Lock> {
private lockTtl_: number = lockDefaultTtl;
private lockTtl_: number = defaultLockTtl;
protected get tableName(): string {
return 'locks';
@@ -17,11 +17,7 @@ export default class LockModel extends BaseModel<Lock> {
return UuidType.Native;
}
// TODO: validate lock when acquiring and releasing
// TODO: test "should allow exclusive lock if the sync locks have expired"
// TODO: test "should not allow exclusive lock if there are sync locks"
private get lockTtl() {
public get lockTtl() {
return this.lockTtl_;
}
@@ -31,12 +27,34 @@ export default class LockModel extends BaseModel<Lock> {
return v ? JSON.parse(v) : [];
}
private async acquireSyncLock(userId: Uuid, clientType: string, clientId: string): Promise<Lock> {
protected async validate(lock: Lock): Promise<Lock> {
if (![LockType.Sync, LockType.Exclusive].includes(lock.type)) throw new ErrorUnprocessableEntity(`Invalid lock type: ${lock.type}`);
if (![LockClientType.Desktop, LockClientType.Mobile, LockClientType.Cli].includes(lock.clientType)) throw new ErrorUnprocessableEntity(`Invalid lock client type: ${lock.clientType}`);
if (lock.clientId.length > 64) throw new ErrorUnprocessableEntity(`Invalid client ID length: ${lock.clientId}`);
return lock;
}
private expireLocks(locks: Lock[]): Lock[] {
const cutOffTime = Date.now() - this.lockTtl;
const output: Lock[] = [];
for (const lock of locks) {
if (lock.updatedTime > cutOffTime) {
output.push(lock);
}
}
return output;
}
private async acquireSyncLock(userId: Uuid, clientType: LockClientType, clientId: string): Promise<Lock> {
const userKey = `locks::${userId}`;
let output: Lock = null;
await this.models().keyValue().readThenWrite(userKey, async (value: Value) => {
let locks: Lock[] = value ? JSON.parse(value as string) : [];
locks = this.expireLocks(locks);
const exclusiveLock = activeLock(locks, new Date(), this.lockTtl, LockType.Exclusive);
@@ -71,12 +89,13 @@ export default class LockModel extends BaseModel<Lock> {
return output;
}
private async acquireExclusiveLock(userId: Uuid, clientType: string, clientId: string): Promise<Lock> {
private async acquireExclusiveLock(userId: Uuid, clientType: LockClientType, clientId: string): Promise<Lock> {
const userKey = `locks::${userId}`;
let output: Lock = null;
await this.models().keyValue().readThenWrite(userKey, async (value: Value) => {
let locks: Lock[] = value ? JSON.parse(value as string) : [];
locks = this.expireLocks(locks);
const exclusiveLock = activeLock(locks, new Date(), this.lockTtl, LockType.Exclusive);
@@ -122,7 +141,9 @@ export default class LockModel extends BaseModel<Lock> {
return output;
}
public async acquireLock(userId: Uuid, type: LockType, clientType: string, clientId: string): Promise<Lock> {
public async acquireLock(userId: Uuid, type: LockType, clientType: LockClientType, clientId: string): Promise<Lock> {
await this.validate({ type, clientType, clientId });
if (type === LockType.Sync) {
return this.acquireSyncLock(userId, clientType, clientId);
} else {
@@ -130,15 +151,18 @@ export default class LockModel extends BaseModel<Lock> {
}
}
public async releaseLock(userId: Uuid, lockType: LockType, clientType: string, clientId: string) {
public async releaseLock(userId: Uuid, type: LockType, clientType: LockClientType, clientId: string) {
await this.validate({ type, clientType, clientId });
const userKey = `locks::${userId}`;
await this.models().keyValue().readThenWrite(userKey, async (value: Value) => {
const locks: Lock[] = value ? JSON.parse(value as string) : [];
let locks: Lock[] = value ? JSON.parse(value as string) : [];
locks = this.expireLocks(locks);
for (let i = locks.length - 1; i >= 0; i--) {
const lock = locks[i];
if (lock.type === lockType && lock.clientType === clientType && lock.clientId === clientId) {
if (lock.type === type && lock.clientType === clientType && lock.clientId === clientId) {
locks.splice(i, 1);
}
}

View File

@@ -11,7 +11,6 @@ import { requestDeltaPagination, requestPagination } from '../../models/utils/pa
import { AclAction } from '../../models/BaseModel';
import { safeRemove } from '../../utils/fileUtils';
import { formatBytes, MB } from '../../utils/bytes';
import lockHandler from './utils/items/lockHandler';
const router = new Router(RouteType.Api);
@@ -43,9 +42,6 @@ export async function putItemContents(path: SubPath, ctx: AppContext, isBatch: b
try {
const buffer = filePath ? await fs.readFile(filePath) : Buffer.alloc(0);
const lockResult = await lockHandler(path, ctx, buffer);
if (lockResult.handled) return lockResult.response;
// This end point can optionally set the associated jop_share_id field. It
// is only useful when uploading resource blob (under .resource folder)
// since they can't have metadata. Note, Folder and Resource items all
@@ -108,13 +104,8 @@ router.del('api/items/:id', async (path: SubPath, ctx: AppContext) => {
if (ctx.joplin.env !== 'dev') throw new ErrorMethodNotAllowed('Deleting the root is not allowed');
await ctx.joplin.models.item().deleteAll(ctx.joplin.owner.id);
} else {
// const item = await itemFromPath(ctx.joplin.owner.id, ctx.joplin.models.item(), path);
// await ctx.joplin.models.item().checkIfAllowed(ctx.joplin.owner, AclAction.Delete, item);
const lockResult = await lockHandler(path, ctx);
if (lockResult.handled) return lockResult.response;
const item = await itemFromPath(ctx.joplin.owner.id, ctx.joplin.models.item(), path);
await ctx.joplin.models.item().checkIfAllowed(ctx.joplin.owner, AclAction.Delete, item);
await ctx.joplin.models.item().deleteForUser(ctx.joplin.owner.id, item);
}
} catch (error) {
@@ -146,9 +137,6 @@ router.get('api/items/:id/delta', async (_path: SubPath, ctx: AppContext) => {
});
router.get('api/items/:id/children', async (path: SubPath, ctx: AppContext) => {
const lockResult = await lockHandler(path, ctx);
if (lockResult.handled) return lockResult.response;
const itemModel = ctx.joplin.models.item();
const parentName = itemModel.pathToName(path.id);
const result = await itemModel.children(ctx.joplin.owner.id, parentName, requestPagination(ctx.query));

View File

@@ -1,4 +1,4 @@
import { lockNameToObject, LockType } from '@joplin/lib/services/synchronizer/LockHandler';
import { LockType, LockClientType, lockNameToObject } from '@joplin/lib/services/synchronizer/LockHandler';
import { bodyFields } from '../../utils/requestUtils';
import Router from '../../utils/Router';
import { SubPath } from '../../utils/routeUtils';
@@ -8,7 +8,7 @@ const router = new Router(RouteType.Api);
interface PostFields {
type: LockType;
clientType: string;
clientType: LockClientType;
clientId: string;
}

View File

@@ -1,47 +0,0 @@
import config from '../../../../config';
import { PaginatedItems } from '../../../../models/ItemModel';
import { Item } from '../../../../services/database/types';
import { getApi, putApi } from '../../../../utils/testing/apiUtils';
import { beforeAllDb, afterAllTests, beforeEachDb, createUserAndSession, models } from '../../../../utils/testing/testUtils';
describe('items/lockHandlers', function() {
beforeAll(async () => {
await beforeAllDb('items/lockHandlers');
});
afterAll(async () => {
await afterAllTests();
});
beforeEach(async () => {
await beforeEachDb();
config().buildInLocksEnabled = true;
});
test('should save locks to the key-value store', async function() {
const { session, user } = await createUserAndSession(1);
const lockName = 'locks/exclusive_cli_12cb74fa9de644958b2ccbc772cb4e29.json';
const now = Date.now();
const result: Item = await putApi(session.id, `items/root:/${lockName}:/content`, { testing: true });
expect(result.name).toBe(lockName);
expect(result.updated_time).toBeGreaterThanOrEqual(now);
expect(result.id).toBe(null);
const values = await models().keyValue().all();
expect(values.length).toBe(1);
expect(values[0].key).toBe(`locks::${user.id}`);
const value = JSON.parse(values[0].value);
expect(value[lockName].name).toBe(lockName);
expect(value[lockName].updated_time).toBeGreaterThanOrEqual(now);
const getResult: PaginatedItems = await getApi(session.id, 'items/root:/locks/*:children');
console.info(getResult);
expect(getResult.items[0].name).toBe(result.name);
expect(getResult.items[0].updated_time).toBe(result.updated_time);
});
});

View File

@@ -1,98 +0,0 @@
import config from '../../../../config';
import { PaginatedItems } from '../../../../models/ItemModel';
import { Value } from '../../../../models/KeyValueModel';
import { Item } from '../../../../services/database/types';
import { ErrorBadRequest } from '../../../../utils/errors';
import { SubPath } from '../../../../utils/routeUtils';
import { AppContext } from '../../../../utils/types';
interface LockHandlerResult {
handled: boolean;
response: any;
}
const lockHandler = async (path: SubPath, ctx: AppContext, requestBody: Buffer = null): Promise<LockHandlerResult | null> => {
if (!config().buildInLocksEnabled) return { handled: false, response: null };
if (!path.id || !path.id.startsWith('root:/locks/')) return { handled: false, response: null };
const ownerId = ctx.joplin.owner.id;
const models = ctx.joplin.models;
const userKey = `locks::${ownerId}`;
// PUT /api/items/root:/locks/exclusive_cli_12cb74fa9de644958b2ccbc772cb4e29.json:/content
if (ctx.method === 'PUT') {
const itemName = models.item().pathToName(path.id);
const now = Date.now();
await models.keyValue().readThenWrite(userKey, async (value: Value) => {
const output = value ? JSON.parse(value as string) : {};
output[itemName] = {
name: itemName,
updated_time: now,
jop_updated_time: now,
content: requestBody.toString(),
};
return JSON.stringify(output);
});
return {
handled: true,
response: {
[itemName]: {
item: {
name: itemName,
updated_time: now,
id: null,
},
error: null,
},
},
};
}
// DELETE /api/items/root:/locks/exclusive_cli_12cb74fa9de644958b2ccbc772cb4e29.json:
if (ctx.method === 'DELETE') {
const itemName = models.item().pathToName(path.id);
await models.keyValue().readThenWrite(userKey, async (value: Value) => {
const output = value ? JSON.parse(value as string) : {};
delete output[itemName];
return JSON.stringify(output);
});
return {
handled: true,
response: null,
};
}
// GET /api/items/root:/locks/*:/children
if (ctx.method === 'GET' && path.id === 'root:/locks/*:') {
const result = await models.keyValue().value<string>(userKey);
const obj: Record<string, Item> = result ? JSON.parse(result) : {};
const items: Item[] = [];
for (const name of Object.keys(obj)) {
items.push(obj[name]);
}
const page: PaginatedItems = {
has_more: false,
items,
};
return {
handled: true,
response: page,
};
}
throw new ErrorBadRequest(`Unhandled lock path: ${path.id}`);
};
export default lockHandler;

View File

@@ -258,16 +258,6 @@ export interface Item extends WithDates, WithUuid {
owner_id?: Uuid;
}
export interface Lock {
id?: Uuid;
user_id?: Uuid;
type?: number;
client_type?: string;
client_id?: Uuid;
updated_time?: string;
created_time?: string;
}
export const databaseSchema: DatabaseTables = {
sessions: {
id: { type: 'string' },
@@ -440,14 +430,5 @@ export const databaseSchema: DatabaseTables = {
jop_updated_time: { type: 'string' },
owner_id: { type: 'string' },
},
locks: {
id: { type: 'string' },
user_id: { type: 'string' },
type: { type: 'number' },
client_type: { type: 'string' },
client_id: { type: 'string' },
updated_time: { type: 'string' },
created_time: { type: 'string' },
},
};
// AUTO-GENERATED-TYPES

View File

@@ -115,7 +115,6 @@ export interface Config {
businessEmail: string;
isJoplinCloud: boolean;
cookieSecure: boolean;
buildInLocksEnabled: boolean;
}
export enum HttpMethod {