1
0
mirror of https://github.com/laurent22/joplin.git synced 2026-01-17 00:33:59 +02:00

Compare commits

..

6 Commits

Author SHA1 Message Date
Laurent Cozic
9a536d9b5e bug 2021-11-01 16:32:56 +00:00
Laurent Cozic
0929699d7d update 2021-11-01 13:18:06 +00:00
Laurent Cozic
cc9cdd0bec tests 2021-11-01 09:23:05 +00:00
Laurent Cozic
08050f6d28 sqlite 2021-11-01 08:50:37 +00:00
Laurent Cozic
63cbdd9a62 native locks 2021-11-01 08:24:51 +00:00
Laurent Cozic
c49174124e Doc: Added forum assets 2021-11-01 07:57:21 +00:00
41 changed files with 8116 additions and 318 deletions

View File

@@ -1314,9 +1314,6 @@ packages/lib/services/interop/InteropService_Exporter_Jex.js.map
packages/lib/services/interop/InteropService_Exporter_Md.d.ts
packages/lib/services/interop/InteropService_Exporter_Md.js
packages/lib/services/interop/InteropService_Exporter_Md.js.map
packages/lib/services/interop/InteropService_Exporter_Md.test.d.ts
packages/lib/services/interop/InteropService_Exporter_Md.test.js
packages/lib/services/interop/InteropService_Exporter_Md.test.js.map
packages/lib/services/interop/InteropService_Exporter_Md_frontmatter.d.ts
packages/lib/services/interop/InteropService_Exporter_Md_frontmatter.js
packages/lib/services/interop/InteropService_Exporter_Md_frontmatter.js.map

3
.gitignore vendored
View File

@@ -1297,9 +1297,6 @@ packages/lib/services/interop/InteropService_Exporter_Jex.js.map
packages/lib/services/interop/InteropService_Exporter_Md.d.ts
packages/lib/services/interop/InteropService_Exporter_Md.js
packages/lib/services/interop/InteropService_Exporter_Md.js.map
packages/lib/services/interop/InteropService_Exporter_Md.test.d.ts
packages/lib/services/interop/InteropService_Exporter_Md.test.js
packages/lib/services/interop/InteropService_Exporter_Md.test.js.map
packages/lib/services/interop/InteropService_Exporter_Md_frontmatter.d.ts
packages/lib/services/interop/InteropService_Exporter_Md_frontmatter.js
packages/lib/services/interop/InteropService_Exporter_Md_frontmatter.js.map

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.6 KiB

View File

@@ -1,12 +1,12 @@
{
"name": "@joplin/app-desktop",
"version": "2.5.12",
"version": "2.5.10",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@joplin/app-desktop",
"version": "2.5.12",
"version": "2.5.10",
"license": "MIT",
"dependencies": {
"@electron/remote": "^2.0.1",

View File

@@ -1,6 +1,6 @@
{
"name": "@joplin/app-desktop",
"version": "2.5.12",
"version": "2.5.10",
"description": "Joplin for Desktop",
"main": "main.js",
"private": true,

View File

@@ -492,13 +492,13 @@
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Joplin/Joplin.entitlements;
CURRENT_PROJECT_VERSION = 77;
CURRENT_PROJECT_VERSION = 74;
DEVELOPMENT_TEAM = A9BXAFS6CT;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = Joplin/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 9.0;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
MARKETING_VERSION = 12.5.3;
MARKETING_VERSION = 12.5.0;
OTHER_LDFLAGS = (
"$(inherited)",
"-ObjC",
@@ -521,12 +521,12 @@
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CODE_SIGN_ENTITLEMENTS = Joplin/Joplin.entitlements;
CURRENT_PROJECT_VERSION = 77;
CURRENT_PROJECT_VERSION = 74;
DEVELOPMENT_TEAM = A9BXAFS6CT;
INFOPLIST_FILE = Joplin/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 9.0;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
MARKETING_VERSION = 12.5.3;
MARKETING_VERSION = 12.5.0;
OTHER_LDFLAGS = (
"$(inherited)",
"-ObjC",
@@ -667,14 +667,14 @@
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CODE_SIGN_ENTITLEMENTS = ShareExtension/ShareExtension.entitlements;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 77;
CURRENT_PROJECT_VERSION = 74;
DEBUG_INFORMATION_FORMAT = dwarf;
DEVELOPMENT_TEAM = A9BXAFS6CT;
GCC_C_LANGUAGE_STANDARD = gnu11;
INFOPLIST_FILE = ShareExtension/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 9.0;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @executable_path/../../Frameworks";
MARKETING_VERSION = 12.5.3;
MARKETING_VERSION = 12.5.0;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
PRODUCT_BUNDLE_IDENTIFIER = net.cozic.joplin.ShareExtension;
@@ -698,14 +698,14 @@
CODE_SIGN_ENTITLEMENTS = ShareExtension/ShareExtension.entitlements;
CODE_SIGN_STYLE = Automatic;
COPY_PHASE_STRIP = NO;
CURRENT_PROJECT_VERSION = 77;
CURRENT_PROJECT_VERSION = 74;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
DEVELOPMENT_TEAM = A9BXAFS6CT;
GCC_C_LANGUAGE_STANDARD = gnu11;
INFOPLIST_FILE = ShareExtension/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 9.0;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @executable_path/../../Frameworks";
MARKETING_VERSION = 12.5.3;
MARKETING_VERSION = 12.5.0;
MTL_FAST_MATH = YES;
PRODUCT_BUNDLE_IDENTIFIER = net.cozic.joplin.ShareExtension;
PRODUCT_NAME = "$(TARGET_NAME)";

View File

@@ -317,7 +317,7 @@ PODS:
- React
- RNSecureRandom (1.0.0-rc.0):
- React
- RNShare (7.2.1):
- RNShare (5.1.5):
- React-Core
- RNVectorIcons (7.1.0):
- React
@@ -540,7 +540,7 @@ SPEC CHECKSUMS:
RNFS: 2bd9eb49dc82fa9676382f0585b992c424cd59df
RNQuickAction: 6d404a869dc872cde841ad3147416a670d13fa93
RNSecureRandom: 1f19ad1492f7ed416b8fc79e92216a1f73f13a4c
RNShare: edd621a71124961e29a7ba43a84bd1c6f9980d88
RNShare: 9cdd23357981cf4dee275eb79239e860dccc0faf
RNVectorIcons: bc69e6a278b14842063605de32bec61f0b251a59
Yoga: 575c581c63e0d35c9a83f4b46d01d63abc1100ac

File diff suppressed because it is too large Load Diff

View File

@@ -50,7 +50,7 @@
"react-native-quick-actions": "^0.3.13",
"react-native-rsa-native": "^2.0.4",
"react-native-securerandom": "^1.0.0-rc.0",
"react-native-share": "^7.2.1",
"react-native-share": "^5.1.5",
"react-native-side-menu": "^1.1.3",
"react-native-sqlite-storage": "^5.0.0",
"react-native-vector-icons": "^7.1.0",

View File

@@ -253,8 +253,12 @@ export default class JoplinServerApi {
const output = await loadResponseJson();
return output;
} catch (error) {
if (error.code !== 404) {
// Don't print error info for file not found (handled by the
// driver), or lock-acquisition errors because it's handled by
// LockHandler.
if (![404, 'hasExclusiveLock', 'hasSyncLock'].includes(error.code)) {
logger.warn(this.requestToCurl_(url, fetchOptions));
logger.warn('Code:', error.code);
logger.warn(error);
}

View File

@@ -1,5 +1,5 @@
import Logger from './Logger';
import LockHandler, { LockType } from './services/synchronizer/LockHandler';
import LockHandler, { hasActiveLock, LockType } from './services/synchronizer/LockHandler';
import Setting from './models/Setting';
import shim from './shim';
import MigrationHandler from './services/synchronizer/MigrationHandler';
@@ -298,10 +298,13 @@ export default class Synchronizer {
}
async lockErrorStatus_() {
const hasActiveExclusiveLock = await this.lockHandler().hasActiveLock(LockType.Exclusive);
const locks = await this.lockHandler().locks();
const currentDate = await this.lockHandler().currentDate();
const hasActiveExclusiveLock = await hasActiveLock(locks, currentDate, this.lockHandler().lockTtl, LockType.Exclusive);
if (hasActiveExclusiveLock) return 'hasExclusiveLock';
const hasActiveSyncLock = await this.lockHandler().hasActiveLock(LockType.Sync, this.appType_, this.clientId_);
const hasActiveSyncLock = await hasActiveLock(locks, currentDate, this.lockHandler().lockTtl, LockType.Sync, this.appType_, this.clientId_);
if (!hasActiveSyncLock) return 'syncLockGone';
return '';

View File

@@ -2,6 +2,7 @@ import { MultiPutItem } from './file-api';
import JoplinError from './JoplinError';
import JoplinServerApi from './JoplinServerApi';
import { trimSlashes } from './path-utils';
import { Lock, LockType } from './services/synchronizer/LockHandler';
// All input paths should be in the format: "path/to/file". This is converted to
// "root:/path/to/file:" when doing the API call.
@@ -40,6 +41,10 @@ export default class FileApiDriverJoplinServer {
return true;
}
public get supportsLocks() {
return true;
}
public requestRepeatCount() {
return 3;
}
@@ -196,6 +201,22 @@ export default class FileApiDriverJoplinServer {
throw new Error('Not supported');
}
public async acquireLock(type: LockType, clientType: string, clientId: string): Promise<Lock> {
return this.api().exec('POST', 'api/locks', null, {
type,
clientType: clientType,
clientId: clientId,
});
}
public async releaseLock(type: LockType, clientType: string, clientId: string) {
await this.api().exec('DELETE', `api/locks/${type}_${clientType}_${clientId}`);
}
public async listLocks() {
return this.api().exec('GET', 'api/locks');
}
public async clearRoot(path: string) {
const response = await this.list(path);
@@ -203,6 +224,8 @@ export default class FileApiDriverJoplinServer {
await this.delete(item.path);
}
await this.api().exec('POST', 'api/debug', null, { action: 'clearKeyValues' });
if (response.has_more) throw new Error('has_more support not implemented');
}
}

View File

@@ -5,6 +5,7 @@ import time from './time';
const { isHidden } = require('./path-utils');
import JoplinError from './JoplinError';
import { Lock, LockType } from './services/synchronizer/LockHandler';
const ArrayUtils = require('./ArrayUtils');
const { sprintf } = require('sprintf-js');
const Mutex = require('async-mutex').Mutex;
@@ -36,7 +37,7 @@ export interface RemoteItem {
export interface PaginatedList {
items: RemoteItem[];
has_more: boolean;
hasMore: boolean;
context: any;
}
@@ -130,6 +131,10 @@ class FileApi {
return !!this.driver().supportsAccurateTimestamp;
}
public get supportsLocks(): boolean {
return !!this.driver().supportsLocks;
}
async fetchRemoteDateOffset_() {
const tempFile = `${this.tempDirName()}/timeCheck${Math.round(Math.random() * 1000000)}.txt`;
const startTime = Date.now();
@@ -349,6 +354,22 @@ class FileApi {
logger.debug(`delta ${this.fullPath(path)}`);
return tryAndRepeat(() => this.driver_.delta(this.fullPath(path), options), this.requestRepeatCount());
}
public async acquireLock(type: LockType, clientType: string, clientId: string): Promise<Lock> {
if (!this.supportsLocks) throw new Error('Sync target does not support built-in locks');
return tryAndRepeat(() => this.driver_.acquireLock(type, clientType, clientId), this.requestRepeatCount());
}
public async releaseLock(type: LockType, clientType: string, clientId: string) {
if (!this.supportsLocks) throw new Error('Sync target does not support built-in locks');
return tryAndRepeat(() => this.driver_.releaseLock(type, clientType, clientId), this.requestRepeatCount());
}
public async listLocks() {
if (!this.supportsLocks) throw new Error('Sync target does not support built-in locks');
return tryAndRepeat(() => this.driver_.listLocks(), this.requestRepeatCount());
}
}
function basicDeltaContextFromOptions_(options: any) {

View File

@@ -1,15 +1,15 @@
import * as fs from 'fs-extra';
import { setupDatabaseAndSynchronizer, switchClient, exportDir, supportDir } from '../../testing/test-utils.js';
import InteropService_Exporter_Md from '../../services/interop/InteropService_Exporter_Md';
import BaseModel from '../../BaseModel';
import Folder from '../../models/Folder';
import Resource from '../../models/Resource';
import Note from '../../models/Note';
import shim from '../../shim';
import { MarkupToHtml } from '@joplin/renderer';
import { NoteEntity, ResourceEntity } from '../database/types.js';
import InteropService from './InteropService.js';
import { fileExtension } from '../../path-utils.js';
/* eslint-disable no-unused-vars */
const fs = require('fs-extra');
const { setupDatabaseAndSynchronizer, switchClient, exportDir, supportDir } = require('../../testing/test-utils.js');
const InteropService_Exporter_Md = require('../../services/interop/InteropService_Exporter_Md').default;
const BaseModel = require('../../BaseModel').default;
const Folder = require('../../models/Folder').default;
const Resource = require('../../models/Resource').default;
const Note = require('../../models/Note').default;
const shim = require('../../shim').default;
const { MarkupToHtml } = require('@joplin/renderer');
describe('interop/InteropService_Exporter_Md', function() {
@@ -33,8 +33,8 @@ describe('interop/InteropService_Exporter_Md', function() {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir());
const itemsToExport: any[] = [];
const queueExportItem = (itemType: number, itemOrId: any) => {
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
@@ -59,13 +59,13 @@ describe('interop/InteropService_Exporter_Md', function() {
queueExportItem(BaseModel.TYPE_NOTE, note3);
queueExportItem(BaseModel.TYPE_RESOURCE, (await Note.linkedResourceIds(note3.body))[0]);
expect(!exporter.context() && !(exporter.context().notePaths || Object.keys(exporter.context().notePaths).length)).toBe(false);
expect(!exporter.context() && !(exporter.context().notePaths || Object.keys(exporter.context().notePaths).length)).toBe(false, 'Context should be empty before processing.');
await exporter.processItem(Folder.modelType(), folder1);
await exporter.processItem(Folder.modelType(), folder2);
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
expect(Object.keys(exporter.context().notePaths).length).toBe(3);
expect(Object.keys(exporter.context().notePaths).length).toBe(3, 'There should be 3 note paths in the context.');
expect(exporter.context().notePaths[note1.id]).toBe('folder1/note1.md');
expect(exporter.context().notePaths[note2.id]).toBe('folder1/note2.md');
expect(exporter.context().notePaths[note3.id]).toBe('folder2/note3.html');
@@ -75,8 +75,8 @@ describe('interop/InteropService_Exporter_Md', function() {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir());
const itemsToExport: any[] = [];
const queueExportItem = (itemType: number, itemOrId: any) => {
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
@@ -110,9 +110,9 @@ describe('interop/InteropService_Exporter_Md', function() {
await exporter.processResource(resource1, Resource.fullPath(resource1));
await exporter.processResource(resource2, Resource.fullPath(resource2));
expect(!exporter.context() && !(exporter.context().destResourcePaths || Object.keys(exporter.context().destResourcePaths).length)).toBe(false);
expect(!exporter.context() && !(exporter.context().destResourcePaths || Object.keys(exporter.context().destResourcePaths).length)).toBe(false, 'Context should be empty before processing.');
expect(Object.keys(exporter.context().destResourcePaths).length).toBe(2);
expect(Object.keys(exporter.context().destResourcePaths).length).toBe(2, 'There should be 2 resource paths in the context.');
expect(exporter.context().destResourcePaths[resource1.id]).toBe(`${exportDir()}/_resources/photo.jpg`);
expect(exporter.context().destResourcePaths[resource2.id]).toBe(`${exportDir()}/_resources/photo-1.jpg`);
}));
@@ -121,8 +121,8 @@ describe('interop/InteropService_Exporter_Md', function() {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir());
const itemsToExport: any[] = [];
const queueExportItem = (itemType: number, itemOrId: any) => {
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
@@ -139,7 +139,7 @@ describe('interop/InteropService_Exporter_Md', function() {
await exporter.processItem(Folder.modelType(), folder1);
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
expect(Object.keys(exporter.context().notePaths).length).toBe(2);
expect(Object.keys(exporter.context().notePaths).length).toBe(2, 'There should be 2 note paths in the context.');
expect(exporter.context().notePaths[note1.id]).toBe('folder1/note1.md');
expect(exporter.context().notePaths[note1_2.id]).toBe('folder1/note1-1.md');
}));
@@ -148,8 +148,8 @@ describe('interop/InteropService_Exporter_Md', function() {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir());
const itemsToExport: any[] = [];
const queueExportItem = (itemType: number, itemOrId: any) => {
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
@@ -167,7 +167,7 @@ describe('interop/InteropService_Exporter_Md', function() {
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
expect(Object.keys(exporter.context().notePaths).length).toBe(1);
expect(Object.keys(exporter.context().notePaths).length).toBe(1, 'There should be 1 note paths in the context.');
expect(exporter.context().notePaths[note1.id]).toBe('folder1/note1-1.md');
}));
@@ -175,8 +175,8 @@ describe('interop/InteropService_Exporter_Md', function() {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir());
const itemsToExport: any[] = [];
const queueExportItem = (itemType: number, itemOrId: any) => {
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
@@ -204,16 +204,16 @@ describe('interop/InteropService_Exporter_Md', function() {
await exporter.processResource(resource1, Resource.fullPath(resource1));
await exporter.processResource(resource2, Resource.fullPath(resource2));
expect(await shim.fsDriver().exists(`${exportDir()}/_resources/photo.jpg`)).toBe(true);
expect(await shim.fsDriver().exists(`${exportDir()}/_resources/photo-1.jpg`)).toBe(true);
expect(await shim.fsDriver().exists(`${exportDir()}/_resources/photo.jpg`)).toBe(true, 'Resource file should be copied to _resources directory.');
expect(await shim.fsDriver().exists(`${exportDir()}/_resources/photo-1.jpg`)).toBe(true, 'Resource file should be copied to _resources directory.');
}));
it('should create folders in fs', (async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir());
const itemsToExport: any[] = [];
const queueExportItem = (itemType: number, itemOrId: any) => {
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
@@ -234,17 +234,17 @@ describe('interop/InteropService_Exporter_Md', function() {
await exporter.prepareForProcessingItemType(BaseModel.TYPE_NOTE, itemsToExport);
await exporter.processItem(Note.modelType(), note2);
expect(await shim.fsDriver().exists(`${exportDir()}/folder1`)).toBe(true);
expect(await shim.fsDriver().exists(`${exportDir()}/folder1/folder2`)).toBe(true);
expect(await shim.fsDriver().exists(`${exportDir()}/folder1/folder3`)).toBe(true);
expect(await shim.fsDriver().exists(`${exportDir()}/folder1`)).toBe(true, 'Folder should be created in filesystem.');
expect(await shim.fsDriver().exists(`${exportDir()}/folder1/folder2`)).toBe(true, 'Folder should be created in filesystem.');
expect(await shim.fsDriver().exists(`${exportDir()}/folder1/folder3`)).toBe(true, 'Folder should be created in filesystem.');
}));
it('should save notes in fs', (async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir());
const itemsToExport: any[] = [];
const queueExportItem = (itemType: number, itemOrId: any) => {
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
@@ -271,17 +271,17 @@ describe('interop/InteropService_Exporter_Md', function() {
await exporter.processItem(Note.modelType(), note2);
await exporter.processItem(Note.modelType(), note3);
expect(await shim.fsDriver().exists(`${exportDir()}/${exporter.context().notePaths[note1.id]}`)).toBe(true);
expect(await shim.fsDriver().exists(`${exportDir()}/${exporter.context().notePaths[note2.id]}`)).toBe(true);
expect(await shim.fsDriver().exists(`${exportDir()}/${exporter.context().notePaths[note3.id]}`)).toBe(true);
expect(await shim.fsDriver().exists(`${exportDir()}/${exporter.context().notePaths[note1.id]}`)).toBe(true, 'File should be saved in filesystem.');
expect(await shim.fsDriver().exists(`${exportDir()}/${exporter.context().notePaths[note2.id]}`)).toBe(true, 'File should be saved in filesystem.');
expect(await shim.fsDriver().exists(`${exportDir()}/${exporter.context().notePaths[note3.id]}`)).toBe(true, 'File should be saved in filesystem.');
}));
it('should replace resource ids with relative paths', (async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir());
const itemsToExport: any[] = [];
const queueExportItem = (itemType: number, itemOrId: any) => {
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
@@ -325,7 +325,7 @@ describe('interop/InteropService_Exporter_Md', function() {
await exporter.processResource(resource2, Resource.fullPath(resource2));
await exporter.processResource(resource3, Resource.fullPath(resource3));
await exporter.processResource(resource4, Resource.fullPath(resource3));
const context: any = {
const context = {
resourcePaths: {},
};
context.resourcePaths[resource1.id] = 'resource1.jpg';
@@ -343,25 +343,25 @@ describe('interop/InteropService_Exporter_Md', function() {
const note3_body = await shim.fsDriver().readFile(`${exportDir()}/${exporter.context().notePaths[note3.id]}`);
const note4_body = await shim.fsDriver().readFile(`${exportDir()}/${exporter.context().notePaths[note4.id]}`);
expect(note1_body).toContain('](../_resources/photo.jpg)');
expect(note2_body).toContain('](../../_resources/photo-1.jpg)');
expect(note3_body).toContain('<img src="../../_resources/photo-2.jpg" alt="alt">');
expect(note4_body).toContain('](../../_resources/photo-3.jpg "title")');
expect(note1_body).toContain('](../_resources/photo.jpg)', 'Resource id should be replaced with a relative path.');
expect(note2_body).toContain('](../../_resources/photo-1.jpg)', 'Resource id should be replaced with a relative path.');
expect(note3_body).toContain('<img src="../../_resources/photo-2.jpg" alt="alt">', 'Resource id should be replaced with a relative path.');
expect(note4_body).toContain('](../../_resources/photo-3.jpg "title")', 'Resource id should be replaced with a relative path.');
}));
it('should replace note ids with relative paths', (async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir());
const itemsToExport: any[] = [];
const queueExportItem = (itemType: number, itemOrId: any) => {
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
});
};
const changeNoteBodyAndReload = async (note: NoteEntity, newBody: string) => {
const changeNoteBodyAndReload = async (note, newBody) => {
note.body = newBody;
await Note.save(note);
return await Note.load(note.id);
@@ -395,18 +395,18 @@ describe('interop/InteropService_Exporter_Md', function() {
const note2_body = await shim.fsDriver().readFile(`${exportDir()}/${exporter.context().notePaths[note2.id]}`);
const note3_body = await shim.fsDriver().readFile(`${exportDir()}/${exporter.context().notePaths[note3.id]}`);
expect(note1_body).toContain('](../folder3/note3.md)');
expect(note2_body).toContain('](../../folder3/note3.md)');
expect(note2_body).toContain('](../../folder1/note1.md)');
expect(note3_body).toContain('](../folder1/folder2/note2.md)');
expect(note1_body).toContain('](../folder3/note3.md)', 'Note id should be replaced with a relative path.');
expect(note2_body).toContain('](../../folder3/note3.md)', 'Resource id should be replaced with a relative path.');
expect(note2_body).toContain('](../../folder1/note1.md)', 'Resource id should be replaced with a relative path.');
expect(note3_body).toContain('](../folder1/folder2/note2.md)', 'Resource id should be replaced with a relative path.');
}));
it('should url encode relative note links', (async () => {
const exporter = new InteropService_Exporter_Md();
await exporter.init(exportDir());
const itemsToExport: any[] = [];
const queueExportItem = (itemType: number, itemOrId: any) => {
const itemsToExport = [];
const queueExportItem = (itemType, itemOrId) => {
itemsToExport.push({
type: itemType,
itemOrId: itemOrId,
@@ -425,26 +425,6 @@ describe('interop/InteropService_Exporter_Md', function() {
await exporter.processItem(Note.modelType(), note2);
const note2_body = await shim.fsDriver().readFile(`${exportDir()}/${exporter.context().notePaths[note2.id]}`);
expect(note2_body).toContain('[link](../folder%20with%20space1/note1%20name%20with%20space.md)');
expect(note2_body).toContain('[link](../folder%20with%20space1/note1%20name%20with%20space.md)', 'Whitespace in URL should be encoded');
}));
it('should preserve resource file extension', (async () => {
const folder = await Folder.save({ title: 'testing' });
const note = await Note.save({ title: 'mynote', parent_id: folder.id });
await shim.attachFileToNote(note, `${supportDir}/photo.jpg`);
const resource: ResourceEntity = (await Resource.all())[0];
await Resource.save({ id: resource.id, title: 'veryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitleveryverylongtitle.jpg' });
const service = InteropService.instance();
await service.export({
path: exportDir(),
format: 'md',
});
const resourceFilename = (await fs.readdir(`${exportDir()}/_resources`))[0];
expect(fileExtension(resourceFilename)).toBe('jpg');
}));
});

View File

@@ -143,7 +143,7 @@ export default class InteropService_Exporter_Md extends InteropService_Exporter_
if (resource.filename) {
fileName = resource.filename;
} else if (resource.title) {
fileName = friendlySafeFilename(resource.title, null, true);
fileName = friendlySafeFilename(resource.title);
}
// Fall back on the resource filename saved in the users resource folder

View File

@@ -122,7 +122,6 @@ describe('interop/InteropService_Exporter_Md_frontmatter', function() {
const content = await exportAndLoad(`${exportDir()}/folder1/Source_title.md`);
expect(content).toContain('title: |-\n Source\n title');
}));
test('should not export coordinates if they\'re not available', (async () => {
const folder1 = await Folder.save({ title: 'folder1' });
await Note.save({ title: 'Coordinates', body: '**ma note**', parent_id: folder1.id });

View File

@@ -3,6 +3,7 @@ import shim from '../../shim';
import JoplinError from '../../JoplinError';
import time from '../../time';
import { FileApi } from '../../file-api';
const { fileExtension, filename } = require('../../path-utils');
export enum LockType {
@@ -12,12 +13,68 @@ export enum LockType {
}
export interface Lock {
id?: string;
type: LockType;
clientType: string;
clientId: string;
updatedTime?: number;
}
function lockIsActive(lock: Lock, currentDate: Date, lockTtl: number): boolean {
return currentDate.getTime() - lock.updatedTime < lockTtl;
}
export function lockNameToObject(name: string, updatedTime: number = null): Lock {
const p = name.split('_');
return {
type: p[0] as LockType,
clientType: p[1],
clientId: p[2],
updatedTime: updatedTime,
};
}
export function hasActiveLock(locks: Lock[], currentDate: Date, lockTtl: number, lockType: LockType, clientType: string = null, clientId: string = null) {
const lock = activeLock(locks, currentDate, lockTtl, lockType, clientType, clientId);
return !!lock;
}
// Finds if there's an active lock for this clientType and clientId and returns it.
// If clientType and clientId are not specified, returns the first active lock
// of that type instead.
export function activeLock(locks: Lock[], currentDate: Date, lockTtl: number, lockType: LockType, clientType: string = null, clientId: string = null) {
if (lockType === LockType.Exclusive) {
const activeLocks = locks
.slice()
.filter((lock: Lock) => lockIsActive(lock, currentDate, lockTtl) && lock.type === lockType)
.sort((a: Lock, b: Lock) => {
if (a.updatedTime === b.updatedTime) {
return a.clientId < b.clientId ? -1 : +1;
}
return a.updatedTime < b.updatedTime ? -1 : +1;
});
if (!activeLocks.length) return null;
const lock = activeLocks[0];
if (clientType && clientType !== lock.clientType) return null;
if (clientId && clientId !== lock.clientId) return null;
return lock;
} else if (lockType === LockType.Sync) {
for (const lock of locks) {
if (lock.type !== lockType) continue;
if (clientType && lock.clientType !== clientType) continue;
if (clientId && lock.clientId !== clientId) continue;
if (lockIsActive(lock, currentDate, lockTtl)) return lock;
}
return null;
}
throw new Error(`Unsupported lock type: ${lockType}`);
}
export interface AcquireLockOptions {
// In theory, a client that tries to acquire an exclusive lock shouldn't
// also have a sync lock. It can however happen when the app is closed
@@ -55,14 +112,16 @@ export interface LockHandlerOptions {
lockTtl?: number;
}
export const lockDefaultTtl = 1000 * 60 * 3;
export default class LockHandler {
private api_: any = null;
private api_: FileApi = null;
private refreshTimers_: RefreshTimers = {};
private autoRefreshInterval_: number = 1000 * 60;
private lockTtl_: number = 1000 * 60 * 3;
private lockTtl_: number = lockDefaultTtl;
constructor(api: any, options: LockHandlerOptions = null) {
public constructor(api: FileApi, options: LockHandlerOptions = null) {
if (!options) options = {};
this.api_ = api;
@@ -80,6 +139,10 @@ export default class LockHandler {
this.lockTtl_ = v;
}
public get useBuiltInLocks() {
return this.api_.supportsLocks;
}
private lockFilename(lock: Lock) {
return `${[lock.type, lock.clientType, lock.clientId].join('_')}.json`;
}
@@ -97,17 +160,15 @@ export default class LockHandler {
}
private lockFileToObject(file: any): Lock {
const p = filename(file.path).split('_');
return {
type: p[0],
clientType: p[1],
clientId: p[2],
updatedTime: file.updated_time,
};
return lockNameToObject(filename(file.path), file.updated_time);
}
async locks(lockType: LockType = null): Promise<Lock[]> {
if (this.useBuiltInLocks) {
const locks = (await this.api_.listLocks()).items;
return locks;
}
const result = await this.api_.list(Dirnames.Locks);
if (result.hasMore) throw new Error('hasMore not handled'); // Shouldn't happen anyway
@@ -123,51 +184,6 @@ export default class LockHandler {
return output;
}
private lockIsActive(lock: Lock, currentDate: Date): boolean {
return currentDate.getTime() - lock.updatedTime < this.lockTtl;
}
async hasActiveLock(lockType: LockType, clientType: string = null, clientId: string = null) {
const lock = await this.activeLock(lockType, clientType, clientId);
return !!lock;
}
// Finds if there's an active lock for this clientType and clientId and returns it.
// If clientType and clientId are not specified, returns the first active lock
// of that type instead.
async activeLock(lockType: LockType, clientType: string = null, clientId: string = null) {
const locks = await this.locks(lockType);
const currentDate = await this.api_.remoteDate();
if (lockType === LockType.Exclusive) {
const activeLocks = locks
.slice()
.filter((lock: Lock) => this.lockIsActive(lock, currentDate))
.sort((a: Lock, b: Lock) => {
if (a.updatedTime === b.updatedTime) {
return a.clientId < b.clientId ? -1 : +1;
}
return a.updatedTime < b.updatedTime ? -1 : +1;
});
if (!activeLocks.length) return null;
const activeLock = activeLocks[0];
if (clientType && clientType !== activeLock.clientType) return null;
if (clientId && clientId !== activeLock.clientId) return null;
return activeLock;
} else if (lockType === LockType.Sync) {
for (const lock of locks) {
if (clientType && lock.clientType !== clientType) continue;
if (clientId && lock.clientId !== clientId) continue;
if (this.lockIsActive(lock, currentDate)) return lock;
}
return null;
}
throw new Error(`Unsupported lock type: ${lockType}`);
}
private async saveLock(lock: Lock) {
await this.api_.put(this.lockFilePath(lock), JSON.stringify(lock));
}
@@ -178,12 +194,17 @@ export default class LockHandler {
}
private async acquireSyncLock(clientType: string, clientId: string): Promise<Lock> {
if (this.useBuiltInLocks) return this.api_.acquireLock(LockType.Sync, clientType, clientId);
try {
let isFirstPass = true;
while (true) {
const locks = await this.locks();
const currentDate = await this.currentDate();
const [exclusiveLock, syncLock] = await Promise.all([
this.activeLock(LockType.Exclusive),
this.activeLock(LockType.Sync, clientType, clientId),
activeLock(locks, currentDate, this.lockTtl, LockType.Exclusive),
activeLock(locks, currentDate, this.lockTtl, LockType.Sync, clientType, clientId),
]);
if (exclusiveLock) {
@@ -222,6 +243,8 @@ export default class LockHandler {
}
private async acquireExclusiveLock(clientType: string, clientId: string, options: AcquireLockOptions = null): Promise<Lock> {
if (this.useBuiltInLocks) return this.api_.acquireLock(LockType.Exclusive, clientType, clientId);
// The logic to acquire an exclusive lock, while avoiding race conditions is as follow:
//
// - Check if there is a lock file present
@@ -252,9 +275,12 @@ export default class LockHandler {
try {
while (true) {
const locks = await this.locks();
const currentDate = await this.currentDate();
const [activeSyncLock, activeExclusiveLock] = await Promise.all([
this.activeLock(LockType.Sync),
this.activeLock(LockType.Exclusive),
activeLock(locks, currentDate, this.lockTtl, LockType.Sync),
activeLock(locks, currentDate, this.lockTtl, LockType.Exclusive),
]);
if (activeSyncLock) {
@@ -299,7 +325,11 @@ export default class LockHandler {
return [lock.type, lock.clientType, lock.clientId].join('_');
}
startAutoLockRefresh(lock: Lock, errorHandler: Function): string {
public async currentDate() {
return this.api_.remoteDate();
}
public startAutoLockRefresh(lock: Lock, errorHandler: Function): string {
const handle = this.autoLockRefreshHandle(lock);
if (this.refreshTimers_[handle]) {
throw new Error(`There is already a timer refreshing this lock: ${handle}`);
@@ -321,10 +351,11 @@ export default class LockHandler {
this.refreshTimers_[handle].inProgress = true;
let error = null;
const hasActiveLock = await this.hasActiveLock(lock.type, lock.clientType, lock.clientId);
if (!this.refreshTimers_[handle]) return defer(); // Timeout has been cleared
if (!hasActiveLock) {
const locks = await this.locks(lock.type);
if (!hasActiveLock(locks, await this.currentDate(), this.lockTtl, lock.type, lock.clientType, lock.clientId)) {
// If the previous lock has expired, we shouldn't try to acquire a new one. This is because other clients might have performed
// in the meantime operations that invalidates the current operation. For example, another client might have upgraded the
// sync target in the meantime, so any active operation should be cancelled here. Or if the current client was upgraded
@@ -384,6 +415,11 @@ export default class LockHandler {
}
public async releaseLock(lockType: LockType, clientType: string, clientId: string) {
if (this.useBuiltInLocks) {
await this.api_.releaseLock(lockType, clientType, clientId);
return;
}
await this.api_.delete(this.lockFilePath({
type: lockType,
clientType: clientType,

View File

@@ -1,4 +1,4 @@
import LockHandler, { LockType, LockHandlerOptions, Lock } from '../../services/synchronizer/LockHandler';
import LockHandler, { LockType, LockHandlerOptions, Lock, activeLock } from '../../services/synchronizer/LockHandler';
import { isNetworkSyncTarget, fileApi, setupDatabaseAndSynchronizer, synchronizer, switchClient, msleep, expectThrow, expectNotThrow } from '../../testing/test-utils';
// For tests with memory of file system we can use low intervals to make the tests faster.
@@ -46,6 +46,8 @@ describe('synchronizer_LockHandler', function() {
}));
it('should not use files that are not locks', (async () => {
if (lockHandler().useBuiltInLocks) return; // Doesn't make sense with built-in locks
await fileApi().put('locks/desktop.ini', 'a');
await fileApi().put('locks/exclusive.json', 'a');
await fileApi().put('locks/garbage.json', 'a');
@@ -75,10 +77,10 @@ describe('synchronizer_LockHandler', function() {
it('should auto-refresh a lock', (async () => {
const handler = newLockHandler({ autoRefreshInterval: 100 * timeoutMultipler });
const lock = await handler.acquireLock(LockType.Sync, 'desktop', '111');
const lockBefore = await handler.activeLock(LockType.Sync, 'desktop', '111');
const lockBefore = activeLock(await handler.locks(), new Date(), handler.lockTtl, LockType.Sync, 'desktop', '111');
handler.startAutoLockRefresh(lock, () => {});
await msleep(500 * timeoutMultipler);
const lockAfter = await handler.activeLock(LockType.Sync, 'desktop', '111');
const lockAfter = activeLock(await handler.locks(), new Date(), handler.lockTtl, LockType.Sync, 'desktop', '111');
expect(lockAfter.updatedTime).toBeGreaterThan(lockBefore.updatedTime);
handler.stopAutoLockRefresh(lock);
}));
@@ -95,11 +97,14 @@ describe('synchronizer_LockHandler', function() {
autoLockError = error;
});
await msleep(250 * timeoutMultipler);
try {
await msleep(250 * timeoutMultipler);
expect(autoLockError.code).toBe('lockExpired');
handler.stopAutoLockRefresh(lock);
expect(autoLockError).toBeTruthy();
expect(autoLockError.code).toBe('lockExpired');
} finally {
handler.stopAutoLockRefresh(lock);
}
}));
it('should not allow sync locks if there is an exclusive lock', (async () => {
@@ -112,6 +117,7 @@ describe('synchronizer_LockHandler', function() {
it('should not allow exclusive lock if there are sync locks', (async () => {
const lockHandler = newLockHandler({ lockTtl: 1000 * 60 * 60 });
if (lockHandler.useBuiltInLocks) return; // Tested server side
await lockHandler.acquireLock(LockType.Sync, 'mobile', '111');
await lockHandler.acquireLock(LockType.Sync, 'mobile', '222');
@@ -123,6 +129,7 @@ describe('synchronizer_LockHandler', function() {
it('should allow exclusive lock if the sync locks have expired', (async () => {
const lockHandler = newLockHandler({ lockTtl: 500 * timeoutMultipler });
if (lockHandler.useBuiltInLocks) return; // Tested server side
await lockHandler.acquireLock(LockType.Sync, 'mobile', '111');
await lockHandler.acquireLock(LockType.Sync, 'mobile', '222');
@@ -138,74 +145,44 @@ describe('synchronizer_LockHandler', function() {
const lockHandler = newLockHandler();
{
const lock1: Lock = { type: LockType.Exclusive, clientId: '1', clientType: 'd' };
const lock2: Lock = { type: LockType.Exclusive, clientId: '2', clientType: 'd' };
await lockHandler.saveLock_(lock1);
await msleep(100);
await lockHandler.saveLock_(lock2);
const locks: Lock[] = [
{
type: LockType.Exclusive,
clientId: '1',
clientType: 'd',
updatedTime: Date.now(),
},
];
const activeLock = await lockHandler.activeLock(LockType.Exclusive);
expect(activeLock.clientId).toBe('1');
await msleep(100);
locks.push({
type: LockType.Exclusive,
clientId: '2',
clientType: 'd',
updatedTime: Date.now(),
});
const lock = activeLock(locks, new Date(), lockHandler.lockTtl, LockType.Exclusive);
expect(lock.clientId).toBe('1');
}
}));
it('should ignore locks by same client when trying to acquire exclusive lock', (async () => {
const lockHandler = newLockHandler();
await lockHandler.acquireLock(LockType.Sync, 'desktop', '111');
await expectThrow(async () => {
await lockHandler.acquireLock(LockType.Exclusive, 'desktop', '111', { clearExistingSyncLocksFromTheSameClient: false });
}, 'hasSyncLock');
await expectNotThrow(async () => {
await lockHandler.acquireLock(LockType.Exclusive, 'desktop', '111', { clearExistingSyncLocksFromTheSameClient: true });
});
const activeLock = await lockHandler.activeLock(LockType.Exclusive);
expect(activeLock.clientId).toBe('111');
}));
// it('should not have race conditions', (async () => {
// it('should ignore locks by same client when trying to acquire exclusive lock', (async () => {
// const lockHandler = newLockHandler();
// const clients = [];
// for (let i = 0; i < 20; i++) {
// clients.push({
// id: 'client' + i,
// type: 'desktop',
// });
// }
// await lockHandler.acquireLock(LockType.Sync, 'desktop', '111');
// for (let loopIndex = 0; loopIndex < 1000; loopIndex++) {
// const promises:Promise<void | Lock>[] = [];
// for (let clientIndex = 0; clientIndex < clients.length; clientIndex++) {
// const client = clients[clientIndex];
// await expectThrow(async () => {
// await lockHandler.acquireLock(LockType.Exclusive, 'desktop', '111', { clearExistingSyncLocksFromTheSameClient: false });
// }, 'hasSyncLock');
// promises.push(
// lockHandler.acquireLock(LockType.Exclusive, client.type, client.id).catch(() => {})
// );
// await expectNotThrow(async () => {
// await lockHandler.acquireLock(LockType.Exclusive, 'desktop', '111', { clearExistingSyncLocksFromTheSameClient: true });
// });
// // if (gotLock) {
// // await msleep(100);
// // const locks = await lockHandler.locks(LockType.Exclusive);
// // console.info('=======================================');
// // console.info(locks);
// // lockHandler.releaseLock(LockType.Exclusive, client.type, client.id);
// // }
// // await msleep(500);
// }
// const result = await Promise.all(promises);
// const locks = result.filter((lock:any) => !!lock);
// expect(locks.length).toBe(1);
// const lock:Lock = locks[0] as Lock;
// const allLocks = await lockHandler.locks();
// console.info('================================', allLocks);
// lockHandler.releaseLock(LockType.Exclusive, lock.clientType, lock.clientId);
// }
// const lock = activeLock(await lockHandler.locks(), new Date(), lockHandler.lockTtl, LockType.Exclusive);
// expect(lock.clientId).toBe('111');
// }));
});

View File

@@ -12,7 +12,7 @@ export async function allNotesFolders() {
async function remoteItemsByTypes(types: number[]) {
const list = await fileApi().list('', { includeDirs: false, syncItemsOnly: true });
if (list.has_more) throw new Error('Not implemented!!!');
if (list.hasMore) throw new Error('Not implemented!!!');
const files = list.items;
const output = [];

View File

@@ -1,12 +1,12 @@
{
"name": "@joplin/server",
"version": "2.5.10",
"version": "2.5.9",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@joplin/server",
"version": "2.5.10",
"version": "2.5.9",
"dependencies": {
"@fortawesome/fontawesome-free": "^5.15.1",
"@koa/cors": "^3.1.0",

View File

@@ -1,6 +1,6 @@
{
"name": "@joplin/server",
"version": "2.5.10",
"version": "2.5.9",
"private": true,
"scripts": {
"start-dev": "npm run build && nodemon --config nodemon.json --ext ts,js,mustache,css,tsx dist/app.js --env dev",
@@ -11,7 +11,7 @@
"devDropTables": "node dist/app.js --env dev --drop-tables",
"devDropDb": "node dist/app.js --env dev --drop-db",
"start": "node dist/app.js",
"generateTypes": "rm -f db-buildTypes.sqlite && npm run start -- --migrate-latest --env buildTypes && node dist/tools/generateTypes.js && mv db-buildTypes.sqlite schema.sqlite",
"generateTypes": "rm -f db-buildTypes.sqlite && npm run start -- --env buildTypes migrate latest && node dist/tools/generateTypes.js && mv db-buildTypes.sqlite schema.sqlite",
"tsc": "tsc --project tsconfig.json",
"test": "jest --verbose=false",
"test-ci": "npm run test",

Binary file not shown.

View File

@@ -22,6 +22,7 @@ export interface EnvVariables {
ERROR_STACK_TRACES?: string;
COOKIES_SECURE?: string;
RUNNING_IN_DOCKER?: string;
BUILTIN_LOCKS_ENABLED?: string;
// ==================================================
// URL config
@@ -211,6 +212,7 @@ export async function initConfig(envType: Env, env: EnvVariables, overrides: any
supportName: env.SUPPORT_NAME || appName,
businessEmail: env.BUSINESS_EMAIL || supportEmail,
cookieSecure: env.COOKIES_SECURE === '1',
buildInLocksEnabled: envReadBool(env.BUILTIN_LOCKS_ENABLED, false),
...overrides,
};
}

View File

@@ -116,6 +116,10 @@ export const clientType = (db: DbConnection): DatabaseConfigClient => {
return db.client.config.client;
};
export const returningSupported = (db: DbConnection) => {
return clientType(db) === DatabaseConfigClient.PostgreSQL;
};
export const isPostgres = (db: DbConnection) => {
return clientType(db) === DatabaseConfigClient.PostgreSQL;
};

View File

@@ -0,0 +1,23 @@
import { Knex } from 'knex';
import { DbConnection } from '../db';
export async function up(db: DbConnection): Promise<any> {
await db.schema.createTable('locks', (table: Knex.CreateTableBuilder) => {
table.uuid('id').unique().primary().notNullable();
table.string('user_id', 32).notNullable();
table.integer('type', 2).notNullable();
table.string('client_type', 32).notNullable();
table.string('client_id', 32).notNullable();
table.bigInteger('updated_time').notNullable();
table.bigInteger('created_time').notNullable();
});
await db.schema.alterTable('locks', (table: Knex.CreateTableBuilder) => {
table.index('user_id');
table.index('created_time');
});
}
export async function down(db: DbConnection): Promise<any> {
await db.schema.dropTable('locks');
}

View File

@@ -95,6 +95,10 @@ export default abstract class BaseModel<T> {
return this.db_;
}
protected get dbRead(): DbConnection {
return this.db;
}
protected get defaultFields(): string[] {
if (!this.defaultFields_.length) {
this.defaultFields_ = Object.keys(databaseSchema[this.tableName]);

View File

@@ -1,4 +1,6 @@
import { returningSupported } from '../db';
import { KeyValue } from '../services/database/types';
import { msleep } from '../utils/time';
import BaseModel from './BaseModel';
export enum ValueType {
@@ -6,7 +8,9 @@ export enum ValueType {
String = 2,
}
type Value = number | string;
export type Value = number | string;
export type ReadThenWriteHandler = (value: Value)=> Promise<Value>;
export default class KeyValueModel extends BaseModel<KeyValue> {
@@ -57,6 +61,45 @@ export default class KeyValueModel extends BaseModel<KeyValue> {
return this.unserializeValue(row.type, row.value) as any;
}
public async readThenWrite(key: string, handler: ReadThenWriteHandler) {
if (!returningSupported(this.db)) {
// While inside a transaction SQlite should lock the whole database
// file, which should allow atomic read then write.
await this.withTransaction(async () => {
const value: any = await this.value(key);
const newValue = await handler(value);
await this.setValue(key, newValue);
}, 'KeyValueModel::readThenWrite');
return;
}
let loopCount = 0;
while (true) {
const row: KeyValue = await this.db(this.tableName).where('key', '=', key).first();
const newValue = await handler(row ? row.value : null);
let previousValue: Value = null;
if (row) {
previousValue = row.value;
} else {
await this.setValue(key, newValue);
previousValue = newValue;
}
const updatedRows = await this
.db(this.tableName)
.update({ value: newValue }, ['id'])
.where('key', '=', key)
.where('value', '=', previousValue);
if (updatedRows.length) return;
loopCount++;
if (loopCount >= 10) throw new Error(`Could not update key: ${key}`);
await msleep(10000 * Math.random());
}
}
public async deleteValue(key: string): Promise<void> {
await this.db(this.tableName).where('key', '=', key).delete();
}
@@ -65,4 +108,8 @@ export default class KeyValueModel extends BaseModel<KeyValue> {
throw new Error('Call ::deleteValue()');
}
public async deleteAll(): Promise<void> {
await this.db(this.tableName).delete();
}
}

View File

@@ -0,0 +1,150 @@
import BaseModel, { UuidType } from './BaseModel';
import { Uuid } from '../services/database/types';
import { Lock, LockType, lockDefaultTtl, activeLock } from '@joplin/lib/services/synchronizer/LockHandler';
import { Value } from './KeyValueModel';
import { ErrorConflict } from '../utils/errors';
import uuidgen from '../utils/uuidgen';
export default class LockModel extends BaseModel<Lock> {
private lockTtl_: number = lockDefaultTtl;
protected get tableName(): string {
return 'locks';
}
protected uuidType(): UuidType {
return UuidType.Native;
}
// TODO: validate lock when acquiring and releasing
// TODO: test "should allow exclusive lock if the sync locks have expired"
// TODO: test "should not allow exclusive lock if there are sync locks"
private get lockTtl() {
return this.lockTtl_;
}
public async allLocks(userId: Uuid): Promise<Lock[]> {
const userKey = `locks::${userId}`;
const v = await this.models().keyValue().value<string>(userKey);
return v ? JSON.parse(v) : [];
}
private async acquireSyncLock(userId: Uuid, clientType: string, clientId: string): Promise<Lock> {
const userKey = `locks::${userId}`;
let output: Lock = null;
await this.models().keyValue().readThenWrite(userKey, async (value: Value) => {
let locks: Lock[] = value ? JSON.parse(value as string) : [];
const exclusiveLock = activeLock(locks, new Date(), this.lockTtl, LockType.Exclusive);
if (exclusiveLock) {
throw new ErrorConflict(`Cannot acquire lock because there is already an exclusive lock for client: ${exclusiveLock.clientType} #${exclusiveLock.clientId}`, 'hasExclusiveLock');
}
const syncLock = activeLock(locks, new Date(), this.lockTtl, LockType.Sync, clientType, clientId);
if (syncLock) {
output = {
...syncLock,
updatedTime: Date.now(),
};
locks = locks.map(l => l.id === syncLock.id ? output : l);
} else {
output = {
id: uuidgen(),
type: LockType.Sync,
clientId,
clientType,
updatedTime: Date.now(),
};
locks.push(output);
}
return JSON.stringify(locks);
});
return output;
}
private async acquireExclusiveLock(userId: Uuid, clientType: string, clientId: string): Promise<Lock> {
const userKey = `locks::${userId}`;
let output: Lock = null;
await this.models().keyValue().readThenWrite(userKey, async (value: Value) => {
let locks: Lock[] = value ? JSON.parse(value as string) : [];
const exclusiveLock = activeLock(locks, new Date(), this.lockTtl, LockType.Exclusive);
if (exclusiveLock) {
if (exclusiveLock.clientId === clientId) {
locks = locks.filter(l => l.id !== exclusiveLock.id);
output = {
...exclusiveLock,
updatedTime: Date.now(),
};
locks.push(output);
return JSON.stringify(locks);
} else {
throw new ErrorConflict(`Cannot acquire lock because there is already an exclusive lock for client: ${exclusiveLock.clientType} #${exclusiveLock.clientId}`, 'hasExclusiveLock');
}
}
const syncLock = activeLock(locks, new Date(), this.lockTtl, LockType.Sync);
if (syncLock) {
if (syncLock.clientId === clientId) {
locks = locks.filter(l => l.id !== syncLock.id);
} else {
throw new ErrorConflict(`Cannot acquire exclusive lock because there is an active sync lock for client: ${syncLock.clientType} #${syncLock.clientId}`, 'hasSyncLock');
}
}
output = {
id: uuidgen(),
type: LockType.Exclusive,
clientId,
clientType,
updatedTime: Date.now(),
};
locks.push(output);
return JSON.stringify(locks);
});
return output;
}
public async acquireLock(userId: Uuid, type: LockType, clientType: string, clientId: string): Promise<Lock> {
if (type === LockType.Sync) {
return this.acquireSyncLock(userId, clientType, clientId);
} else {
return this.acquireExclusiveLock(userId, clientType, clientId);
}
}
public async releaseLock(userId: Uuid, lockType: LockType, clientType: string, clientId: string) {
const userKey = `locks::${userId}`;
await this.models().keyValue().readThenWrite(userKey, async (value: Value) => {
const locks: Lock[] = value ? JSON.parse(value as string) : [];
for (let i = locks.length - 1; i >= 0; i--) {
const lock = locks[i];
if (lock.type === lockType && lock.clientType === clientType && lock.clientId === clientId) {
locks.splice(i, 1);
}
}
return JSON.stringify(locks);
});
}
}

View File

@@ -72,6 +72,7 @@ import SubscriptionModel from './SubscriptionModel';
import UserFlagModel from './UserFlagModel';
import EventModel from './EventModel';
import { Config } from '../utils/types';
import LockModel from './LockModel';
export class Models {
@@ -147,6 +148,10 @@ export class Models {
return new EventModel(this.db_, newModelFactory, this.config_);
}
public lock() {
return new LockModel(this.db_, newModelFactory, this.config_);
}
}
export default function newModelFactory(db: DbConnection, config: Config): Models {

View File

@@ -2,9 +2,10 @@ import config from '../../config';
import { clearDatabase, createTestUsers, CreateTestUsersOptions } from '../../tools/debugTools';
import { bodyFields } from '../../utils/requestUtils';
import Router from '../../utils/Router';
import { RouteType } from '../../utils/types';
import { Env, RouteType } from '../../utils/types';
import { SubPath } from '../../utils/routeUtils';
import { AppContext } from '../../utils/types';
import { ErrorForbidden } from '../../utils/errors';
const router = new Router(RouteType.Api);
@@ -17,7 +18,10 @@ interface Query {
}
router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
if (config().env !== Env.Dev) throw new ErrorForbidden();
const query: Query = (await bodyFields(ctx.req)) as Query;
const models = ctx.joplin.models;
console.info(`Action: ${query.action}`);
@@ -33,6 +37,10 @@ router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
if (query.action === 'clearDatabase') {
await clearDatabase(ctx.joplin.db);
}
if (query.action === 'clearKeyValues') {
await models.keyValue().deleteAll();
}
});
export default router;

View File

@@ -11,6 +11,7 @@ import { requestDeltaPagination, requestPagination } from '../../models/utils/pa
import { AclAction } from '../../models/BaseModel';
import { safeRemove } from '../../utils/fileUtils';
import { formatBytes, MB } from '../../utils/bytes';
import lockHandler from './utils/items/lockHandler';
const router = new Router(RouteType.Api);
@@ -42,6 +43,9 @@ export async function putItemContents(path: SubPath, ctx: AppContext, isBatch: b
try {
const buffer = filePath ? await fs.readFile(filePath) : Buffer.alloc(0);
const lockResult = await lockHandler(path, ctx, buffer);
if (lockResult.handled) return lockResult.response;
// This end point can optionally set the associated jop_share_id field. It
// is only useful when uploading resource blob (under .resource folder)
// since they can't have metadata. Note, Folder and Resource items all
@@ -104,8 +108,13 @@ router.del('api/items/:id', async (path: SubPath, ctx: AppContext) => {
if (ctx.joplin.env !== 'dev') throw new ErrorMethodNotAllowed('Deleting the root is not allowed');
await ctx.joplin.models.item().deleteAll(ctx.joplin.owner.id);
} else {
// const item = await itemFromPath(ctx.joplin.owner.id, ctx.joplin.models.item(), path);
// await ctx.joplin.models.item().checkIfAllowed(ctx.joplin.owner, AclAction.Delete, item);
const lockResult = await lockHandler(path, ctx);
if (lockResult.handled) return lockResult.response;
const item = await itemFromPath(ctx.joplin.owner.id, ctx.joplin.models.item(), path);
await ctx.joplin.models.item().checkIfAllowed(ctx.joplin.owner, AclAction.Delete, item);
await ctx.joplin.models.item().deleteForUser(ctx.joplin.owner.id, item);
}
} catch (error) {
@@ -137,6 +146,9 @@ router.get('api/items/:id/delta', async (_path: SubPath, ctx: AppContext) => {
});
router.get('api/items/:id/children', async (path: SubPath, ctx: AppContext) => {
const lockResult = await lockHandler(path, ctx);
if (lockResult.handled) return lockResult.response;
const itemModel = ctx.joplin.models.item();
const parentName = itemModel.pathToName(path.id);
const result = await itemModel.children(ctx.joplin.owner.id, parentName, requestPagination(ctx.query));

View File

@@ -0,0 +1,32 @@
import { lockNameToObject, LockType } from '@joplin/lib/services/synchronizer/LockHandler';
import { bodyFields } from '../../utils/requestUtils';
import Router from '../../utils/Router';
import { SubPath } from '../../utils/routeUtils';
import { AppContext, RouteType } from '../../utils/types';
const router = new Router(RouteType.Api);
interface PostFields {
type: LockType;
clientType: string;
clientId: string;
}
router.post('api/locks', async (_path: SubPath, ctx: AppContext) => {
const fields = await bodyFields<PostFields>(ctx.req);
return ctx.joplin.models.lock().acquireLock(ctx.joplin.owner.id, fields.type, fields.clientType, fields.clientId);
});
router.del('api/locks/:id', async (path: SubPath, ctx: AppContext) => {
const lock = lockNameToObject(path.id);
await ctx.joplin.models.lock().releaseLock(ctx.joplin.owner.id, lock.type, lock.clientType, lock.clientId);
});
router.get('api/locks', async (_path: SubPath, ctx: AppContext) => {
return {
items: await ctx.joplin.models.lock().allLocks(ctx.joplin.owner.id),
has_more: false,
};
});
export default router;

View File

@@ -0,0 +1,47 @@
import config from '../../../../config';
import { PaginatedItems } from '../../../../models/ItemModel';
import { Item } from '../../../../services/database/types';
import { getApi, putApi } from '../../../../utils/testing/apiUtils';
import { beforeAllDb, afterAllTests, beforeEachDb, createUserAndSession, models } from '../../../../utils/testing/testUtils';
describe('items/lockHandlers', function() {
beforeAll(async () => {
await beforeAllDb('items/lockHandlers');
});
afterAll(async () => {
await afterAllTests();
});
beforeEach(async () => {
await beforeEachDb();
config().buildInLocksEnabled = true;
});
test('should save locks to the key-value store', async function() {
const { session, user } = await createUserAndSession(1);
const lockName = 'locks/exclusive_cli_12cb74fa9de644958b2ccbc772cb4e29.json';
const now = Date.now();
const result: Item = await putApi(session.id, `items/root:/${lockName}:/content`, { testing: true });
expect(result.name).toBe(lockName);
expect(result.updated_time).toBeGreaterThanOrEqual(now);
expect(result.id).toBe(null);
const values = await models().keyValue().all();
expect(values.length).toBe(1);
expect(values[0].key).toBe(`locks::${user.id}`);
const value = JSON.parse(values[0].value);
expect(value[lockName].name).toBe(lockName);
expect(value[lockName].updated_time).toBeGreaterThanOrEqual(now);
const getResult: PaginatedItems = await getApi(session.id, 'items/root:/locks/*:children');
console.info(getResult);
expect(getResult.items[0].name).toBe(result.name);
expect(getResult.items[0].updated_time).toBe(result.updated_time);
});
});

View File

@@ -0,0 +1,98 @@
import config from '../../../../config';
import { PaginatedItems } from '../../../../models/ItemModel';
import { Value } from '../../../../models/KeyValueModel';
import { Item } from '../../../../services/database/types';
import { ErrorBadRequest } from '../../../../utils/errors';
import { SubPath } from '../../../../utils/routeUtils';
import { AppContext } from '../../../../utils/types';
interface LockHandlerResult {
handled: boolean;
response: any;
}
const lockHandler = async (path: SubPath, ctx: AppContext, requestBody: Buffer = null): Promise<LockHandlerResult | null> => {
if (!config().buildInLocksEnabled) return { handled: false, response: null };
if (!path.id || !path.id.startsWith('root:/locks/')) return { handled: false, response: null };
const ownerId = ctx.joplin.owner.id;
const models = ctx.joplin.models;
const userKey = `locks::${ownerId}`;
// PUT /api/items/root:/locks/exclusive_cli_12cb74fa9de644958b2ccbc772cb4e29.json:/content
if (ctx.method === 'PUT') {
const itemName = models.item().pathToName(path.id);
const now = Date.now();
await models.keyValue().readThenWrite(userKey, async (value: Value) => {
const output = value ? JSON.parse(value as string) : {};
output[itemName] = {
name: itemName,
updated_time: now,
jop_updated_time: now,
content: requestBody.toString(),
};
return JSON.stringify(output);
});
return {
handled: true,
response: {
[itemName]: {
item: {
name: itemName,
updated_time: now,
id: null,
},
error: null,
},
},
};
}
// DELETE /api/items/root:/locks/exclusive_cli_12cb74fa9de644958b2ccbc772cb4e29.json:
if (ctx.method === 'DELETE') {
const itemName = models.item().pathToName(path.id);
await models.keyValue().readThenWrite(userKey, async (value: Value) => {
const output = value ? JSON.parse(value as string) : {};
delete output[itemName];
return JSON.stringify(output);
});
return {
handled: true,
response: null,
};
}
// GET /api/items/root:/locks/*:/children
if (ctx.method === 'GET' && path.id === 'root:/locks/*:') {
const result = await models.keyValue().value<string>(userKey);
const obj: Record<string, Item> = result ? JSON.parse(result) : {};
const items: Item[] = [];
for (const name of Object.keys(obj)) {
items.push(obj[name]);
}
const page: PaginatedItems = {
has_more: false,
items,
};
return {
handled: true,
response: page,
};
}
throw new ErrorBadRequest(`Unhandled lock path: ${path.id}`);
};
export default lockHandler;

View File

@@ -22,7 +22,7 @@ async function renderItem(context: AppContext, item: Item, share: Share): Promis
}
function createContentDispositionHeader(filename: string) {
const encoded = encodeURIComponent(friendlySafeFilename(filename, null, true));
const encoded = encodeURIComponent(friendlySafeFilename(filename));
return `attachment; filename*=UTF-8''${encoded}; filename="${encoded}"`;
}

View File

@@ -10,6 +10,7 @@ import apiSessions from './api/sessions';
import apiShares from './api/shares';
import apiShareUsers from './api/share_users';
import apiUsers from './api/users';
import apiLocks from './api/locks';
import indexChanges from './index/changes';
import indexHelp from './index/help';
@@ -41,6 +42,7 @@ const routes: Routers = {
'api/share_users': apiShareUsers,
'api/shares': apiShares,
'api/users': apiUsers,
'api/locks': apiLocks,
'changes': indexChanges,
'home': indexHome,

View File

@@ -258,6 +258,16 @@ export interface Item extends WithDates, WithUuid {
owner_id?: Uuid;
}
export interface Lock {
id?: Uuid;
user_id?: Uuid;
type?: number;
client_type?: string;
client_id?: Uuid;
updated_time?: string;
created_time?: string;
}
export const databaseSchema: DatabaseTables = {
sessions: {
id: { type: 'string' },
@@ -430,5 +440,14 @@ export const databaseSchema: DatabaseTables = {
jop_updated_time: { type: 'string' },
owner_id: { type: 'string' },
},
locks: {
id: { type: 'string' },
user_id: { type: 'string' },
type: { type: 'number' },
client_type: { type: 'string' },
client_id: { type: 'string' },
updated_time: { type: 'string' },
created_time: { type: 'string' },
},
};
// AUTO-GENERATED-TYPES

View File

@@ -79,8 +79,8 @@ export class ErrorUnprocessableEntity extends ApiError {
export class ErrorConflict extends ApiError {
public static httpCode: number = 409;
public constructor(message: string = 'Conflict') {
super(message, ErrorConflict.httpCode);
public constructor(message: string = 'Conflict', code: string = undefined) {
super(message, ErrorConflict.httpCode, code);
Object.setPrototypeOf(this, ErrorConflict.prototype);
}
}

View File

@@ -115,6 +115,7 @@ export interface Config {
businessEmail: string;
isJoplinCloud: boolean;
cookieSecure: boolean;
buildInLocksEnabled: boolean;
}
export enum HttpMethod {

View File

@@ -1,10 +1,5 @@
# Joplin Server Changelog
## [server-v2.5.10](https://github.com/laurent22/joplin/releases/tag/server-v2.5.10) - 2021-11-02T14:45:54Z
- New: Add unique constraint on name and owner ID of items table (f7a18ba)
- Fixed: Fixed issue that could cause server to return empty items in some rare cases (99ea4b7)
## [server-v2.5.9](https://github.com/laurent22/joplin/releases/tag/server-v2.5.9) - 2021-10-28T19:43:41Z
- Improved: Remove session expiration for now (4a2af32)