Merge branch 'dev' into release-1.1

ReactNativeClient/lib/BaseApplication.js

@@ -44,6 +44,9 @@ const KvStore = require('lib/services/KvStore');
 const MigrationService = require('lib/services/MigrationService');
 const { toSystemSlashes } = require('lib/path-utils.js');
 
+// const ntpClient = require('lib/vendor/ntp-client');
+// ntpClient.dgram = require('dgram');
+
 class BaseApplication {
 	constructor() {
 		this.logger_ = new Logger();
@@ -673,6 +676,7 @@ class BaseApplication {
 		reg.dispatch = () => {};
 
 		BaseService.logger_ = this.logger_;
+		// require('lib/ntpDate').setLogger(reg.logger());
 
 		this.dbLogger_.addTarget('file', { path: `${profileDir}/log-database.txt` });
 		this.dbLogger_.setLevel(initArgs.logLevel);

ReactNativeClient/lib/file-api-driver-dropbox.js

@@ -44,7 +44,7 @@ class FileApiDriverDropbox {
 	metadataToStat_(md, path) {
 		const output = {
 			path: path,
-			updated_time: md.server_modified ? new Date(md.server_modified) : new Date(),
+			updated_time: md.server_modified ? (new Date(md.server_modified)).getTime() : Date.now(),
 			isDir: md['.tag'] === 'folder',
 		};
 

ReactNativeClient/lib/file-api.js

@@ -6,6 +6,7 @@ const JoplinError = require('lib/JoplinError');
 const ArrayUtils = require('lib/ArrayUtils');
 const { time } = require('lib/time-utils.js');
 const { sprintf } = require('sprintf-js');
+const Mutex = require('async-mutex').Mutex;
 
 function requestCanBeRepeated(error) {
 	const errorCode = typeof error === 'object' && error.code ? error.code : null;
@@ -57,6 +58,65 @@ class FileApi {
 		this.tempDirName_ = null;
 		this.driver_.fileApi_ = this;
 		this.requestRepeatCount_ = null; // For testing purpose only - normally this value should come from the driver
+		this.remoteDateOffset_ = 0;
+		this.remoteDateNextCheckTime_ = 0;
+		this.remoteDateMutex_ = new Mutex();
+	}
+
+
+	async fetchRemoteDateOffset_() {
+		const tempFile = `${this.tempDirName()}/timeCheck${Math.round(Math.random() * 1000000)}.txt`;
+		const startTime = Date.now();
+		await this.put(tempFile, 'timeCheck');
+
+		// Normally it should be possible to read the file back immediately but
+		// just in case, read it in a loop.
+		const loopStartTime = Date.now();
+		let stat = null;
+		while (Date.now() - loopStartTime < 5000) {
+			stat = await this.stat(tempFile);
+			if (stat) break;
+			await time.msleep(200);
+		}
+
+		if (!stat) throw new Error('Timed out trying to get sync target clock time');
+
+		this.delete(tempFile); // No need to await for this call
+
+		const endTime = Date.now();
+		const expectedTime = Math.round((endTime + startTime) / 2);
+		return stat.updated_time - expectedTime;
+	}
+
+	// Approximates the current time on the sync target. It caches the time offset to
+	// improve performance.
+	async remoteDate() {
+		const shouldSyncTime = () => {
+			return !this.remoteDateNextCheckTime_ || Date.now() > this.remoteDateNextCheckTime_;
+		};
+
+		if (shouldSyncTime()) {
+			const release = await this.remoteDateMutex_.acquire();
+
+			try {
+				// Another call might have refreshed the time while we were waiting for the mutex,
+				// so check again if we need to refresh.
+				if (shouldSyncTime()) {
+					this.remoteDateOffset_ = await this.fetchRemoteDateOffset_();
+					// The sync target clock should rarely change but the device one might,
+					// so we need to refresh relatively frequently.
+					this.remoteDateNextCheckTime_ = Date.now() + 10 * 60 * 1000;
+				}
+			} catch (error) {
+				this.logger().warn('Could not retrieve remote date - defaulting to device date:', error);
+				this.remoteDateOffset_ = 0;
+				this.remoteDateNextCheckTime_ = Date.now() + 60 * 1000;
+			} finally {
+				release();
+			}
+		}
+
+		return new Date(Date.now() + this.remoteDateOffset_);
 	}
 
 	// Ideally all requests repeating should be done at the FileApi level to remove duplicate code in the drivers, but

ReactNativeClient/lib/models/Setting.js

@@ -422,7 +422,7 @@ class Setting extends BaseModel {
 				value: false,
 				type: Setting.TYPE_BOOL,
 				section: 'note',
-				public: true,
+				public: mobilePlatform === 'ios',
 				appTypes: ['mobile'],
 				label: () => 'Opt-in to the editor beta',
 				description: () => 'This beta adds list continuation, Markdown preview, and Markdown shortcuts. If you find bugs, please report them in the Discourse forum.',

ReactNativeClient/lib/ntpDate.ts

@@ -0,0 +1,59 @@
+const ntpClient = require('lib/vendor/ntp-client');
+const { Logger } = require('lib/logger.js');
+const Mutex = require('async-mutex').Mutex;
+
+let nextSyncTime = 0;
+let timeOffset = 0;
+let logger = new Logger();
+
+const fetchingTimeMutex = new Mutex();
+
+const server = {
+	domain: 'pool.ntp.org',
+	port: 123,
+};
+
+async function networkTime():Promise<Date> {
+	return new Promise(function(resolve:Function, reject:Function) {
+		ntpClient.getNetworkTime(server.domain, server.port, function(error:any, date:Date) {
+			if (error) {
+				reject(error);
+				return;
+			}
+
+			resolve(date);
+		});
+	});
+}
+
+function shouldSyncTime() {
+	return !nextSyncTime || Date.now() > nextSyncTime;
+}
+
+export function setLogger(v:any) {
+	logger = v;
+}
+
+export default async function():Promise<Date> {
+	if (shouldSyncTime()) {
+		const release = await fetchingTimeMutex.acquire();
+
+		try {
+			if (shouldSyncTime()) {
+				const date = await networkTime();
+				nextSyncTime = Date.now() + 60 * 1000;
+				timeOffset = date.getTime() - Date.now();
+			}
+		} catch (error) {
+			logger.warn('Could not get NTP time - falling back to device time:', error);
+			// Fallback to device time since
+			// most of the time it's actually correct
+			nextSyncTime = Date.now() + 20 * 1000;
+			timeOffset = 0;
+		} finally {
+			release();
+		}
+	}
+
+	return new Date(Date.now() + timeOffset);
+}

ReactNativeClient/lib/services/searchengine/SearchEngine.js

@@ -434,7 +434,6 @@ class SearchEngine {
 	}
 
 	async parseQuery(query, fuzzy = false) {
-		// fuzzy = false;
 		const trimQuotes = (str) => str.startsWith('"') ? str.substr(1, str.length - 2) : str;
 
 		let allTerms = [];
@@ -453,18 +452,22 @@ class SearchEngine {
 		const fuzzyScore = [];
 		let numFuzzyMatches = [];
 		let terms = null;
-		if (fuzzy) {
-			const fuzzyText = await this.fuzzifier(textTerms.filter(x => !x.quoted).map(x => trimQuotes(x.value)));
-			const fuzzyTitle = await this.fuzzifier(titleTerms.map(x => trimQuotes(x.value)));
-			const fuzzyBody = await this.fuzzifier(bodyTerms.map(x => trimQuotes(x.value)));
-			const phraseSearches = textTerms.filter(x => x.quoted).map(x => x.value);
 
-			// Save number of matches we got for each word
-			// fuzzifier() is currently set to return at most 3 matches)
+		if (fuzzy) {
+			const fuzzyText = await this.fuzzifier(textTerms.filter(x => !(x.quoted || x.wildcard)).map(x => trimQuotes(x.value)));
+			const fuzzyTitle = await this.fuzzifier(titleTerms.filter(x => !x.wildcard).map(x => trimQuotes(x.value)));
+			const fuzzyBody = await this.fuzzifier(bodyTerms.filter(x => !x.wildcard).map(x => trimQuotes(x.value)));
+
+			const phraseTextSearch = textTerms.filter(x => x.quoted);
+			const wildCardSearch = textTerms.concat(titleTerms).concat(bodyTerms).filter(x => x.wildcard);
+
+			// Save number of fuzzy matches we got for each word
+			// fuzzifier() is currently set to return at most 3 matches
 			// We need to know which fuzzy words go together so that we can filter out notes that don't contain a required word.
 			numFuzzyMatches = fuzzyText.concat(fuzzyTitle).concat(fuzzyBody).map(x => x.length);
-			for (let i = 0; i < phraseSearches.length; i++) {
-				numFuzzyMatches.push(1); // Phrase searches are preserved without fuzzification
+			for (let i = 0; i < phraseTextSearch.length + wildCardSearch.length; i++) {
+				// Phrase searches and wildcard searches are preserved without fuzzification (A single match)
+				numFuzzyMatches.push(1);
 			}
 
 			const mergedFuzzyText = [].concat.apply([], fuzzyText);
@@ -474,18 +477,33 @@ class SearchEngine {
 			const fuzzyTextTerms = mergedFuzzyText.map(x => { return { name: 'text', value: x.word, negated: false, score: x.score }; });
 			const fuzzyTitleTerms = mergedFuzzyTitle.map(x => { return { name: 'title', value: x.word, negated: false, score: x.score }; });
 			const fuzzyBodyTerms = mergedFuzzyBody.map(x => { return { name: 'body', value: x.word, negated: false, score: x.score }; });
-			const phraseTextTerms = phraseSearches.map(x => { return { name: 'text', value: x, negated: false, score: 0 }; });
 
+			// Remove previous text, title and body and replace with fuzzy versions
 			allTerms = allTerms.filter(x => (x.name !== 'text' && x.name !== 'title' && x.name !== 'body'));
 
-			allFuzzyTerms = allTerms.concat(fuzzyTextTerms).concat(fuzzyTitleTerms).concat(fuzzyBodyTerms).concat(phraseTextTerms);
+			// The order matters here!
+			// The text goes first, then title, then body, then phrase and finally wildcard
+			// This is because it needs to match with numFuzzyMathches.
+			allFuzzyTerms = allTerms.concat(fuzzyTextTerms).concat(fuzzyTitleTerms).concat(fuzzyBodyTerms).concat(phraseTextSearch).concat(wildCardSearch);
 
 			const allTextTerms = allFuzzyTerms.filter(x => x.name === 'title' || x.name === 'body' || x.name === 'text');
 			for (let i = 0; i < allTextTerms.length; i++) {
+				// Phrase searches and wildcard searches will get a fuzziness score of zero.
+				// This means that they will go first in the sort order (Even if there are other words with matches in the title)
+				// Undesirable?
 				fuzzyScore.push(allFuzzyTerms[i].score ? allFuzzyTerms[i].score : 0);
 			}
 
-			terms = { _: fuzzyTextTerms.concat(phraseTextTerms).map(x =>trimQuotes(x.value)), 'title': fuzzyTitleTerms.map(x =>trimQuotes(x.value)), 'body': fuzzyBodyTerms.map(x =>trimQuotes(x.value)) };
+			const wildCardTextTerms = wildCardSearch.filter(x => x.name === 'text').map(x =>trimQuotes(x.value));
+			const wildCardTitleTerms = wildCardSearch.filter(x => x.name === 'title').map(x =>trimQuotes(x.value));
+			const wildCardBodyTerms = wildCardSearch.filter(x => x.name === 'body').map(x =>trimQuotes(x.value));
+			const phraseTextTerms = phraseTextSearch.map(x => trimQuotes(x.value));
+
+			terms = {
+				_: fuzzyTextTerms.map(x => trimQuotes(x.value)).concat(phraseTextTerms).concat(wildCardTextTerms),
+				title: fuzzyTitleTerms.map(x => trimQuotes(x.value)).concat(wildCardTitleTerms),
+				body: fuzzyBodyTerms.map(x => trimQuotes(x.value)).concat(wildCardBodyTerms),
+			};
 		} else {
 			const nonNegatedTextTerms = textTerms.length + titleTerms.length + bodyTerms.length;
 			for (let i = 0; i < nonNegatedTextTerms; i++) {

ReactNativeClient/lib/services/searchengine/filterParser.ts

@@ -4,6 +4,7 @@ interface Term {
 	value: string
 	negated: boolean
 	quoted?: boolean
+	wildcard?: boolean
 }
 
 const makeTerm = (name: string, value: string): Term => {
@@ -82,13 +83,13 @@ const parseQuery = (query: string): Term[] => {
 			}
 
 			if (name === 'tag' || name === 'notebook' || name === 'resource' || name === 'sourceurl') {
-				result.push({ name, value: value.replace(/[*]/g, '%'), negated }); // for wildcard search
+				result.push({ name, value: trimQuotes(value.replace(/[*]/g, '%')), negated }); // for wildcard search
 			} else if (name === 'title' || name === 'body') {
 				// Trim quotes since we don't support phrase query here
 				// eg. Split title:"hello world" to title:hello title:world
 				const values = trimQuotes(value).split(/[\s-_]+/);
 				values.forEach(value => {
-					result.push({ name, value, negated });
+					result.push({ name, value, negated, wildcard: value.indexOf('*') >= 0 });
 				});
 			} else {
 				result.push({ name, value, negated });
@@ -97,9 +98,21 @@ const parseQuery = (query: string): Term[] => {
 			// Every word is quoted if not already.
 			// By quoting the word, FTS match query will take care of removing dashes and other word seperators.
 			if (value.startsWith('-')) {
-				result.push({ name: 'text', value: quote(value.slice(1)) , negated: true, quoted: quoted(value) });
+				result.push({
+					name: 'text',
+					value: quote(value.slice(1)),
+					negated: true,
+					quoted: quoted(value),
+					wildcard: value.indexOf('*') >= 0,
+				});
 			} else {
-				result.push({ name: 'text', value: quote(value), negated: false, quoted: quoted(value) });
+				result.push({
+					name: 'text',
+					value: quote(value),
+					negated: false,
+					quoted: quoted(value),
+					wildcard: value.indexOf('*') >= 0,
+				});
 			}
 		}
 	}

ReactNativeClient/lib/services/synchronizer/LockHandler.ts

@@ -1,4 +1,5 @@
 import { Dirnames } from './utils/types';
+
 const JoplinError = require('lib/JoplinError');
 const { time } = require('lib/time-utils');
 const { fileExtension, filename } = require('lib/path-utils.js');
@@ -98,8 +99,8 @@ export default class LockHandler {
 		return output;
 	}
 
-	private lockIsActive(lock:Lock):boolean {
-		return Date.now() - lock.updatedTime < this.lockTtl;
+	private lockIsActive(lock:Lock, currentDate:Date):boolean {
+		return currentDate.getTime() - lock.updatedTime < this.lockTtl;
 	}
 
 	async hasActiveLock(lockType:LockType, clientType:string = null, clientId:string = null) {
@@ -112,11 +113,12 @@ export default class LockHandler {
 	// of that type instead.
 	async activeLock(lockType:LockType, clientType:string = null, clientId:string = null) {
 		const locks = await this.locks(lockType);
+		const currentDate = await this.api_.remoteDate();
 
 		if (lockType === LockType.Exclusive) {
 			const activeLocks = locks
 				.slice()
-				.filter((lock:Lock) => this.lockIsActive(lock))
+				.filter((lock:Lock) => this.lockIsActive(lock, currentDate))
 				.sort((a:Lock, b:Lock) => {
 					if (a.updatedTime === b.updatedTime) {
 						return a.clientId < b.clientId ? -1 : +1;
@@ -134,7 +136,7 @@ export default class LockHandler {
 			for (const lock of locks) {
 				if (clientType && lock.clientType !== clientType) continue;
 				if (clientId && lock.clientId !== clientId) continue;
-				if (this.lockIsActive(lock)) return lock;
+				if (this.lockIsActive(lock, currentDate)) return lock;
 			}
 			return null;
 		}
@@ -290,6 +292,12 @@ export default class LockHandler {
 			if (!this.refreshTimers_[handle]) return defer(); // Timeout has been cleared
 
 			if (!hasActiveLock) {
+				// If the previous lock has expired, we shouldn't try to acquire a new one. This is because other clients might have performed
+				// in the meantime operations that invalidates the current operation. For example, another client might have upgraded the
+				// sync target in the meantime, so any active operation should be cancelled here. Or if the current client was upgraded
+				// the sync target, another client might have synced since then, making any cached data invalid.
+				// In some cases it should be safe to re-acquire a lock but adding support for this would make the algorithm more complex
+				// without much benefits.
 				error = new JoplinError('Lock has expired', 'lockExpired');
 			} else {
 				try {

ReactNativeClient/lib/services/synchronizer/MigrationHandler.ts

@@ -90,9 +90,11 @@ export default class MigrationHandler extends BaseService {
 		// it the lock handler will break. So we create the directory now.
 		// Also if the sync target version is 0, it means it's a new one so we need the
 		// lock folder first before doing anything else.
+		// Temp folder is needed too to get remoteDate() call to work.
 		if (syncTargetInfo.version === 0 || syncTargetInfo.version === 1) {
-			this.logger().info('MigrationHandler: Sync target version is 0 or 1 - creating "locks" directory:', syncTargetInfo);
+			this.logger().info('MigrationHandler: Sync target version is 0 or 1 - creating "locks" and "temp" directory:', syncTargetInfo);
 			await this.api_.mkdir(Dirnames.Locks);
+			await this.api_.mkdir(Dirnames.Temp);
 		}
 
 		this.logger().info('MigrationHandler: Acquiring exclusive lock');

ReactNativeClient/lib/synchronizer.js

@@ -305,6 +305,8 @@ class Synchronizer {
 		let syncLock = null;
 
 		try {
+			this.api().setTempDirName(Dirnames.Temp);
+
 			try {
 				const syncTargetInfo = await this.migrationHandler().checkCanSync();
 
@@ -321,8 +323,6 @@ class Synchronizer {
 				throw error;
 			}
 
-			this.api().setTempDirName(Dirnames.Temp);
-
 			syncLock = await this.lockHandler().acquireLock('sync', this.appType_, this.clientId_);
 
 			this.lockHandler().startAutoLockRefresh(syncLock, (error) => {

ReactNativeClient/lib/vendor/ntp-client.js

@@ -0,0 +1,155 @@
+/*
+ * ntp-client
+ * https://github.com/moonpyk/node-ntp-client
+ *
+ * Copyright (c) 2013 Clément Bourgeois
+ * Licensed under the MIT license.
+ */
+
+// ----------------------------------------------------------------------------------------
+// 2020-08-09: We vendor the package because although it works
+// it has several bugs and is currently unmaintained
+// ----------------------------------------------------------------------------------------
+
+const Buffer = require('buffer').Buffer;
+
+(function (exports) {
+    "use strict";
+
+    exports.defaultNtpPort = 123;
+    exports.defaultNtpServer = "pool.ntp.org";
+
+    exports.dgram = null;
+
+    /**
+     * Amount of acceptable time to await for a response from the remote server.
+     * Configured default to 10 seconds.
+     */
+    exports.ntpReplyTimeout = 10000;
+
+    /**
+     * Fetches the current NTP Time from the given server and port.
+     * @param {string} server IP/Hostname of the remote NTP Server
+     * @param {number} port Remote NTP Server port number
+     * @param {function(Object, Date)} callback(err, date) Async callback for
+     * the result date or eventually error.
+     */
+    exports.getNetworkTime = function (server, port, callback) {
+        if (callback === null || typeof callback !== "function") {
+            return;
+        }
+
+        server = server || exports.defaultNtpServer;
+        port = port || exports.defaultNtpPort;
+
+        if (!exports.dgram) throw new Error('dgram package has not been set!!');
+
+        var client = exports.dgram.createSocket("udp4"),
+            ntpData = new Buffer(48);
+
+        // RFC 2030 -> LI = 0 (no warning, 2 bits), VN = 3 (IPv4 only, 3 bits), Mode = 3 (Client Mode, 3 bits) -> 1 byte
+        // -> rtol(LI, 6) ^ rotl(VN, 3) ^ rotl(Mode, 0)
+        // -> = 0x00 ^ 0x18 ^ 0x03
+        ntpData[0] = 0x1B;
+
+        for (var i = 1; i < 48; i++) {
+            ntpData[i] = 0;
+        }
+
+        // Some errors can happen before/after send() or cause send() to was impossible.
+        // Some errors will also be given to the send() callback.
+        // We keep a flag, therefore, to prevent multiple callbacks.
+        // NOTE : the error callback is not generalised, as the client has to lose the connection also, apparently.
+        var errorFired = false;
+
+        function closeClient(client) {
+            try {
+                client.close();
+            } catch (error) {
+                // Doesn't mater if it could not be closed
+            }
+        }
+
+        var timeout = setTimeout(function () {
+            closeClient(client);
+
+            if (errorFired) {
+                return;
+            }
+            callback(new Error("Timeout waiting for NTP response."), null);
+            errorFired = true;
+        }, exports.ntpReplyTimeout);
+
+        client.on('error', function (err) {
+            clearTimeout(timeout);
+
+            if (errorFired) {
+                return;
+            }
+
+            callback(err, null);
+            errorFired = true;
+        });
+
+        // NOTE: To make it work in React Native (Android), a port need to be bound
+        // before calling client.send()
+
+        // client.bind(5555, '0.0.0.0', function() {
+            client.send(ntpData, 0, ntpData.length, port, server, function (err) {
+                if (err) {
+                    clearTimeout(timeout);
+                    if (errorFired) {
+                        return;
+                    }
+                    callback(err, null);
+                    errorFired = true;
+                    closeClient(client);
+                    return;
+                }
+
+                client.once('message', function (msg) {
+                    clearTimeout(timeout);
+                    closeClient(client);
+
+                    // Offset to get to the "Transmit Timestamp" field (time at which the reply
+                    // departed the server for the client, in 64-bit timestamp format."
+                    var offsetTransmitTime = 40,
+                        intpart = 0,
+                        fractpart = 0;
+
+                    // Get the seconds part
+                    for (var i = 0; i <= 3; i++) {
+                        intpart = 256 * intpart + msg[offsetTransmitTime + i];
+                    }
+
+                    // Get the seconds fraction
+                    for (i = 4; i <= 7; i++) {
+                        fractpart = 256 * fractpart + msg[offsetTransmitTime + i];
+                    }
+
+                    var milliseconds = (intpart * 1000 + (fractpart * 1000) / 0x100000000);
+
+                    // **UTC** time
+                    var date = new Date("Jan 01 1900 GMT");
+                    date.setUTCMilliseconds(date.getUTCMilliseconds() + milliseconds);
+
+                    callback(null, date);
+                });
+            });
+        // });
+    };
+
+    exports.demo = function (argv) {
+        exports.getNetworkTime(
+            exports.defaultNtpServer,
+            exports.defaultNtpPort,
+            function (err, date) {
+                if (err) {
+                    console.error(err);
+                    return;
+                }
+
+                console.log(date);
+            });
+    };
+}(exports));

ReactNativeClient/root.js

@@ -401,6 +401,7 @@ async function initialize(dispatch, messageHandler) {
 	reg.setShowErrorMessageBoxHandler((message) => { alert(message); });
 
 	BaseService.logger_ = mainLogger;
+	// require('lib/ntpDate').setLogger(reg.logger());
 
 	reg.logger().info('====================================');
 	reg.logger().info(`Starting application ${Setting.value('appId')} (${Setting.value('env')})`);