All: Optimised Nextcloud sync delta functionality

CliClient/tests/ArrayUtils.js

@@ -29,4 +29,19 @@ describe('Encryption', function() {
 		done();
 	});
 
+	it('should find items using binary search', async (done) => {
+		let items = ['aaa', 'ccc', 'bbb'];
+		expect(ArrayUtils.binarySearch(items, 'bbb')).toBe(-1); // Array not sorted!
+		items.sort();
+		expect(ArrayUtils.binarySearch(items, 'bbb')).toBe(1);
+		expect(ArrayUtils.binarySearch(items, 'ccc')).toBe(2);
+		expect(ArrayUtils.binarySearch(items, 'oops')).toBe(-1);
+		expect(ArrayUtils.binarySearch(items, 'aaa')).toBe(0);
+
+		items = [];
+		expect(ArrayUtils.binarySearch(items, 'aaa')).toBe(-1);
+
+		done();
+	});
+
 });

CliClient/tests/synchronizer.js

@@ -19,7 +19,7 @@ process.on('unhandledRejection', (reason, p) => {
 	console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
 });
 
-jasmine.DEFAULT_TIMEOUT_INTERVAL = 15000; // The first test is slow because the database needs to be built
+jasmine.DEFAULT_TIMEOUT_INTERVAL = 35000; // The first test is slow because the database needs to be built
 
 async function allItems() {
 	let folders = await Folder.all();
@@ -459,7 +459,7 @@ describe('Synchronizer', function() {
 		let unconflictedNotes = await Note.unconflictedNotes();
 
 		expect(unconflictedNotes.length).toBe(0);
-			}));
+	}));
 
 	it('should handle conflict when remote folder is deleted then local folder is renamed', asyncTest(async () => {
 		let folder1 = await Folder.save({ title: "folder1" });
@@ -489,7 +489,7 @@ describe('Synchronizer', function() {
 		let items = await allItems();
 
 		expect(items.length).toBe(1);
-			}));
+	}));
 
 	it('should allow duplicate folder titles', asyncTest(async () => {
 		let localF1 = await Folder.save({ title: "folder" });
@@ -575,10 +575,12 @@ describe('Synchronizer', function() {
 	}
 
 	it('should sync tags', asyncTest(async () => {
-		await shoudSyncTagTest(false);	}));
+		await shoudSyncTagTest(false);
+	}));
 
 	it('should sync encrypted tags', asyncTest(async () => {
-		await shoudSyncTagTest(true);	}));
+		await shoudSyncTagTest(true);
+	}));
 
 	it('should not sync notes with conflicts', asyncTest(async () => {
 		let f1 = await Folder.save({ title: "folder" });

CliClient/tests/test-utils.js

@@ -51,12 +51,12 @@ SyncTargetRegistry.addClass(SyncTargetFilesystem);
 SyncTargetRegistry.addClass(SyncTargetOneDrive);
 SyncTargetRegistry.addClass(SyncTargetNextcloud);
 
-//const syncTargetId_ = SyncTargetRegistry.nameToId('nextcloud');
+const syncTargetId_ = SyncTargetRegistry.nameToId('nextcloud');
-const syncTargetId_ = SyncTargetRegistry.nameToId('memory');
+//const syncTargetId_ = SyncTargetRegistry.nameToId('memory');
 //const syncTargetId_ = SyncTargetRegistry.nameToId('filesystem');
 const syncDir = __dirname + '/../tests/sync';
 
-const sleepTime = syncTargetId_ == SyncTargetRegistry.nameToId('filesystem') ? 1001 : 400;
+const sleepTime = syncTargetId_ == SyncTargetRegistry.nameToId('filesystem') ? 1001 : 10;//400;
 
 console.info('Testing with sync target: ' + SyncTargetRegistry.idToName(syncTargetId_));
 

ReactNativeClient/lib/ArrayUtils.js

@@ -13,4 +13,28 @@ ArrayUtils.removeElement = function(array, element) {
 	return array;
 }
 
+// https://stackoverflow.com/a/10264318/561309
+ArrayUtils.binarySearch = function(items, value) {
+
+	var startIndex  = 0,
+		stopIndex   = items.length - 1,
+		middle      = Math.floor((stopIndex + startIndex)/2);
+
+	while(items[middle] != value && startIndex < stopIndex){
+
+		//adjust search area
+		if (value < items[middle]){
+			stopIndex = middle - 1;
+		} else if (value > items[middle]){
+			startIndex = middle + 1;
+		}
+
+		//recalculate middle
+		middle = Math.floor((stopIndex + startIndex)/2);
+	}
+
+	//make sure it's the right value
+	return (items[middle] != value) ? -1 : middle;
+}
+
 module.exports = ArrayUtils;

ReactNativeClient/lib/WebDavApi.js

@@ -109,7 +109,7 @@ class WebDavApi {
 		return this.valueFromJson(json, keys, 'array');
 	}
 
-	async execPropFind(path, depth, fields = null) {
+	async execPropFind(path, depth, fields = null, options = null) {
 		if (fields === null) fields = ['d:getlastmodified'];
 
 		let fieldsXml = '';
@@ -131,7 +131,7 @@ class WebDavApi {
 				</d:prop>
 			</d:propfind>`;
 
-		return this.exec('PROPFIND', path, body, { 'Depth': depth });
+		return this.exec('PROPFIND', path, body, { 'Depth': depth }, options);
 	}
 
 	// curl -u admin:123456 'http://nextcloud.local/remote.php/dav/files/admin/' -X PROPFIND --data '<?xml version="1.0" encoding="UTF-8"?>

ReactNativeClient/lib/file-api-driver-webdav.js

@@ -2,6 +2,8 @@ const BaseItem = require('lib/models/BaseItem.js');
 const { time } = require('lib/time-utils.js');
 const { basicDelta } = require('lib/file-api');
 const { rtrimSlashes, ltrimSlashes } = require('lib/path-utils.js');
+const Entities = require('html-entities').AllHtmlEntities;
+const html_entity_decode = (new Entities()).decode;
 
 class FileApiDriverWebDav { 
 
@@ -45,21 +47,6 @@ class FileApiDriverWebDav {
 		};
 	}
 
-	statsFromResources_(resources) {
-		const relativeBaseUrl = this.api().relativeBaseUrl();
-		let output = [];
-		for (let i = 0; i < resources.length; i++) {
-			const resource = resources[i];
-			const href = this.api().stringFromJson(resource, ['d:href', 0]);
-			if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible
-			const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length)));
-			if (path === '') continue; // The list of resources includes the root dir too, which we don't want
-			const stat = this.statFromResource_(resources[i], path);
-			output.push(stat);
-		}
-		return output;
-	}
-
 	async setTimestamp(path, timestampMs) {
 		throw new Error('Not implemented'); // Not needed anymore
 	}
@@ -74,15 +61,150 @@ class FileApiDriverWebDav {
 	}
 
 	async list(path, options) {
-		const result = await this.api().execPropFind(path, 1, [
+		const relativeBaseUrl = this.api().relativeBaseUrl();
-			'd:getlastmodified',
-			'd:resourcetype',
-		]);
 
-		const resources = this.api().arrayFromJson(result, ['d:multistatus', 'd:response']);
+		// function parsePropFindXml(xmlString) {
+		// 	return new Promise(async (resolve, reject) => {
+		// 		const saxOptions = {};
+		// 		const saxParser = require('sax').parser(false, { position: false });
+
+		// 		let stats = [];
+		// 		let currentStat = null;
+		// 		let currentText = '';
+
+		// 		// When this is on, the tags from the bloated XML string are replaced by shorter ones, 
+		// 		// which makes parsing about 25% faster. However it's a bit of a hack so keep it as
+		// 		// an option so that it can be disabled if it causes problems.
+		// 		const optimizeXml = true;
+
+		// 		const tagResponse = optimizeXml ? 'd:r' : 'd:response';
+		// 		const tagGetLastModified = optimizeXml ? 'd:glm' : 'd:getlastmodified';
+		// 		const tagPropStat = optimizeXml ? 'd:ps' : 'd:propstat';
+		// 		const replaceUrls = optimizeXml;
+
+		// 		saxParser.onerror = function (error) {
+		// 			reject(new Error(e.toString()));
+		// 		};
+
+		// 		saxParser.ontext = function (t) {
+		// 			currentText += t;
+		// 		};
+
+		// 		saxParser.onopentag = function (node) {
+		// 			const tagName = node.name.toLowerCase();
+
+		// 			currentText = '';
+
+		// 			if (tagName === tagResponse) {
+		// 				currentStat = { isDir: false };
+		// 			}
+		// 		};
+
+		// 		saxParser.onclosetag = function(tagName) {
+		// 			tagName = tagName.toLowerCase();
+					
+		// 			if (tagName === tagResponse) {
+		// 				if (currentStat.path) { // The list of resources includes the root dir too, which we don't want
+		// 					if (!currentStat.updated_time) throw new Error('Resource does not have a getlastmodified prop');
+		// 					stats.push(currentStat);
+		// 				}
+		// 				currentStat = null;
+		// 			}
+
+		// 			if (tagName === 'd:href') {
+		// 				const href = currentText;
+
+		// 				if (replaceUrls) {
+		// 					currentStat.path = rtrimSlashes(ltrimSlashes(href));
+		// 				} else {
+		// 					if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible
+		// 					currentStat.path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length)));
+		// 				}
+		// 			}
+
+		// 			if (tagName === tagGetLastModified) {
+		// 				const lastModifiedDate = new Date(currentText);
+		// 				if (isNaN(lastModifiedDate.getTime())) throw new Error('Invalid date: ' + currentText);
+		// 				currentStat.updated_time = lastModifiedDate.getTime();
+		// 				currentStat.created_time = currentStat.updated_time;
+		// 			}
+
+		// 			if (tagName === 'd:collection') {
+		// 				currentStat.isDir = true;
+		// 			}
+
+		// 			currentText = '';
+		// 		}
+
+		// 		saxParser.onend = function () {
+		// 			resolve(stats);
+		// 		};
+
+		// 		if (optimizeXml) {
+		// 			xmlString = xmlString.replace(/<d:status>HTTP\/1\.1 200 OK<\/d:status>/ig, ''); 
+		// 			xmlString = xmlString.replace(/<d:resourcetype\/>/ig, ''); 
+		// 			xmlString = xmlString.replace(/d:getlastmodified/ig, tagGetLastModified); 
+		// 			xmlString = xmlString.replace(/d:response/ig, tagResponse); 
+		// 			xmlString = xmlString.replace(/d:propstat/ig, tagPropStat); 
+		// 			if (replaceUrls) xmlString = xmlString.replace(new RegExp(relativeBaseUrl, 'gi'), ''); 
+		// 		}
+
+		// 		let idx = 0;
+		// 		let size = 1024 * 100;
+		// 		while (true) {
+		// 			sub = xmlString.substr(idx, size);
+		// 			if (!sub.length) break;
+		// 			saxParser.write(sub);
+		// 			idx += size;
+		// 			//await time.msleep(500);
+		// 		}
+
+		// 		saxParser.close();
+
+		// 		//saxParser.write(xmlString).close();
+		// 	});
+		// }
+
+		// For performance reasons, the response of the PROPFIND call is manually parsed with a regex below
+		// instead of being processed by xml2json like the other WebDAV responses. This is over 2 times faster
+		// and it means the mobile app does not freeze during sync. 
+
+		async function parsePropFindXml2(xmlString) {
+			const regex = /<d:response>[\S\s]*?<d:href>([\S\s]*?)<\/d:href>[\S\s]*?<d:getlastmodified>(.*?)<\/d:getlastmodified>/g;
+
+			let output = [];
+			let match = null;
+
+			while (match = regex.exec(xmlString)) {
+				const href = html_entity_decode(match[1]);
+				if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible
+				const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length)));
+
+				if (!path) continue; // The list of resources includes the root dir too, which we don't want
+
+				const lastModifiedDate = new Date(match[2]);
+				if (isNaN(lastModifiedDate.getTime())) throw new Error('Invalid date: ' + match[2]);
+
+				output.push({
+					path: path,
+					updated_time: lastModifiedDate.getTime(),
+					created_time: lastModifiedDate.getTime(),
+					isDir: !BaseItem.isSystemPath(path),
+				});
+			}
+
+			return output;
+		}
+
+		const resultXml = await this.api().execPropFind(path, 1, [
+			'd:getlastmodified',
+			//'d:resourcetype', // Include this to use parsePropFindXml()
+		], { responseFormat: 'text' });
+
+		const stats = await parsePropFindXml2(resultXml);
 
 		return {
-			items: this.statsFromResources_(resources),
+			items: stats,
 			hasMore: false,
 			context: null,
 		};

ReactNativeClient/lib/file-api.js

@@ -3,6 +3,7 @@ const { Logger } = require('lib/logger.js');
 const { shim } = require('lib/shim');
 const BaseItem = require('lib/models/BaseItem.js');
 const JoplinError = require('lib/JoplinError');
+const ArrayUtils = require('lib/ArrayUtils');
 
 class FileApi {
 
@@ -167,6 +168,8 @@ async function basicDelta(path, getDirStatFn, options) {
 		newContext.statsCache.sort(function(a, b) {
 			return a.updated_time - b.updated_time;
 		});
+		newContext.statIdsCache = newContext.statsCache.map((item) => BaseItem.pathToId(item.path));
+		newContext.statIdsCache.sort(); // Items must be sorted to use binary search below
 	}
 
 	let output = [];
@@ -210,16 +213,8 @@ async function basicDelta(path, getDirStatFn, options) {
 		if (output.length + deletedItems.length >= outputLimit) break;
 
 		const itemId = itemIds[i];
-		let found = false;
-		for (let j = 0; j < newContext.statsCache.length; j++) {
-			const item = newContext.statsCache[j];
-			if (BaseItem.pathToId(item.path) == itemId) {
-				found = true;
-				break;
-			}
-		}
 
-		if (!found) {
+		if (ArrayUtils.binarySearch(newContext.statIdsCache, itemId) < 0) {
 			deletedItems.push({
 				path: BaseItem.systemPath(itemId),
 				isDeleted: true,

ReactNativeClient/package-lock.json

@@ -5390,9 +5390,9 @@
       }
     },
     "sax": {
-      "version": "1.1.6",
+      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/sax/-/sax-1.1.6.tgz",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
-      "integrity": "sha1-XWFr6KXmB9VOEUr65Vt+ry/MMkA="
+      "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
     },
     "semver": {
       "version": "5.4.1",
@@ -6255,7 +6255,7 @@
       "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
       "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
       "requires": {
-        "sax": "1.1.6",
+        "sax": "1.2.4",
         "xmlbuilder": "9.0.4"
       },
       "dependencies": {
@@ -6287,6 +6287,13 @@
       "integrity": "sha1-0lciS+g5PqrL+DfvIn/Y7CWzaIg=",
       "requires": {
         "sax": "1.1.6"
+      },
+      "dependencies": {
+        "sax": {
+          "version": "1.1.6",
+          "resolved": "https://registry.npmjs.org/sax/-/sax-1.1.6.tgz",
+          "integrity": "sha1-XWFr6KXmB9VOEUr65Vt+ry/MMkA="
+        }
       }
     },
     "xmldom": {