diff --git a/CliClient/tests/ArrayUtils.js b/CliClient/tests/ArrayUtils.js index c894b3d94..f5c860f7a 100644 --- a/CliClient/tests/ArrayUtils.js +++ b/CliClient/tests/ArrayUtils.js @@ -29,4 +29,19 @@ describe('Encryption', function() { done(); }); + it('should find items using binary search', async (done) => { + let items = ['aaa', 'ccc', 'bbb']; + expect(ArrayUtils.binarySearch(items, 'bbb')).toBe(-1); // Array not sorted! + items.sort(); + expect(ArrayUtils.binarySearch(items, 'bbb')).toBe(1); + expect(ArrayUtils.binarySearch(items, 'ccc')).toBe(2); + expect(ArrayUtils.binarySearch(items, 'oops')).toBe(-1); + expect(ArrayUtils.binarySearch(items, 'aaa')).toBe(0); + + items = []; + expect(ArrayUtils.binarySearch(items, 'aaa')).toBe(-1); + + done(); + }); + }); \ No newline at end of file diff --git a/CliClient/tests/synchronizer.js b/CliClient/tests/synchronizer.js index 1ffba3e74..fbf1ea1f5 100644 --- a/CliClient/tests/synchronizer.js +++ b/CliClient/tests/synchronizer.js @@ -19,7 +19,7 @@ process.on('unhandledRejection', (reason, p) => { console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); }); -jasmine.DEFAULT_TIMEOUT_INTERVAL = 15000; // The first test is slow because the database needs to be built +jasmine.DEFAULT_TIMEOUT_INTERVAL = 35000; // The first test is slow because the database needs to be built async function allItems() { let folders = await Folder.all(); @@ -459,7 +459,7 @@ describe('Synchronizer', function() { let unconflictedNotes = await Note.unconflictedNotes(); expect(unconflictedNotes.length).toBe(0); - })); + })); it('should handle conflict when remote folder is deleted then local folder is renamed', asyncTest(async () => { let folder1 = await Folder.save({ title: "folder1" }); @@ -489,7 +489,7 @@ describe('Synchronizer', function() { let items = await allItems(); expect(items.length).toBe(1); - })); + })); it('should allow duplicate folder titles', asyncTest(async () => { let localF1 = await Folder.save({ title: "folder" }); @@ -575,10 +575,12 @@ describe('Synchronizer', function() { } it('should sync tags', asyncTest(async () => { - await shoudSyncTagTest(false); })); + await shoudSyncTagTest(false); + })); it('should sync encrypted tags', asyncTest(async () => { - await shoudSyncTagTest(true); })); + await shoudSyncTagTest(true); + })); it('should not sync notes with conflicts', asyncTest(async () => { let f1 = await Folder.save({ title: "folder" }); diff --git a/CliClient/tests/test-utils.js b/CliClient/tests/test-utils.js index e269bd3df..b390c34f7 100644 --- a/CliClient/tests/test-utils.js +++ b/CliClient/tests/test-utils.js @@ -51,12 +51,12 @@ SyncTargetRegistry.addClass(SyncTargetFilesystem); SyncTargetRegistry.addClass(SyncTargetOneDrive); SyncTargetRegistry.addClass(SyncTargetNextcloud); -//const syncTargetId_ = SyncTargetRegistry.nameToId('nextcloud'); -const syncTargetId_ = SyncTargetRegistry.nameToId('memory'); +const syncTargetId_ = SyncTargetRegistry.nameToId('nextcloud'); +//const syncTargetId_ = SyncTargetRegistry.nameToId('memory'); //const syncTargetId_ = SyncTargetRegistry.nameToId('filesystem'); const syncDir = __dirname + '/../tests/sync'; -const sleepTime = syncTargetId_ == SyncTargetRegistry.nameToId('filesystem') ? 1001 : 400; +const sleepTime = syncTargetId_ == SyncTargetRegistry.nameToId('filesystem') ? 1001 : 10;//400; console.info('Testing with sync target: ' + SyncTargetRegistry.idToName(syncTargetId_)); diff --git a/ReactNativeClient/lib/ArrayUtils.js b/ReactNativeClient/lib/ArrayUtils.js index 06f5bd090..48276ec29 100644 --- a/ReactNativeClient/lib/ArrayUtils.js +++ b/ReactNativeClient/lib/ArrayUtils.js @@ -13,4 +13,28 @@ ArrayUtils.removeElement = function(array, element) { return array; } +// https://stackoverflow.com/a/10264318/561309 +ArrayUtils.binarySearch = function(items, value) { + + var startIndex = 0, + stopIndex = items.length - 1, + middle = Math.floor((stopIndex + startIndex)/2); + + while(items[middle] != value && startIndex < stopIndex){ + + //adjust search area + if (value < items[middle]){ + stopIndex = middle - 1; + } else if (value > items[middle]){ + startIndex = middle + 1; + } + + //recalculate middle + middle = Math.floor((stopIndex + startIndex)/2); + } + + //make sure it's the right value + return (items[middle] != value) ? -1 : middle; +} + module.exports = ArrayUtils; \ No newline at end of file diff --git a/ReactNativeClient/lib/WebDavApi.js b/ReactNativeClient/lib/WebDavApi.js index d007141b1..84f13c180 100644 --- a/ReactNativeClient/lib/WebDavApi.js +++ b/ReactNativeClient/lib/WebDavApi.js @@ -109,7 +109,7 @@ class WebDavApi { return this.valueFromJson(json, keys, 'array'); } - async execPropFind(path, depth, fields = null) { + async execPropFind(path, depth, fields = null, options = null) { if (fields === null) fields = ['d:getlastmodified']; let fieldsXml = ''; @@ -131,7 +131,7 @@ class WebDavApi { `; - return this.exec('PROPFIND', path, body, { 'Depth': depth }); + return this.exec('PROPFIND', path, body, { 'Depth': depth }, options); } // curl -u admin:123456 'http://nextcloud.local/remote.php/dav/files/admin/' -X PROPFIND --data ' diff --git a/ReactNativeClient/lib/file-api-driver-webdav.js b/ReactNativeClient/lib/file-api-driver-webdav.js index 63f607d53..460d70ae1 100644 --- a/ReactNativeClient/lib/file-api-driver-webdav.js +++ b/ReactNativeClient/lib/file-api-driver-webdav.js @@ -2,6 +2,8 @@ const BaseItem = require('lib/models/BaseItem.js'); const { time } = require('lib/time-utils.js'); const { basicDelta } = require('lib/file-api'); const { rtrimSlashes, ltrimSlashes } = require('lib/path-utils.js'); +const Entities = require('html-entities').AllHtmlEntities; +const html_entity_decode = (new Entities()).decode; class FileApiDriverWebDav { @@ -45,21 +47,6 @@ class FileApiDriverWebDav { }; } - statsFromResources_(resources) { - const relativeBaseUrl = this.api().relativeBaseUrl(); - let output = []; - for (let i = 0; i < resources.length; i++) { - const resource = resources[i]; - const href = this.api().stringFromJson(resource, ['d:href', 0]); - if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible - const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length))); - if (path === '') continue; // The list of resources includes the root dir too, which we don't want - const stat = this.statFromResource_(resources[i], path); - output.push(stat); - } - return output; - } - async setTimestamp(path, timestampMs) { throw new Error('Not implemented'); // Not needed anymore } @@ -74,15 +61,150 @@ class FileApiDriverWebDav { } async list(path, options) { - const result = await this.api().execPropFind(path, 1, [ - 'd:getlastmodified', - 'd:resourcetype', - ]); + const relativeBaseUrl = this.api().relativeBaseUrl(); - const resources = this.api().arrayFromJson(result, ['d:multistatus', 'd:response']); + // function parsePropFindXml(xmlString) { + // return new Promise(async (resolve, reject) => { + // const saxOptions = {}; + // const saxParser = require('sax').parser(false, { position: false }); + + // let stats = []; + // let currentStat = null; + // let currentText = ''; + + // // When this is on, the tags from the bloated XML string are replaced by shorter ones, + // // which makes parsing about 25% faster. However it's a bit of a hack so keep it as + // // an option so that it can be disabled if it causes problems. + // const optimizeXml = true; + + // const tagResponse = optimizeXml ? 'd:r' : 'd:response'; + // const tagGetLastModified = optimizeXml ? 'd:glm' : 'd:getlastmodified'; + // const tagPropStat = optimizeXml ? 'd:ps' : 'd:propstat'; + // const replaceUrls = optimizeXml; + + // saxParser.onerror = function (error) { + // reject(new Error(e.toString())); + // }; + + // saxParser.ontext = function (t) { + // currentText += t; + // }; + + // saxParser.onopentag = function (node) { + // const tagName = node.name.toLowerCase(); + + // currentText = ''; + + // if (tagName === tagResponse) { + // currentStat = { isDir: false }; + // } + // }; + + // saxParser.onclosetag = function(tagName) { + // tagName = tagName.toLowerCase(); + + // if (tagName === tagResponse) { + // if (currentStat.path) { // The list of resources includes the root dir too, which we don't want + // if (!currentStat.updated_time) throw new Error('Resource does not have a getlastmodified prop'); + // stats.push(currentStat); + // } + // currentStat = null; + // } + + // if (tagName === 'd:href') { + // const href = currentText; + + // if (replaceUrls) { + // currentStat.path = rtrimSlashes(ltrimSlashes(href)); + // } else { + // if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible + // currentStat.path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length))); + // } + // } + + // if (tagName === tagGetLastModified) { + // const lastModifiedDate = new Date(currentText); + // if (isNaN(lastModifiedDate.getTime())) throw new Error('Invalid date: ' + currentText); + // currentStat.updated_time = lastModifiedDate.getTime(); + // currentStat.created_time = currentStat.updated_time; + // } + + // if (tagName === 'd:collection') { + // currentStat.isDir = true; + // } + + // currentText = ''; + // } + + // saxParser.onend = function () { + // resolve(stats); + // }; + + // if (optimizeXml) { + // xmlString = xmlString.replace(/HTTP\/1\.1 200 OK<\/d:status>/ig, ''); + // xmlString = xmlString.replace(//ig, ''); + // xmlString = xmlString.replace(/d:getlastmodified/ig, tagGetLastModified); + // xmlString = xmlString.replace(/d:response/ig, tagResponse); + // xmlString = xmlString.replace(/d:propstat/ig, tagPropStat); + // if (replaceUrls) xmlString = xmlString.replace(new RegExp(relativeBaseUrl, 'gi'), ''); + // } + + // let idx = 0; + // let size = 1024 * 100; + // while (true) { + // sub = xmlString.substr(idx, size); + // if (!sub.length) break; + // saxParser.write(sub); + // idx += size; + // //await time.msleep(500); + // } + + // saxParser.close(); + + // //saxParser.write(xmlString).close(); + // }); + // } + + // For performance reasons, the response of the PROPFIND call is manually parsed with a regex below + // instead of being processed by xml2json like the other WebDAV responses. This is over 2 times faster + // and it means the mobile app does not freeze during sync. + + async function parsePropFindXml2(xmlString) { + const regex = /[\S\s]*?([\S\s]*?)<\/d:href>[\S\s]*?(.*?)<\/d:getlastmodified>/g; + + let output = []; + let match = null; + + while (match = regex.exec(xmlString)) { + const href = html_entity_decode(match[1]); + if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible + const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length))); + + if (!path) continue; // The list of resources includes the root dir too, which we don't want + + const lastModifiedDate = new Date(match[2]); + if (isNaN(lastModifiedDate.getTime())) throw new Error('Invalid date: ' + match[2]); + + output.push({ + path: path, + updated_time: lastModifiedDate.getTime(), + created_time: lastModifiedDate.getTime(), + isDir: !BaseItem.isSystemPath(path), + }); + } + + return output; + } + + const resultXml = await this.api().execPropFind(path, 1, [ + 'd:getlastmodified', + //'d:resourcetype', // Include this to use parsePropFindXml() + ], { responseFormat: 'text' }); + + const stats = await parsePropFindXml2(resultXml); return { - items: this.statsFromResources_(resources), + items: stats, hasMore: false, context: null, }; diff --git a/ReactNativeClient/lib/file-api.js b/ReactNativeClient/lib/file-api.js index e4e349980..5ec29fe7b 100644 --- a/ReactNativeClient/lib/file-api.js +++ b/ReactNativeClient/lib/file-api.js @@ -3,6 +3,7 @@ const { Logger } = require('lib/logger.js'); const { shim } = require('lib/shim'); const BaseItem = require('lib/models/BaseItem.js'); const JoplinError = require('lib/JoplinError'); +const ArrayUtils = require('lib/ArrayUtils'); class FileApi { @@ -167,6 +168,8 @@ async function basicDelta(path, getDirStatFn, options) { newContext.statsCache.sort(function(a, b) { return a.updated_time - b.updated_time; }); + newContext.statIdsCache = newContext.statsCache.map((item) => BaseItem.pathToId(item.path)); + newContext.statIdsCache.sort(); // Items must be sorted to use binary search below } let output = []; @@ -210,16 +213,8 @@ async function basicDelta(path, getDirStatFn, options) { if (output.length + deletedItems.length >= outputLimit) break; const itemId = itemIds[i]; - let found = false; - for (let j = 0; j < newContext.statsCache.length; j++) { - const item = newContext.statsCache[j]; - if (BaseItem.pathToId(item.path) == itemId) { - found = true; - break; - } - } - if (!found) { + if (ArrayUtils.binarySearch(newContext.statIdsCache, itemId) < 0) { deletedItems.push({ path: BaseItem.systemPath(itemId), isDeleted: true, diff --git a/ReactNativeClient/package-lock.json b/ReactNativeClient/package-lock.json index 76efb49c8..39c4671e3 100644 --- a/ReactNativeClient/package-lock.json +++ b/ReactNativeClient/package-lock.json @@ -5390,9 +5390,9 @@ } }, "sax": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.1.6.tgz", - "integrity": "sha1-XWFr6KXmB9VOEUr65Vt+ry/MMkA=" + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "semver": { "version": "5.4.1", @@ -6255,7 +6255,7 @@ "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", "requires": { - "sax": "1.1.6", + "sax": "1.2.4", "xmlbuilder": "9.0.4" }, "dependencies": { @@ -6287,6 +6287,13 @@ "integrity": "sha1-0lciS+g5PqrL+DfvIn/Y7CWzaIg=", "requires": { "sax": "1.1.6" + }, + "dependencies": { + "sax": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.1.6.tgz", + "integrity": "sha1-XWFr6KXmB9VOEUr65Vt+ry/MMkA=" + } } }, "xmldom": {