self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-24 10:27:10 +02:00
Code Issues Releases Activity

All: Optimised Nextcloud sync delta functionality

Browse Source
This commit is contained in:
Laurent Cozic 2018-01-29 20:51:14 +00:00
parent 0dba2821b6
commit 5cb5ccc781
8 changed files with 209 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
CliClient/tests/ArrayUtils.js
View File

@ -29,4 +29,19 @@ describe('Encryption', function() {
done();
});
it('should find items using binary search', async (done) => {
let items = ['aaa', 'ccc', 'bbb'];
expect(ArrayUtils.binarySearch(items, 'bbb')).toBe(-1); // Array not sorted!
items.sort();
expect(ArrayUtils.binarySearch(items, 'bbb')).toBe(1);
expect(ArrayUtils.binarySearch(items, 'ccc')).toBe(2);
expect(ArrayUtils.binarySearch(items, 'oops')).toBe(-1);
expect(ArrayUtils.binarySearch(items, 'aaa')).toBe(0);
items = [];
expect(ArrayUtils.binarySearch(items, 'aaa')).toBe(-1);
done();
});
});

8
CliClient/tests/synchronizer.js
View File

@ -19,7 +19,7 @@ process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
jasmine.DEFAULT_TIMEOUT_INTERVAL = 15000; // The first test is slow because the database needs to be built
jasmine.DEFAULT_TIMEOUT_INTERVAL = 35000; // The first test is slow because the database needs to be built
async function allItems() {
let folders = await Folder.all();
@ -575,10 +575,12 @@ describe('Synchronizer', function() {
}
it('should sync tags', asyncTest(async () => {
await shoudSyncTagTest(false); }));
await shoudSyncTagTest(false);
}));
it('should sync encrypted tags', asyncTest(async () => {
await shoudSyncTagTest(true); }));
await shoudSyncTagTest(true);
}));
it('should not sync notes with conflicts', asyncTest(async () => {
let f1 = await Folder.save({ title: "folder" });

6
CliClient/tests/test-utils.js
View File

@ -51,12 +51,12 @@ SyncTargetRegistry.addClass(SyncTargetFilesystem);
SyncTargetRegistry.addClass(SyncTargetOneDrive);
SyncTargetRegistry.addClass(SyncTargetNextcloud);
//const syncTargetId_ = SyncTargetRegistry.nameToId('nextcloud');
const syncTargetId_ = SyncTargetRegistry.nameToId('memory');
const syncTargetId_ = SyncTargetRegistry.nameToId('nextcloud');
//const syncTargetId_ = SyncTargetRegistry.nameToId('memory');
//const syncTargetId_ = SyncTargetRegistry.nameToId('filesystem');
const syncDir = __dirname + '/../tests/sync';
const sleepTime = syncTargetId_ == SyncTargetRegistry.nameToId('filesystem') ? 1001 : 400;
const sleepTime = syncTargetId_ == SyncTargetRegistry.nameToId('filesystem') ? 1001 : 10;//400;
console.info('Testing with sync target: ' + SyncTargetRegistry.idToName(syncTargetId_));

24
ReactNativeClient/lib/ArrayUtils.js
View File

@ -13,4 +13,28 @@ ArrayUtils.removeElement = function(array, element) {
return array;
}
// https://stackoverflow.com/a/10264318/561309
ArrayUtils.binarySearch = function(items, value) {
var startIndex = 0,
stopIndex = items.length - 1,
middle = Math.floor((stopIndex + startIndex)/2);
while(items[middle] != value && startIndex < stopIndex){
//adjust search area
if (value < items[middle]){
stopIndex = middle - 1;
} else if (value > items[middle]){
startIndex = middle + 1;
}
//recalculate middle
middle = Math.floor((stopIndex + startIndex)/2);
}
//make sure it's the right value
return (items[middle] != value) ? -1 : middle;
}
module.exports = ArrayUtils;

4
ReactNativeClient/lib/WebDavApi.js
View File

@ -109,7 +109,7 @@ class WebDavApi {
return this.valueFromJson(json, keys, 'array');
}
async execPropFind(path, depth, fields = null) {
async execPropFind(path, depth, fields = null, options = null) {
if (fields === null) fields = ['d:getlastmodified'];
let fieldsXml = '';
@ -131,7 +131,7 @@ class WebDavApi {
</d:prop>
</d:propfind>`;
return this.exec('PROPFIND', path, body, { 'Depth': depth });
return this.exec('PROPFIND', path, body, { 'Depth': depth }, options);
}
// curl -u admin:123456 'http://nextcloud.local/remote.php/dav/files/admin/' -X PROPFIND --data '<?xml version="1.0" encoding="UTF-8"?>

164
ReactNativeClient/lib/file-api-driver-webdav.js
View File

@ -2,6 +2,8 @@ const BaseItem = require('lib/models/BaseItem.js');
const { time } = require('lib/time-utils.js');
const { basicDelta } = require('lib/file-api');
const { rtrimSlashes, ltrimSlashes } = require('lib/path-utils.js');
const Entities = require('html-entities').AllHtmlEntities;
const html_entity_decode = (new Entities()).decode;
class FileApiDriverWebDav {
@ -45,21 +47,6 @@ class FileApiDriverWebDav {
};
}
statsFromResources_(resources) {
const relativeBaseUrl = this.api().relativeBaseUrl();
let output = [];
for (let i = 0; i < resources.length; i++) {
const resource = resources[i];
const href = this.api().stringFromJson(resource, ['d:href', 0]);
if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible
const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length)));
if (path === '') continue; // The list of resources includes the root dir too, which we don't want
const stat = this.statFromResource_(resources[i], path);
output.push(stat);
}
return output;
}
async setTimestamp(path, timestampMs) {
throw new Error('Not implemented'); // Not needed anymore
}
@ -74,15 +61,150 @@ class FileApiDriverWebDav {
}
async list(path, options) {
const result = await this.api().execPropFind(path, 1, [
'd:getlastmodified',
'd:resourcetype',
]);
const relativeBaseUrl = this.api().relativeBaseUrl();
const resources = this.api().arrayFromJson(result, ['d:multistatus', 'd:response']);
// function parsePropFindXml(xmlString) {
// return new Promise(async (resolve, reject) => {
// const saxOptions = {};
// const saxParser = require('sax').parser(false, { position: false });
// let stats = [];
// let currentStat = null;
// let currentText = '';
// // When this is on, the tags from the bloated XML string are replaced by shorter ones,
// // which makes parsing about 25% faster. However it's a bit of a hack so keep it as
// // an option so that it can be disabled if it causes problems.
// const optimizeXml = true;
// const tagResponse = optimizeXml ? 'd:r' : 'd:response';
// const tagGetLastModified = optimizeXml ? 'd:glm' : 'd:getlastmodified';
// const tagPropStat = optimizeXml ? 'd:ps' : 'd:propstat';
// const replaceUrls = optimizeXml;
// saxParser.onerror = function (error) {
// reject(new Error(e.toString()));
// };
// saxParser.ontext = function (t) {
// currentText += t;
// };
// saxParser.onopentag = function (node) {
// const tagName = node.name.toLowerCase();
// currentText = '';
// if (tagName === tagResponse) {
// currentStat = { isDir: false };
// }
// };
// saxParser.onclosetag = function(tagName) {
// tagName = tagName.toLowerCase();
// if (tagName === tagResponse) {
// if (currentStat.path) { // The list of resources includes the root dir too, which we don't want
// if (!currentStat.updated_time) throw new Error('Resource does not have a getlastmodified prop');
// stats.push(currentStat);
// }
// currentStat = null;
// }
// if (tagName === 'd:href') {
// const href = currentText;
// if (replaceUrls) {
// currentStat.path = rtrimSlashes(ltrimSlashes(href));
// } else {
// if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible
// currentStat.path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length)));
// }
// }
// if (tagName === tagGetLastModified) {
// const lastModifiedDate = new Date(currentText);
// if (isNaN(lastModifiedDate.getTime())) throw new Error('Invalid date: ' + currentText);
// currentStat.updated_time = lastModifiedDate.getTime();
// currentStat.created_time = currentStat.updated_time;
// }
// if (tagName === 'd:collection') {
// currentStat.isDir = true;
// }
// currentText = '';
// }
// saxParser.onend = function () {
// resolve(stats);
// };
// if (optimizeXml) {
// xmlString = xmlString.replace(/<d:status>HTTP\/1\.1 200 OK<\/d:status>/ig, '');
// xmlString = xmlString.replace(/<d:resourcetype\/>/ig, '');
// xmlString = xmlString.replace(/d:getlastmodified/ig, tagGetLastModified);
// xmlString = xmlString.replace(/d:response/ig, tagResponse);
// xmlString = xmlString.replace(/d:propstat/ig, tagPropStat);
// if (replaceUrls) xmlString = xmlString.replace(new RegExp(relativeBaseUrl, 'gi'), '');
// }
// let idx = 0;
// let size = 1024 * 100;
// while (true) {
// sub = xmlString.substr(idx, size);
// if (!sub.length) break;
// saxParser.write(sub);
// idx += size;
// //await time.msleep(500);
// }
// saxParser.close();
// //saxParser.write(xmlString).close();
// });
// }
// For performance reasons, the response of the PROPFIND call is manually parsed with a regex below
// instead of being processed by xml2json like the other WebDAV responses. This is over 2 times faster
// and it means the mobile app does not freeze during sync.
async function parsePropFindXml2(xmlString) {
const regex = /<d:response>[\S\s]*?<d:href>([\S\s]*?)<\/d:href>[\S\s]*?<d:getlastmodified>(.*?)<\/d:getlastmodified>/g;
let output = [];
let match = null;
while (match = regex.exec(xmlString)) {
const href = html_entity_decode(match[1]);
if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible
const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length)));
if (!path) continue; // The list of resources includes the root dir too, which we don't want
const lastModifiedDate = new Date(match[2]);
if (isNaN(lastModifiedDate.getTime())) throw new Error('Invalid date: ' + match[2]);
output.push({
path: path,
updated_time: lastModifiedDate.getTime(),
created_time: lastModifiedDate.getTime(),
isDir: !BaseItem.isSystemPath(path),
});
}
return output;
}
const resultXml = await this.api().execPropFind(path, 1, [
'd:getlastmodified',
//'d:resourcetype', // Include this to use parsePropFindXml()
], { responseFormat: 'text' });
const stats = await parsePropFindXml2(resultXml);
return {
items: this.statsFromResources_(resources),
items: stats,
hasMore: false,
context: null,
};

13
ReactNativeClient/lib/file-api.js
View File

@ -3,6 +3,7 @@ const { Logger } = require('lib/logger.js');
const { shim } = require('lib/shim');
const BaseItem = require('lib/models/BaseItem.js');
const JoplinError = require('lib/JoplinError');
const ArrayUtils = require('lib/ArrayUtils');
class FileApi {
@ -167,6 +168,8 @@ async function basicDelta(path, getDirStatFn, options) {
newContext.statsCache.sort(function(a, b) {
return a.updated_time - b.updated_time;
});
newContext.statIdsCache = newContext.statsCache.map((item) => BaseItem.pathToId(item.path));
newContext.statIdsCache.sort(); // Items must be sorted to use binary search below
}
let output = [];
@ -210,16 +213,8 @@ async function basicDelta(path, getDirStatFn, options) {
if (output.length + deletedItems.length >= outputLimit) break;
const itemId = itemIds[i];
let found = false;
for (let j = 0; j < newContext.statsCache.length; j++) {
const item = newContext.statsCache[j];
if (BaseItem.pathToId(item.path) == itemId) {
found = true;
break;
}
}
if (!found) {
if (ArrayUtils.binarySearch(newContext.statIdsCache, itemId) < 0) {
deletedItems.push({
path: BaseItem.systemPath(itemId),
isDeleted: true,

15
ReactNativeClient/package-lock.json generated
View File

@ -5390,9 +5390,9 @@
}
},
"sax": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.1.6.tgz",
"integrity": "sha1-XWFr6KXmB9VOEUr65Vt+ry/MMkA="
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
},
"semver": {
"version": "5.4.1",
@ -6255,7 +6255,7 @@
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
"integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
"requires": {
"sax": "1.1.6",
"sax": "1.2.4",
"xmlbuilder": "9.0.4"
},
"dependencies": {
@ -6287,6 +6287,13 @@
"integrity": "sha1-0lciS+g5PqrL+DfvIn/Y7CWzaIg=",
"requires": {
"sax": "1.1.6"
},
"dependencies": {
"sax": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.1.6.tgz",
"integrity": "sha1-XWFr6KXmB9VOEUr65Vt+ry/MMkA="
}
}
},
"xmldom": {