From 1cc27f2509dc2680e7d606b832ada6e9cfc46b5e Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Thu, 25 Jan 2018 19:01:14 +0000 Subject: [PATCH] Got Nextcloud sync to work in Electron --- CliClient/app/main.js | 30 +++++++-- ElectronClient/app/package-lock.json | 16 +++++ ElectronClient/app/package.json | 3 +- ReactNativeClient/lib/BaseApplication.js | 2 + ReactNativeClient/lib/BaseSyncTarget.js | 4 ++ ReactNativeClient/lib/SyncTargetNextcloud.js | 65 +++++++++++++++++++ ReactNativeClient/lib/SyncTargetOneDrive.js | 12 ++-- ReactNativeClient/lib/WebDavApi.js | 61 +++++++++++++---- .../lib/components/shared/side-menu-shared.js | 17 +++-- .../lib/file-api-driver-webdav.js | 52 +++++++++++---- ReactNativeClient/lib/file-api.js | 8 ++- ReactNativeClient/lib/models/Setting.js | 3 + ReactNativeClient/lib/path-utils.js | 10 ++- 13 files changed, 234 insertions(+), 49 deletions(-) create mode 100644 ReactNativeClient/lib/SyncTargetNextcloud.js diff --git a/CliClient/app/main.js b/CliClient/app/main.js index 3c257ca28..250976b01 100644 --- a/CliClient/app/main.js +++ b/CliClient/app/main.js @@ -72,18 +72,36 @@ process.stdout.on('error', function( err ) { // async function main() { // const WebDavApi = require('lib/WebDavApi'); -// const api = new WebDavApi('http://nextcloud.local/remote.php/dav/files/admin/Joplin', { username: 'admin', password: '123456' }); +// const api = new WebDavApi('http://nextcloud.local/remote.php/dav/files/admin/Joplin', { username: 'admin', password: '1234567' }); // const { FileApiDriverWebDav } = new require('lib/file-api-driver-webdav'); // const driver = new FileApiDriverWebDav(api); -// //await driver.stat('testing.txt'); -// const stat = await driver.stat('testing.txt'); +// const stat = await driver.stat(''); // console.info(stat); -// //await api.execPropFind(''); +// // const stat = await driver.stat('testing.txt'); +// // console.info(stat); -// //const stat = await driver.stat('testing.txt'); -// //console.info(stat); + +// // const content = await driver.get('testing.txta'); +// // console.info(content); + +// // const content = await driver.get('testing.txta', { target: 'file', path: '/var/www/joplin/CliClient/testing-file.txt' }); +// // console.info(content); + +// // const content = await driver.mkdir('newdir5'); +// // console.info(content); + +// //await driver.put('myfile4.md', 'this is my content'); + +// // await driver.put('testimg.jpg', null, { source: 'file', path: '/mnt/d/test.jpg' }); + +// // await driver.delete('myfile4.md'); + +// // const deltaResult = await driver.delta('', { +// // allItemIdsHandler: () => { return []; } +// // }); +// // console.info(deltaResult); // } // main().catch((error) => { console.error(error); }); diff --git a/ElectronClient/app/package-lock.json b/ElectronClient/app/package-lock.json index 02ea987da..ebcbcbd1a 100644 --- a/ElectronClient/app/package-lock.json +++ b/ElectronClient/app/package-lock.json @@ -5334,6 +5334,22 @@ "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=", "dev": true }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": "1.2.4", + "xmlbuilder": "9.0.4" + }, + "dependencies": { + "xmlbuilder": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.4.tgz", + "integrity": "sha1-UZy0ymhtAFqEINNJbz8MruzKWA8=" + } + } + }, "xmlbuilder": { "version": "8.2.2", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-8.2.2.tgz", diff --git a/ElectronClient/app/package.json b/ElectronClient/app/package.json index 22cd60b0e..315e4b521 100644 --- a/ElectronClient/app/package.json +++ b/ElectronClient/app/package.json @@ -84,6 +84,7 @@ "string-padding": "^1.0.2", "string-to-stream": "^1.1.0", "tcp-port-used": "^0.1.2", - "uuid": "^3.1.0" + "uuid": "^3.1.0", + "xml2js": "^0.4.19" } } diff --git a/ReactNativeClient/lib/BaseApplication.js b/ReactNativeClient/lib/BaseApplication.js index a6cbead1d..5dc05dcba 100644 --- a/ReactNativeClient/lib/BaseApplication.js +++ b/ReactNativeClient/lib/BaseApplication.js @@ -26,12 +26,14 @@ const SyncTargetRegistry = require('lib/SyncTargetRegistry.js'); const SyncTargetFilesystem = require('lib/SyncTargetFilesystem.js'); const SyncTargetOneDrive = require('lib/SyncTargetOneDrive.js'); const SyncTargetOneDriveDev = require('lib/SyncTargetOneDriveDev.js'); +const SyncTargetNextcloud = require('lib/SyncTargetNextcloud.js'); const EncryptionService = require('lib/services/EncryptionService'); const DecryptionWorker = require('lib/services/DecryptionWorker'); SyncTargetRegistry.addClass(SyncTargetFilesystem); SyncTargetRegistry.addClass(SyncTargetOneDrive); SyncTargetRegistry.addClass(SyncTargetOneDriveDev); +SyncTargetRegistry.addClass(SyncTargetNextcloud); class BaseApplication { diff --git a/ReactNativeClient/lib/BaseSyncTarget.js b/ReactNativeClient/lib/BaseSyncTarget.js index b72d1d8f5..c1e594a30 100644 --- a/ReactNativeClient/lib/BaseSyncTarget.js +++ b/ReactNativeClient/lib/BaseSyncTarget.js @@ -30,6 +30,10 @@ class BaseSyncTarget { return false; } + authRouteName() { + return null; + } + static id() { throw new Error('id() not implemented'); } diff --git a/ReactNativeClient/lib/SyncTargetNextcloud.js b/ReactNativeClient/lib/SyncTargetNextcloud.js new file mode 100644 index 000000000..1bd8aa854 --- /dev/null +++ b/ReactNativeClient/lib/SyncTargetNextcloud.js @@ -0,0 +1,65 @@ +const BaseSyncTarget = require('lib/BaseSyncTarget.js'); +const { _ } = require('lib/locale.js'); +const Setting = require('lib/models/Setting.js'); +const { FileApi } = require('lib/file-api.js'); +const { Synchronizer } = require('lib/synchronizer.js'); +const WebDavApi = require('lib/WebDavApi'); +const { FileApiDriverWebDav } = new require('lib/file-api-driver-webdav'); + +class SyncTargetNextcloud extends BaseSyncTarget { + + static id() { + return 5; + } + + constructor(db, options = null) { + super(db, options); + // this.authenticated_ = false; + } + + static targetName() { + return 'nextcloud'; + } + + static label() { + return _('Nextcloud'); + } + + isAuthenticated() { + return true; + //return this.authenticated_; + } + + async initFileApi() { + const options = { + baseUrl: () => Setting.value('sync.5.path'), + username: () => Setting.value('sync.5.username'), + password: () => Setting.value('sync.5.password'), + }; + + //const api = new WebDavApi('http://nextcloud.local/remote.php/dav/files/admin/Joplin', { username: 'admin', password: '123456' }); + const api = new WebDavApi(options); + const driver = new FileApiDriverWebDav(api); + + // this.authenticated_ = true; + // try { + // await driver.stat(''); + // } catch (error) { + // console.info(error); + // this.authenticated_ = false; + // if (error.code !== 401) throw error; + // } + + const fileApi = new FileApi('', driver); + fileApi.setSyncTargetId(SyncTargetNextcloud.id()); + fileApi.setLogger(this.logger()); + return fileApi; + } + + async initSynchronizer() { + return new Synchronizer(this.db(), await this.fileApi(), Setting.value('appType')); + } + +} + +module.exports = SyncTargetNextcloud; \ No newline at end of file diff --git a/ReactNativeClient/lib/SyncTargetOneDrive.js b/ReactNativeClient/lib/SyncTargetOneDrive.js index 03d0c62dc..50f0a04eb 100644 --- a/ReactNativeClient/lib/SyncTargetOneDrive.js +++ b/ReactNativeClient/lib/SyncTargetOneDrive.js @@ -9,15 +9,15 @@ const { FileApiDriverOneDrive } = require('lib/file-api-driver-onedrive.js'); class SyncTargetOneDrive extends BaseSyncTarget { + static id() { + return 3; + } + constructor(db, options = null) { super(db, options); this.api_ = null; } - static id() { - return 3; - } - static targetName() { return 'onedrive'; } @@ -38,6 +38,10 @@ class SyncTargetOneDrive extends BaseSyncTarget { return parameters().oneDrive; } + authRouteName() { + return 'OneDriveLogin'; + } + api() { if (this.api_) return this.api_; diff --git a/ReactNativeClient/lib/WebDavApi.js b/ReactNativeClient/lib/WebDavApi.js index 7e5c1d756..e6d8dc0e7 100644 --- a/ReactNativeClient/lib/WebDavApi.js +++ b/ReactNativeClient/lib/WebDavApi.js @@ -2,6 +2,8 @@ const { Logger } = require('lib/logger.js'); const { shim } = require('lib/shim.js'); const parseXmlString = require('xml2js').parseString; const JoplinError = require('lib/JoplinError'); +const urlParser = require("url"); +const { rtrimSlashes, ltrimSlashes } = require('lib/path-utils.js'); // Note that the d: namespace (the DAV namespace) is specific to Nextcloud. The RFC for example uses "D:" however // we make all the tags and attributes lowercase so we handle both the Nextcloud style and RFC. Hopefully other @@ -11,10 +13,11 @@ const JoplinError = require('lib/JoplinError'); class WebDavApi { - constructor(baseUrl, options) { + constructor(options) { this.logger_ = new Logger(); - this.baseUrl_ = baseUrl.replace(/\/+$/, ""); // Remove last trailing slashes this.options_ = options; + + } setLogger(l) { @@ -26,12 +29,17 @@ class WebDavApi { } authToken() { - if (!this.options_.username || !this.options_.password) return null; - return (new Buffer(this.options_.username + ':' + this.options_.password)).toString('base64'); + if (!this.options_.username() || !this.options_.password()) return null; + return (new Buffer(this.options_.username() + ':' + this.options_.password())).toString('base64'); } baseUrl() { - return this.baseUrl_; + return this.options_.baseUrl(); + } + + relativeBaseUrl() { + const url = urlParser.parse(this.baseUrl(), true); + return url.path; } async xmlToJson(xml) { @@ -60,11 +68,16 @@ class WebDavApi { }); } - valueFromJson(json, keys, type) { + valueFromJson(json, keys, type) { let output = json; + for (let i = 0; i < keys.length; i++) { const key = keys[i]; - if (!output || !output[key]) return null; + + // console.info(key, typeof key, typeof output, typeof output === 'object' && (key in output), Array.isArray(output)); + + if (typeof key === 'number' && !Array.isArray(output)) return null; + if (typeof key === 'string' && (typeof output !== 'object' || !(key in output))) return null; output = output[key]; } @@ -78,6 +91,10 @@ class WebDavApi { return null; } + if (type === 'array') { + return Array.isArray(output) ? output : null; + } + return null; } @@ -89,7 +106,11 @@ class WebDavApi { return this.valueFromJson(json, keys, 'object'); } - async execPropFind(path, fields = null) { + arrayFromJson(json, keys) { + return this.valueFromJson(json, keys, 'array'); + } + + async execPropFind(path, depth, fields = null) { if (fields === null) fields = ['d:getlastmodified']; let fieldsXml = ''; @@ -111,7 +132,7 @@ class WebDavApi { `; - return this.exec('PROPFIND', path, body); + return this.exec('PROPFIND', path, body, { 'Depth': depth }); } // curl -u admin:123456 'http://nextcloud.local/remote.php/dav/files/admin/' -X PROPFIND --data ' @@ -151,33 +172,45 @@ class WebDavApi { const responseText = await response.text(); + // Gives a shorter response for error messages. Useful for cases where a full HTML page is accidentally loaded instead of + // JSON. That way the error message will still show there's a problem but without filling up the log or screen. + const shortResponseText = () => { + return (responseText + '').substr(0, 1024); + } + let responseJson_ = null; const loadResponseJson = async () => { if (!responseText) return null; if (responseJson_) return responseJson_; responseJson_ = await this.xmlToJson(responseText); - if (!responseJson_) throw new JoplinError('Cannot parse JSON response: ' + responseText, response.status); + if (!responseJson_) throw new JoplinError('Cannot parse JSON response: ' + shortResponseText(), response.status); return responseJson_; } if (!response.ok) { // When using fetchBlob we only get a string (not xml or json) back - if (options.target === 'file') throw new JoplinError(responseText, response.status); + if (options.target === 'file') throw new JoplinError(shortResponseText(), response.status); const json = await loadResponseJson(); if (json['d:error']) { const code = json['d:error']['s:exception'] ? json['d:error']['s:exception'].join(' ') : response.status; - const message = json['d:error']['s:message'] ? json['d:error']['s:message'].join("\n") : responseText; + const message = json['d:error']['s:message'] ? json['d:error']['s:message'].join("\n") : shortResponseText(); throw new JoplinError(message + ' (' + code + ')', response.status); } - throw new JoplinError(responseText, response.status); + throw new JoplinError(shortResponseText(), response.status); } if (options.responseFormat === 'text') return responseText; - return await loadResponseJson(); + const output = await loadResponseJson(); + + // Check that we didn't get for example an HTML page (as an error) instead of the JSON response + // null responses are possible, for example for DELETE calls + if (output !== null && typeof output === 'object' && !('d:multistatus' in output)) throw new Error('Not a valid JSON response: ' + shortResponseText()); + + return output; } } diff --git a/ReactNativeClient/lib/components/shared/side-menu-shared.js b/ReactNativeClient/lib/components/shared/side-menu-shared.js index 8cedc59c0..5cb87ec0a 100644 --- a/ReactNativeClient/lib/components/shared/side-menu-shared.js +++ b/ReactNativeClient/lib/components/shared/side-menu-shared.js @@ -36,12 +36,17 @@ shared.synchronize_press = async function(comp) { const action = comp.props.syncStarted ? 'cancel' : 'start'; - if (!reg.syncTarget().isAuthenticated()) { - comp.props.dispatch({ - type: 'NAV_GO', - routeName: 'OneDriveLogin', - }); - return 'auth'; + if (!reg.syncTarget().isAuthenticated()) { + if (reg.syncTarget().authRouteName()) { + comp.props.dispatch({ + type: 'NAV_GO', + routeName: reg.syncTarget().authRouteName(), + }); + return 'auth'; + } + + reg.logger().info('Not authentified with sync target - please check your credential.'); + return 'error'; } let sync = null; diff --git a/ReactNativeClient/lib/file-api-driver-webdav.js b/ReactNativeClient/lib/file-api-driver-webdav.js index e09536546..b7aa370ec 100644 --- a/ReactNativeClient/lib/file-api-driver-webdav.js +++ b/ReactNativeClient/lib/file-api-driver-webdav.js @@ -1,5 +1,7 @@ const BaseItem = require('lib/models/BaseItem.js'); const { time } = require('lib/time-utils.js'); +const { basicDelta } = require('lib/file-api'); +const { rtrimSlashes, ltrimSlashes } = require('lib/path-utils.js'); class FileApiDriverWebDav { @@ -12,19 +14,25 @@ class FileApiDriverWebDav { } async stat(path) { - const result = await this.api().execPropFind(path, [ - 'd:getlastmodified', - 'd:resourcetype', - ]); + try { + const result = await this.api().execPropFind(path, 0, [ + 'd:getlastmodified', + 'd:resourcetype', + ]); - return this.metadataFromStat_(result, path); + const resource = this.api().objectFromJson(result, ['d:multistatus', 'd:response', 0]); + return this.statFromResource_(resource, path); + } catch (error) { + if (error.code === 404) return null; + throw error; + } } - metadataFromStat_(stat, path) { - const isCollection = this.api().stringFromJson(stat, ['d:multistatus', 'd:response', 0, 'd:propstat', 0, 'd:prop', 0, 'd:resourcetype', 0, 'd:collection', 0]); - const lastModifiedString = this.api().stringFromJson(stat, ['d:multistatus', 'd:response', 0, 'd:propstat', 0, 'd:prop', 0, 'd:getlastmodified', 0]); + statFromResource_(resource, path) { + const isCollection = this.api().stringFromJson(resource, ['d:propstat', 0, 'd:prop', 0, 'd:resourcetype', 0, 'd:collection', 0]); + const lastModifiedString = this.api().stringFromJson(resource, ['d:propstat', 0, 'd:prop', 0, 'd:getlastmodified', 0]); - if (!lastModifiedString) throw new Error('Could not get lastModified date: ' + JSON.stringify(stat)); + if (!lastModifiedString) throw new Error('Could not get lastModified date: ' + JSON.stringify(resource)); const lastModifiedDate = new Date(lastModifiedString); if (isNaN(lastModifiedDate.getTime())) throw new Error('Invalid date: ' + lastModifiedString); @@ -37,11 +45,17 @@ class FileApiDriverWebDav { }; } - metadataFromStats_(stats) { + statsFromResources_(resources) { + const relativeBaseUrl = this.api().relativeBaseUrl(); let output = []; - for (let i = 0; i < stats.length; i++) { - const mdStat = this.metadataFromStat_(stats[i]); - output.push(mdStat); + for (let i = 0; i < resources.length; i++) { + const resource = resources[i]; + const href = this.api().stringFromJson(resource, ['d:href', 0]); + if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible + const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length))); + if (path === '') continue; // The list of resources includes the root dir too, which we don't want + const stat = this.statFromResource_(resources[i], path); + output.push(stat); } return output; } @@ -51,7 +65,17 @@ class FileApiDriverWebDav { } async delta(path, options) { - + const getDirStats = async (path) => { + const result = await this.api().execPropFind(path, 1, [ + 'd:getlastmodified', + 'd:resourcetype', + ]); + + const resources = this.api().arrayFromJson(result, ['d:multistatus', 'd:response']); + return this.statsFromResources_(resources); + }; + + return await basicDelta(path, getDirStats, options); } async list(path, options) { diff --git a/ReactNativeClient/lib/file-api.js b/ReactNativeClient/lib/file-api.js index 4c25d33af..71a001473 100644 --- a/ReactNativeClient/lib/file-api.js +++ b/ReactNativeClient/lib/file-api.js @@ -139,9 +139,9 @@ function basicDeltaContextFromOptions_(options) { } // This is the basic delta algorithm, which can be used in case the cloud service does not have -// a built-on delta API. OneDrive and Dropbox have one for example, but Nextcloud and obviously +// a built-in delta API. OneDrive and Dropbox have one for example, but Nextcloud and obviously // the file system do not. -async function basicDelta(path, getStatFn, options) { +async function basicDelta(path, getDirStatFn, options) { const outputLimit = 1000; const itemIds = await options.allItemIdsHandler(); if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided'); @@ -156,7 +156,7 @@ async function basicDelta(path, getStatFn, options) { // Stats are cached until all items have been processed (until hasMore is false) if (newContext.statsCache === null) { - newContext.statsCache = await getStatFn(path); + newContext.statsCache = await getDirStatFn(path); newContext.statsCache.sort(function(a, b) { return a.updated_time - b.updated_time; }); @@ -196,6 +196,8 @@ async function basicDelta(path, getStatFn, options) { if (output.length >= outputLimit) break; } + // Find out which items have been deleted on the sync target by comparing the items + // we have to the items on the target. let deletedItems = []; for (let i = 0; i < itemIds.length; i++) { if (output.length + deletedItems.length >= outputLimit) break; diff --git a/ReactNativeClient/lib/models/Setting.js b/ReactNativeClient/lib/models/Setting.js index 68472b5d1..e20e43637 100644 --- a/ReactNativeClient/lib/models/Setting.js +++ b/ReactNativeClient/lib/models/Setting.js @@ -82,6 +82,9 @@ class Setting extends BaseModel { return SyncTargetRegistry.idAndLabelPlainObject(); }}, 'sync.2.path': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('filesystem') }, public: true, label: () => _('Directory to synchronise with (absolute path)'), description: () => _('The path to synchronise with when file system synchronisation is enabled. See `sync.target`.') }, + 'sync.5.path': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('nextcloud') }, public: true, label: () => _('Nexcloud directory URL to synchronise with') }, + 'sync.5.username': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('nextcloud') }, public: true, label: () => _('Nexcloud username') }, + 'sync.5.password': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('nextcloud') }, public: true, label: () => _('Nexcloud password') }, 'sync.3.auth': { value: '', type: Setting.TYPE_STRING, public: false }, 'sync.4.auth': { value: '', type: Setting.TYPE_STRING, public: false }, 'sync.1.context': { value: '', type: Setting.TYPE_STRING, public: false }, diff --git a/ReactNativeClient/lib/path-utils.js b/ReactNativeClient/lib/path-utils.js index ded050740..5dfbbebd2 100644 --- a/ReactNativeClient/lib/path-utils.js +++ b/ReactNativeClient/lib/path-utils.js @@ -45,4 +45,12 @@ function toSystemSlashes(path, os) { return path.replace(/\\/g, "/"); } -module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFileExtension, toSystemSlashes }; \ No newline at end of file +function rtrimSlashes(path) { + return path.replace(/\/+$/, ''); +} + +function ltrimSlashes(path) { + return path.replace(/^\/+/, ''); +} + +module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFileExtension, toSystemSlashes, rtrimSlashes, ltrimSlashes }; \ No newline at end of file