Got Nextcloud sync to work in Electron

CliClient/app/main.js

@@ -72,18 +72,36 @@ process.stdout.on('error', function( err ) {
 
 // async function main() {
 // 	const WebDavApi = require('lib/WebDavApi');
-// 	const api = new WebDavApi('http://nextcloud.local/remote.php/dav/files/admin/Joplin', { username: 'admin', password: '123456' });
+// 	const api = new WebDavApi('http://nextcloud.local/remote.php/dav/files/admin/Joplin', { username: 'admin', password: '1234567' });
 // 	const { FileApiDriverWebDav } = new require('lib/file-api-driver-webdav');
 // 	const driver = new FileApiDriverWebDav(api);
 
-// 	//await driver.stat('testing.txt');
-// 	const stat = await driver.stat('testing.txt');
+// 	const stat = await driver.stat('');
 // 	console.info(stat);
 
-// 	//await api.execPropFind('');
-
 // 	// const stat = await driver.stat('testing.txt');
 // 	// console.info(stat);
+
+
+// 	// const content = await driver.get('testing.txta');
+// 	// console.info(content);
+
+// 	// const content = await driver.get('testing.txta', { target: 'file', path: '/var/www/joplin/CliClient/testing-file.txt' });
+// 	// console.info(content);
+
+// 	// const content = await driver.mkdir('newdir5');
+// 	// console.info(content);
+
+// 	//await driver.put('myfile4.md', 'this is my content');
+
+// 	// await driver.put('testimg.jpg', null, { source: 'file', path: '/mnt/d/test.jpg' });
+
+// 	// await driver.delete('myfile4.md');
+
+// 	// const deltaResult = await driver.delta('', {
+// 	// 	allItemIdsHandler: () => { return []; }
+// 	// });
+// 	// console.info(deltaResult);
 // }
 
 // main().catch((error) => { console.error(error); });

ElectronClient/app/package-lock.json

@@ -5334,6 +5334,22 @@
       "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=",
       "dev": true
     },
+    "xml2js": {
+      "version": "0.4.19",
+      "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
+      "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
+      "requires": {
+        "sax": "1.2.4",
+        "xmlbuilder": "9.0.4"
+      },
+      "dependencies": {
+        "xmlbuilder": {
+          "version": "9.0.4",
+          "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.4.tgz",
+          "integrity": "sha1-UZy0ymhtAFqEINNJbz8MruzKWA8="
+        }
+      }
+    },
     "xmlbuilder": {
       "version": "8.2.2",
       "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-8.2.2.tgz",

ElectronClient/app/package.json

@@ -84,6 +84,7 @@
     "string-padding": "^1.0.2",
     "string-to-stream": "^1.1.0",
     "tcp-port-used": "^0.1.2",
-    "uuid": "^3.1.0"
+    "uuid": "^3.1.0",
+    "xml2js": "^0.4.19"
   }
 }

ReactNativeClient/lib/BaseApplication.js

@@ -26,12 +26,14 @@ const SyncTargetRegistry = require('lib/SyncTargetRegistry.js');
 const SyncTargetFilesystem = require('lib/SyncTargetFilesystem.js');
 const SyncTargetOneDrive = require('lib/SyncTargetOneDrive.js');
 const SyncTargetOneDriveDev = require('lib/SyncTargetOneDriveDev.js');
+const SyncTargetNextcloud = require('lib/SyncTargetNextcloud.js');
 const EncryptionService = require('lib/services/EncryptionService');
 const DecryptionWorker = require('lib/services/DecryptionWorker');
 
 SyncTargetRegistry.addClass(SyncTargetFilesystem);
 SyncTargetRegistry.addClass(SyncTargetOneDrive);
 SyncTargetRegistry.addClass(SyncTargetOneDriveDev);
+SyncTargetRegistry.addClass(SyncTargetNextcloud);
 
 class BaseApplication {
 

ReactNativeClient/lib/BaseSyncTarget.js

@@ -30,6 +30,10 @@ class BaseSyncTarget {
 		return false;
 	}
 
+	authRouteName() {
+		return null;
+	}
+
 	static id() {
 		throw new Error('id() not implemented');
 	}

ReactNativeClient/lib/SyncTargetNextcloud.js

@@ -0,0 +1,65 @@
+const BaseSyncTarget = require('lib/BaseSyncTarget.js');
+const { _ } = require('lib/locale.js');
+const Setting = require('lib/models/Setting.js');
+const { FileApi } = require('lib/file-api.js');
+const { Synchronizer } = require('lib/synchronizer.js');
+const WebDavApi = require('lib/WebDavApi');
+const { FileApiDriverWebDav } = new require('lib/file-api-driver-webdav');
+
+class SyncTargetNextcloud extends BaseSyncTarget {
+
+	static id() {
+		return 5;
+	}
+
+	constructor(db, options = null) {
+		super(db, options);
+		// this.authenticated_ = false;
+	}
+
+	static targetName() {
+		return 'nextcloud';
+	}
+
+	static label() {
+		return _('Nextcloud');
+	}
+
+	isAuthenticated() {
+		return true;
+		//return this.authenticated_;
+	}
+
+	async initFileApi() {
+		const options = {
+			baseUrl: () => Setting.value('sync.5.path'),
+			username: () => Setting.value('sync.5.username'),
+			password: () => Setting.value('sync.5.password'),
+		};
+
+		//const api = new WebDavApi('http://nextcloud.local/remote.php/dav/files/admin/Joplin', { username: 'admin', password: '123456' });
+		const api = new WebDavApi(options);
+		const driver = new FileApiDriverWebDav(api);
+
+		// this.authenticated_ = true;
+		// try {
+		// 	await driver.stat('');
+		// } catch (error) {
+		// 	console.info(error);
+		// 	this.authenticated_ = false;
+		// 	if (error.code !== 401) throw error;
+		// }
+
+		const fileApi = new FileApi('', driver);
+		fileApi.setSyncTargetId(SyncTargetNextcloud.id());
+		fileApi.setLogger(this.logger());
+		return fileApi;
+	}
+
+	async initSynchronizer() {
+		return new Synchronizer(this.db(), await this.fileApi(), Setting.value('appType'));
+	}
+
+}
+
+module.exports = SyncTargetNextcloud;

ReactNativeClient/lib/SyncTargetOneDrive.js

@@ -9,15 +9,15 @@ const { FileApiDriverOneDrive } = require('lib/file-api-driver-onedrive.js');
 
 class SyncTargetOneDrive extends BaseSyncTarget {
 
+	static id() {
+		return 3;
+	}
+
 	constructor(db, options = null) {
 		super(db, options);
 		this.api_ = null;
 	}
 
-	static id() {
-		return 3;
-	}
-
 	static targetName() {
 		return 'onedrive';
 	}
@@ -38,6 +38,10 @@ class SyncTargetOneDrive extends BaseSyncTarget {
 		return parameters().oneDrive;
 	}
 
+	authRouteName() {
+		return 'OneDriveLogin';
+	}
+
 	api() {
 		if (this.api_) return this.api_;
 

ReactNativeClient/lib/WebDavApi.js

@@ -2,6 +2,8 @@ const { Logger } = require('lib/logger.js');
 const { shim } = require('lib/shim.js');
 const parseXmlString = require('xml2js').parseString;
 const JoplinError = require('lib/JoplinError');
+const urlParser = require("url");
+const { rtrimSlashes, ltrimSlashes } = require('lib/path-utils.js');
 
 // Note that the d: namespace (the DAV namespace) is specific to Nextcloud. The RFC for example uses "D:" however
 // we make all the tags and attributes lowercase so we handle both the Nextcloud style and RFC. Hopefully other
@@ -11,10 +13,11 @@ const JoplinError = require('lib/JoplinError');
 
 class WebDavApi {
 
-	constructor(baseUrl, options) {
+	constructor(options) {
 		this.logger_ = new Logger();
-		this.baseUrl_ = baseUrl.replace(/\/+$/, ""); // Remove last trailing slashes
 		this.options_ = options;
+
+		
 	}
 
 	setLogger(l) {
@@ -26,12 +29,17 @@ class WebDavApi {
 	}
 
 	authToken() {
-		if (!this.options_.username || !this.options_.password) return null;
-		return (new Buffer(this.options_.username + ':' + this.options_.password)).toString('base64');
+		if (!this.options_.username() || !this.options_.password()) return null;
+		return (new Buffer(this.options_.username() + ':' + this.options_.password())).toString('base64');
 	}
 
 	baseUrl() {
-		return this.baseUrl_;
+		return this.options_.baseUrl();
+	}
+
+	relativeBaseUrl() {
+		const url = urlParser.parse(this.baseUrl(), true);
+		return url.path;
 	}
 
 	async xmlToJson(xml) {
@@ -62,9 +70,14 @@ class WebDavApi {
 
 	valueFromJson(json, keys, type) {	
 		let output = json;
+
 		for (let i = 0; i < keys.length; i++) {
 			const key = keys[i];
-			if (!output || !output[key]) return null;
+
+			// console.info(key, typeof key, typeof output, typeof output === 'object' && (key in output), Array.isArray(output));
+
+			if (typeof key === 'number' && !Array.isArray(output)) return null;
+			if (typeof key === 'string' && (typeof output !== 'object' || !(key in output))) return null;
 			output = output[key];
 		}
 
@@ -78,6 +91,10 @@ class WebDavApi {
 			return null;
 		}
 
+		if (type === 'array') {
+			return Array.isArray(output) ? output : null;
+		}
+
 		return null;
 	}
 
@@ -89,7 +106,11 @@ class WebDavApi {
 		return this.valueFromJson(json, keys, 'object');
 	}
 
-	async execPropFind(path, fields = null) {
+	arrayFromJson(json, keys) {
+		return this.valueFromJson(json, keys, 'array');
+	}
+
+	async execPropFind(path, depth, fields = null) {
 		if (fields === null) fields = ['d:getlastmodified'];
 
 		let fieldsXml = '';
@@ -111,7 +132,7 @@ class WebDavApi {
 				</d:prop>
 			</d:propfind>`;
 
-		return this.exec('PROPFIND', path, body);
+		return this.exec('PROPFIND', path, body, { 'Depth': depth });
 	}
 
 	// curl -u admin:123456 'http://nextcloud.local/remote.php/dav/files/admin/' -X PROPFIND --data '<?xml version="1.0" encoding="UTF-8"?>
@@ -151,33 +172,45 @@ class WebDavApi {
 
 		const responseText = await response.text();
 
+		// Gives a shorter response for error messages. Useful for cases where a full HTML page is accidentally loaded instead of
+		// JSON. That way the error message will still show there's a problem but without filling up the log or screen.
+		const shortResponseText = () => {
+			return (responseText + '').substr(0, 1024);
+		}
+
 		let responseJson_ = null;
 		const loadResponseJson = async () => {
 			if (!responseText) return null;
 			if (responseJson_) return responseJson_;
 			responseJson_ = await this.xmlToJson(responseText);
-			if (!responseJson_) throw new JoplinError('Cannot parse JSON response: ' + responseText, response.status);
+			if (!responseJson_) throw new JoplinError('Cannot parse JSON response: ' + shortResponseText(), response.status);
 			return responseJson_;
 		}
 
 		if (!response.ok) {
 			// When using fetchBlob we only get a string (not xml or json) back
-			if (options.target === 'file') throw new JoplinError(responseText, response.status);
+			if (options.target === 'file') throw new JoplinError(shortResponseText(), response.status);
 
 			const json = await loadResponseJson();
 
 			if (json['d:error']) {
 				const code = json['d:error']['s:exception'] ? json['d:error']['s:exception'].join(' ') : response.status;
-				const message = json['d:error']['s:message'] ? json['d:error']['s:message'].join("\n") : responseText;
+				const message = json['d:error']['s:message'] ? json['d:error']['s:message'].join("\n") : shortResponseText();
 				throw new JoplinError(message + ' (' + code + ')', response.status);
 			}
 
-			throw new JoplinError(responseText, response.status);
+			throw new JoplinError(shortResponseText(), response.status);
 		}
 		
 		if (options.responseFormat === 'text') return responseText;
 
-		return await loadResponseJson();
+		const output = await loadResponseJson();
+
+		// Check that we didn't get for example an HTML page (as an error) instead of the JSON response
+		// null responses are possible, for example for DELETE calls
+		if (output !== null && typeof output === 'object' && !('d:multistatus' in output)) throw new Error('Not a valid JSON response: ' + shortResponseText());
+
+		return output;
 	}
 
 }

ReactNativeClient/lib/components/shared/side-menu-shared.js

@@ -37,13 +37,18 @@ shared.synchronize_press = async function(comp) {
 	const action = comp.props.syncStarted ? 'cancel' : 'start';
 
 	if (!reg.syncTarget().isAuthenticated()) {
+		if (reg.syncTarget().authRouteName()) {
 			comp.props.dispatch({
 				type: 'NAV_GO',
-			routeName: 'OneDriveLogin',
+				routeName: reg.syncTarget().authRouteName(),
 			});
 			return 'auth';
 		}
 
+		reg.logger().info('Not authentified with sync target - please check your credential.');
+		return 'error';
+	}
+
 	let sync = null;
 	try {
 		sync = await reg.syncTarget().synchronizer();

ReactNativeClient/lib/file-api-driver-webdav.js

@@ -1,5 +1,7 @@
 const BaseItem = require('lib/models/BaseItem.js');
 const { time } = require('lib/time-utils.js');
+const { basicDelta } = require('lib/file-api');
+const { rtrimSlashes, ltrimSlashes } = require('lib/path-utils.js');
 
 class FileApiDriverWebDav { 
 
@@ -12,19 +14,25 @@ class FileApiDriverWebDav {
 	}
 
 	async stat(path) {
-		const result = await this.api().execPropFind(path, [
+		try {
+			const result = await this.api().execPropFind(path, 0, [
 				'd:getlastmodified',
 				'd:resourcetype',
 			]);
 
-		return this.metadataFromStat_(result, path);
+			const resource = this.api().objectFromJson(result, ['d:multistatus', 'd:response', 0]);
+			return this.statFromResource_(resource, path);
+		} catch (error) {
+			if (error.code === 404) return null;
+			throw error;
+		}
 	}
 
-	metadataFromStat_(stat, path) {
-		const isCollection = this.api().stringFromJson(stat, ['d:multistatus', 'd:response', 0, 'd:propstat', 0, 'd:prop', 0, 'd:resourcetype', 0, 'd:collection', 0]);
-		const lastModifiedString = this.api().stringFromJson(stat, ['d:multistatus', 'd:response', 0, 'd:propstat', 0, 'd:prop', 0, 'd:getlastmodified', 0]);
+	statFromResource_(resource, path) {
+		const isCollection = this.api().stringFromJson(resource, ['d:propstat', 0, 'd:prop', 0, 'd:resourcetype', 0, 'd:collection', 0]);
+		const lastModifiedString = this.api().stringFromJson(resource, ['d:propstat', 0, 'd:prop', 0, 'd:getlastmodified', 0]);
 
-		if (!lastModifiedString) throw new Error('Could not get lastModified date: ' + JSON.stringify(stat));
+		if (!lastModifiedString) throw new Error('Could not get lastModified date: ' + JSON.stringify(resource));
 
 		const lastModifiedDate = new Date(lastModifiedString);
 		if (isNaN(lastModifiedDate.getTime())) throw new Error('Invalid date: ' + lastModifiedString);
@@ -37,11 +45,17 @@ class FileApiDriverWebDav {
 		};
 	}
 
-	metadataFromStats_(stats) {
+	statsFromResources_(resources) {
+		const relativeBaseUrl = this.api().relativeBaseUrl();
 		let output = [];
-		for (let i = 0; i < stats.length; i++) {
-			const mdStat = this.metadataFromStat_(stats[i]);
-			output.push(mdStat);
+		for (let i = 0; i < resources.length; i++) {
+			const resource = resources[i];
+			const href = this.api().stringFromJson(resource, ['d:href', 0]);
+			if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible
+			const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length)));
+			if (path === '') continue; // The list of resources includes the root dir too, which we don't want
+			const stat = this.statFromResource_(resources[i], path);
+			output.push(stat);
 		}
 		return output;
 	}
@@ -51,7 +65,17 @@ class FileApiDriverWebDav {
 	}
 
 	async delta(path, options) {
+		const getDirStats = async (path) => {
+			const result = await this.api().execPropFind(path, 1, [
+				'd:getlastmodified',
+				'd:resourcetype',
+			]);
 
+			const resources = this.api().arrayFromJson(result, ['d:multistatus', 'd:response']);
+			return this.statsFromResources_(resources);
+		};
+
+		return await basicDelta(path, getDirStats, options);
 	}
 
 	async list(path, options) {

ReactNativeClient/lib/file-api.js

@@ -139,9 +139,9 @@ function basicDeltaContextFromOptions_(options) {
 }
 
 // This is the basic delta algorithm, which can be used in case the cloud service does not have
-// a built-on delta API. OneDrive and Dropbox have one for example, but Nextcloud and obviously
+// a built-in delta API. OneDrive and Dropbox have one for example, but Nextcloud and obviously
 // the file system do not.
-async function basicDelta(path, getStatFn, options) {
+async function basicDelta(path, getDirStatFn, options) {
 	const outputLimit = 1000;
 	const itemIds = await options.allItemIdsHandler();
 	if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided');
@@ -156,7 +156,7 @@ async function basicDelta(path, getStatFn, options) {
 
 	// Stats are cached until all items have been processed (until hasMore is false)
 	if (newContext.statsCache === null) {
-		newContext.statsCache = await getStatFn(path);
+		newContext.statsCache = await getDirStatFn(path);
 		newContext.statsCache.sort(function(a, b) {
 			return a.updated_time - b.updated_time;
 		});
@@ -196,6 +196,8 @@ async function basicDelta(path, getStatFn, options) {
 		if (output.length >= outputLimit) break;
 	}
 
+	// Find out which items have been deleted on the sync target by comparing the items
+	// we have to the items on the target.
 	let deletedItems = [];
 	for (let i = 0; i < itemIds.length; i++) {
 		if (output.length + deletedItems.length >= outputLimit) break;

ReactNativeClient/lib/models/Setting.js

@@ -82,6 +82,9 @@ class Setting extends BaseModel {
 				return SyncTargetRegistry.idAndLabelPlainObject();
 			}},
 			'sync.2.path': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('filesystem') }, public: true, label: () => _('Directory to synchronise with (absolute path)'), description: () => _('The path to synchronise with when file system synchronisation is enabled. See `sync.target`.') },
+			'sync.5.path': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('nextcloud') }, public: true, label: () => _('Nexcloud directory URL to synchronise with') },
+			'sync.5.username': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('nextcloud') }, public: true, label: () => _('Nexcloud username') },
+			'sync.5.password': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('nextcloud') }, public: true, label: () => _('Nexcloud password') },
 			'sync.3.auth': { value: '', type: Setting.TYPE_STRING, public: false },
 			'sync.4.auth': { value: '', type: Setting.TYPE_STRING, public: false },
 			'sync.1.context': { value: '', type: Setting.TYPE_STRING, public: false },

ReactNativeClient/lib/path-utils.js

@@ -45,4 +45,12 @@ function toSystemSlashes(path, os) {
 	return path.replace(/\\/g, "/");
 }
 
-module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFileExtension, toSystemSlashes };
+function rtrimSlashes(path) {
+	return path.replace(/\/+$/, '');
+}
+
+function ltrimSlashes(path) {
+	return path.replace(/^\/+/, '');
+}
+
+module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFileExtension, toSystemSlashes, rtrimSlashes, ltrimSlashes };