All: Made WebDAV driver more generic to support services other than Nextcloud and added WebDAV sync target

ReactNativeClient/lib/BaseApplication.js

@@ -27,6 +27,7 @@ const SyncTargetFilesystem = require('lib/SyncTargetFilesystem.js');
 const SyncTargetOneDrive = require('lib/SyncTargetOneDrive.js');
 const SyncTargetOneDriveDev = require('lib/SyncTargetOneDriveDev.js');
 const SyncTargetNextcloud = require('lib/SyncTargetNextcloud.js');
+const SyncTargetWebDAV = require('lib/SyncTargetWebDAV.js');
 const EncryptionService = require('lib/services/EncryptionService');
 const DecryptionWorker = require('lib/services/DecryptionWorker');
 
@@ -34,6 +35,7 @@ SyncTargetRegistry.addClass(SyncTargetFilesystem);
 SyncTargetRegistry.addClass(SyncTargetOneDrive);
 SyncTargetRegistry.addClass(SyncTargetOneDriveDev);
 SyncTargetRegistry.addClass(SyncTargetNextcloud);
+SyncTargetRegistry.addClass(SyncTargetWebDAV);
 
 class BaseApplication {
 

ReactNativeClient/lib/SyncTargetWebDAV.js

@@ -0,0 +1,52 @@
+const BaseSyncTarget = require('lib/BaseSyncTarget.js');
+const { _ } = require('lib/locale.js');
+const Setting = require('lib/models/Setting.js');
+const { FileApi } = require('lib/file-api.js');
+const { Synchronizer } = require('lib/synchronizer.js');
+const WebDavApi = require('lib/WebDavApi');
+const { FileApiDriverWebDav } = require('lib/file-api-driver-webdav');
+
+class SyncTargetWebDAV extends BaseSyncTarget {
+
+	static id() {
+		return 6;
+	}
+
+	constructor(db, options = null) {
+		super(db, options);
+	}
+
+	static targetName() {
+		return 'webdav';
+	}
+
+	static label() {
+		return _('WebDAV (Beta)');
+	}
+
+	isAuthenticated() {
+		return true;
+	}
+
+	async initFileApi() {
+		const options = {
+			baseUrl: () => Setting.value('sync.6.path'),
+			username: () => Setting.value('sync.6.username'),
+			password: () => Setting.value('sync.6.password'),
+		};
+
+		const api = new WebDavApi(options);
+		const driver = new FileApiDriverWebDav(api);
+		const fileApi = new FileApi('', driver);
+		fileApi.setSyncTargetId(SyncTargetWebDAV.id());
+		fileApi.setLogger(this.logger());
+		return fileApi;
+	}
+
+	async initSynchronizer() {
+		return new Synchronizer(this.db(), await this.fileApi(), Setting.value('appType'));
+	}
+
+}
+
+module.exports = SyncTargetWebDAV;

ReactNativeClient/lib/WebDavApi.js

@@ -42,18 +42,34 @@ class WebDavApi {
 	}
 
 	async xmlToJson(xml) {
+		let davNamespaces = []; // Yes, there can be more than one... xmlns:a="DAV:" xmlns:D="DAV:"
 
 		const nameProcessor = (name) => {
-			// const idx = name.indexOf(':');
+			if (name.indexOf('xmlns:') !== 0) {
-			// if (idx >= 0) {
+				// Check if the current name is within the DAV namespace. If it is, normalise it
-			// 	if (name.indexOf('xmlns:') !== 0) name = name.substr(idx + 1);
+				// by moving it to the "d:" namespace, which is what all the functions are using.
-			// }
+				const p = name.split(':');
+				if (p.length == 2) {
+					const ns = p[0];
+					if (davNamespaces.indexOf(ns) >= 0) {
+						name = 'd:' + p[1];
+					}
+				}
+			}
 			return name.toLowerCase();
 		};
 
+		const attrValueProcessor = (value, name) => {
+			if (value.toLowerCase() === 'dav:') {
+				const p = name.split(':');
+				davNamespaces.push(p[p.length - 1]);
+			}
+		}
+
 		const options = {
 			tagNameProcessors: [nameProcessor],
 			attrNameProcessors: [nameProcessor],
+			attrValueProcessors: [attrValueProcessor]
 		}
 
 		return new Promise((resolve, reject) => {
@@ -81,6 +97,14 @@ class WebDavApi {
 		}
 
 		if (type === 'string') {
+			// If the XML has not attribute the value is directly a string
+			// If the XML node has attributes, the value is under "_".
+			// Eg for this XML, the string will be under {"_":"Thu, 01 Feb 2018 17:24:05 GMT"}:
+			// <a:getlastmodified b:dt="dateTime.rfc1123">Thu, 01 Feb 2018 17:24:05 GMT</a:getlastmodified>
+			// For this XML, the value will be "Thu, 01 Feb 2018 17:24:05 GMT"
+			// <a:getlastmodified>Thu, 01 Feb 2018 17:24:05 GMT</a:getlastmodified>
+
+			if (typeof output === 'object' && '_' in output) output = output['_'];
 			if (typeof output !== 'string') return null;
 			return output;
 		}
@@ -151,7 +175,7 @@ class WebDavApi {
 
 		if (authToken) headers['Authorization'] = 'Basic ' + authToken;
 
-		if (typeof body === 'string') headers['Content-length'] = body.length;
+		if (typeof body === 'string') headers['Content-Length'] = body.length;
 
 		const fetchOptions = {};
 		fetchOptions.headers = headers;
@@ -163,7 +187,7 @@ class WebDavApi {
 
 		let response = null;
 
-		// console.info('WebDAV', method + ' ' + path, headers, options);
+		// console.info('WebDAV', method + ' ' + url, headers, options);
 
 		if (options.source == 'file' && (method == 'POST' || method == 'PUT')) {
 			response = await shim.uploadBlob(url, fetchOptions);
@@ -202,7 +226,7 @@ class WebDavApi {
 				throw new JoplinError(method + ' ' + path + ': ' + message + ' (' + code + ')', response.status);
 			}
 
-			throw new JoplinError(shortResponseText(), response.status);
+			throw new JoplinError(method + ' ' + path + ': ' + shortResponseText(), response.status);
 		}
 		
 		if (options.responseFormat === 'text') return responseText;

ReactNativeClient/lib/components/screen-header.js

@@ -372,7 +372,7 @@ class ScreenHeaderComponent extends Component {
 					if (mustSelect) output.push({ label: _('Move to notebook...'), value: null });
 					for (let i = 0; i < this.props.folders.length; i++) {
 						let f = this.props.folders[i];
-						output.push({ label: f.title, value: f.id });
+						output.push({ label: Folder.displayTitle(f), value: f.id });
 					}
 					output.sort((a, b) => {
 						if (a.value === null) return -1;

ReactNativeClient/lib/file-api-driver-webdav.js

@@ -6,6 +6,7 @@ const Entities = require('html-entities').AllHtmlEntities;
 const html_entity_decode = (new Entities()).decode;
 const { shim } = require('lib/shim');
 const { basename } = require('lib/path-utils');
+const JoplinError = require('lib/JoplinError');
 
 class FileApiDriverWebDav { 
 
@@ -67,8 +68,41 @@ class FileApiDriverWebDav {
 		return await basicDelta(path, getDirStats, options);
 	}
 
-	async list(path, options) {
+	// A file href, as found in the result of a PROPFIND, can be either an absolute URL or a
+	// relative URL (an absolute URL minus the protocol and domain), while the sync algorithm
+	// works with paths relative to the base URL.
+	hrefToRelativePath_(href, baseUrl, relativeBaseUrl) {
+		let output = '';
+		if (href.indexOf(baseUrl) === 0) {
+			output = href.substr(baseUrl.length);
+		} else if (href.indexOf(relativeBaseUrl) === 0) {
+			output = href.substr(relativeBaseUrl.length);
+		} else {
+			throw new Error('href ' + href + ' not in baseUrl ' + baseUrl + ' nor relativeBaseUrl ' + relativeBaseUrl);
+		}
+
+		return rtrimSlashes(ltrimSlashes(output));
+	}
+
+	statsFromResources_(resources) {
 		const relativeBaseUrl = this.api().relativeBaseUrl();
+		const baseUrl = this.api().baseUrl();
+		let output = [];
+		for (let i = 0; i < resources.length; i++) {
+			const resource = resources[i];
+			const href = this.api().stringFromJson(resource, ['d:href', 0]);
+			const path = this.hrefToRelativePath_(href, baseUrl, relativeBaseUrl);
+			// if (href.indexOf(relativeBaseUrl) !== 0) throw new Error('Path "' + href + '" not inside base URL: ' + relativeBaseUrl);
+			// const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length)));
+			if (path === '') continue; // The list of resources includes the root dir too, which we don't want
+			const stat = this.statFromResource_(resources[i], path);
+			output.push(stat);
+		}
+		return output;
+	}
+
+	async list(path, options) {
+		// const relativeBaseUrl = this.api().relativeBaseUrl();
 
 		// function parsePropFindXml(xmlString) {
 		// 	return new Promise(async (resolve, reject) => {
@@ -176,39 +210,53 @@ class FileApiDriverWebDav {
 		// instead of being processed by xml2json like the other WebDAV responses. This is over 2 times faster
 		// and it means the mobile app does not freeze during sync. 
 
-		async function parsePropFindXml2(xmlString) {
+		// async function parsePropFindXml2(xmlString) {
-			const regex = /<d:response>[\S\s]*?<d:href>([\S\s]*?)<\/d:href>[\S\s]*?<d:getlastmodified>(.*?)<\/d:getlastmodified>/g;
+		// 	const regex = /<d:response>[\S\s]*?<d:href>([\S\s]*?)<\/d:href>[\S\s]*?<d:getlastmodified>(.*?)<\/d:getlastmodified>/g;
 
-			let output = [];
+		// 	let output = [];
-			let match = null;
+		// 	let match = null;
 
-			while (match = regex.exec(xmlString)) {
+		// 	while (match = regex.exec(xmlString)) {
-				const href = html_entity_decode(match[1]);
+		// 		const href = html_entity_decode(match[1]);
-				if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible
+		// 		if (href.indexOf(relativeBaseUrl) < 0) throw new Error('Path not inside base URL: ' + relativeBaseUrl); // Normally not possible
-				const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length)));
+		// 		const path = rtrimSlashes(ltrimSlashes(href.substr(relativeBaseUrl.length)));
 
-				if (!path) continue; // The list of resources includes the root dir too, which we don't want
+		// 		if (!path) continue; // The list of resources includes the root dir too, which we don't want
 
-				const lastModifiedDate = new Date(match[2]);
+		// 		const lastModifiedDate = new Date(match[2]);
-				if (isNaN(lastModifiedDate.getTime())) throw new Error('Invalid date: ' + match[2]);
+		// 		if (isNaN(lastModifiedDate.getTime())) throw new Error('Invalid date: ' + match[2]);
 
-				output.push({
+		// 		output.push({
-					path: path,
+		// 			path: path,
-					updated_time: lastModifiedDate.getTime(),
+		// 			updated_time: lastModifiedDate.getTime(),
-					created_time: lastModifiedDate.getTime(),
+		// 			created_time: lastModifiedDate.getTime(),
-					isDir: !BaseItem.isSystemPath(path),
+		// 			isDir: !BaseItem.isSystemPath(path),
-				});
+		// 		});
-			}
+		// 	}
 
-			return output;
+		// 	return output;
-		}
+		// }
 
-		const resultXml = await this.api().execPropFind(path, 1, [
+		// const resultXml = await this.api().execPropFind(path, 1, [
+		// 	'd:getlastmodified',
+		// 	//'d:resourcetype', // Include this to use parsePropFindXml()
+		// ], { responseFormat: 'text' });
+
+		// const stats = await parsePropFindXml2(resultXml);
+
+		// return {
+		// 	items: stats,
+		// 	hasMore: false,
+		// 	context: null,
+		// };
+
+		const result = await this.api().execPropFind(path, 1, [
 			'd:getlastmodified',
-			//'d:resourcetype', // Include this to use parsePropFindXml()
+			'd:resourcetype',
-		], { responseFormat: 'text' });
+		]);
 
-		const stats = await parsePropFindXml2(resultXml);
+		const resources = this.api().arrayFromJson(result, ['d:multistatus', 'd:response']);
+		const stats = this.statsFromResources_(resources)
 
 		return {
 			items: stats,
@@ -221,7 +269,13 @@ class FileApiDriverWebDav {
 		if (!options) options = {};
 		if (!options.responseFormat) options.responseFormat = 'text';
 		try {
-			return await this.api().exec('GET', path, null, null, options);
+			const response = await this.api().exec('GET', path, null, null, options);
+
+			// This is awful but instead of a 404 Not Found, Microsoft IIS returns an HTTP code 200
+			// with a response body "The specified file doesn't exist." for non-existing files,
+			// so we need to check for this.
+			if (response === "The specified file doesn't exist.") throw new JoplinError(response, 404);
+			return response;
 		} catch (error) {
 			if (error.code !== 404) throw error;
 		}
@@ -231,7 +285,17 @@ class FileApiDriverWebDav {
 		try {
 			await this.api().exec('MKCOL', path);
 		} catch (error) {
-			if (error.code !== 405) throw error; // 405 means that the collection already exists (Method Not Allowed)
+			if (error.code === 405) return; // 405 means that the collection already exists (Method Not Allowed)
+
+			// 409 should only be returned if a parent path does not exists (eg. when trying to create a/b/c when a/b does not exist)
+			// however non-compliant servers (eg. Microsoft IIS) also return this code when the directory already exists. So here, if
+			// we get this code, verify that indeed the directory already exists and exit if it does.
+			if (error.code === 409) {
+				const stat = await this.stat(path);
+				if (stat) return;
+			}
+			
+			throw error;
 		}
 	}
 
@@ -275,6 +339,7 @@ class FileApiDriverWebDav {
 	async move(oldPath, newPath) {
 		await this.api().exec('MOVE', oldPath, null, {
 			'Destination': this.api().baseUrl() + '/' + newPath,
+			'Overwrite': 'T',
 		});
 	}
 

ReactNativeClient/lib/models/Setting.js

@@ -106,6 +106,11 @@ class Setting extends BaseModel {
 			'sync.5.path': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('nextcloud') }, public: true, label: () => _('Nexcloud WebDAV URL') },
 			'sync.5.username': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('nextcloud') }, public: true, label: () => _('Nexcloud username') },
 			'sync.5.password': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('nextcloud') }, public: true, label: () => _('Nexcloud password'), secure: true },
+
+			'sync.6.path': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('webdav') }, public: true, label: () => _('WebDAV URL') },
+			'sync.6.username': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('webdav') }, public: true, label: () => _('WebDAV username') },
+			'sync.6.password': { value: '', type: Setting.TYPE_STRING, show: (settings) => { return settings['sync.target'] == SyncTargetRegistry.nameToId('webdav') }, public: true, label: () => _('WebDAV password'), secure: true },
+
 			'sync.3.auth': { value: '', type: Setting.TYPE_STRING, public: false },
 			'sync.4.auth': { value: '', type: Setting.TYPE_STRING, public: false },
 			'sync.1.context': { value: '', type: Setting.TYPE_STRING, public: false },

ReactNativeClient/lib/synchronizer.js

@@ -450,7 +450,7 @@ class Synchronizer {
 					if (!BaseItem.isSystemPath(remote.path)) continue; // The delta API might return things like the .sync, .resource or the root folder
 
 					const loadContent = async () => {
-						content = await this.api().get(path);
+						let content = await this.api().get(path);
 						if (!content) return null;
 						return await BaseItem.unserialize(content);
 					}

ReactNativeClient/root.js

@@ -52,9 +52,11 @@ const SyncTargetOneDrive = require('lib/SyncTargetOneDrive.js');
 const SyncTargetFilesystem = require('lib/SyncTargetFilesystem.js');
 const SyncTargetOneDriveDev = require('lib/SyncTargetOneDriveDev.js');
 const SyncTargetNextcloud = require('lib/SyncTargetNextcloud.js');
+const SyncTargetWebDAV = require('lib/SyncTargetWebDAV.js');
 SyncTargetRegistry.addClass(SyncTargetOneDrive);
 SyncTargetRegistry.addClass(SyncTargetOneDriveDev);
 SyncTargetRegistry.addClass(SyncTargetNextcloud);
+SyncTargetRegistry.addClass(SyncTargetWebDAV);
 
 // Disabled because not fully working
 //SyncTargetRegistry.addClass(SyncTargetFilesystem);