All: Refactored filesystem sync driver to support mobile

CliClient/app/main.js

@@ -23,12 +23,14 @@ const { Logger } = require('lib/logger.js');
 const { FsDriverNode } = require('lib/fs-driver-node.js');
 const { shimInit } = require('lib/shim-init-node.js');
 const { _ } = require('lib/locale.js');
+const { FileApiDriverLocal } = require('lib/file-api-driver-local.js');
 const EncryptionService = require('lib/services/EncryptionService');
 
 const fsDriver = new FsDriverNode();
 Logger.fsDriver_ = fsDriver;
 Resource.fsDriver_ = fsDriver;
 EncryptionService.fsDriver_ = fsDriver;
+FileApiDriverLocal.fsDriver_ = fsDriver;
 
 // That's not good, but it's to avoid circular dependency issues
 // in the BaseItem class.

CliClient/tests/test-utils.js

@@ -38,6 +38,7 @@ const fsDriver = new FsDriverNode();
 Logger.fsDriver_ = fsDriver;
 Resource.fsDriver_ = fsDriver;
 EncryptionService.fsDriver_ = fsDriver;
+FileApiDriverLocal.fsDriver_ = fsDriver;
 
 const logDir = __dirname + '/../tests/logs';
 fs.mkdirpSync(logDir, 0o755);
@@ -142,25 +143,6 @@ async function setupDatabase(id = null) {
 
 	BaseModel.db_ = databases_[id];
 	await Setting.load();
-	//return setupDatabase(id);
-
-
-
-	// return databases_[id].open({ name: filePath }).then(() => {
-	// 	BaseModel.db_ = databases_[id];
-	// 	return setupDatabase(id);
-	// });
-
-
-	// return fs.unlink(filePath).catch(() => {
-	// 	// Don't care if the file doesn't exist
-	// }).then(() => {
-	// 	databases_[id] = new JoplinDatabase(new DatabaseDriverNode());
-	// 	return databases_[id].open({ name: filePath }).then(() => {
-	// 		BaseModel.db_ = databases_[id];
-	// 		return setupDatabase(id);
-	// 	});
-	// });
 }
 
 function resourceDir(id = null) {

ElectronClient/app/main-html.js

@@ -17,11 +17,13 @@ const { FsDriverNode } = require('lib/fs-driver-node.js');
 const { shimInit } = require('lib/shim-init-node.js');
 const EncryptionService = require('lib/services/EncryptionService');
 const { bridge } = require('electron').remote.require('./bridge');
+const { FileApiDriverLocal } = require('lib/file-api-driver-local.js');
 
 const fsDriver = new FsDriverNode();
 Logger.fsDriver_ = fsDriver;
 Resource.fsDriver_ = fsDriver;
 EncryptionService.fsDriver_ = fsDriver;
+FileApiDriverLocal.fsDriver_ = fsDriver;
 
 // That's not good, but it's to avoid circular dependency issues
 // in the BaseItem class.

ReactNativeClient/lib/file-api-driver-local.js

@@ -26,66 +26,55 @@ class FileApiDriverLocal {
 		return output;
 	}
 
-	stat(path) {
+	fsDriver() {
-		return new Promise((resolve, reject) => {
+		if (!FileApiDriverLocal.fsDriver_) throw new Error('FileApiDriverLocal.fsDriver_ not set!');
-			fs.stat(path, (error, s) => {
+		return FileApiDriverLocal.fsDriver_;
-				if (error) {
-					if (error.code == 'ENOENT') {
-						resolve(null);
-					} else {
-						reject(this.fsErrorToJsError_(error));
-					}
-					return;
-				}
-				resolve(this.metadataFromStats_(path, s));
-			});			
-		});
 	}
 
-	statTimeToTimestampMs_(time) {
+	async stat(path) {
-		let m = moment(time, 'YYYY-MM-DDTHH:mm:ss.SSSZ');
+		try {
-		if (!m.isValid()) {
+			const s = await this.fsDriver().stat(path);
-			throw new Error('Invalid date: ' + time);
+			return this.metadataFromStat_(s);
+		} catch (error) {
+			if (error.code == 'ENOENT') return null;
+			throw this.fsErrorToJsError_(error);
 		}
-		return m.toDate().getTime();
 	}
 
-	metadataFromStats_(path, stats) {
+	metadataFromStat_(stat) {
 		return {
-			path: path,
+			path: stat.path,
-			created_time: this.statTimeToTimestampMs_(stats.birthtime),
+			created_time: stat.birthtime.getTime(),
-			updated_time: this.statTimeToTimestampMs_(stats.mtime),
+			updated_time: stat.mtime.getTime(),
-			created_time_orig: stats.birthtime,
+			created_time_orig: stat.birthtime,
-			updated_time_orig: stats.mtime,
+			updated_time_orig: stat.mtime,
-			isDir: stats.isDirectory(),
+			isDir: stat.isDirectory(),
 		};
 	}
 
-	setTimestamp(path, timestampMs) {
+	metadataFromStats_(stats) {
-		return new Promise((resolve, reject) => {
+		let output = [];
-			let t = Math.floor(timestampMs / 1000);
+		for (let i = 0; i < stats.length; i++) {
-			fs.utimes(path, t, t, (error) => {
+			const mdStat = this.metadataFromStat_(stats[i]);
-				if (error) {
+			output.push(mdStat);
-					reject(this.fsErrorToJsError_(error));
+		}
-					return;
+		return output;
-				}
+	}
-				resolve();
+
-			});
+	async setTimestamp(path, timestampMs) {
-		});
+		try {
+			await this.fsDriver().setTimestamp(path, new Date(timestampMs));
+		} catch (error) {
+			throw this.fsErrorToJsError_(error);
+		}
 	}
 
 	async delta(path, options) {
 		const itemIds = await options.allItemIdsHandler();
 
 		try {
-			let items = await fs.readdir(path);
+			const stats = await this.fsDriver().readDirStats(path);
-			let output = [];
+			let output = this.metadataFromStats_(stats);
-			for (let i = 0; i < items.length; i++) {
-				let stat = await this.stat(path + '/' + items[i]);
-				if (!stat) continue; // Has been deleted between the readdir() call and now
-				stat.path = items[i];
-				output.push(stat);
-			}
 
 			if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided');
 
@@ -123,14 +112,8 @@ class FileApiDriverLocal {
 
 	async list(path, options) {
 		try {
-			let items = await fs.readdir(path);
+			const stats = await this.fsDriver().readDirStats(path);
-			let output = [];
+			const output = this.metadataFromStats_(stats);
-			for (let i = 0; i < items.length; i++) {
-				let stat = await this.stat(path + '/' + items[i]);
-				if (!stat) continue; // Has been deleted between the readdir() call and now
-				stat.path = items[i];
-				output.push(stat);
-			}
 
 			return {
 				items: output,
@@ -147,9 +130,11 @@ class FileApiDriverLocal {
 
 		try {
 			if (options.target === 'file') {
-				output = await fs.copy(path, options.path, { overwrite: true });
+				//output = await fs.copy(path, options.path, { overwrite: true });
+				output = await this.fsDriver().copy(path, options.path);
 			} else {
-				output = await fs.readFile(path, options.encoding);
+				//output = await fs.readFile(path, options.encoding);
+				output = await this.fsDriver().readFile(path, options.encoding);
 			}
 		} catch (error) {
 			if (error.code == 'ENOENT') return null;
@@ -159,78 +144,107 @@ class FileApiDriverLocal {
 		return output;
 	}
 
-	mkdir(path) {
+	async mkdir(path) {
-		return new Promise((resolve, reject) => {
+		if (await this.fsDriver().exists(path)) return;
-			fs.exists(path, (exists) => {
+
-				if (exists) {
+		try {
-					resolve();
+			await this.fsDriver().mkdir(path);
-					return;
+		} catch (error) {
-				}
+			throw this.fsErrorToJsError_(error);
+		}
+
+		// return new Promise((resolve, reject) => {
+		// 	fs.exists(path, (exists) => {
+		// 		if (exists) {
+		// 			resolve();
+		// 			return;
+		// 		}
 			
-				fs.mkdirp(path, (error) => {
+		// 		fs.mkdirp(path, (error) => {
-					if (error) {
+		// 			if (error) {
-						reject(this.fsErrorToJsError_(error));
+		// 				reject(this.fsErrorToJsError_(error));
-					} else {
+		// 			} else {
-						resolve();
+		// 				resolve();
-					}
+		// 			}
-				});
+		// 		});
-			});
+		// 	});
-		});
+		// });
 	}
 
 	async put(path, content, options = null) {
 		if (!options) options = {};
 
-		if (options.source === 'file') content = await fs.readFile(options.path);
+		try {
+			if (options.source === 'file') {
+				await this.fsDriver().copy(options.path, path);
+				return;
+			}
+		
+			await this.fsDriver().writeFile(path, content, 'utf8');
+		} catch (error) {
+			throw this.fsErrorToJsError_(error);
+		}
 
-		return new Promise((resolve, reject) => {
+		// if (!options) options = {};
-			fs.writeFile(path, content, function(error) {
+
-				if (error) {
+		// if (options.source === 'file') content = await fs.readFile(options.path);
-					reject(this.fsErrorToJsError_(error));
+
-				} else {
+		// return new Promise((resolve, reject) => {
-					resolve();
+		// 	fs.writeFile(path, content, function(error) {
-				}
+		// 		if (error) {
-			});
+		// 			reject(this.fsErrorToJsError_(error));
-		});
+		// 		} else {
+		// 			resolve();
+		// 		}
+		// 	});
+		// });
 	}
 
-	delete(path) {
+	async delete(path) {
-		return new Promise((resolve, reject) => {
+		try {
-			fs.unlink(path, function(error) {
+			await this.fsDriver().unlink(path);
-				if (error) {
+		} catch (error) {
-					if (error && error.code == 'ENOENT') {
+			throw this.fsErrorToJsError_(error);
-						// File doesn't exist - it's fine
+		}
-						resolve();
+
-					} else {
+		// return new Promise((resolve, reject) => {
-						reject(this.fsErrorToJsError_(error));
+		// 	fs.unlink(path, function(error) {
-					}
+		// 		if (error) {
-				} else {
+		// 			if (error && error.code == 'ENOENT') {
-					resolve();
+		// 				// File doesn't exist - it's fine
-				}
+		// 				resolve();
-			});
+		// 			} else {
-		});
+		// 				reject(this.fsErrorToJsError_(error));
+		// 			}
+		// 		} else {
+		// 			resolve();
+		// 		}
+		// 	});
+		// });
 	}
 
 	async move(oldPath, newPath) {
-		let lastError = null;
+		return this.fsDriver().move(oldPath, newPath);
-		
-		for (let i = 0; i < 5; i++) {
-			try {
-				let output = await fs.move(oldPath, newPath, { overwrite: true });
-				return output;
-			} catch (error) {
-				lastError = error;
-				// Normally cannot happen with the `overwrite` flag but sometime it still does.
-				// In this case, retry.
-				if (error.code == 'EEXIST') {
-					await time.sleep(1);
-					continue;
-				}
-				throw this.fsErrorToJsError_(error);
-			}
-		}
 
-		throw lastError;
+		// let lastError = null;
+		
+		// for (let i = 0; i < 5; i++) {
+		// 	try {
+		// 		let output = await fs.move(oldPath, newPath, { overwrite: true });
+		// 		return output;
+		// 	} catch (error) {
+		// 		lastError = error;
+		// 		// Normally cannot happen with the `overwrite` flag but sometime it still does.
+		// 		// In this case, retry.
+		// 		if (error.code == 'EEXIST') {
+		// 			await time.sleep(1);
+		// 			continue;
+		// 		}
+		// 		throw this.fsErrorToJsError_(error);
+		// 	}
+		// }
+
+		// throw lastError;
 	}
 
 	format() {

ReactNativeClient/lib/fs-driver-node.js

@@ -1,4 +1,5 @@
 const fs = require('fs-extra');
+const { time } = require('lib/time-utils.js');
 
 class FsDriverNode {
 
@@ -15,14 +16,62 @@ class FsDriverNode {
 		return fs.writeFile(path, buffer);
 	}
 
-	move(source, dest) {
+	writeFile(path, string, encoding = 'base64') {
-		return fs.move(source, dest, { overwrite: true });
+		return fs.writeFile(path, string, { encoding: encoding });
+	}
+
+	async move(source, dest) {
+		let lastError = null;
+
+		for (let i = 0; i < 5; i++) {
+			try {
+				const output = await fs.move(source, dest, { overwrite: true });
+				return output;
+			} catch (error) {
+				lastError = error;
+				// Normally cannot happen with the `overwrite` flag but sometime it still does.
+				// In this case, retry.
+				if (error.code == 'EEXIST') {
+					await time.sleep(1);
+					continue;
+				}
+				throw this.fsErrorToJsError_(error);
+			}
+		}
+
+		throw lastError;
 	}
 
 	exists(path) {
 		return fs.pathExists(path);
 	}
 
+	async mkdir(path) {
+		return fs.mkdirp(path);
+	}
+
+	async stat(path) {
+		const s = await fs.stat(path);
+		s.path = path;
+		return s;
+	}
+
+	async setTimestamp(path, timestampDate) {
+		return fs.utimes(path, timestampDate, timestampDate);
+	}
+
+	async readDirStats(path) {
+		let items = await fs.readdir(path);
+		let output = [];
+		for (let i = 0; i < items.length; i++) {
+			let stat = await this.stat(path + '/' + items[i]);
+			if (!stat) continue; // Has been deleted between the readdir() call and now
+			stat.path = stat.path.substr(path.length + 1);
+			output.push(stat);
+		}
+		return output;
+	}
+
 	open(path, mode) {
 		return fs.open(path, mode);
 	}
@@ -31,8 +80,14 @@ class FsDriverNode {
 		return fs.close(handle);
 	}
 
-	readFile(path) {
+	readFile(path, encoding = 'utf8') {
-		return fs.readFile(path);
+		if (encoding === 'Buffer') return fs.readFile(path); // Returns the raw buffer
+		return fs.readFile(path, encoding);
+	}
+
+	// Always overwrite destination
+	async copy(source, dest) {
+		return fs.copy(source, dest, { overwrite: true });
 	}
 
 	async unlink(path) {

ReactNativeClient/lib/fs-driver-rn.js

@@ -10,6 +10,10 @@ class FsDriverRN {
 		return RNFS.appendFile(path, string, encoding);
 	}
 
+	writeFile(path, string, encoding = 'base64') {
+		return RNFS.writeFile(path, string, encoding);
+	}
+
 	writeBinaryFile(path, content) {
 		throw new Error('Not implemented');
 	}
@@ -22,11 +26,30 @@ class FsDriverRN {
 		return RNFS.exists(path);
 	}
 
+	async mkdir(path) {
+		return fs.mkdir(path);
+	}
+
+	async stat(path) {
+		const r = await RNFS.stat(path);
+		// Returns a format compatible with Node.js format
+		return {
+			birthtime: r.ctime, // Confusingly, "ctime" normally means "change time" but here it's used as "creation time"
+			mtime: r.mtime,
+			isDirectory: () => return r.isDirectory(),
+			path: path,
+		};  
+	}
+
+	async setTimestamp(path, timestampDate) {
+		return RNFS.touch(path, timestampDate);
+	}
+
 	async open(path, mode) {
 		// Note: RNFS.read() doesn't provide any way to know if the end of file has been reached.
 		// So instead we stat the file here and use stat.size to manually check for end of file.
 		// Bug: https://github.com/itinance/react-native-fs/issues/342
-		const stat = await RNFS.stat(path);
+		const stat = await this.stat(path);
 		return {
 			path: path,
 			offset: 0,
@@ -39,8 +62,23 @@ class FsDriverRN {
 		return null;
 	}
 
-	readFile(path) {
+	readFile(path, encoding = 'utf8') {
-		throw new Error('Not implemented');
+		if (encoding === 'Buffer') throw new Error('Raw buffer output not supported for FsDriverRN.readFile');
+		return RNFS.readFile(path, encoding);
+	}
+
+	// Always overwrite destination
+	async copy(source, dest) {
+		let retry = false;
+		try {
+			await RNFS.copyFile(source, dest);
+		} catch (error) {
+			// On iOS it will throw an error if the file already exist
+			retry = true;
+			await this.unlink(dest);
+		}
+
+		if (retry) await RNFS.copyFile(source, dest);
 	}
 
 	async unlink(path) {

ReactNativeClient/lib/models/Resource.js

@@ -120,7 +120,7 @@ class Resource extends BaseItem {
 	}
 
 	static async content(resource) {
-		return this.fsDriver().readFile(this.fullPath(resource));
+		return this.fsDriver().readFile(this.fullPath(resource), 'Buffer');
 	}
 
 	static setContent(resource, content) {