Merge branch 'master' into webdav

CliClient/tests/synchronizer.js

@@ -74,11 +74,11 @@ async function localItemsSameAsRemote(locals, expect) {
 			expect(!!remote).toBe(true);
 			if (!remote) continue;
 
-			if (syncTargetId() == SyncTargetRegistry.nameToId('filesystem')) {
-				expect(remote.updated_time).toBe(Math.floor(dbItem.updated_time / 1000) * 1000);
-			} else {
-				expect(remote.updated_time).toBe(dbItem.updated_time);
-			}
+			// if (syncTargetId() == SyncTargetRegistry.nameToId('filesystem')) {
+			// 	expect(remote.updated_time).toBe(Math.floor(dbItem.updated_time / 1000) * 1000);
+			// } else {
+			// 	expect(remote.updated_time).toBe(dbItem.updated_time);
+			// }
 
 			let remoteContent = await fileApi().get(path);
 			remoteContent = dbItem.type_ == BaseModel.TYPE_NOTE ? await Note.unserialize(remoteContent) : await Folder.unserialize(remoteContent);
@@ -683,12 +683,12 @@ describe('Synchronizer', function() {
 
 		await switchClient(2);
 
-		synchronizer().debugFlags_ = ['cancelDeltaLoop2'];		
+		synchronizer().testingHooks_ = ['cancelDeltaLoop2'];		
 		let context = await synchronizer().start();
 		let notes = await Note.all();
 		expect(notes.length).toBe(0);
 
-		synchronizer().debugFlags_ = [];
+		synchronizer().testingHooks_ = [];
 		await synchronizer().start({ context: context });
 		notes = await Note.all();
 		expect(notes.length).toBe(1);
@@ -702,9 +702,9 @@ describe('Synchronizer', function() {
 		let disabledItems = await BaseItem.syncDisabledItems(syncTargetId());
 		expect(disabledItems.length).toBe(0);
 		await Note.save({ id: noteId, title: "un mod", });
-		synchronizer().debugFlags_ = ['rejectedByTarget'];
+		synchronizer().testingHooks_ = ['rejectedByTarget'];
 		await synchronizer().start();
-		synchronizer().debugFlags_ = [];
+		synchronizer().testingHooks_ = [];
 		await synchronizer().start(); // Another sync to check that this item is now excluded from sync
 
 		await switchClient(2);

ElectronClient/app/gui/EncryptionConfigScreen.jsx

@@ -121,7 +121,7 @@ class EncryptionConfigScreenComponent extends React.Component {
 			}
 		}
 
-		const decryptedItemsInfo = this.props.encryptionEnabled ? <p style={theme.textStyle}>{shared.decryptedStatText(this)}</p> : null;
+		const decryptedItemsInfo = <p style={theme.textStyle}>{shared.decryptedStatText(this)}</p>;
 		const toggleButton = <button onClick={() => { onToggleButtonClick() }}>{this.props.encryptionEnabled ? _('Disable encryption') : _('Enable encryption')}</button>
 
 		let masterKeySection = null;

ElectronClient/app/main-html.js

@@ -3,6 +3,15 @@
 // Make it possible to require("/lib/...") without specifying full path
 require('app-module-path').addPath(__dirname);
 
+// Disable React message in console "Download the React DevTools for a better development experience"
+// https://stackoverflow.com/questions/42196819/disable-hide-download-the-react-devtools#42196820
+__REACT_DEVTOOLS_GLOBAL_HOOK__ = {
+	supportsFiber: true,
+	inject: function() {},
+	onCommitFiberRoot: function() {},
+	onCommitFiberUnmount: function() {},
+};
+
 const { app } = require('./app.js');
 const Folder = require('lib/models/Folder.js');
 const Resource = require('lib/models/Resource.js');

ReactNativeClient/lib/MdToHtml.js

@@ -73,7 +73,7 @@ class MdToHtml {
 
 	renderImage_(attrs, options) {
 		const loadResource = async (id) => {
-			console.info('Loading resource: ' + id);
+			// console.info('Loading resource: ' + id);
 
 			// Initially set to to an empty object to make
 			// it clear that it is being loaded. Otherwise

ReactNativeClient/lib/file-api-driver-local.js

@@ -65,21 +65,89 @@ class FileApiDriverLocal {
 		}
 	}
 
+	contextFromOptions_(options) {
+		let output = {
+			timestamp: 0,
+			filesAtTimestamp: [],
+			statsCache: null,
+		};
+
+		if (!options || !options.context) return output;
+		const d = new Date(options.context.timestamp);
+
+		output.timestamp = isNaN(d.getTime()) ? 0 : options.context.timestamp;
+		output.filesAtTimestamp = Array.isArray(options.context.filesAtTimestamp) ? options.context.filesAtTimestamp.slice() : [];
+		output.statsCache = options.context && options.context.statsCache ? options.context.statsCache : null;
+
+		return output;
+	}
+
 	async delta(path, options) {
+		const outputLimit = 1000;
 		const itemIds = await options.allItemIdsHandler();
 
 		try {
+			const context = this.contextFromOptions_(options);
+
+			let newContext = {
+				timestamp: context.timestamp,
+				filesAtTimestamp: context.filesAtTimestamp.slice(),
+				statsCache: context.statsCache,
+			};
+
+			// Stats are cached until all items have been processed (until hasMore is false)
+			if (newContext.statsCache === null) {
 				const stats = await this.fsDriver().readDirStats(path);
-			let output = this.metadataFromStats_(stats);
+				newContext.statsCache = this.metadataFromStats_(stats);
+				newContext.statsCache.sort(function(a, b) {
+					return a.updated_time - b.updated_time;
+				});
+			}
+
+			let output = [];
+
+			// Find out which files have been changed since the last time. Note that we keep
+			// both the timestamp of the most recent change, *and* the items that exactly match
+			// this timestamp. This to handle cases where an item is modified while this delta
+			// function is running. For example:
+			// t0: Item 1 is changed
+			// t0: Sync items - run delta function
+			// t0: While delta() is running, modify Item 2
+			// Since item 2 was modified within the same millisecond, it would be skipped in the
+			// next sync if we relied exclusively on a timestamp.
+			for (let i = 0; i < newContext.statsCache.length; i++) {
+				const stat = newContext.statsCache[i];
+
+				if (stat.isDir) continue;
+
+				if (stat.updated_time < context.timestamp) continue;
+
+				// Special case for items that exactly match the timestamp
+				if (stat.updated_time === context.timestamp) {
+					if (context.filesAtTimestamp.indexOf(stat.path) >= 0) continue;
+				}
+
+				if (stat.updated_time > newContext.timestamp) {
+					newContext.timestamp = stat.updated_time;
+					newContext.filesAtTimestamp = [];
+				}
+
+				newContext.filesAtTimestamp.push(stat.path);
+				output.push(stat);
+
+				if (output.length >= outputLimit) break;
+			}
 
 			if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided');
 
 			let deletedItems = [];
 			for (let i = 0; i < itemIds.length; i++) {
+				if (output.length + deletedItems.length >= outputLimit) break;
+
 				const itemId = itemIds[i];
 				let found = false;
-				for (let j = 0; j < output.length; j++) {
-					const item = output[j];
+				for (let j = 0; j < newContext.statsCache.length; j++) {
+					const item = newContext.statsCache[j];
 					if (BaseItem.pathToId(item.path) == itemId) {
 						found = true;
 						break;
@@ -96,9 +164,12 @@ class FileApiDriverLocal {
 
 			output = output.concat(deletedItems);
 
+			const hasMore = output.length >= outputLimit;
+			if (!hasMore) newContext.statsCache = null;
+
 			return {
-				hasMore: false,
-				context: null,
+				hasMore: hasMore,
+				context: newContext,
 				items: output,
 			};
 		} catch(error) {

ReactNativeClient/lib/file-api.js

@@ -1,5 +1,7 @@
 const { isHidden } = require('lib/path-utils.js');
 const { Logger } = require('lib/logger.js');
+const { shim } = require('lib/shim');
+const JoplinError = require('lib/JoplinError');
 
 class FileApi {
 
@@ -10,6 +12,10 @@ class FileApi {
 		this.syncTargetId_ = null;
 	}
 
+	fsDriver() {
+		return shim.fsDriver();
+	}
+
 	driver() {
 		return this.driver_;
 	}
@@ -83,8 +89,13 @@ class FileApi {
 		return this.driver_.get(this.fullPath_(path), options);
 	}
 
-	put(path, content, options = null) {
-		this.logger().debug('put ' + this.fullPath_(path));
+	async put(path, content, options = null) {
+		this.logger().debug('put ' + this.fullPath_(path), options);
+		
+		if (options && options.source === 'file') {
+			if (!await this.fsDriver().exists(options.path)) throw new JoplinError('File not found: ' + options.path, 'fileNotFound');
+		}
+
 		return this.driver_.put(this.fullPath_(path), content, options);
 	}
 

ReactNativeClient/lib/shim-init-node.js

@@ -4,14 +4,20 @@ const { GeolocationNode } = require('lib/geolocation-node.js');
 const { FileApiDriverLocal } = require('lib/file-api-driver-local.js');
 const { time } = require('lib/time-utils.js');
 const { setLocale, defaultLocale, closestSupportedLocale } = require('lib/locale.js');
+const { FsDriverNode } = require('lib/fs-driver-node.js');
 
 function shimInit() {
-	shim.fs = fs;
+	shim.fsDriver = () => { throw new Error('Not implemented') }
 	shim.FileApiDriverLocal = FileApiDriverLocal;
 	shim.Geolocation = GeolocationNode;
 	shim.FormData = require('form-data');
 	shim.sjclModule = require('lib/vendor/sjcl.js');
 
+	shim.fsDriver = () => {
+		if (!shim.fsDriver_) shim.fsDriver_ = new FsDriverNode();
+		return shim.fsDriver_;
+	}
+
 	shim.randomBytes = async (count) => {
 		const buffer = require('crypto').randomBytes(count);
 		return Array.from(buffer);

ReactNativeClient/lib/shim-init-react.js

@@ -3,6 +3,7 @@ const { GeolocationReact } = require('lib/geolocation-react.js');
 const { PoorManIntervals } = require('lib/poor-man-intervals.js');
 const RNFetchBlob = require('react-native-fetch-blob').default;
 const { generateSecureRandom }  = require('react-native-securerandom');
+const FsDriverRN = require('lib/fs-driver-rn.js').FsDriverRN;
 
 function shimInit() {
 	shim.Geolocation = GeolocationReact;
@@ -10,6 +11,11 @@ function shimInit() {
 	shim.clearInterval = PoorManIntervals.clearInterval;
 	shim.sjclModule = require('lib/vendor/sjcl-rn.js');
 
+	shim.fsDriver = () => {
+		if (!shim.fsDriver_) shim.fsDriver_ = new FsDriverRN();
+		return shim.fsDriver_;
+	}
+
 	shim.randomBytes = async (count) => {
 		const randomBytes = await generateSecureRandom(count);
 		let temp = [];

ReactNativeClient/lib/shim.js

@@ -107,7 +107,7 @@ shim.fetchWithRetry = async function(fetchFn, options = null) {
 shim.nativeFetch_ = typeof fetch !== 'undefined' ? fetch : null;
 shim.fetch = () => { throw new Error('Not implemented'); }
 shim.FormData = typeof FormData !== 'undefined' ? FormData : null;
-shim.fs = null;
+shim.fsDriver = () => { throw new Error('Not implemented') }
 shim.FileApiDriverLocal = null;
 shim.readLocalFileBase64 = (path) => { throw new Error('Not implemented'); }
 shim.uploadBlob = () => { throw new Error('Not implemented'); }

ReactNativeClient/lib/synchronizer.js

@@ -10,7 +10,7 @@ const { time } = require('lib/time-utils.js');
 const { Logger } = require('lib/logger.js');
 const { _ } = require('lib/locale.js');
 const { shim } = require('lib/shim.js');
-const moment = require('moment');
+const JoplinError = require('lib/JoplinError');
 
 class Synchronizer {
 
@@ -27,7 +27,7 @@ class Synchronizer {
 
 		// Debug flags are used to test certain hard-to-test conditions
 		// such as cancelling in the middle of a loop.
-		this.debugFlags_ = [];
+		this.testingHooks_ = [];
 
 		this.onProgress_ = function(s) {};
 		this.progressReport_ = {};
@@ -279,7 +279,7 @@ class Synchronizer {
 							const localResourceContentPath = result.path;
 							await this.api().put(remoteContentPath, null, { path: localResourceContentPath, source: 'file' });
 						} catch (error) {
-							if (error && error.code === 'rejectedByTarget') {
+							if (error && ['rejectedByTarget', 'fileNotFound'].indexOf(error.code) >= 0) {
 								await handleCannotSyncItem(syncTargetId, local, error.message);
 								action = null;
 							} else {
@@ -303,11 +303,7 @@ class Synchronizer {
 
 						let canSync = true;
 						try {
-							if (this.debugFlags_.indexOf('rejectedByTarget') >= 0) {
-								const error = new Error('Testing rejectedByTarget');
-								error.code = 'rejectedByTarget';
-								throw error;
-							}
+							if (this.testingHooks_.indexOf('rejectedByTarget') >= 0) throw new JoplinError('Testing rejectedByTarget', 'rejectedByTarget');
 							const content = await ItemClass.serializeForSync(local);
 							await this.api().put(path, content);
 						} catch (error) {
@@ -334,7 +330,12 @@ class Synchronizer {
 						// change is uniquely identified. Leaving it like this for now.
 
 						if (canSync) {
-							await this.api().setTimestamp(path, local.updated_time);
+							// 2018-01-21: Setting timestamp is not needed because the delta() logic doesn't rely
+							// on it (instead it uses a more reliable `context` object) and the itemsThatNeedSync loop
+							// above also doesn't use it because it fetches the whole remote object and read the
+							// more reliable 'updated_time' property. Basically remote.updated_time is deprecated.
+
+							// await this.api().setTimestamp(path, local.updated_time);
 							await ItemClass.saveSyncTime(syncTargetId, local, local.updated_time);
 						}
 
@@ -449,7 +450,7 @@ class Synchronizer {
 				this.logSyncOperation('fetchingTotal', null, null, 'Fetching delta items from sync target', remotes.length);
 
 				for (let i = 0; i < remotes.length; i++) {
-					if (this.cancelling() || this.debugFlags_.indexOf('cancelDeltaLoop2') >= 0) {
+					if (this.cancelling() || this.testingHooks_.indexOf('cancelDeltaLoop2') >= 0) {
 						hasCancelled = true;
 						break;
 					}