Merge branch 'sync-limits'

CliClient/tests/synchronizer.js

@@ -629,4 +629,32 @@ describe('Synchronizer', function() {
 		done();
 	});
 
+	it('items should skip items that cannot be synced', async (done) => {
+		let folder1 = await Folder.save({ title: "folder1" });
+		let note1 = await Note.save({ title: "un", is_todo: 1, parent_id: folder1.id });
+		const noteId = note1.id;
+		await synchronizer().start();
+		let disabledItems = await BaseItem.syncDisabledItems();
+		expect(disabledItems.length).toBe(0);
+		await Note.save({ id: noteId, title: "un mod", });
+		synchronizer().debugFlags_ = ['cannotSync'];
+		await synchronizer().start();
+		synchronizer().debugFlags_ = [];
+		await synchronizer().start(); // Another sync to check that this item is now excluded from sync
+
+		await switchClient(2);
+
+		await synchronizer().start();
+		let notes = await Note.all();
+		expect(notes.length).toBe(1);
+		expect(notes[0].title).toBe('un');
+
+		await switchClient(1);
+
+		disabledItems = await BaseItem.syncDisabledItems();
+		expect(disabledItems.length).toBe(1);
+
+		done();
+	});
+
 });

CliClient/tests/test-utils.js

@@ -43,8 +43,9 @@ const syncDir = __dirname + '/../tests/sync';
 const sleepTime = syncTargetId_ == SyncTargetRegistry.nameToId('filesystem') ? 1001 : 400;
 
 const logger = new Logger();
+logger.addTarget('console');
 logger.addTarget('file', { path: logDir + '/log.txt' });
-logger.setLevel(Logger.LEVEL_DEBUG);
+logger.setLevel(Logger.LEVEL_WARN);
 
 BaseItem.loadClass('Note', Note);
 BaseItem.loadClass('Folder', Folder);

ElectronClient/app/app.js

@@ -259,6 +259,14 @@ class Application extends BaseApplication {
 			}, {
 				label: _('Tools'),
 				submenu: [{
+					label: _('Synchronisation status'),
+					click: () => {
+						this.dispatch({
+							type: 'NAV_GO',
+							routeName: 'Status',
+						});
+					}
+				},{
 					label: _('Options'),
 					click: () => {
 						this.dispatch({

ElectronClient/app/gui/MainScreen.jsx

@@ -229,8 +229,8 @@ class MainScreenComponent extends React.Component {
 		}
 	}
 
-	styles(themeId, width, height) {
-		const styleKey = themeId + '_' + width + '_' + height;
+	styles(themeId, width, height, messageBoxVisible) {
+		const styleKey = themeId + '_' + width + '_' + height + '_' + messageBoxVisible;
 		if (styleKey === this.styleKey_) return this.styles_;
 
 		const theme = themeStyle(themeId);
@@ -239,12 +239,21 @@ class MainScreenComponent extends React.Component {
 
 		this.styles_ = {};
 
-		const rowHeight = height - theme.headerHeight;
-
 		this.styles_.header = {
 			width: width,
 		};
 
+		this.styles_.messageBox = {
+			width: width,
+			height: 30,
+			display: 'flex',
+			alignItems: 'center',
+			paddingLeft: 10,
+			backgroundColor: theme.warningBackgroundColor,
+		}
+
+		const rowHeight = height - theme.headerHeight - (messageBoxVisible ? this.styles_.messageBox.height : 0);
+
 		this.styles_.sideBar = {
 			width: Math.floor(layoutUtils.size(width * .2, 150, 300)),
 			height: rowHeight,
@@ -280,7 +289,8 @@ class MainScreenComponent extends React.Component {
 		const folders = this.props.folders;
 		const notes = this.props.notes;
 
-		const styles = this.styles(this.props.theme, style.width, style.height);
+		const styles = this.styles(this.props.theme, style.width, style.height, true);
+		const theme = themeStyle(this.props.theme);
 
 		const headerButtons = [];
 
@@ -325,6 +335,21 @@ class MainScreenComponent extends React.Component {
 			}
 		}
 
+		const onViewDisabledItemsClick = () => {
+			this.props.dispatch({
+				type: 'NAV_GO',
+				routeName: 'Status',
+			});
+		}
+
+		const messageComp = this.props.hasDisabledSyncItems ? (
+			<div style={styles.messageBox}>
+				<span style={theme.textStyle}>
+					{_('Some items cannot be synchronised.')} <a href="#" onClick={() => { onViewDisabledItemsClick() }}>{_('View them now')}</a>
+				</span>
+			</div>
+		) : null;
+
 		return (
 			<div style={style}>
 				<PromptDialog
@@ -339,6 +364,7 @@ class MainScreenComponent extends React.Component {
 					buttons={promptOptions && ('buttons' in promptOptions) ? promptOptions.buttons : null}
 					inputType={promptOptions && ('inputType' in promptOptions) ? promptOptions.inputType : null} />
 				<Header style={styles.header} showBackButton={false} buttons={headerButtons} />
+				{messageComp}
 				<SideBar style={styles.sideBar} />
 				<NoteList style={styles.noteList} />
 				<NoteText style={styles.noteText} visiblePanes={this.props.noteVisiblePanes} />
@@ -355,6 +381,7 @@ const mapStateToProps = (state) => {
 		noteVisiblePanes: state.noteVisiblePanes,
 		folders: state.folders,
 		notes: state.notes,
+		hasDisabledSyncItems: state.hasDisabledSyncItems,
 	};
 };
 

ElectronClient/app/gui/Root.jsx

@@ -8,6 +8,7 @@ const { Setting } = require('lib/models/setting.js');
 
 const { MainScreen } = require('./MainScreen.min.js');
 const { OneDriveLoginScreen } = require('./OneDriveLoginScreen.min.js');
+const { StatusScreen } = require('./StatusScreen.min.js');
 const { ImportScreen } = require('./ImportScreen.min.js');
 const { ConfigScreen } = require('./ConfigScreen.min.js');
 const { Navigator } = require('./Navigator.min.js');
@@ -75,6 +76,7 @@ class RootComponent extends React.Component {
 			OneDriveLogin: { screen: OneDriveLoginScreen, title: () => _('OneDrive Login') },
 			Import: { screen: ImportScreen, title: () => _('Import') },
 			Config: { screen: ConfigScreen, title: () => _('Options') },
+			Status: { screen: StatusScreen, title: () => _('Synchronisation Status') },
 		};
 
 		return (

ElectronClient/app/gui/StatusScreen.jsx

@@ -0,0 +1,118 @@
+const React = require('react');
+const { connect } = require('react-redux');
+const { reg } = require('lib/registry.js');
+const { Setting } = require('lib/models/setting.js');
+const { bridge } = require('electron').remote.require('./bridge');
+const { Header } = require('./Header.min.js');
+const { themeStyle } = require('../theme.js');
+const { _ } = require('lib/locale.js');
+const { ReportService } = require('lib/services/report.js');
+
+class StatusScreenComponent extends React.Component {
+
+	constructor() {
+		super();
+		this.state = {
+			report: [],
+		};
+	}
+
+	componentWillMount() {
+		this.resfreshScreen();
+	}
+
+	async resfreshScreen() {
+		const service = new ReportService();
+		const report = await service.status(Setting.value('sync.target'));
+		this.setState({ report: report });
+	}
+
+	render() {
+		const theme = themeStyle(this.props.theme);
+		const style = this.props.style;
+
+		const headerStyle = {
+			width: style.width,
+		};
+
+		const containerPadding = 10;
+
+		const containerStyle = {
+			padding: containerPadding,
+			overflowY: 'auto',
+			height: style.height - theme.headerHeight - containerPadding * 2,
+		};
+
+		function renderSectionTitleHtml(key, title) {
+			return <h2 key={'section_' + key} style={theme.h2Style}>{title}</h2>
+		}
+
+		function renderSectionHtml(key, section) {
+			let itemsHtml = [];
+
+			itemsHtml.push(renderSectionTitleHtml(section.title, section.title));
+
+			for (let n in section.body) {
+				if (!section.body.hasOwnProperty(n)) continue;
+				itemsHtml.push(<div style={theme.textStyle} key={'item_' + n}>{section.body[n]}</div>);
+			}
+
+			return (
+				<div key={key}>
+					{itemsHtml}
+				</div>
+			);
+		}
+
+		function renderBodyHtml(report) {
+			let output = [];
+			let baseStyle = {
+				paddingLeft: 6,
+				paddingRight: 6,
+				paddingTop: 2,
+				paddingBottom: 2,
+				flex: 0,
+				color: theme.color,
+				fontSize: theme.fontSize,
+			};
+
+			let sectionsHtml = [];
+
+			for (let i = 0; i < report.length; i++) {
+				let section = report[i];
+				if (!section.body.length) continue;
+				sectionsHtml.push(renderSectionHtml(i, section));
+			}
+
+			return (
+				<div>
+					{sectionsHtml}
+				</div>
+			);
+		}
+
+		let body = renderBodyHtml(this.state.report);
+		
+		return (
+			<div style={style}>
+				<Header style={headerStyle} />
+				<div style={containerStyle}>
+					{body}
+				</div>
+			</div>
+		);
+	}
+
+}
+
+const mapStateToProps = (state) => {
+	return {
+		theme: state.settings.theme,
+		settings: state.settings,
+		locale: state.settings.locale,
+	};
+};
+
+const StatusScreen = connect(mapStateToProps)(StatusScreenComponent);
+
+module.exports = { StatusScreen };

ElectronClient/app/style.css

@@ -9,6 +9,19 @@ body, textarea {
 	overflow: hidden;
 }
 
+table {
+	border-collapse: collapse;
+}
+
+table th {
+	text-align: left;
+}
+
+table td, table th {
+	padding: .5em;
+	border: 1px solid #ccc;
+}
+
 /* By default, the Ice Editor displays invalid characters, such as non-breaking spaces
    as red boxes, but since those are actually valid characters and common in imported
    Evernote data, we hide them here. */

ElectronClient/app/theme.js

@@ -25,6 +25,8 @@ const globalStyle = {
 	selectedColor2: "#5A4D70",
 	colorError2: "#ff6c6c",
 
+	warningBackgroundColor: "#FFD08D",
+
 	headerHeight: 35,
 	headerButtonHPadding: 6,
 
@@ -69,6 +71,9 @@ globalStyle.textStyle2 = Object.assign({}, globalStyle.textStyle, {
 	color: globalStyle.color2,
 });
 
+globalStyle.h2Style = Object.assign({}, globalStyle.textStyle);
+globalStyle.h2Style.fontSize *= 1.3;
+
 let themeCache_ = {};
 
 function themeStyle(theme) {

ReactNativeClient/lib/file-api-driver-onedrive.js

@@ -123,15 +123,27 @@ class FileApiDriverOneDrive {
 		return this.makeItem_(item);
 	}
 
-	put(path, content, options = null) {
+	async put(path, content, options = null) {
 		if (!options) options = {};
 
-		if (options.source == 'file') {
-			return this.api_.exec('PUT', this.makePath_(path) + ':/content', null, null, options);
-		} else {
-			options.headers = { 'Content-Type': 'text/plain' };
-			return this.api_.exec('PUT', this.makePath_(path) + ':/content', null, content, options);
+		let response = null;
+
+		try {
+			if (options.source == 'file') {
+				response = await this.api_.exec('PUT', this.makePath_(path) + ':/content', null, null, options);
+			} else {
+				options.headers = { 'Content-Type': 'text/plain' };
+				response = await this.api_.exec('PUT', this.makePath_(path) + ':/content', null, content, options);
+			}
+		} catch (error) {
+			if (error && error.code === 'BadRequest' && error.message === 'Maximum request length exceeded.') {
+				error.code = 'cannotSync';
+				error.message = 'Resource exceeds OneDrive max file size (4MB)';
+			}
+			throw error;
 		}
+
+		return response;
 	}
 
 	delete(path) {

ReactNativeClient/lib/joplin-database.js

@@ -202,7 +202,7 @@ class JoplinDatabase extends Database {
 		// default value and thus might cause problems. In that case, the default value
 		// must be set in the synchronizer too.
 
-		const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7];
+		const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8];
 
 		let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion);
 		// currentVersionIndex < 0 if for the case where an old version of Joplin used with a newer
@@ -265,6 +265,11 @@ class JoplinDatabase extends Database {
 				queries.push('ALTER TABLE resources ADD COLUMN file_extension TEXT NOT NULL DEFAULT ""');
 			}
 
+			if (targetVersion == 8) {
+				queries.push('ALTER TABLE sync_items ADD COLUMN sync_disabled INT NOT NULL DEFAULT "0"');
+				queries.push('ALTER TABLE sync_items ADD COLUMN sync_disabled_reason TEXT NOT NULL DEFAULT ""');
+			}
+
 			queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] });
 			await this.transactionExecBatch(queries);
 

ReactNativeClient/lib/models/base-item.js

@@ -339,6 +339,7 @@ class BaseItem extends BaseModel {
 					JOIN sync_items s ON s.item_id = items.id
 					WHERE sync_target = %d
 					AND s.sync_time < items.updated_time
+					AND s.sync_disabled = 0
 					%s
 					LIMIT %d
 				`,
@@ -382,7 +383,21 @@ class BaseItem extends BaseModel {
 		throw new Error('Invalid type: ' + type);
 	}
 
-	static updateSyncTimeQueries(syncTarget, item, syncTime) {
+	static async syncDisabledItems(syncTargetId) {
+		const rows = await this.db().selectAll('SELECT * FROM sync_items WHERE sync_disabled = 1 AND sync_target = ?', [syncTargetId]);
+		let output = [];
+		for (let i = 0; i < rows.length; i++) {
+			const item = await this.loadItem(rows[i].item_type, rows[i].item_id);
+			if (!item) continue; // The referenced item no longer exist
+			output.push({
+				syncInfo: rows[i],
+				item: item,
+			});
+		}
+		return output;
+	}
+
+	static updateSyncTimeQueries(syncTarget, item, syncTime, syncDisabled = false, syncDisabledReason = '') {
 		const itemType = item.type_;
 		const itemId = item.id;
 		if (!itemType || !itemId || syncTime === undefined) throw new Error('Invalid parameters in updateSyncTimeQueries()');
@@ -393,8 +408,8 @@ class BaseItem extends BaseModel {
 				params: [syncTarget, itemType, itemId],
 			},
 			{
-				sql: 'INSERT INTO sync_items (sync_target, item_type, item_id, sync_time) VALUES (?, ?, ?, ?)',
-				params: [syncTarget, itemType, itemId, syncTime],
+				sql: 'INSERT INTO sync_items (sync_target, item_type, item_id, sync_time, sync_disabled, sync_disabled_reason) VALUES (?, ?, ?, ?, ?, ?)',
+				params: [syncTarget, itemType, itemId, syncTime, syncDisabled ? 1 : 0, syncDisabledReason + ''],
 			}
 		];
 	}
@@ -404,6 +419,12 @@ class BaseItem extends BaseModel {
 		return this.db().transactionExecBatch(queries);
 	}
 
+	static async saveSyncDisabled(syncTargetId, item, syncDisabledReason) {
+		const syncTime = 'sync_time' in item ? item.sync_time : 0;
+		const queries = this.updateSyncTimeQueries(syncTargetId, item, syncTime, true, syncDisabledReason);
+		return this.db().transactionExecBatch(queries);
+	}
+
 	// When an item is deleted, its associated sync_items data is not immediately deleted for
 	// performance reason. So this function is used to look for these remaining sync_items and
 	// delete them.

ReactNativeClient/lib/models/note.js

@@ -126,7 +126,11 @@ class Note extends BaseItem {
 			let r = null;
 			r = noteFieldComp(a.user_updated_time, b.user_updated_time); if (r) return r;
 			r = noteFieldComp(a.user_created_time, b.user_created_time); if (r) return r;
-			r = noteFieldComp(a.title.toLowerCase(), b.title.toLowerCase()); if (r) return r;
+
+			const titleA = a.title ? a.title.toLowerCase() : '';
+			const titleB = b.title ? b.title.toLowerCase() : '';
+			r = noteFieldComp(titleA, titleB); if (r) return r;
+			
 			return noteFieldComp(a.id, b.id);
 		}
 

ReactNativeClient/lib/reducer.js

@@ -25,7 +25,8 @@ const defaultState = {
 	searchQuery: '',
 	settings: {},
 	appState: 'starting',
-	windowContentSize: { width: 0, height: 0 },
+	//windowContentSize: { width: 0, height: 0 },
+	hasDisabledSyncItems: false,
 };
 
 // When deleting a note, tag or folder
@@ -395,6 +396,12 @@ const reducer = (state = defaultState, action) => {
 				newState.appState = action.state;
 				break;
 
+			case 'SYNC_HAS_DISABLED_SYNC_ITEMS':
+
+				newState = Object.assign({}, state);
+				newState.hasDisabledSyncItems = true;
+				break;
+
 		}
 	} catch (error) {
 		error.message = 'In reducer: ' + error.message + ' Action: ' + JSON.stringify(action);

ReactNativeClient/lib/services/report.js

@@ -109,8 +109,21 @@ class ReportService {
 	async status(syncTarget) {
 		let r = await this.syncStatus(syncTarget);
 		let sections = [];
+		let section = null;
 
-		let section = { title: _('Sync status (synced items / total items)'), body: [] };
+		const disabledItems = await BaseItem.syncDisabledItems(syncTarget);
+
+		if (disabledItems.length) {
+			section = { title: _('Items that cannot be synchronised'), body: [] };
+
+			for (let i = 0; i < disabledItems.length; i++) {
+				const row = disabledItems[i];
+				section.body.push(_('"%s": "%s"', row.item.title, row.syncInfo.sync_disabled_reason));
+			}
+			sections.push(section);
+		}
+
+		section = { title: _('Sync status (synced items / total items)'), body: [] };
 
 		for (let n in r.items) {
 			if (!r.items.hasOwnProperty(n)) continue;
@@ -138,16 +151,19 @@ class ReportService {
 
 		sections.push(section);
 
-		section = { title: _('Coming alarms'), body: [] };
-
 		const alarms = await Alarm.allDue();
-		for (let i = 0; i < alarms.length; i++) {
-			const alarm = alarms[i];
-			const note = await Note.load(alarm.note_id);
-			section.body.push(_('On %s: %s', time.formatMsToLocal(alarm.trigger_time), note.title));
-		}
 
-		sections.push(section);
+		if (alarms.length) {
+			section = { title: _('Coming alarms'), body: [] };
+
+			for (let i = 0; i < alarms.length; i++) {
+				const alarm = alarms[i];
+				const note = await Note.load(alarm.note_id);
+				section.body.push(_('On %s: %s', time.formatMsToLocal(alarm.trigger_time), note.title));
+			}
+
+			sections.push(section);
+		}
 
 		return sections;
 	}

ReactNativeClient/lib/synchronizer.js

@@ -253,22 +253,36 @@ class Synchronizer {
 
 					this.logSyncOperation(action, local, remote, reason);
 
+					const handleCannotSyncItem = async (syncTargetId, item, cannotSyncReason) => {
+						await ItemClass.saveSyncDisabled(syncTargetId, item, cannotSyncReason);
+						this.dispatch({ type: 'SYNC_HAS_DISABLED_SYNC_ITEMS' });
+					}
+
 					if (local.type_ == BaseModel.TYPE_RESOURCE && (action == 'createRemote' || (action == 'itemConflict' && remote))) {
 						let remoteContentPath = this.resourceDirName_ + '/' + local.id;
-						// TODO: handle node and mobile in the same way
-						if (shim.isNode()) {
-							let resourceContent = '';
-							try {
-								resourceContent = await Resource.content(local);
-							} catch (error) {
-								error.message = 'Cannot read resource content: ' + local.id + ': ' + error.message;
-								this.logger().error(error);
-								this.progressReport_.errors.push(error);
+						try {
+							// TODO: handle node and mobile in the same way
+							if (shim.isNode()) {
+								let resourceContent = '';
+								try {
+									resourceContent = await Resource.content(local);
+								} catch (error) {
+									error.message = 'Cannot read resource content: ' + local.id + ': ' + error.message;
+									this.logger().error(error);
+									this.progressReport_.errors.push(error);
+								}
+								await this.api().put(remoteContentPath, resourceContent);
+							} else {
+								const localResourceContentPath = Resource.fullPath(local);
+								await this.api().put(remoteContentPath, null, { path: localResourceContentPath, source: 'file' });
+							}
+						} catch (error) {
+							if (error && error.code === 'cannotSync') {
+								await handleCannotSyncItem(syncTargetId, local, error.message);
+								action = null;
+							} else {
+								throw error;
 							}
-							await this.api().put(remoteContentPath, resourceContent);
-						} else {
-							const localResourceContentPath = Resource.fullPath(local);
-							await this.api().put(remoteContentPath, null, { path: localResourceContentPath, source: 'file' });
 						}
 					}
 
@@ -285,9 +299,27 @@ class Synchronizer {
 						// await this.api().setTimestamp(tempPath, local.updated_time);
 						// await this.api().move(tempPath, path);
 
-						await this.api().put(path, content);
-						await this.api().setTimestamp(path, local.updated_time);
-						await ItemClass.saveSyncTime(syncTargetId, local, time.unixMs());
+						let canSync = true;
+						try {
+							if (this.debugFlags_.indexOf('cannotSync') >= 0) {
+								const error = new Error('Testing cannotSync');
+								error.code = 'cannotSync';
+								throw error;
+							}
+							await this.api().put(path, content);
+						} catch (error) {
+							if (error && error.code === 'cannotSync') {
+								await handleCannotSyncItem(syncTargetId, local, error.message);
+								canSync = false;
+							} else {
+								throw error;
+							}
+						}
+
+						if (canSync) {
+							await this.api().setTimestamp(path, local.updated_time);
+							await ItemClass.saveSyncTime(syncTargetId, local, time.unixMs());
+						}
 
 					} else if (action == 'itemConflict') {