All: Decryption worker and handling of missing master key passwords

CliClient/app/gui/FolderListWidget.js

@@ -1,6 +1,6 @@
-const Folder = require('lib/models/folder.js').Folder;
+const Folder = require('lib/models/Folder.js');
-const Tag = require('lib/models/tag.js').Tag;
+const Tag = require('lib/models/Tag.js');
-const BaseModel = require('lib/base-model.js').BaseModel;
+const BaseModel = require('lib/BaseModel.js');
 const ListWidget = require('tkwidgets/ListWidget.js');
 const _ = require('lib/locale.js')._;
 

CliClient/app/gui/NoteListWidget.js

@@ -1,4 +1,4 @@
-const Note = require('lib/models/note.js').Note;
+const Note = require('lib/models/Note.js');
 const ListWidget = require('tkwidgets/ListWidget.js');
 
 class NoteListWidget extends ListWidget {

CliClient/app/gui/NoteMetadataWidget.js

@@ -1,4 +1,4 @@
-const Note = require('lib/models/note.js').Note;
+const Note = require('lib/models/Note.js');
 const TextWidget = require('tkwidgets/TextWidget.js');
 
 class NoteMetadataWidget extends TextWidget {

CliClient/app/gui/NoteWidget.js

@@ -1,4 +1,4 @@
-const Note = require('lib/models/note.js').Note;
+const Note = require('lib/models/Note.js');
 const TextWidget = require('tkwidgets/TextWidget.js');
 
 class NoteWidget extends TextWidget {

ElectronClient/app/gui/MainScreen.jsx

@@ -288,8 +288,7 @@ class MainScreenComponent extends React.Component {
 		const promptOptions = this.state.promptOptions;
 		const folders = this.props.folders;
 		const notes = this.props.notes;
-		const messageBoxVisible = this.props.hasDisabledSyncItems;
+		const messageBoxVisible = this.props.hasDisabledSyncItems || this.props.missingMasterKeys.length;
-
 		const styles = this.styles(this.props.theme, style.width, style.height, messageBoxVisible);
 		const theme = themeStyle(this.props.theme);
 
@@ -343,13 +342,31 @@ class MainScreenComponent extends React.Component {
 			});
 		}
 
-		const messageComp = messageBoxVisible ? (
+		const onViewMasterKeysClick = () => {
+			this.props.dispatch({
+				type: 'NAV_GO',
+				routeName: 'MasterKeys',
+			});
+		}
+
+		let messageComp = null;
+
+		if (messageBoxVisible) {
+			let msg = null;
+			if (this.props.hasDisabledSyncItems) {
+				msg = <span>{_('Some items cannot be synchronised.')} <a href="#" onClick={() => { onViewDisabledItemsClick() }}>{_('View them now')}</a></span>
+			} else if (this.props.missingMasterKeys.length) {
+				msg = <span>{_('Some items cannot be decrypted.')} <a href="#" onClick={() => { onViewMasterKeysClick() }}>{_('Set the password')}</a></span>
+			}
+
+			messageComp = (
 				<div style={styles.messageBox}>
 					<span style={theme.textStyle}>
-					{_('Some items cannot be synchronised.')} <a href="#" onClick={() => { onViewDisabledItemsClick() }}>{_('View them now')}</a>
+						{msg}
 					</span>
 				</div>
-		) : null;
+			);
+		}
 
 		return (
 			<div style={style}>
@@ -383,6 +400,7 @@ const mapStateToProps = (state) => {
 		folders: state.folders,
 		notes: state.notes,
 		hasDisabledSyncItems: state.hasDisabledSyncItems,
+		missingMasterKeys: state.missingMasterKeys,
 	};
 };
 

ElectronClient/app/gui/MasterKeysScreen.jsx

@@ -0,0 +1,64 @@
+const React = require('react');
+const { connect } = require('react-redux');
+const MasterKeys = require('lib/models/MasterKey');
+const { Header } = require('./Header.min.js');
+const { themeStyle } = require('../theme.js');
+const { _ } = require('lib/locale.js');
+const { time } = require('lib/time-utils.js');
+
+class MasterKeysScreenComponent extends React.Component {
+
+	renderMasterKey(mk) {
+		return (
+			<tr key={mk.id}>
+				<td>{mk.id}</td>
+				<td>{mk.source_application}</td>
+				<td>{time.formatMsToLocal(mk.created_time)}</td>
+				<td>{time.formatMsToLocal(mk.update_time)}</td>
+			</tr>
+		);
+	}
+
+	render() {
+		const style = this.props.style;
+		const theme = themeStyle(this.props.theme);
+		const masterKeys = this.props.masterKeys;
+
+		const headerStyle = {
+			width: style.width,
+		};
+
+		const mkComps = [];
+
+		for (let i = 0; i < masterKeys.length; i++) {
+			const mk = masterKeys[i];
+			mkComps.push(this.renderMasterKey(mk));
+		}
+
+		return (
+			<div>
+				<Header style={headerStyle} />
+				<table>
+					<tbody>
+						<tr>
+							<th>{_('ID')}</th><th>{_('Source')}</th><th>{_('Created')}</th><th>{_('Updated')}</th>
+						</tr>
+						{mkComps}
+					</tbody>
+				</table>
+			</div>
+		);
+	}
+
+}
+
+const mapStateToProps = (state) => {
+	return {
+		theme: state.settings.theme,
+		masterKeys: state.masterKeys,
+	};
+};
+
+const MasterKeysScreen = connect(mapStateToProps)(MasterKeysScreenComponent);
+
+module.exports = { MasterKeysScreen };

ReactNativeClient/lib/BaseApplication.js

@@ -27,6 +27,7 @@ const SyncTargetFilesystem = require('lib/SyncTargetFilesystem.js');
 const SyncTargetOneDrive = require('lib/SyncTargetOneDrive.js');
 const SyncTargetOneDriveDev = require('lib/SyncTargetOneDriveDev.js');
 const EncryptionService = require('lib/services/EncryptionService');
+const DecryptionWorker = require('lib/services/DecryptionWorker');
 
 SyncTargetRegistry.addClass(SyncTargetFilesystem);
 SyncTargetRegistry.addClass(SyncTargetOneDrive);
@@ -307,6 +308,7 @@ class BaseApplication {
 		FoldersScreenUtils.dispatch = this.store().dispatch;
 		reg.dispatch = this.store().dispatch;
 		BaseSyncTarget.dispatch = this.store().dispatch;
+		DecryptionWorker.instance().dispatch = this.store().dispatch;
 	}
 
 	async readFlagsFromFile(flagPath) {
@@ -394,7 +396,9 @@ class BaseApplication {
 		}
 
 		BaseItem.encryptionService_ = EncryptionService.instance();
+		DecryptionWorker.encryptionService_ = EncryptionService.instance();
 		await EncryptionService.instance().loadMasterKeysFromSettings();
+		DecryptionWorker.instance().start();
 
 		let currentFolderId = Setting.value('activeFolderId');
 		let currentFolder = null;

ReactNativeClient/lib/joplin-database.js

@@ -271,12 +271,24 @@ class JoplinDatabase extends Database {
 			}
 
 			if (targetVersion == 9) {
-				queries.push('CREATE TABLE master_keys (id TEXT PRIMARY KEY, created_time INT NOT NULL, updated_time INT NOT NULL, encryption_method INT NOT NULL, checksum TEXT NOT NULL, content TEXT NOT NULL);');
+				const newTableSql = `
+					CREATE TABLE master_keys (
+						id TEXT PRIMARY KEY,
+						created_time INT NOT NULL,
+						updated_time INT NOT NULL,
+						source_application TEXT NOT NULL,
+						encryption_method INT NOT NULL,
+						checksum TEXT NOT NULL,
+						content TEXT NOT NULL
+					);
+				`;
+				queries.push(this.sqlStringToLines(newTableSql)[0]);
 				const tableNames = ['notes', 'folders', 'tags', 'note_tags', 'resources'];
 				for (let i = 0; i < tableNames.length; i++) {
 					const n = tableNames[i];
 					queries.push('ALTER TABLE ' + n + ' ADD COLUMN encryption_cipher_text TEXT NOT NULL DEFAULT ""');
 					queries.push('ALTER TABLE ' + n + ' ADD COLUMN encryption_applied INT NOT NULL DEFAULT 0');
+					queries.push('CREATE INDEX ' + n + '_encryption_applied ON ' + n + ' (encryption_applied)');
 				}
 			}
 

ReactNativeClient/lib/models/BaseItem.js

@@ -249,14 +249,17 @@ class BaseItem extends BaseModel {
 		return temp.join("\n\n");
 	}
 
+	static encryptionService() {
+		if (!this.encryptionService_) throw new Error('BaseItem.encryptionService_ is not set!!');
+		return this.encryptionService_;
+	}
+
 	static async serializeForSync(item) {
 		const ItemClass = this.itemClass(item);
 		let serialized = await ItemClass.serialize(item);
 		if (!Setting.value('encryption.enabled') || !ItemClass.encryptionSupported()) return serialized;
 
-		if (!BaseItem.encryptionService_) throw new Error('BaseItem.encryptionService_ is not set!!');
+		const cipherText = await this.encryptionService().encryptString(serialized);
-
-		const cipherText = await BaseItem.encryptionService_.encryptString(serialized);
 
 		const reducedItem = Object.assign({}, item);
 
@@ -284,7 +287,7 @@ class BaseItem extends BaseModel {
 		if (!item.encryption_cipher_text) throw new Error('Item is not encrypted: ' + item.id);
 
 		const ItemClass = this.itemClass(item);
-		const plainText = await BaseItem.encryptionService_.decryptString(item.encryption_cipher_text);
+		const plainText = await this.encryptionService().decryptString(item.encryption_cipher_text);
 
 		// Note: decryption does not count has a change, so don't update any timestamp
 		const plainItem = await ItemClass.unserialize(plainText);
@@ -339,6 +342,39 @@ class BaseItem extends BaseModel {
 		return output;
 	}
 
+	static async itemsThatNeedDecryption(exclusions = [], limit = 100) {
+		const classNames = this.encryptableItemClassNames();
+
+		for (let i = 0; i < classNames.length; i++) {
+			const className = classNames[i];
+			const ItemClass = this.getClass(className);
+
+			const whereSql = ['encryption_applied = 1'];
+			if (exclusions.length) whereSql.push('id NOT IN ("' + exclusions.join('","') + '")');
+
+			const sql = sprintf(`
+				SELECT *
+				FROM %s
+				WHERE %s
+				LIMIT %d
+				`,
+				this.db().escapeField(ItemClass.tableName()),
+				whereSql.join(' AND '),
+				limit
+			);
+
+			const items = await ItemClass.modelSelectAll(sql);
+
+			if (i >= classNames.length - 1) {
+				return { hasMore: items.length >= limit, items: items };
+			} else {
+				if (items.length) return { hasMore: true, items: items };
+			}
+		}
+
+		throw new Error('Unreachable');
+	}
+
 	static async itemsThatNeedSync(syncTarget, limit = 100) {
 		const classNames = this.syncItemClassNames();
 
@@ -419,6 +455,16 @@ class BaseItem extends BaseModel {
 		});
 	}
 
+	static encryptableItemClassNames() {
+		const temp = this.syncItemClassNames();
+		let output = [];
+		for (let i = 0; i < temp.length; i++) {
+			if (temp[i] === 'MasterKey') continue;
+			output.push(temp[i]);
+		}
+		return output;
+	}
+
 	static syncItemTypes() {
 		return BaseItem.syncItemDefinitions_.map((def) => {
 			return def.type;

ReactNativeClient/lib/reducer.js

@@ -9,6 +9,7 @@ const defaultState = {
 	folders: [],
 	tags: [],
 	masterKeys: [],
+	missingMasterKeys: [],
 	searches: [],
 	selectedNoteIds: [],
 	selectedFolderId: null,
@@ -369,6 +370,14 @@ const reducer = (state = defaultState, action) => {
 				newState.masterKeys = action.items;
 				break;
 
+			case 'MASTERKEY_ADD_MISSING':
+
+				if (state.missingMasterKeys.indexOf(action.id) < 0) {
+					newState = Object.assign({}, state);
+					newState.missingMasterKeys.push(action.id);
+				}
+				break;
+
 			case 'SYNC_STARTED':
 
 				newState = Object.assign({}, state);

ReactNativeClient/lib/services/DecryptionWorker.js

@@ -1,13 +1,57 @@
+const BaseItem = require('lib/models/BaseItem');
+
 class DecryptionWorker {
 
 	constructor() {
 		this.state_ = 'idle';
+
+		this.dispatch = (action) => {
+			console.warn('DecryptionWorker.dispatch is not defined');
+		};
 	}
 
-	start() {
+	static instance() {
+		if (this.instance_) return this.instance_;
+		this.instance_ = new DecryptionWorker();
+		return this.instance_;
+	}
+
+	static encryptionService() {
+		if (!this.encryptionService_) throw new Error('DecryptionWorker.encryptionService_ is not set!!');
+		return this.encryptionService_;
+	}
+
+	async start() {
 		if (this.state_ !== 'idle') return;
 
 		this.state_ = 'started';
+
+		let excludedIds = [];
+
+		while (true) {
+			const result = await BaseItem.itemsThatNeedDecryption(excludedIds);
+			const items = result.items;
+
+			for (let i = 0; i < items.length; i++) {
+				const item = items[i];
+				const ItemClass = BaseItem.itemClass(item);
+				try {
+					await ItemClass.decrypt(item);
+				} catch (error) {
+					if (error.code === 'missingMasterKey') {
+						excludedIds.push(item.id);
+						this.dispatch({
+							type: 'MASTERKEY_ADD_MISSING',
+							id: error.masterKeyId,
+						});
+						continue;
+					}
+					throw error;
+				}
+			}
+
+			if (!result.hasMore) break;
+		}
 	}
 
 }

ReactNativeClient/lib/services/EncryptionService.js

@@ -33,6 +33,7 @@ class EncryptionService {
 		for (let i = 0; i < masterKeys.length; i++) {
 			const mk = masterKeys[i];
 			const password = passwords[mk.id];
+			if (this.isMasterKeyLoaded(mk.id)) continue;
 			if (!password) continue;
 
 			await this.loadMasterKey(mk, password, activeMasterKeyId === mk.id);
@@ -56,6 +57,10 @@ class EncryptionService {
 		return this.activeMasterKeyId_;
 	}
 
+	isMasterKeyLoaded(id) {
+		return !!this.loadedMasterKeys_[id];
+	}
+
 	async loadMasterKey(model, password, makeActive = false) {
 		if (!model.id) throw new Error('Master key does not have an ID - save it first');
 		this.loadedMasterKeys_[model.id] = await this.decryptMasterKey(model, password);
@@ -67,7 +72,12 @@ class EncryptionService {
 	}
 
 	loadedMasterKey(id) {
-		if (!this.loadedMasterKeys_[id]) throw new Error('Master key is not loaded: ' + id);
+		if (!this.loadedMasterKeys_[id]) {
+			const error = new Error('Master key is not loaded: ' + id);
+			error.code = 'missingMasterKey';
+			error.masterKeyId = id;
+			throw error;
+		}
 		return this.loadedMasterKeys_[id];
 	}
 
@@ -105,6 +115,7 @@ class EncryptionService {
 		return {
 			created_time: now,
 			updated_time: now,
+			source_application: Setting.value('appId'),
 			encryption_method: encryptionMethod,
 			checksum: checksum,
 			content: cipherText,