self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-12-17 23:27:48 +02:00
Code Issues Releases Activity

refactor

Browse Source
This commit is contained in:
Laurent Cozic
2021-06-25 14:51:36 +01:00
parent 0d930128c4
commit 400ede122d
23 changed files with 426 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
.eslintignore
View File

@@ -935,6 +935,9 @@ packages/lib/markdownUtils2.test.js.map
packages/lib/markupLanguageUtils.d.ts
packages/lib/markupLanguageUtils.js
packages/lib/markupLanguageUtils.js.map
packages/lib/migrations/40.d.ts
packages/lib/migrations/40.js
packages/lib/migrations/40.js.map
packages/lib/models/Alarm.d.ts
packages/lib/models/Alarm.js
packages/lib/models/Alarm.js.map
@@ -1079,6 +1082,9 @@ packages/lib/services/KvStore.js.map
packages/lib/services/MigrationService.d.ts
packages/lib/services/MigrationService.js
packages/lib/services/MigrationService.js.map
packages/lib/services/MigrationService.test.d.ts
packages/lib/services/MigrationService.test.js
packages/lib/services/MigrationService.test.js.map
packages/lib/services/NavService.d.ts
packages/lib/services/NavService.js
packages/lib/services/NavService.js.map
@@ -1439,9 +1445,6 @@ packages/lib/services/synchronizer/LockHandler.js.map
packages/lib/services/synchronizer/MigrationHandler.d.ts
packages/lib/services/synchronizer/MigrationHandler.js
packages/lib/services/synchronizer/MigrationHandler.js.map
packages/lib/services/synchronizer/SyncTargetInfoHandler.d.ts
packages/lib/services/synchronizer/SyncTargetInfoHandler.js
packages/lib/services/synchronizer/SyncTargetInfoHandler.js.map
packages/lib/services/synchronizer/Synchronizer.basics.test.d.ts
packages/lib/services/synchronizer/Synchronizer.basics.test.js
packages/lib/services/synchronizer/Synchronizer.basics.test.js.map
@@ -1475,6 +1478,9 @@ packages/lib/services/synchronizer/migrations/1.js.map
packages/lib/services/synchronizer/migrations/2.d.ts
packages/lib/services/synchronizer/migrations/2.js
packages/lib/services/synchronizer/migrations/2.js.map
packages/lib/services/synchronizer/syncTargetInfoUtils.d.ts
packages/lib/services/synchronizer/syncTargetInfoUtils.js
packages/lib/services/synchronizer/syncTargetInfoUtils.js.map
packages/lib/services/synchronizer/synchronizer_LockHandler.test.d.ts
packages/lib/services/synchronizer/synchronizer_LockHandler.test.js
packages/lib/services/synchronizer/synchronizer_LockHandler.test.js.map
@@ -1493,6 +1499,9 @@ packages/lib/services/synchronizer/utils/types.js.map
packages/lib/shim.d.ts
packages/lib/shim.js
packages/lib/shim.js.map
packages/lib/testing/syncTargetUtils.d.ts
packages/lib/testing/syncTargetUtils.js
packages/lib/testing/syncTargetUtils.js.map
packages/lib/testing/test-utils-synchronizer.d.ts
packages/lib/testing/test-utils-synchronizer.js
packages/lib/testing/test-utils-synchronizer.js.map

15
.gitignore vendored
View File

@@ -921,6 +921,9 @@ packages/lib/markdownUtils2.test.js.map
packages/lib/markupLanguageUtils.d.ts
packages/lib/markupLanguageUtils.js
packages/lib/markupLanguageUtils.js.map
packages/lib/migrations/40.d.ts
packages/lib/migrations/40.js
packages/lib/migrations/40.js.map
packages/lib/models/Alarm.d.ts
packages/lib/models/Alarm.js
packages/lib/models/Alarm.js.map
@@ -1065,6 +1068,9 @@ packages/lib/services/KvStore.js.map
packages/lib/services/MigrationService.d.ts
packages/lib/services/MigrationService.js
packages/lib/services/MigrationService.js.map
packages/lib/services/MigrationService.test.d.ts
packages/lib/services/MigrationService.test.js
packages/lib/services/MigrationService.test.js.map
packages/lib/services/NavService.d.ts
packages/lib/services/NavService.js
packages/lib/services/NavService.js.map
@@ -1425,9 +1431,6 @@ packages/lib/services/synchronizer/LockHandler.js.map
packages/lib/services/synchronizer/MigrationHandler.d.ts
packages/lib/services/synchronizer/MigrationHandler.js
packages/lib/services/synchronizer/MigrationHandler.js.map
packages/lib/services/synchronizer/SyncTargetInfoHandler.d.ts
packages/lib/services/synchronizer/SyncTargetInfoHandler.js
packages/lib/services/synchronizer/SyncTargetInfoHandler.js.map
packages/lib/services/synchronizer/Synchronizer.basics.test.d.ts
packages/lib/services/synchronizer/Synchronizer.basics.test.js
packages/lib/services/synchronizer/Synchronizer.basics.test.js.map
@@ -1461,6 +1464,9 @@ packages/lib/services/synchronizer/migrations/1.js.map
packages/lib/services/synchronizer/migrations/2.d.ts
packages/lib/services/synchronizer/migrations/2.js
packages/lib/services/synchronizer/migrations/2.js.map
packages/lib/services/synchronizer/syncTargetInfoUtils.d.ts
packages/lib/services/synchronizer/syncTargetInfoUtils.js
packages/lib/services/synchronizer/syncTargetInfoUtils.js.map
packages/lib/services/synchronizer/synchronizer_LockHandler.test.d.ts
packages/lib/services/synchronizer/synchronizer_LockHandler.test.js
packages/lib/services/synchronizer/synchronizer_LockHandler.test.js.map
@@ -1479,6 +1485,9 @@ packages/lib/services/synchronizer/utils/types.js.map
packages/lib/shim.d.ts
packages/lib/shim.js
packages/lib/shim.js.map
packages/lib/testing/syncTargetUtils.d.ts
packages/lib/testing/syncTargetUtils.js
packages/lib/testing/syncTargetUtils.js.map
packages/lib/testing/test-utils-synchronizer.d.ts
packages/lib/testing/test-utils-synchronizer.js
packages/lib/testing/test-utils-synchronizer.js.map

4
packages/app-cli/app/command-e2ee.ts
View File

@@ -1,5 +1,5 @@
import EncryptionService from '@joplin/lib/services/EncryptionService';
import { disableEncryption, generateMasterKeyAndEnableEncryption, loadMasterKeysFromSettings } from '@joplin/lib/services/e2ee/utils';
import { setupAndDisableEncryption, generateMasterKeyAndEnableEncryption, loadMasterKeysFromSettings } from '@joplin/lib/services/e2ee/utils';
import DecryptionWorker from '@joplin/lib/services/DecryptionWorker';
import BaseItem from '@joplin/lib/models/BaseItem';
import Setting from '@joplin/lib/models/Setting';
@@ -99,7 +99,7 @@ class Command extends BaseCommand {
}
if (args.command === 'disable') {
await disableEncryption();
await setupAndDisableEncryption();
return;
}

4
packages/app-desktop/gui/EncryptionConfigScreen.tsx
View File

@@ -2,7 +2,7 @@ const React = require('react');
const { connect } = require('react-redux');
import Setting from '@joplin/lib/models/Setting';
import EncryptionService from '@joplin/lib/services/EncryptionService';
import { disableEncryption, generateMasterKeyAndEnableEncryption } from '@joplin/lib/services/e2ee/utils';
import { setupAndDisableEncryption, generateMasterKeyAndEnableEncryption } from '@joplin/lib/services/e2ee/utils';
import time from '@joplin/lib/time';
import shim from '@joplin/lib/shim';
import { themeStyle } from '@joplin/lib/theme';
@@ -180,7 +180,7 @@ class EncryptionConfigScreenComponent extends React.Component<Props, any> {
try {
if (isEnabled) {
await disableEncryption();
await setupAndDisableEncryption();
} else {
await generateMasterKeyAndEnableEncryption(EncryptionService.instance(), answer);
}

4
packages/app-mobile/components/screens/encryption-config.tsx
View File

@@ -6,7 +6,7 @@ import EncryptionService from '@joplin/lib/services/EncryptionService';
import time from '@joplin/lib/time';
import { _ } from '@joplin/lib/locale';
import { MasterKeyEntity } from '@joplin/lib/services/database/types';
import { disableEncryption, generateMasterKeyAndEnableEncryption } from '@joplin/lib/services/e2ee/utils';
import { setupAndDisableEncryption, generateMasterKeyAndEnableEncryption } from '@joplin/lib/services/e2ee/utils';
const { connect } = require('react-redux');
const { ScreenHeader } = require('../screen-header.js');
const { BaseScreenComponent } = require('../base-screen.js');
@@ -218,7 +218,7 @@ class EncryptionConfigScreenComponent extends BaseScreenComponent {
if (!ok) return;
try {
await disableEncryption();
await setupAndDisableEncryption();
} catch (error) {
await dialogs.error(this, error.message);
}

6
packages/lib/JoplinDatabase.ts
View File

@@ -343,7 +343,7 @@ export default class JoplinDatabase extends Database {
// must be set in the synchronizer too.
// Note: v16 and v17 don't do anything. They were used to debug an issue.
const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39];
const existingDatabaseVersions = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40];
let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion);
@@ -892,6 +892,10 @@ export default class JoplinDatabase extends Database {
queries.push('ALTER TABLE `notes` ADD COLUMN conflict_original_id TEXT NOT NULL DEFAULT ""');
}
if (targetVersion == 40) {
queries.push(this.addMigrationFile(40));
}
const updateVersionQuery = { sql: 'UPDATE version SET version = ?', params: [targetVersion] };
queries.push(updateVersionQuery);

75
packages/lib/Synchronizer.ts
View File

@@ -11,18 +11,17 @@ import Note from './models/Note';
import Resource from './models/Resource';
import ItemChange from './models/ItemChange';
import ResourceLocalState from './models/ResourceLocalState';
import MasterKey from './models/MasterKey';
import BaseModel from './BaseModel';
import time from './time';
import ResourceService from './services/ResourceService';
import EncryptionService from './services/EncryptionService';
import { enableEncryption, loadMasterKeysFromSettings } from './services/e2ee/utils';
import JoplinError from './JoplinError';
import ShareService from './services/share/ShareService';
import TaskQueue from './TaskQueue';
import ItemUploader from './services/synchronizer/ItemUploader';
import { FileApi } from './file-api';
import SyncTargetInfoHandler from './services/synchronizer/SyncTargetInfoHandler';
import { localSyncTargetInfo, remoteSyncTargetInfo, setLocalSyncTargetInfo, syncTargetInfoEquals, setRemoteSyncTargetInfo, mergeSyncTargetInfos, activeMasterKey } from './services/synchronizer/syncTargetInfoUtils';
import { setupAndEnableEncryption, setupAndDisableEncryption } from './services/e2ee/utils';
const { sprintf } = require('sprintf-js');
const { Dirnames } = require('./services/synchronizer/utils/types');
@@ -75,7 +74,6 @@ export default class Synchronizer {
private clientId_: string;
private lockHandler_: LockHandler;
private migrationHandler_: MigrationHandler;
private syncTargetInfoHandler_: SyncTargetInfoHandler;
private encryptionService_: EncryptionService = null;
private resourceService_: ResourceService = null;
private syncTargetIsLocked_: boolean = false;
@@ -136,16 +134,10 @@ export default class Synchronizer {
private migrationHandler() {
if (this.migrationHandler_) return this.migrationHandler_;
this.migrationHandler_ = new MigrationHandler(this.api(), this.syncTargetInfoHandler(), this.lockHandler(), this.appType_, this.clientId_);
this.migrationHandler_ = new MigrationHandler(this.api(), this.lockHandler(), this.appType_, this.clientId_);
return this.migrationHandler_;
}
public syncTargetInfoHandler() {
if (this.syncTargetInfoHandler_) return this.syncTargetInfoHandler_;
this.syncTargetInfoHandler_ = new SyncTargetInfoHandler(this.api());
return this.syncTargetInfoHandler_;
}
maxResourceSize() {
if (this.maxResourceSize_ !== null) return this.maxResourceSize_;
return this.appType_ === 'mobile' ? 100 * 1000 * 1000 : Infinity;
@@ -375,9 +367,6 @@ export default class Synchronizer {
this.syncTargetIsLocked_ = false;
this.cancelling_ = false;
const masterKeysBefore = await MasterKey.count();
let hasAutoEnabledEncryption = false;
const synchronizationId = time.unixMs().toString();
const outputContext = Object.assign({}, lastContext);
@@ -424,18 +413,50 @@ export default class Synchronizer {
this.api().setTempDirName(Dirnames.Temp);
try {
const syncTargetInfoService = new SyncTargetInfoHandler(this.api());
const remoteInfo = await remoteSyncTargetInfo(this.api());
this.logger().info('Sync target info:', remoteInfo);
await this.migrationHandler().checkCanSync();
const syncTargetInfo = await syncTargetInfoService.info();
this.logger().info('Sync target info:', syncTargetInfo);
if (!syncTargetInfo.version) {
if (!remoteInfo.version) {
this.logger().info('Sync target is new - setting it up...');
await this.migrationHandler().upgrade(Setting.value('syncVersion'));
} else {
this.logger().info('Sync target is already setup - checking it...');
await this.migrationHandler().checkCanSync(remoteInfo);
const localInfo = localSyncTargetInfo();
if (!syncTargetInfoEquals(localInfo, remoteInfo)) {
// TODO: if e2ee changed - need to enable/disable encryption
const newInfo = mergeSyncTargetInfos(localInfo, remoteInfo);
await setRemoteSyncTargetInfo(this.api(), newInfo);
setLocalSyncTargetInfo(newInfo);
if (localInfo.e2ee !== remoteInfo.e2ee) {
if (newInfo.e2ee) {
const mk = activeMasterKey(newInfo);
await setupAndEnableEncryption(mk);
} else {
await setupAndDisableEncryption();
}
}
} else {
// Set it to remote anyway so that timestamps are the same
setLocalSyncTargetInfo(remoteInfo);
}
}
// const syncTargetInfoService = new SyncTargetInfoHandler(this.api());
// await this.migrationHandler().checkCanSync();
// const syncTargetInfo = await syncTargetInfoService.info();
// this.logger().info('Sync target info:', syncTargetInfo);
// if (!syncTargetInfo.version) {
// this.logger().info('Sync target is new - setting it up...');
// await this.migrationHandler().upgrade(Setting.value('syncVersion'));
// }
} catch (error) {
if (error.code === 'outdatedSyncTarget') {
Setting.setValue('sync.upgradeState', Setting.SYNC_UPGRADE_STATE_SHOULD_DO);
@@ -461,6 +482,7 @@ export default class Synchronizer {
if (syncSteps.indexOf('update_remote') >= 0) {
const donePaths: string[] = [];
const completeItemProcessing = (path: string) => {
donePaths.push(path);
};
@@ -925,15 +947,6 @@ export default class Synchronizer {
if (creatingOrUpdatingResource) this.dispatch({ type: 'SYNC_CREATED_OR_UPDATED_RESOURCE', id: content.id });
if (!hasAutoEnabledEncryption && content.type_ === BaseModel.TYPE_MASTER_KEY && !masterKeysBefore) {
hasAutoEnabledEncryption = true;
this.logger().info('One master key was downloaded and none was previously available: automatically enabling encryption');
this.logger().info('Using master key: ', content.id);
await enableEncryption(content);
await loadMasterKeysFromSettings(this.encryptionService());
this.logger().info('Encryption has been enabled with downloaded master key as active key. However, note that no password was initially supplied. It will need to be provided by user.');
}
if (content.encryption_applied) this.dispatch({ type: 'SYNC_GOT_ENCRYPTED_ITEM' });
} else if (action == 'deleteLocal') {
if (local.type_ == BaseModel.TYPE_FOLDER) {

27
packages/lib/migrations/40.ts Normal file
View File

@@ -0,0 +1,27 @@
import JoplinDatabase from '../JoplinDatabase';
import { MigrationScript } from '../models/Migration';
import Setting from '../models/Setting';
import { SyncTargetInfo } from '../services/synchronizer/syncTargetInfoUtils';
const script: MigrationScript = {
exec: async function(db: JoplinDatabase): Promise<void> {
const masterKeys = await db.selectAll('SELECT * FROM master_keys');
const masterKeyMap: Record<string, any> = {};
for (const mk of masterKeys) masterKeyMap[mk.id] = mk;
const syncInfo: SyncTargetInfo = {
version: 2,
e2ee: Setting.valueNoThrow('encryption.enabled', false),
masterKeys: masterKeyMap,
activeMasterKeyId: Setting.valueNoThrow('encryption.activeMasterKeyId', ''),
updatedTime: Date.now(),
};
Setting.setValue('sync.info', JSON.stringify(syncInfo));
},
};
export default script;

22
packages/lib/models/Migration.test.js Normal file
View File

@@ -0,0 +1,22 @@
'use strict';
const __awaiter = (this && this.__awaiter) || function(thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function(resolve) { resolve(value); }); }
return new (P || (P = Promise))(function(resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator['throw'](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, '__esModule', { value: true });
const test_utils_1 = require('../testing/test-utils');
describe('models_Note', function() {
beforeEach((done) => __awaiter(this, void 0, void 0, function* () {
yield test_utils_1.setupDatabaseAndSynchronizer(1);
yield test_utils_1.switchClient(1);
done();
}));
it('should migrate to v40', () => __awaiter(this, void 0, void 0, function* () {
}));
});
// # sourceMappingURL=Migration.test.js.map

9
packages/lib/models/Migration.ts
View File

@@ -1,10 +1,17 @@
import BaseModel from '../BaseModel';
import JoplinDatabase from '../JoplinDatabase';
import migration40 from '../migrations/40';
const migrationScripts: Record<number, any> = {
export interface MigrationScript {
exec(db: JoplinDatabase): Promise<void>;
}
const migrationScripts: Record<number, MigrationScript> = {
20: require('../migrations/20.js'),
27: require('../migrations/27.js'),
33: require('../migrations/33.js'),
35: require('../migrations/35.js'),
40: migration40,
};
export default class Migration extends BaseModel {

15
packages/lib/models/Setting.ts
View File

@@ -315,6 +315,13 @@ class Setting extends BaseModel {
storage: SettingStorage.File,
},
'sync.info': {
value: '',
type: SettingItemType.String,
public: false,
storage: SettingStorage.File,
},
'sync.upgradeState': {
value: Setting.SYNC_UPGRADE_STATE_IDLE,
type: SettingItemType.Int,
@@ -1642,7 +1649,7 @@ class Setting extends BaseModel {
throw new Error(`Unhandled value type: ${md.type}`);
}
static value(key: string) {
public static value(key: string) {
// Need to copy arrays and objects since in setValue(), the old value and new one is compared
// with strict equality and the value is updated only if changed. However if the caller acquire
// and object and change a key, the objects will be detected as equal. By returning a copy
@@ -1673,6 +1680,12 @@ class Setting extends BaseModel {
return copyIfNeeded(md.value);
}
// This function returns the default value if the setting key does not exist.
public static valueNoThrow(key: string, defaultValue: any) {
if (!this.keyExists(key)) return defaultValue;
return this.value(key);
}
static isEnum(key: string) {
const md = this.settingMetadata(key);
return md.isEnum === true;

49
packages/lib/services/MigrationService.test.ts Normal file
View File

@@ -0,0 +1,49 @@
import MasterKey from '../models/MasterKey';
import Setting from '../models/Setting';
import { encryptionService, setupDatabaseAndSynchronizer, switchClient } from '../testing/test-utils';
import MigrationService from './MigrationService';
import { SyncTargetInfo } from './synchronizer/syncTargetInfoUtils';
function migrationService() {
return new MigrationService();
}
describe('MigrationService', function() {
beforeEach(async (done) => {
await setupDatabaseAndSynchronizer(1);
await switchClient(1);
done();
});
it('should migrate to v40 - with keys', async () => {
const startTime = Date.now();
const mk1 = await MasterKey.save(await encryptionService().generateMasterKey('1'));
const mk2 = await MasterKey.save(await encryptionService().generateMasterKey('2'));
Setting.setValue('encryption.enabled', true);
Setting.setValue('encryption.activeMasterKeyId', mk2.id);
await migrationService().runScript(40);
const info: SyncTargetInfo = JSON.parse(Setting.value('sync.info'));
expect(info.e2ee).toBe(true);
expect(info.activeMasterKeyId).toBe(mk2.id);
expect(info.version).toBe(2);
expect(Object.keys(info.masterKeys).sort()).toEqual([mk1.id, mk2.id].sort());
expect(info.updatedTime).toBeGreaterThanOrEqual(startTime);
});
it('should migrate to v40 - empty', async () => {
const startTime = Date.now();
await migrationService().runScript(40);
const info: SyncTargetInfo = JSON.parse(Setting.value('sync.info'));
expect(info.e2ee).toBe(false);
expect(info.activeMasterKeyId).toBe('');
expect(info.version).toBe(2);
expect(Object.keys(info.masterKeys)).toEqual([]);
expect(info.updatedTime).toBeGreaterThanOrEqual(startTime);
});
});

8
packages/lib/services/MigrationService.ts
View File

@@ -5,18 +5,18 @@ export default class MigrationService extends BaseService {
private static instance_: MigrationService;
static instance() {
public static instance() {
if (this.instance_) return this.instance_;
this.instance_ = new MigrationService();
return this.instance_;
}
async runScript(num: number) {
public async runScript(num: number) {
const script = Migration.script(num);
await script.exec();
await script.exec(Migration.db());
}
async run() {
public async run() {
const migrations = await Migration.migrationsToDo();
for (const migration of migrations) {

6
packages/lib/services/ResourceService.test.ts
View File

@@ -7,7 +7,7 @@ import Folder from '../models/Folder';
import Note from '../models/Note';
import Resource from '../models/Resource';
import SearchEngine from '../services/searchengine/SearchEngine';
import { enableEncryption, loadMasterKeysFromSettings } from './e2ee/utils';
import { setupAndEnableEncryption, loadMasterKeysFromSettings } from './e2ee/utils';
describe('services_ResourceService', function() {
@@ -139,7 +139,7 @@ describe('services_ResourceService', function() {
// Eventually R1 is deleted because service thinks that it was at some point associated with a note, but no longer.
const masterKey = await loadEncryptionMasterKey();
await enableEncryption(masterKey, '123456');
await setupAndEnableEncryption(masterKey, '123456');
await loadMasterKeysFromSettings(encryptionService());
const folder1 = await Folder.save({ title: 'folder1' });
const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
@@ -151,7 +151,7 @@ describe('services_ResourceService', function() {
await switchClient(2);
await synchronizer().start();
await enableEncryption(masterKey, '123456');
await setupAndEnableEncryption(masterKey, '123456');
await loadMasterKeysFromSettings(encryptionService());
await decryptionWorker().start();
{

24
packages/lib/services/e2ee/utils.ts
View File

@@ -4,12 +4,16 @@ import MasterKey from '../../models/MasterKey';
import Setting from '../../models/Setting';
import { MasterKeyEntity } from '../database/types';
import EncryptionService from '../EncryptionService';
import { localSyncTargetInfo, setLocalSyncTargetInfo } from '../synchronizer/syncTargetInfoUtils';
const logger = Logger.create('e2ee/utils');
export async function enableEncryption(masterKey: MasterKeyEntity, password: string = null) {
Setting.setValue('encryption.enabled', true);
Setting.setValue('encryption.activeMasterKeyId', masterKey.id);
export async function setupAndEnableEncryption(masterKey: MasterKeyEntity, password: string = null) {
setLocalSyncTargetInfo({
...localSyncTargetInfo(),
e2ee: true,
activeMasterKeyId: masterKey.id,
});
if (password) {
const passwordCache = Setting.value('encryption.passwordCache');
@@ -22,7 +26,7 @@ export async function enableEncryption(masterKey: MasterKeyEntity, password: str
await BaseItem.markAllNonEncryptedForSync();
}
export async function disableEncryption() {
export async function setupAndDisableEncryption() {
// Allow disabling encryption even if some items are still encrypted, because whether E2EE is enabled or disabled
// should not affect whether items will enventually be decrypted or not (DecryptionWorker will still work as
// long as there are encrypted items). Also even if decryption is disabled, it's possible that encrypted items
@@ -31,7 +35,11 @@ export async function disableEncryption() {
// const hasEncryptedItems = await BaseItem.hasEncryptedItems();
// if (hasEncryptedItems) throw new Error(_('Encryption cannot currently be disabled because some items are still encrypted. Please wait for all the items to be decrypted and try again.'));
Setting.setValue('encryption.enabled', false);
setLocalSyncTargetInfo({
...localSyncTargetInfo(),
e2ee: false,
});
// The only way to make sure everything gets decrypted on the sync target is
// to re-sync everything.
await BaseItem.forceSyncAll();
@@ -40,15 +48,17 @@ export async function disableEncryption() {
export async function generateMasterKeyAndEnableEncryption(service: EncryptionService, password: string) {
let masterKey = await service.generateMasterKey(password);
masterKey = await MasterKey.save(masterKey);
await enableEncryption(masterKey, password);
await setupAndEnableEncryption(masterKey, password);
await loadMasterKeysFromSettings(service);
return masterKey;
}
export async function loadMasterKeysFromSettings(service: EncryptionService) {
const syncTargetInfo = localSyncTargetInfo();
const masterKeys = await MasterKey.all();
const passwords = Setting.value('encryption.passwordCache');
const activeMasterKeyId = Setting.value('encryption.activeMasterKeyId');
const activeMasterKeyId = syncTargetInfo.activeMasterKeyId || '';
logger.info(`Trying to load ${masterKeys.length} master keys...`);

34
packages/lib/services/synchronizer/MigrationHandler.ts
View File

@@ -16,29 +16,28 @@ const migrations = [
import Setting from '../../models/Setting';
const { sprintf } = require('sprintf-js');
import JoplinError from '../../JoplinError';
import SyncTargetInfoHandler from './SyncTargetInfoHandler';
import { FileApi } from '../../file-api';
import { remoteSyncTargetInfo, setRemoteSyncTargetInfo, setLocalSyncTargetInfo, SyncTargetInfo } from './syncTargetInfoUtils';
export default class MigrationHandler extends BaseService {
private api_: FileApi = null;
private lockHandler_: LockHandler = null;
private syncTargetInfoHandler_: SyncTargetInfoHandler = null;
private clientType_: string;
private clientId_: string;
constructor(api: FileApi, syncTargetInfoHandler: SyncTargetInfoHandler, lockHandler: LockHandler, clientType: string, clientId: string) {
constructor(api: FileApi, lockHandler: LockHandler, clientType: string, clientId: string) {
super();
this.api_ = api;
this.syncTargetInfoHandler_ = syncTargetInfoHandler;
this.lockHandler_ = lockHandler;
this.clientType_ = clientType;
this.clientId_ = clientId;
}
public async checkCanSync(): Promise<void> {
public async checkCanSync(syncTargetInfo: SyncTargetInfo = null): Promise<void> {
syncTargetInfo = syncTargetInfo || await remoteSyncTargetInfo(this.api_);
const supportedSyncTargetVersion = Setting.value('syncVersion');
const syncTargetInfo = await this.syncTargetInfoHandler_.info();
if (syncTargetInfo.version) {
if (syncTargetInfo.version > supportedSyncTargetVersion) {
@@ -51,10 +50,11 @@ export default class MigrationHandler extends BaseService {
public async upgrade(targetVersion: number = 0) {
const supportedSyncTargetVersion = Setting.value('syncVersion');
const syncTargetInfo = await this.syncTargetInfoHandler_.info();
if (syncTargetInfo.version > supportedSyncTargetVersion) {
throw new JoplinError(sprintf('Sync version of the target (%d) is greater than the version supported by the client (%d). Please upgrade your client.', syncTargetInfo.version, supportedSyncTargetVersion), 'outdatedClient');
const info = await remoteSyncTargetInfo(this.api_);
if (info.version > supportedSyncTargetVersion) {
throw new JoplinError(sprintf('Sync version of the target (%d) is greater than the version supported by the client (%d). Please upgrade your client.', info.version, supportedSyncTargetVersion), 'outdatedClient');
}
// if (supportedSyncTargetVersion !== migrations.length - 1) {
@@ -68,8 +68,8 @@ export default class MigrationHandler extends BaseService {
// Also if the sync target version is 0, it means it's a new one so we need the
// lock folder first before doing anything else.
// Temp folder is needed too to get remoteDate() call to work.
if (syncTargetInfo.version === 0 || syncTargetInfo.version === 1) {
this.logger().info('MigrationHandler: Sync target version is 0 or 1 - creating "locks" and "temp" directory:', syncTargetInfo);
if (info.version === 0 || info.version === 1) {
this.logger().info('MigrationHandler: Sync target version is 0 or 1 - creating "locks" and "temp" directory:', info);
await this.api_.mkdir(Dirnames.Locks);
await this.api_.mkdir(Dirnames.Temp);
}
@@ -84,7 +84,7 @@ export default class MigrationHandler extends BaseService {
this.logger().info('MigrationHandler: Acquired exclusive lock:', exclusiveLock);
try {
for (let newVersion = syncTargetInfo.version + 1; newVersion < migrations.length; newVersion++) {
for (let newVersion = info.version + 1; newVersion < migrations.length; newVersion++) {
if (targetVersion && newVersion > targetVersion) break;
const fromVersion = newVersion - 1;
@@ -99,10 +99,14 @@ export default class MigrationHandler extends BaseService {
await migration(this.api_);
if (autoLockError) throw autoLockError;
await this.syncTargetInfoHandler_.setInfo({
...syncTargetInfo,
const newInfo: SyncTargetInfo = {
...info,
version: newVersion,
});
updatedTime: Date.now(),
};
await setRemoteSyncTargetInfo(this.api_, newInfo);
setLocalSyncTargetInfo(newInfo);
this.logger().info(`MigrationHandler: Done migrating from version ${fromVersion} to version ${newVersion}`);
} catch (error) {

52
packages/lib/services/synchronizer/SyncTargetInfoHandler.ts
View File

@@ -1,52 +0,0 @@
import { FileApi } from '../../file-api';
interface SyncTargetInfo {
version: number;
}
export default class SyncTargetInfoHandler {
private api_: FileApi = null;
private info_: SyncTargetInfo = null;
public constructor(api: FileApi) {
this.api_ = api;
}
public async info(): Promise<SyncTargetInfo> {
if (this.info_) return this.info_;
this.info_ = await this.fetchSyncTargetInfo();
return this.info_;
}
private serializeSyncTargetInfo(info: SyncTargetInfo): string {
return JSON.stringify(info);
}
private unserializeSyncTargetInfo(info: string): SyncTargetInfo {
return JSON.parse(info);
}
public async setInfo(info: SyncTargetInfo): Promise<void> {
this.info_ = info;
await this.api_.put('info.json', this.serializeSyncTargetInfo(info));
}
private async fetchSyncTargetInfo(): Promise<SyncTargetInfo> {
const syncTargetInfoText = await this.api_.get('info.json');
// Returns version 0 if the sync target is empty
let output: SyncTargetInfo = { version: 0 };
if (syncTargetInfoText) {
output = this.unserializeSyncTargetInfo(syncTargetInfoText);
if (!output.version) throw new Error('Missing "version" field in info.json');
} else {
const oldVersion = await this.api_.get('.sync/version.txt');
if (oldVersion) output = { version: 1 };
}
return output;
}
}

18
packages/lib/services/synchronizer/Synchronizer.e2ee.test.ts
View File

@@ -10,7 +10,7 @@ import MasterKey from '../../models/MasterKey';
import BaseItem from '../../models/BaseItem';
import { ResourceEntity } from '../database/types';
import Synchronizer from '../../Synchronizer';
import { disableEncryption, enableEncryption, loadMasterKeysFromSettings } from '../e2ee/utils';
import { setupAndDisableEncryption, setupAndEnableEncryption, loadMasterKeysFromSettings } from '../e2ee/utils';
let insideBeforeEach = false;
@@ -142,7 +142,7 @@ describe('Synchronizer.e2ee', function() {
// Then enable encryption and sync again
let masterKey = await encryptionService().generateMasterKey('123456');
masterKey = await MasterKey.save(masterKey);
await enableEncryption(masterKey, '123456');
await setupAndEnableEncryption(masterKey, '123456');
await loadMasterKeysFromSettings(encryptionService());
await synchronizerStart();
@@ -168,7 +168,7 @@ describe('Synchronizer.e2ee', function() {
let allEncrypted = await allSyncTargetItemsEncrypted();
expect(allEncrypted).toBe(true);
await disableEncryption();
await setupAndDisableEncryption();
await synchronizerStart();
allEncrypted = await allSyncTargetItemsEncrypted();
@@ -194,7 +194,7 @@ describe('Synchronizer.e2ee', function() {
// If we try to disable encryption now, it should throw an error because some items are
// currently encrypted. They must be decrypted first so that they can be sent as
// plain text to the sync target.
// let hasThrown = await checkThrowAsync(async () => await disableEncryption());
// let hasThrown = await checkThrowAsync(async () => await setupAndDisableEncryption());
// expect(hasThrown).toBe(true);
// Now supply the password, and decrypt the items
@@ -203,7 +203,7 @@ describe('Synchronizer.e2ee', function() {
await decryptionWorker().start();
// Try to disable encryption again
const hasThrown = await checkThrowAsync(async () => await disableEncryption());
const hasThrown = await checkThrowAsync(async () => await setupAndDisableEncryption());
expect(hasThrown).toBe(false);
// If we sync now the target should receive the decrypted items
@@ -250,7 +250,7 @@ describe('Synchronizer.e2ee', function() {
expect(await allSyncTargetItemsEncrypted()).toBe(false);
const masterKey = await loadEncryptionMasterKey();
await enableEncryption(masterKey, '123456');
await setupAndEnableEncryption(masterKey, '123456');
await loadMasterKeysFromSettings(encryptionService());
await synchronizerStart();
@@ -265,7 +265,7 @@ describe('Synchronizer.e2ee', function() {
const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
await shim.attachFileToNote(note1, `${supportDir}/photo.jpg`);
const masterKey = await loadEncryptionMasterKey();
await enableEncryption(masterKey, '123456');
await setupAndEnableEncryption(masterKey, '123456');
await loadMasterKeysFromSettings(encryptionService());
await synchronizerStart();
@@ -277,7 +277,7 @@ describe('Synchronizer.e2ee', function() {
const note1 = await Note.save({ title: 'note' });
await shim.attachFileToNote(note1, `${supportDir}/photo.jpg`);
const masterKey = await loadEncryptionMasterKey();
await enableEncryption(masterKey, '123456');
await setupAndEnableEncryption(masterKey, '123456');
await loadMasterKeysFromSettings(encryptionService());
await synchronizerStart();
expect(await allSyncTargetItemsEncrypted()).toBe(true);
@@ -312,7 +312,7 @@ describe('Synchronizer.e2ee', function() {
const note = await Note.save({ title: 'ma note' });
const masterKey = await loadEncryptionMasterKey();
await enableEncryption(masterKey, '123456');
await setupAndEnableEncryption(masterKey, '123456');
await loadMasterKeysFromSettings(encryptionService());
await synchronizerStart();

4
packages/lib/services/synchronizer/Synchronizer.revisions.test.ts
View File

@@ -3,7 +3,7 @@ import BaseModel from '../../BaseModel';
import { synchronizerStart, revisionService, setupDatabaseAndSynchronizer, synchronizer, switchClient, encryptionService, loadEncryptionMasterKey, decryptionWorker } from '../../testing/test-utils';
import Note from '../../models/Note';
import Revision from '../../models/Revision';
import { enableEncryption, loadMasterKeysFromSettings } from '../e2ee/utils';
import { setupAndEnableEncryption, loadMasterKeysFromSettings } from '../e2ee/utils';
describe('Synchronizer.revisions', function() {
@@ -165,7 +165,7 @@ describe('Synchronizer.revisions', function() {
await Note.save({ title: 'ma note', updated_time: dateInPast, created_time: dateInPast }, { autoTimestamp: false });
const masterKey = await loadEncryptionMasterKey();
await enableEncryption(masterKey, '123456');
await setupAndEnableEncryption(masterKey, '123456');
await loadMasterKeysFromSettings(encryptionService());
await synchronizerStart();

1
packages/lib/services/synchronizer/gui/useSyncTargetUpgrade.ts
View File

@@ -27,7 +27,6 @@ export default function useSyncTargetUpgrade(): SyncTargetUpgradeResult {
reg.logger().info('useSyncTargetUpgrade: Create migration handler...');
const migrationHandler = new MigrationHandler(
synchronizer.api(),
synchronizer.syncTargetInfoHandler(),
synchronizer.lockHandler(),
Setting.value('appType'),
Setting.value('clientId')

151
packages/lib/services/synchronizer/syncTargetInfoUtils.ts Normal file
View File

@@ -0,0 +1,151 @@
import { FileApi } from '../../file-api';
// import Logger from '../../Logger';
import Setting from '../../models/Setting';
import { MasterKeyEntity } from '../database/types';
const ArrayUtils = require('../../ArrayUtils');
// const logger = Logger.create('SyncTargetInfoHandler');
export interface SyncTargetInfo {
version: number;
e2ee: boolean;
updatedTime: number;
masterKeys: Record<string, MasterKeyEntity>;
activeMasterKeyId: string;
}
function serializeSyncTargetInfo(info: SyncTargetInfo): string {
return JSON.stringify(info);
}
function unserializeSyncTargetInfo(info: string): SyncTargetInfo {
return JSON.parse(info);
}
// export function setLocalSyncTargetInfoProp(key:string, value:any):void {
// const info = localSyncTargetInfo();
// if (!info) throw new Error('Local sync target info has not been set!');
// if (!(key in info)) throw new Error('Invalid sync target info key: ' + key);
// setLocalSyncTargetInfo({
// ...info,
// [key]: value,
// });
// }
// export function localSyncTargetInfoProp(key:string):any {
// const info = localSyncTargetInfo();
// if (!info) throw new Error('Local sync target info has not been set!');
// if (!(key in info)) throw new Error('Invalid sync target info key: ' + key);
// return (info as any)[key];
// }
export function setLocalSyncTargetInfo(info: SyncTargetInfo) {
Setting.setValue('sync.info', serializeSyncTargetInfo(info));
}
export function localSyncTargetInfo(mustExist: boolean = true): SyncTargetInfo | null {
const info = Setting.value('sync.info');
if (mustExist && !info) throw new Error('Sync info is not set');
return unserializeSyncTargetInfo(info);
}
function validateInfo(info: SyncTargetInfo) {
if (!info.version) throw new Error('Missing "version" field in info.json');
}
export function syncTargetInfoEquals(info1: SyncTargetInfo, info2: SyncTargetInfo): boolean {
if (info1.e2ee !== info2.e2ee) return false;
if (info1.version !== info2.version) return false;
const mks1 = info1.masterKeys || {};
const mks2 = info2.masterKeys || {};
if (Object.keys(mks1).length !== Object.keys(mks2).length) return false;
for (const [id, mk1] of Object.entries(mks1)) {
const mk2 = mks2[id];
if (!mk2) return false;
if (mk1.updated_time !== mk2.updated_time) return false;
}
return true;
}
export function mergeSyncTargetInfos(info1: SyncTargetInfo, info2: SyncTargetInfo): SyncTargetInfo {
const baseInfo = info1.updatedTime > info2.updatedTime ? info1 : info2;
const newInfo: SyncTargetInfo = { ...baseInfo };
const masterKeyIds = ArrayUtils.unique(
Object.keys(info1.masterKeys ? info1.masterKeys : {}).concat(
Object.keys(info2.masterKeys ? info2.masterKeys : {})
)
);
const mergedMasterKeys: Record<string, MasterKeyEntity> = {};
for (const id of masterKeyIds) {
const mk1 = info1.masterKeys[id] || { updated_time: 0 };
const mk2 = info2.masterKeys[id] || { updated_time: 0 };
mergedMasterKeys[id] = mk1.updated_time > mk2.updated_time ? mk1 : mk2;
}
newInfo.masterKeys = mergedMasterKeys;
return newInfo;
}
export async function remoteSyncTargetInfo(api: FileApi): Promise<SyncTargetInfo> {
const syncTargetInfoText = await api.get('info.json');
const defaultFields: SyncTargetInfo = {
version: 0,
e2ee: false,
updatedTime: Date.now(),
masterKeys: {},
activeMasterKeyId: '',
};
// Returns version 0 if the sync target is empty
let output: SyncTargetInfo = defaultFields;
if (syncTargetInfoText) {
output = unserializeSyncTargetInfo(syncTargetInfoText);
validateInfo(output);
} else {
const oldVersion = await api.get('.sync/version.txt');
if (oldVersion) output = { ...defaultFields, version: 1 };
}
return output;
}
export async function setRemoteSyncTargetInfo(api: FileApi, info: SyncTargetInfo) {
await api.put('info.json', serializeSyncTargetInfo(info));
}
// -----------------------------------------------------------------------
// Utility functions to manipulate the SyncTargetInfo data
// -----------------------------------------------------------------------
export function activeMasterKey(info: SyncTargetInfo): MasterKeyEntity {
if (!info.activeMasterKeyId) return null;
// Sanity check - but shouldn't happen because the key is saved at the same
// time as the active master key is set.
if (!info.masterKeys[info.activeMasterKeyId]) throw new Error('Active master key is not present in info.json');
return info.masterKeys[info.activeMasterKeyId];
}
export function enableEncryption(enable: boolean = true) {
setLocalSyncTargetInfo({
...localSyncTargetInfo(),
e2ee: enable,
});
}
export function disableEncryption() {
enableEncryption(false);
}

20
packages/lib/services/synchronizer/synchronizer_MigrationHandler.test.ts
View File

@@ -9,23 +9,16 @@ import { Dirnames } from '../../services/synchronizer/utils/types';
import { setSyncTargetName, fileApi, synchronizer, decryptionWorker, encryptionService, setupDatabaseAndSynchronizer, switchClient, expectThrow, expectNotThrow } from '../../testing/test-utils';
const { deploySyncTargetSnapshot, testData, checkTestData } = require('../../testing/syncTargetUtils');
import { deploySyncTargetSnapshot, testData, checkTestData } from '../../testing/syncTargetUtils';
import Setting from '../../models/Setting';
import MasterKey from '../../models/MasterKey';
import SyncTargetInfoHandler from './SyncTargetInfoHandler';
import { loadMasterKeysFromSettings } from '../e2ee/utils';
import { remoteSyncTargetInfo } from './syncTargetInfoUtils';
const specTimeout = 60000 * 10; // Nextcloud tests can be slow
let lockHandler_: LockHandler = null;
let migrationHandler_: MigrationHandler = null;
let syncTargetInfo_: SyncTargetInfoHandler = null;
function syncTargetInfoHandler(): SyncTargetInfoHandler {
if (syncTargetInfo_) return syncTargetInfo_;
syncTargetInfo_ = new SyncTargetInfoHandler(fileApi());
return syncTargetInfo_;
}
function lockHandler(): LockHandler {
if (lockHandler_) return lockHandler_;
@@ -35,7 +28,7 @@ function lockHandler(): LockHandler {
function migrationHandler(clientId: string = 'abcd'): MigrationHandler {
if (migrationHandler_) return migrationHandler_;
migrationHandler_ = new MigrationHandler(fileApi(), syncTargetInfoHandler(), lockHandler(), 'desktop', clientId);
migrationHandler_ = new MigrationHandler(fileApi(), lockHandler(), 'desktop', clientId);
return migrationHandler_;
}
@@ -75,7 +68,6 @@ describe('synchronizer_MigrationHandler', function() {
previousSyncTargetName = setSyncTargetName('filesystem');
lockHandler_ = null;
migrationHandler_ = null;
syncTargetInfo_ = null;
await setupDatabaseAndSynchronizer(1);
await setupDatabaseAndSynchronizer(2);
await switchClient(1);
@@ -113,14 +105,14 @@ describe('synchronizer_MigrationHandler', function() {
it(`should migrate (${migrationVersion})`, (async () => {
await deploySyncTargetSnapshot('normal', migrationVersion - 1);
const info = await syncTargetInfoHandler().info();
const info = await remoteSyncTargetInfo(fileApi());
expect(info.version).toBe(migrationVersion - 1);
// Now, migrate to the new version
await migrationHandler().upgrade(migrationVersion);
// Verify that it has been upgraded
const newInfo = await syncTargetInfoHandler().info();
const newInfo = await remoteSyncTargetInfo(fileApi());
expect(newInfo.version).toBe(migrationVersion);
await migrationTests[migrationVersion]();
@@ -147,7 +139,7 @@ describe('synchronizer_MigrationHandler', function() {
await migrationHandler().upgrade(migrationVersion);
// Verify that it has been upgraded
const newInfo = await syncTargetInfoHandler().info();
const newInfo = await remoteSyncTargetInfo(fileApi());
expect(newInfo.version).toBe(migrationVersion);
await migrationTests[migrationVersion]();

39
packages/lib/testing/syncTargetUtils.js → packages/lib/testing/syncTargetUtils.ts
View File

@@ -1,16 +1,16 @@
const { syncDir, synchronizer, supportDir, loadEncryptionMasterKey, setupDatabaseAndSynchronizer, switchClient } = require('../testing/test-utils.js');
const Setting = require('../models/Setting').default;
const Folder = require('../models/Folder').default;
const Note = require('../models/Note').default;
const Tag = require('../models/Tag').default;
const Resource = require('../models/Resource').default;
const markdownUtils = require('../markdownUtils').default;
const shim = require('../shim').default;
const fs = require('fs-extra');
import { syncDir, synchronizer, supportDir, loadEncryptionMasterKey, setupDatabaseAndSynchronizer, switchClient } from '../testing/test-utils';
import Setting from '../models/Setting';
import Folder from '../models/Folder';
import Note from '../models/Note';
import Tag from '../models/Tag';
import Resource from '../models/Resource';
import markdownUtils from '../markdownUtils';
import shim from '../shim';
import * as fs from 'fs-extra';
const snapshotBaseDir = `${supportDir}/syncTargetSnapshots`;
const testData = {
export const testData = {
folder1: {
subFolder1: {},
subFolder2: {
@@ -36,8 +36,8 @@ const testData = {
},
};
async function createTestData(data) {
async function recurseStruct(s, parentId = '') {
export async function createTestData(data: any) {
async function recurseStruct(s: any, parentId = '') {
for (const n in s) {
if (n.toLowerCase().includes('folder')) {
const folder = await Folder.save({ title: n, parent_id: parentId });
@@ -60,8 +60,8 @@ async function createTestData(data) {
await recurseStruct(data);
}
async function checkTestData(data) {
async function recurseCheck(s) {
export async function checkTestData(data: any) {
async function recurseCheck(s: any) {
for (const n in s) {
const obj = s[n];
@@ -98,13 +98,13 @@ async function checkTestData(data) {
await recurseCheck(data);
}
async function deploySyncTargetSnapshot(syncTargetType, syncVersion) {
export async function deploySyncTargetSnapshot(syncTargetType: string, syncVersion: number) {
const sourceDir = `${snapshotBaseDir}/${syncVersion}/${syncTargetType}`;
await fs.remove(syncDir);
await fs.copy(sourceDir, syncDir);
}
async function main(syncTargetType) {
export async function main(syncTargetType: string) {
const validSyncTargetTypes = ['normal', 'e2ee'];
if (!validSyncTargetTypes.includes(syncTargetType)) throw new Error(`Sync target type must be: ${validSyncTargetTypes.join(', ')}`);
@@ -128,10 +128,3 @@ async function main(syncTargetType) {
console.info(`Sync target snapshot created in: ${destDir}`);
}
module.exports = {
checkTestData,
main,
testData,
deploySyncTargetSnapshot,
};