From aa6348c5c26a7b42ad6dae8a3a0bbfa4fe17084d Mon Sep 17 00:00:00 2001 From: Self Not Found Date: Sun, 27 Oct 2024 04:15:10 +0800 Subject: [PATCH] All: Add new encryption methods based on native crypto libraries (#10696) Co-authored-by: Laurent Cozic Co-authored-by: Henry Heino --- .eslintignore | 5 + .eslintrc.js | 8 + .gitignore | 5 + packages/app-mobile/package.json | 1 + packages/app-mobile/root.tsx | 4 +- packages/app-mobile/services/e2ee/crypto.ts | 123 ++++++++ .../app-mobile/utils/shim-init-react/index.ts | 3 + .../utils/shim-init-react/index.web.ts | 2 + packages/app-mobile/web/mocks/nodeCrypto.js | 2 + packages/app-mobile/web/webpack.config.js | 1 + .../lib/models/settings/builtInMetadata.ts | 11 + .../services/e2ee/EncryptionService.test.ts | 82 ++++- .../lib/services/e2ee/EncryptionService.ts | 182 ++++++++--- packages/lib/services/e2ee/crypto.test.ts | 103 +++++++ packages/lib/services/e2ee/crypto.ts | 113 +++++++ packages/lib/services/e2ee/cryptoShared.ts | 52 ++++ packages/lib/services/e2ee/cryptoTestUtils.ts | 285 ++++++++++++++++++ packages/lib/services/e2ee/types.ts | 43 +++ .../synchronizer/Synchronizer.e2ee.test.ts | 28 +- packages/lib/shim-init-node.ts | 2 + packages/lib/shim.ts | 3 + packages/tools/cspell/dictionary4.txt | 3 +- yarn.lock | 62 +++- 23 files changed, 1064 insertions(+), 59 deletions(-) create mode 100644 packages/app-mobile/services/e2ee/crypto.ts create mode 100644 packages/app-mobile/web/mocks/nodeCrypto.js create mode 100644 packages/lib/services/e2ee/crypto.test.ts create mode 100644 packages/lib/services/e2ee/crypto.ts create mode 100644 packages/lib/services/e2ee/cryptoShared.ts create mode 100644 packages/lib/services/e2ee/cryptoTestUtils.ts diff --git a/.eslintignore b/.eslintignore index a0291f831..b81fbd16e 100644 --- a/.eslintignore +++ b/.eslintignore @@ -738,6 +738,7 @@ packages/app-mobile/services/AlarmServiceDriver.ios.js packages/app-mobile/services/AlarmServiceDriver.web.js packages/app-mobile/services/BackButtonService.js packages/app-mobile/services/e2ee/RSA.react-native.js +packages/app-mobile/services/e2ee/crypto.js packages/app-mobile/services/plugins/PlatformImplementation.js packages/app-mobile/services/profiles/index.js packages/app-mobile/services/voiceTyping/VoiceTyping.js @@ -1109,6 +1110,10 @@ packages/lib/services/debug/populateDatabase.js packages/lib/services/e2ee/EncryptionService.test.js packages/lib/services/e2ee/EncryptionService.js packages/lib/services/e2ee/RSA.node.js +packages/lib/services/e2ee/crypto.test.js +packages/lib/services/e2ee/crypto.js +packages/lib/services/e2ee/cryptoShared.js +packages/lib/services/e2ee/cryptoTestUtils.js packages/lib/services/e2ee/ppk.test.js packages/lib/services/e2ee/ppk.js packages/lib/services/e2ee/ppkTestUtils.js diff --git a/.eslintrc.js b/.eslintrc.js index 3234dda8e..7e4a0632c 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -287,6 +287,14 @@ module.exports = { 'match': true, }, }, + { + selector: 'enumMember', + format: null, + 'filter': { + 'regex': '^(sha1|sha256|sha384|sha512|AES_128_GCM|AES_192_GCM|AES_256_GCM)$', + 'match': true, + }, + }, // ----------------------------------- // INTERFACE diff --git a/.gitignore b/.gitignore index d077aadaf..2737b8c24 100644 --- a/.gitignore +++ b/.gitignore @@ -715,6 +715,7 @@ packages/app-mobile/services/AlarmServiceDriver.ios.js packages/app-mobile/services/AlarmServiceDriver.web.js packages/app-mobile/services/BackButtonService.js packages/app-mobile/services/e2ee/RSA.react-native.js +packages/app-mobile/services/e2ee/crypto.js packages/app-mobile/services/plugins/PlatformImplementation.js packages/app-mobile/services/profiles/index.js packages/app-mobile/services/voiceTyping/VoiceTyping.js @@ -1086,6 +1087,10 @@ packages/lib/services/debug/populateDatabase.js packages/lib/services/e2ee/EncryptionService.test.js packages/lib/services/e2ee/EncryptionService.js packages/lib/services/e2ee/RSA.node.js +packages/lib/services/e2ee/crypto.test.js +packages/lib/services/e2ee/crypto.js +packages/lib/services/e2ee/cryptoShared.js +packages/lib/services/e2ee/cryptoTestUtils.js packages/lib/services/e2ee/ppk.test.js packages/lib/services/e2ee/ppk.js packages/lib/services/e2ee/ppkTestUtils.js diff --git a/packages/app-mobile/package.json b/packages/app-mobile/package.json index a852fdf3c..8acfbfcb4 100644 --- a/packages/app-mobile/package.json +++ b/packages/app-mobile/package.json @@ -62,6 +62,7 @@ "react-native-paper": "5.12.3", "react-native-popup-menu": "0.16.1", "react-native-quick-actions": "0.3.13", + "react-native-quick-crypto": "0.7.1", "react-native-rsa-native": "2.0.5", "react-native-safe-area-context": "4.10.8", "react-native-securerandom": "1.0.1", diff --git a/packages/app-mobile/root.tsx b/packages/app-mobile/root.tsx index 3acae8bbd..d1b2a3637 100644 --- a/packages/app-mobile/root.tsx +++ b/packages/app-mobile/root.tsx @@ -110,6 +110,7 @@ import { loadMasterKeysFromSettings, migrateMasterPassword } from '@joplin/lib/s import { setRSA } from '@joplin/lib/services/e2ee/ppk'; import RSA from './services/e2ee/RSA.react-native'; import { runIntegrationTests as runRsaIntegrationTests } from '@joplin/lib/services/e2ee/ppkTestUtils'; +import { runIntegrationTests as runCryptoIntegrationTests } from '@joplin/lib/services/e2ee/cryptoTestUtils'; import { Theme, ThemeAppearance } from '@joplin/lib/themes/type'; import ProfileSwitcher from './components/ProfileSwitcher/ProfileSwitcher'; import ProfileEditor from './components/ProfileSwitcher/ProfileEditor'; @@ -818,8 +819,9 @@ async function initialize(dispatch: Dispatch) { if (Platform.OS !== 'web') { await runRsaIntegrationTests(); } else { - logger.info('Skipping RSA tests -- not supported on mobile.'); + logger.info('Skipping encryption tests -- not supported on web.'); } + await runCryptoIntegrationTests(); await runOnDeviceFsDriverTests(); } diff --git a/packages/app-mobile/services/e2ee/crypto.ts b/packages/app-mobile/services/e2ee/crypto.ts new file mode 100644 index 000000000..221103cfd --- /dev/null +++ b/packages/app-mobile/services/e2ee/crypto.ts @@ -0,0 +1,123 @@ +import { Crypto, CryptoBuffer, Digest, CipherAlgorithm, EncryptionResult, EncryptionParameters } from '@joplin/lib/services/e2ee/types'; +import QuickCrypto from 'react-native-quick-crypto'; +import { HashAlgorithm } from 'react-native-quick-crypto/lib/typescript/keys'; +import type { CipherGCMOptions, CipherGCM, DecipherGCM } from 'crypto'; +import { + generateNonce as generateNonceShared, + increaseNonce as increaseNonceShared, + setRandomBytesImplementation, +} from '@joplin/lib/services/e2ee/cryptoShared'; + +type DigestNameMap = Record; +const digestNameMap: DigestNameMap = { + [Digest.sha1]: 'sha1', + [Digest.sha256]: 'sha256', + [Digest.sha384]: 'sha384', + [Digest.sha512]: 'sha512', +}; + +const pbkdf2Raw = (password: string, salt: CryptoBuffer, iterations: number, keylen: number, digest: Digest): Promise => { + return new Promise((resolve, reject) => { + QuickCrypto.pbkdf2(password, salt, iterations, keylen, digest as HashAlgorithm, (error, result) => { + if (error) { + reject(error); + } else { + resolve(result); + } + }); + }); +}; + +const encryptRaw = (data: CryptoBuffer, algorithm: CipherAlgorithm, key: CryptoBuffer, iv: CryptoBuffer, authTagLength: number, associatedData: CryptoBuffer) => { + + const cipher = QuickCrypto.createCipheriv(algorithm, key, iv, { authTagLength: authTagLength } as CipherGCMOptions) as CipherGCM; + + cipher.setAAD(associatedData, { plaintextLength: Buffer.byteLength(data) }); + + const encryptedData = [cipher.update(data), cipher.final()]; + const authTag = cipher.getAuthTag(); + + return Buffer.concat([encryptedData[0], encryptedData[1], authTag]); +}; + +const decryptRaw = (data: CryptoBuffer, algorithm: CipherAlgorithm, key: CryptoBuffer, iv: CryptoBuffer, authTagLength: number, associatedData: CryptoBuffer) => { + + const decipher = QuickCrypto.createDecipheriv(algorithm, key, iv, { authTagLength: authTagLength } as CipherGCMOptions) as DecipherGCM; + + const authTag = data.subarray(-authTagLength); + const encryptedData = data.subarray(0, data.byteLength - authTag.byteLength); + decipher.setAuthTag(authTag); + decipher.setAAD(associatedData, { plaintextLength: Buffer.byteLength(data) }); + + try { + return Buffer.concat([decipher.update(encryptedData), decipher.final()]); + } catch (error) { + throw new Error(`Authentication failed! ${error}`); + } +}; + +const crypto: Crypto = { + + randomBytes: async (size: number) => { + return new Promise((resolve, reject) => { + QuickCrypto.randomBytes(size, (error, result) => { + if (error) { + reject(error); + } else { + resolve(result); + } + }); + }); + }, + + digest: async (algorithm: Digest, data: Uint8Array) => { + const hash = QuickCrypto.createHash(digestNameMap[algorithm]); + hash.update(data); + return hash.digest(); + }, + + encrypt: async (password: string, salt: CryptoBuffer, data: CryptoBuffer, encryptionParameters: EncryptionParameters) => { + + // Parameters in EncryptionParameters won't appear in result + const result: EncryptionResult = { + salt: salt.toString('base64'), + iv: '', + ct: '', // cipherText + }; + + // 96 bits IV + // "For IVs, it is recommended that implementations restrict support to the length of 96 bits, to promote interoperability, efficiency, and simplicity of design." - NIST SP 800-38D + const iv = await crypto.randomBytes(12); + + const key = await pbkdf2Raw(password, salt, encryptionParameters.iterationCount, encryptionParameters.keyLength, encryptionParameters.digestAlgorithm); + const encrypted = encryptRaw(data, encryptionParameters.cipherAlgorithm, key, iv, encryptionParameters.authTagLength, encryptionParameters.associatedData); + + result.iv = iv.toString('base64'); + result.ct = encrypted.toString('base64'); + + return result; + }, + + decrypt: async (password: string, data: EncryptionResult, encryptionParameters: EncryptionParameters) => { + + const salt = Buffer.from(data.salt, 'base64'); + const iv = Buffer.from(data.iv, 'base64'); + + const key = await pbkdf2Raw(password, salt, encryptionParameters.iterationCount, encryptionParameters.keyLength, encryptionParameters.digestAlgorithm); + const decrypted = decryptRaw(Buffer.from(data.ct, 'base64'), encryptionParameters.cipherAlgorithm, key, iv, encryptionParameters.authTagLength, encryptionParameters.associatedData); + + return decrypted; + }, + + encryptString: async (password: string, salt: CryptoBuffer, data: string, encoding: BufferEncoding, encryptionParameters: EncryptionParameters) => { + return crypto.encrypt(password, salt, Buffer.from(data, encoding), encryptionParameters); + }, + + generateNonce: generateNonceShared, + + increaseNonce: increaseNonceShared, +}; + +setRandomBytesImplementation(crypto.randomBytes); + +export default crypto; diff --git a/packages/app-mobile/utils/shim-init-react/index.ts b/packages/app-mobile/utils/shim-init-react/index.ts index 27da1025d..bd57fd54e 100644 --- a/packages/app-mobile/utils/shim-init-react/index.ts +++ b/packages/app-mobile/utils/shim-init-react/index.ts @@ -6,6 +6,7 @@ import RNFetchBlob from 'rn-fetch-blob'; import { generateSecureRandom } from 'react-native-securerandom'; import FsDriverRN from '../fs-driver/fs-driver-rn'; import { Linking, Platform } from 'react-native'; +import crypto from '../../services/e2ee/crypto'; const RNExitApp = require('react-native-exit-app').default; export default function shimInit() { @@ -18,6 +19,8 @@ export default function shimInit() { return shim.fsDriver_; }; + shim.crypto = crypto; + shim.randomBytes = async (count: number) => { const randomBytes = await generateSecureRandom(count); const temp = []; diff --git a/packages/app-mobile/utils/shim-init-react/index.web.ts b/packages/app-mobile/utils/shim-init-react/index.web.ts index c92e1e0d0..e0b648a22 100644 --- a/packages/app-mobile/utils/shim-init-react/index.web.ts +++ b/packages/app-mobile/utils/shim-init-react/index.web.ts @@ -5,6 +5,7 @@ import shimInitShared from './shimInitShared'; import FsDriverWeb from '../fs-driver/fs-driver-rn.web'; import { FetchBlobOptions } from '@joplin/lib/types'; import JoplinError from '@joplin/lib/JoplinError'; +import joplinCrypto from '@joplin/lib/services/e2ee/crypto'; const shimInit = () => { type GetLocationOptions = { timeout?: number }; @@ -41,6 +42,7 @@ const shimInit = () => { return fsDriver_; }; shim.fsDriver = fsDriver; + shim.crypto = joplinCrypto; shim.randomBytes = async (count: number) => { const buffer = new Uint8Array(count); diff --git a/packages/app-mobile/web/mocks/nodeCrypto.js b/packages/app-mobile/web/mocks/nodeCrypto.js new file mode 100644 index 000000000..25d0c48dd --- /dev/null +++ b/packages/app-mobile/web/mocks/nodeCrypto.js @@ -0,0 +1,2 @@ + +exports.webcrypto = crypto; diff --git a/packages/app-mobile/web/webpack.config.js b/packages/app-mobile/web/webpack.config.js index 2a00a6b65..0e6f7f53c 100644 --- a/packages/app-mobile/web/webpack.config.js +++ b/packages/app-mobile/web/webpack.config.js @@ -61,6 +61,7 @@ module.exports = { resolve: { alias: { 'react-native$': 'react-native-web', + 'crypto': path.resolve(__dirname, 'mocks/nodeCrypto.js'), // Map some modules that don't work on web to the empty dictionary. 'react-native-fingerprint-scanner': emptyLibraryMock, diff --git a/packages/lib/models/settings/builtInMetadata.ts b/packages/lib/models/settings/builtInMetadata.ts index 3e035f893..d052a8568 100644 --- a/packages/lib/models/settings/builtInMetadata.ts +++ b/packages/lib/models/settings/builtInMetadata.ts @@ -1603,6 +1603,17 @@ const builtInMetadata = (Setting: typeof SettingType) => { // storage: SettingStorage.File, // }, + 'featureFlag.useBetaEncryptionMethod': { + value: false, + type: SettingItemType.Bool, + public: true, + storage: SettingStorage.File, + label: () => 'Use beta encryption', + description: () => 'Set beta encryption methods as the default methods. This applies to all clients and takes effect after restarting the app.', + section: 'sync', + isGlobal: true, + }, + 'sync.allowUnsupportedProviders': { value: -1, type: SettingItemType.Int, diff --git a/packages/lib/services/e2ee/EncryptionService.test.ts b/packages/lib/services/e2ee/EncryptionService.test.ts index 628939fb8..ffeb76c33 100644 --- a/packages/lib/services/e2ee/EncryptionService.test.ts +++ b/packages/lib/services/e2ee/EncryptionService.test.ts @@ -95,9 +95,12 @@ describe('services_EncryptionService', () => { expect(!!masterKey.content).toBe(true); })); - it('should not require a checksum for new master keys', (async () => { + it.each([ + EncryptionMethod.SJCL4, + EncryptionMethod.KeyV1, + ])('should not require a checksum for new master keys', (async (masterKeyEncryptionMethod) => { const masterKey = await service.generateMasterKey('123456', { - encryptionMethod: EncryptionMethod.SJCL4, + encryptionMethod: masterKeyEncryptionMethod, }); expect(!masterKey.checksum).toBe(true); @@ -107,9 +110,12 @@ describe('services_EncryptionService', () => { expect(decryptedMasterKey.length).toBe(512); })); - it('should throw an error if master key decryption fails', (async () => { + it.each([ + EncryptionMethod.SJCL4, + EncryptionMethod.KeyV1, + ])('should throw an error if master key decryption fails', (async (masterKeyEncryptionMethod) => { const masterKey = await service.generateMasterKey('123456', { - encryptionMethod: EncryptionMethod.SJCL4, + encryptionMethod: masterKeyEncryptionMethod, }); const hasThrown = await checkThrowAsync(async () => await service.decryptMasterKeyContent(masterKey, 'wrong')); @@ -148,7 +154,7 @@ describe('services_EncryptionService', () => { // Test that a long string, that is going to be split into multiple chunks, encrypt // and decrypt properly too. let veryLongSecret = ''; - for (let i = 0; i < service.chunkSize() * 3; i++) veryLongSecret += Math.floor(Math.random() * 9); + for (let i = 0; i < service.chunkSize(service.defaultEncryptionMethod()) * 3; i++) veryLongSecret += Math.floor(Math.random() * 9); const cipherText2 = await service.encryptString(veryLongSecret); const plainText2 = await service.decryptString(cipherText2); @@ -212,6 +218,56 @@ describe('services_EncryptionService', () => { expect(hasThrown).toBe(true); })); + it.each([ + EncryptionMethod.SJCL1a, + EncryptionMethod.SJCL1b, + EncryptionMethod.SJCL4, + EncryptionMethod.KeyV1, + EncryptionMethod.FileV1, + EncryptionMethod.StringV1, + ])('should fail to decrypt if ciphertext is not a valid JSON string', (async (jsonCipherTextMethod) => { + const masterKey = await service.generateMasterKey('123456'); + const masterKeyContent = await service.decryptMasterKeyContent(masterKey, '123456'); + + const cipherTextString = await service.encrypt(jsonCipherTextMethod, masterKeyContent, 'e21de21d'); // 'e21de21d' is a valid base64/hex string + + // Check if decryption is working + const plainText = await service.decrypt(jsonCipherTextMethod, masterKeyContent, cipherTextString); + expect(plainText).toBe('e21de21d'); + + // Make invalid JSON + const invalidCipherText = cipherTextString.replace('{', '{,'); + const hasThrown = await checkThrowAsync(async () => await service.decrypt(jsonCipherTextMethod, masterKeyContent, invalidCipherText)); + expect(hasThrown).toBe(true); + })); + + it.each([ + EncryptionMethod.SJCL1a, + EncryptionMethod.SJCL1b, + EncryptionMethod.SJCL4, + EncryptionMethod.KeyV1, + EncryptionMethod.FileV1, + EncryptionMethod.StringV1, + ])('should fail to decrypt if ciphertext authentication failed', (async (authenticatedEncryptionMethod) => { + const masterKey = await service.generateMasterKey('123456'); + const masterKeyContent = await service.decryptMasterKeyContent(masterKey, '123456'); + + const cipherTextObject = JSON.parse(await service.encrypt(authenticatedEncryptionMethod, masterKeyContent, 'e21de21d')); // 'e21de21d' is a valid base64/hex string + expect(cipherTextObject).toHaveProperty('ct'); + const ct = Buffer.from(cipherTextObject['ct'], 'base64'); + + // Should not fail if the binary data of ct is not modified + const oldCipherTextObject = { ...cipherTextObject, ct: ct.toString('base64') }; + const plainText = await service.decrypt(authenticatedEncryptionMethod, masterKeyContent, JSON.stringify(oldCipherTextObject)); + expect(plainText).toBe('e21de21d'); + + // The encrypted data part is changed so it doesn't match the authentication tag. Decryption should fail. + ct[0] ^= 0x55; + const newCipherTextObject = { ...cipherTextObject, ct: ct.toString('base64') }; + const hasThrown = await checkThrowAsync(async () => service.decrypt(authenticatedEncryptionMethod, masterKeyContent, JSON.stringify(newCipherTextObject))); + expect(hasThrown).toBe(true); + })); + it('should encrypt and decrypt notes and folders', (async () => { let masterKey = await service.generateMasterKey('123456'); masterKey = await MasterKey.save(masterKey); @@ -243,7 +299,12 @@ describe('services_EncryptionService', () => { expect(decryptedNote.parent_id).toBe(note.parent_id); })); - it('should encrypt and decrypt files', (async () => { + it.each([ + EncryptionMethod.SJCL1a, + EncryptionMethod.SJCL1b, + EncryptionMethod.FileV1, + EncryptionMethod.StringV1, + ])('should encrypt and decrypt files', (async (fileEncryptionMethod) => { let masterKey = await service.generateMasterKey('123456'); masterKey = await MasterKey.save(masterKey); await service.loadMasterKey(masterKey, '123456', true); @@ -252,6 +313,7 @@ describe('services_EncryptionService', () => { const encryptedPath = `${Setting.value('tempDir')}/photo.crypted`; const decryptedPath = `${Setting.value('tempDir')}/photo.jpg`; + service.defaultFileEncryptionMethod_ = fileEncryptionMethod; await service.encryptFile(sourcePath, encryptedPath); await service.decryptFile(encryptedPath, decryptedPath); @@ -259,7 +321,11 @@ describe('services_EncryptionService', () => { expect(fileContentEqual(sourcePath, decryptedPath)).toBe(true); })); - it('should encrypt invalid UTF-8 data', (async () => { + it.each([ + EncryptionMethod.SJCL1a, + EncryptionMethod.SJCL1b, + EncryptionMethod.StringV1, + ])('should encrypt invalid UTF-8 data', (async (stringEncryptionMethod) => { let masterKey = await service.generateMasterKey('123456'); masterKey = await MasterKey.save(masterKey); @@ -271,7 +337,7 @@ describe('services_EncryptionService', () => { expect(hasThrown).toBe(true); // Now check that the new one fixes the problem - service.defaultEncryptionMethod_ = EncryptionMethod.SJCL1a; + service.defaultEncryptionMethod_ = stringEncryptionMethod; const cipherText = await service.encryptString('🐢🐢🐢'.substr(0, 5)); const plainText = await service.decryptString(cipherText); expect(plainText).toBe('🐢🐢🐢'.substr(0, 5)); diff --git a/packages/lib/services/e2ee/EncryptionService.ts b/packages/lib/services/e2ee/EncryptionService.ts index 4ec7191a5..e67118e7d 100644 --- a/packages/lib/services/e2ee/EncryptionService.ts +++ b/packages/lib/services/e2ee/EncryptionService.ts @@ -1,4 +1,4 @@ -import { MasterKeyEntity } from './types'; +import { CipherAlgorithm, Digest, MasterKeyEntity } from './types'; import Logger from '@joplin/utils/Logger'; import shim from '../../shim'; import Setting from '../../models/Setting'; @@ -10,6 +10,8 @@ const { padLeft } = require('../../string-utils.js'); const logger = Logger.create('EncryptionService'); +const emptyUint8Array = new Uint8Array(0); + function hexPad(s: string, length: number) { return padLeft(s, length, '0'); } @@ -42,6 +44,9 @@ export enum EncryptionMethod { SJCL1a = 5, Custom = 6, SJCL1b = 7, + KeyV1 = 8, + FileV1 = 9, + StringV1 = 10, } export interface EncryptOptions { @@ -65,24 +70,13 @@ export default class EncryptionService { // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied public static fsDriver_: any = null; - // Note: 1 MB is very slow with Node and probably even worse on mobile. - // - // On mobile the time it takes to decrypt increases exponentially for some reason, so it's important - // to have a relatively small size so as not to freeze the app. For example, on Android 7.1 simulator - // with 4.1 GB RAM, it takes this much to decrypt a block; - // - // 50KB => 1000 ms - // 25KB => 250ms - // 10KB => 200ms - // 5KB => 10ms - // - // So making the block 10 times smaller make it 100 times faster! So for now using 5KB. This can be - // changed easily since the chunk size is incorporated into the encrypted data. - private chunkSize_ = 5000; private encryptedMasterKeys_: Map = new Map(); private decryptedMasterKeys_: Map = new Map(); - public defaultEncryptionMethod_ = EncryptionMethod.SJCL1a; // public because used in tests - private defaultMasterKeyEncryptionMethod_ = EncryptionMethod.SJCL4; + public defaultEncryptionMethod_ = Setting.value('featureFlag.useBetaEncryptionMethod') ? EncryptionMethod.StringV1 : EncryptionMethod.SJCL1a; // public because used in tests + public defaultFileEncryptionMethod_ = Setting.value('featureFlag.useBetaEncryptionMethod') ? EncryptionMethod.FileV1 : EncryptionMethod.SJCL1a; // public because used in tests + private defaultMasterKeyEncryptionMethod_ = Setting.value('featureFlag.useBetaEncryptionMethod') ? EncryptionMethod.KeyV1 : EncryptionMethod.SJCL4; + + private encryptionNonce_: Uint8Array = null; private headerTemplates_ = { // Template version 1 @@ -92,6 +86,15 @@ export default class EncryptionService { }, }; + public constructor() { + const crypto = shim.crypto; + crypto.generateNonce(new Uint8Array(36)) + // eslint-disable-next-line promise/prefer-await-to-then + .then((nonce) => this.encryptionNonce_ = nonce) + // eslint-disable-next-line promise/prefer-await-to-then + .catch((error) => logger.error(error)); + } + public static instance() { if (this.instance_) return this.instance_; this.instance_ = new EncryptionService(); @@ -106,14 +109,47 @@ export default class EncryptionService { return this.loadedMasterKeyIds().length; } - public chunkSize() { - return this.chunkSize_; + // Note for methods using SJCL: + // + // 1 MB is very slow with Node and probably even worse on mobile. + // + // On mobile the time it takes to decrypt increases exponentially for some reason, so it's important + // to have a relatively small size so as not to freeze the app. For example, on Android 7.1 simulator + // with 4.1 GB RAM, it takes this much to decrypt a block; + // + // 50KB => 1000 ms + // 25KB => 250ms + // 10KB => 200ms + // 5KB => 10ms + // + // So making the block 10 times smaller make it 100 times faster! So for now using 5KB. This can be + // changed easily since the chunk size is incorporated into the encrypted data. + public chunkSize(method: EncryptionMethod) { + type EncryptionMethodChunkSizeMap = Record; + const encryptionMethodChunkSizeMap: EncryptionMethodChunkSizeMap = { + [EncryptionMethod.SJCL]: 5000, + [EncryptionMethod.SJCL1a]: 5000, + [EncryptionMethod.SJCL1b]: 5000, + [EncryptionMethod.SJCL2]: 5000, + [EncryptionMethod.SJCL3]: 5000, + [EncryptionMethod.SJCL4]: 5000, + [EncryptionMethod.Custom]: 5000, + [EncryptionMethod.KeyV1]: 5000, // Master key is not encrypted by chunks so this value will not be used. + [EncryptionMethod.FileV1]: 131072, // 128k + [EncryptionMethod.StringV1]: 65536, // 64k + }; + + return encryptionMethodChunkSizeMap[method]; } public defaultEncryptionMethod() { return this.defaultEncryptionMethod_; } + public defaultFileEncryptionMethod() { + return this.defaultFileEncryptionMethod_; + } + public setActiveMasterKeyId(id: string) { setActiveMasterKeyId(id); } @@ -322,8 +358,10 @@ export default class EncryptionService { if (!key) throw new Error('Encryption key is required'); const sjcl = shim.sjclModule; + const crypto = shim.crypto; - const handlers: Record string> = { + type EncryptionMethodHandler = (()=> Promise); + const handlers: Record = { // 2020-01-23: Deprecated and no longer secure due to the use og OCB2 mode - do not use. [EncryptionMethod.SJCL]: () => { try { @@ -438,6 +476,47 @@ export default class EncryptionService { } }, + // New encryption method powered by native crypto libraries(node:crypto/react-native-quick-crypto). Using AES-256-GCM and pbkdf2 + // The master key is not directly used. A new data key is generated from the master key and a 256 bits random salt to prevent nonce reuse problem + // 2024-08: Set iteration count in pbkdf2 to 220000 as suggested by OWASP. https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#pbkdf2 + [EncryptionMethod.KeyV1]: async () => { + return JSON.stringify(await crypto.encryptString(key, await crypto.digest(Digest.sha256, this.encryptionNonce_), plainText, 'hex', { + cipherAlgorithm: CipherAlgorithm.AES_256_GCM, + authTagLength: 16, + digestAlgorithm: Digest.sha512, + keyLength: 32, + associatedData: emptyUint8Array, + iterationCount: 220000, + })); + }, + + // New encryption method powered by native crypto libraries(node:crypto/react-native-quick-crypto). Using AES-256-GCM and pbkdf2 + // The master key is not directly used. A new data key is generated from the master key and a 256 bits random salt to prevent nonce reuse problem + // The file content is base64 encoded. Decoding it before encryption to reduce the size overhead. + [EncryptionMethod.FileV1]: async () => { + return JSON.stringify(await crypto.encryptString(key, await crypto.digest(Digest.sha256, this.encryptionNonce_), plainText, 'base64', { + cipherAlgorithm: CipherAlgorithm.AES_256_GCM, + authTagLength: 16, + digestAlgorithm: Digest.sha512, + keyLength: 32, + associatedData: emptyUint8Array, + iterationCount: 3, + })); + }, + + // New encryption method powered by native crypto libraries(node:crypto/react-native-quick-crypto). Using AES-256-GCM and pbkdf2 + // The master key is not directly used. A new data key is generated from the master key and a 256 bits random salt to prevent nonce reuse problem + [EncryptionMethod.StringV1]: async () => { + return JSON.stringify(await crypto.encryptString(key, await crypto.digest(Digest.sha256, this.encryptionNonce_), plainText, 'utf16le', { + cipherAlgorithm: CipherAlgorithm.AES_256_GCM, + authTagLength: 16, + digestAlgorithm: Digest.sha512, + keyLength: 32, + associatedData: emptyUint8Array, + iterationCount: 3, + })); + }, + [EncryptionMethod.Custom]: () => { // This is handled elsewhere but as a sanity check, throw an exception throw new Error('Custom encryption method is not supported here'); @@ -452,19 +531,49 @@ export default class EncryptionService { if (!key) throw new Error('Encryption key is required'); const sjcl = shim.sjclModule; - if (!this.isValidEncryptionMethod(method)) throw new Error(`Unknown decryption method: ${method}`); + const crypto = shim.crypto; + if (method === EncryptionMethod.KeyV1) { + return (await crypto.decrypt(key, JSON.parse(cipherText), { + cipherAlgorithm: CipherAlgorithm.AES_256_GCM, + authTagLength: 16, + digestAlgorithm: Digest.sha512, + keyLength: 32, + associatedData: emptyUint8Array, + iterationCount: 220000, + })).toString('hex'); + } else if (method === EncryptionMethod.FileV1) { + return (await crypto.decrypt(key, JSON.parse(cipherText), { + cipherAlgorithm: CipherAlgorithm.AES_256_GCM, + authTagLength: 16, + digestAlgorithm: Digest.sha512, + keyLength: 32, + associatedData: emptyUint8Array, + iterationCount: 3, + })).toString('base64'); + } else if (method === EncryptionMethod.StringV1) { + return (await crypto.decrypt(key, JSON.parse(cipherText), { + cipherAlgorithm: CipherAlgorithm.AES_256_GCM, + authTagLength: 16, + digestAlgorithm: Digest.sha512, + keyLength: 32, + associatedData: emptyUint8Array, + iterationCount: 3, + })).toString('utf16le'); + } else if (this.isValidSjclEncryptionMethod(method)) { + try { + const output = sjcl.json.decrypt(key, cipherText); - try { - const output = sjcl.json.decrypt(key, cipherText); - - if (method === EncryptionMethod.SJCL1a || method === EncryptionMethod.SJCL1b) { - return unescape(output); - } else { - return output; + if (method === EncryptionMethod.SJCL1a || method === EncryptionMethod.SJCL1b) { + return unescape(output); + } else { + return output; + } + } catch (error) { + // SJCL returns a string as error which means stack trace is missing so convert to an error object here + throw new Error(error.message); } - } catch (error) { - // SJCL returns a string as error which means stack trace is missing so convert to an error object here - throw new Error(error.message); + } else { + throw new Error(`Unknown decryption method: ${method}`); } } @@ -475,6 +584,8 @@ export default class EncryptionService { const method = options.encryptionMethod; const masterKeyId = options.masterKeyId ? options.masterKeyId : this.activeMasterKeyId(); const masterKeyPlainText = (await this.loadedMasterKey(masterKeyId)).plainText; + const chunkSize = this.chunkSize(method); + const crypto = shim.crypto; const header = { encryptionMethod: method, @@ -486,10 +597,10 @@ export default class EncryptionService { let doneSize = 0; while (true) { - const block = await source.read(this.chunkSize_); + const block = await source.read(chunkSize); if (!block) break; - doneSize += this.chunkSize_; + doneSize += chunkSize; if (options.onProgress) options.onProgress({ doneSize: doneSize }); // Wait for a frame so that the app remains responsive in mobile. @@ -497,6 +608,7 @@ export default class EncryptionService { await shim.waitForFrame(); const encrypted = await this.encrypt(method, masterKeyPlainText, block); + await crypto.increaseNonce(this.encryptionNonce_); await destination.append(padLeft(encrypted.length.toString(16), 6, '0')); await destination.append(encrypted); @@ -604,6 +716,8 @@ export default class EncryptionService { } public async encryptFile(srcPath: string, destPath: string, options: EncryptOptions = null) { + options = { encryptionMethod: this.defaultFileEncryptionMethod(), ...options }; + let source = await this.fileReader_(srcPath, 'base64'); let destination = await this.fileWriter_(destPath, 'ascii'); @@ -724,7 +838,7 @@ export default class EncryptionService { return output; } - public isValidEncryptionMethod(method: EncryptionMethod) { + private isValidSjclEncryptionMethod(method: EncryptionMethod) { return [EncryptionMethod.SJCL, EncryptionMethod.SJCL1a, EncryptionMethod.SJCL1b, EncryptionMethod.SJCL2, EncryptionMethod.SJCL3, EncryptionMethod.SJCL4].indexOf(method) >= 0; } diff --git a/packages/lib/services/e2ee/crypto.test.ts b/packages/lib/services/e2ee/crypto.test.ts new file mode 100644 index 000000000..ff39b3110 --- /dev/null +++ b/packages/lib/services/e2ee/crypto.test.ts @@ -0,0 +1,103 @@ +import { afterAllCleanUp, expectNotThrow, setupDatabaseAndSynchronizer, switchClient } from '../../testing/test-utils'; +import { runIntegrationTests } from './cryptoTestUtils'; +import crypto from './crypto'; + +describe('e2ee/crypto', () => { + + beforeEach(async () => { + jest.useRealTimers(); + await setupDatabaseAndSynchronizer(1); + await switchClient(1); + }); + + afterAll(async () => { + await afterAllCleanUp(); + }); + + it('should encrypt and decrypt data from different devices', (async () => { + await expectNotThrow(async () => runIntegrationTests(true)); + })); + + it('should not generate new nonce if counter does not overflow', (async () => { + jest.useFakeTimers(); + + const nonce = await crypto.generateNonce(new Uint8Array(36)); + expect(nonce.subarray(-8)).toEqual(new Uint8Array(8)); + const nonCounterPart = nonce.slice(0, 28); + + jest.advanceTimersByTime(1); + await crypto.increaseNonce(nonce); + // Counter should have expected value + expect(nonce.subarray(-8)).toEqual(new Uint8Array([0, 0, 0, 0, 0, 0, 0, 1])); + // Non-counter part should stay the same + expect(nonce.subarray(0, 28)).toEqual(nonCounterPart); + + jest.advanceTimersByTime(1); + await crypto.increaseNonce(nonce); + // Counter should have expected value + expect(nonce.subarray(-8)).toEqual(new Uint8Array([0, 0, 0, 0, 0, 0, 0, 2])); + // Non-counter part should stay the same + expect(nonce.subarray(0, 28)).toEqual(nonCounterPart); + + jest.advanceTimersByTime(1); + nonce.set(new Uint8Array([248, 249, 250, 251, 252, 253, 254, 255]), 28); + await crypto.increaseNonce(nonce); + // Counter should have expected value + expect(nonce.subarray(-8)).toEqual(new Uint8Array([248, 249, 250, 251, 252, 253, 255, 0])); + // Non-counter part should stay the same + expect(nonce.subarray(0, 28)).toEqual(nonCounterPart); + + jest.advanceTimersByTime(1); + nonce.set(new Uint8Array([249, 250, 251, 252, 253, 254, 255, 255]), 28); + await crypto.increaseNonce(nonce); + // Counter should have expected value + expect(nonce.subarray(-8)).toEqual(new Uint8Array([249, 250, 251, 252, 253, 255, 0, 0])); + // Non-counter part should stay the same + expect(nonce.subarray(0, 28)).toEqual(nonCounterPart); + + jest.advanceTimersByTime(1); + nonce.set(new Uint8Array([253, 254, 255, 255, 255, 255, 255, 255]), 28); + await crypto.increaseNonce(nonce); + // Counter should have expected value + expect(nonce.subarray(-8)).toEqual(new Uint8Array([253, 255, 0, 0, 0, 0, 0, 0])); + // Non-counter part should stay the same + expect(nonce.subarray(0, 28)).toEqual(nonCounterPart); + + jest.advanceTimersByTime(1); + nonce.set(new Uint8Array([254, 255, 255, 255, 255, 255, 255, 255]), 28); + await crypto.increaseNonce(nonce); + // Counter should have expected value + expect(nonce.subarray(-8)).toEqual(new Uint8Array([255, 0, 0, 0, 0, 0, 0, 0])); + // Non-counter part should stay the same + expect(nonce.subarray(0, 28)).toEqual(nonCounterPart); + })); + + it('should generate new nonce if counter overflow', (async () => { + jest.useFakeTimers(); + + const nonce = await crypto.generateNonce(new Uint8Array(36)); + expect(nonce.subarray(-8)).toEqual(new Uint8Array(8)); + const nonCounterPart = nonce.slice(0, 28); + const randomPart = nonce.slice(0, 21); + const timestampPart = nonce.slice(21, 28); + + jest.advanceTimersByTime(1); + await crypto.increaseNonce(nonce); + // Counter should have expected value + expect(nonce.subarray(-8)).toEqual(new Uint8Array([0, 0, 0, 0, 0, 0, 0, 1])); + // Non-counter part should stay the same + expect(nonce.subarray(0, 28)).toEqual(nonCounterPart); + + jest.advanceTimersByTime(1); + nonce.set(new Uint8Array([255, 255, 255, 255, 255, 255, 255, 255]), 28); + await crypto.increaseNonce(nonce); + // Counter should have expected value + expect(nonce.subarray(-8)).toEqual(new Uint8Array([0, 0, 0, 0, 0, 0, 0, 0])); + // Random part should be changed + expect(nonce.subarray(0, 21)).not.toEqual(randomPart); + // Timestamp part should have expected value + expect(nonce[21]).toBe(timestampPart[0] + 2); + expect(nonce.subarray(22, 28)).toEqual(timestampPart.subarray(1)); + })); + +}); diff --git a/packages/lib/services/e2ee/crypto.ts b/packages/lib/services/e2ee/crypto.ts new file mode 100644 index 000000000..2e0da0833 --- /dev/null +++ b/packages/lib/services/e2ee/crypto.ts @@ -0,0 +1,113 @@ +import { Crypto, CryptoBuffer, Digest, EncryptionResult, EncryptionParameters } from './types'; +import { webcrypto } from 'crypto'; +import { Buffer } from 'buffer'; +import { + generateNonce as generateNonceShared, + increaseNonce as increaseNonceShared, + setRandomBytesImplementation, +} from './cryptoShared'; + +const pbkdf2Raw = async (password: string, salt: Uint8Array, iterations: number, keylenBytes: number, digest: Digest) => { + const encoder = new TextEncoder(); + const key = await webcrypto.subtle.importKey( + 'raw', encoder.encode(password), { name: 'PBKDF2' }, false, ['deriveKey'], + ); + return webcrypto.subtle.deriveKey( + { name: 'PBKDF2', salt, iterations, hash: digest }, key, { name: 'AES-GCM', length: keylenBytes * 8 }, false, ['encrypt', 'decrypt'], + ); +}; + +const encryptRaw = async (data: Uint8Array, key: webcrypto.CryptoKey, iv: Uint8Array, authTagLengthBytes: number, additionalData: Uint8Array) => { + return Buffer.from(await webcrypto.subtle.encrypt({ + name: 'AES-GCM', + iv, + additionalData, + tagLength: authTagLengthBytes * 8, + }, key, data)); +}; + +const decryptRaw = async (data: Uint8Array, key: webcrypto.CryptoKey, iv: Uint8Array, authTagLengthBytes: number, associatedData: Uint8Array) => { + return Buffer.from(await webcrypto.subtle.decrypt({ + name: 'AES-GCM', + iv, + additionalData: associatedData, + tagLength: authTagLengthBytes * 8, + }, key, data)); +}; + +const crypto: Crypto = { + + randomBytes: async (size: number) => { + // .getRandomValues has a maximum output size + const maxChunkSize = 65536; + const result = new Uint8Array(size); + + if (size <= maxChunkSize) { + webcrypto.getRandomValues(result); + } else { + const fullSizeChunk = new Uint8Array(maxChunkSize); + const lastChunkSize = size % maxChunkSize; + const maxOffset = size - lastChunkSize; + let offset = 0; + while (offset < maxOffset) { + webcrypto.getRandomValues(fullSizeChunk); + result.set(fullSizeChunk, offset); + offset += maxChunkSize; + } + if (lastChunkSize > 0) { + const lastChunk = webcrypto.getRandomValues(new Uint8Array(lastChunkSize)); + result.set(lastChunk, offset); + } + } + return Buffer.from(result); + }, + + digest: async (algorithm: Digest, data: Uint8Array) => { + return Buffer.from(await webcrypto.subtle.digest(algorithm, data)); + }, + + encrypt: async (password: string, salt: CryptoBuffer, data: CryptoBuffer, encryptionParameters: EncryptionParameters) => { + + // Parameters in EncryptionParameters won't appear in result + const result: EncryptionResult = { + salt: salt.toString('base64'), + iv: '', + ct: '', // cipherText + }; + + // 96 bits IV + // "For IVs, it is recommended that implementations restrict support to the length of 96 bits, to promote interoperability, efficiency, and simplicity of design." - NIST SP 800-38D + const iv = await crypto.randomBytes(12); + + const key = await pbkdf2Raw(password, salt, encryptionParameters.iterationCount, encryptionParameters.keyLength, encryptionParameters.digestAlgorithm); + const encrypted = await encryptRaw(data, key, iv, encryptionParameters.authTagLength, encryptionParameters.associatedData); + + result.iv = iv.toString('base64'); + result.ct = encrypted.toString('base64'); + + return result; + }, + + decrypt: async (password: string, data: EncryptionResult, encryptionParameters: EncryptionParameters) => { + + const salt = Buffer.from(data.salt, 'base64'); + const iv = Buffer.from(data.iv, 'base64'); + + const key = await pbkdf2Raw(password, salt, encryptionParameters.iterationCount, encryptionParameters.keyLength, encryptionParameters.digestAlgorithm); + const decrypted = decryptRaw(Buffer.from(data.ct, 'base64'), key, iv, encryptionParameters.authTagLength, encryptionParameters.associatedData); + + return decrypted; + }, + + encryptString: async (password: string, salt: CryptoBuffer, data: string, encoding: BufferEncoding, encryptionParameters: EncryptionParameters) => { + return crypto.encrypt(password, salt, Buffer.from(data, encoding), encryptionParameters); + }, + + generateNonce: generateNonceShared, + + increaseNonce: increaseNonceShared, +}; + +setRandomBytesImplementation(crypto.randomBytes); + +export default crypto; diff --git a/packages/lib/services/e2ee/cryptoShared.ts b/packages/lib/services/e2ee/cryptoShared.ts new file mode 100644 index 000000000..829d0e508 --- /dev/null +++ b/packages/lib/services/e2ee/cryptoShared.ts @@ -0,0 +1,52 @@ +import { CryptoBuffer } from './types'; + +const nonceCounterLength = 8; +const nonceTimestampLength = 7; + +type RandomBytesImplementation = (size: number)=> Promise; + +let randomBytesImplementation: RandomBytesImplementation = null; + +export const setRandomBytesImplementation = (implementation: RandomBytesImplementation) => { + randomBytesImplementation = implementation; +}; + +export const generateNonce = async (nonce: Uint8Array) => { + const randomLength = nonce.length - nonceTimestampLength - nonceCounterLength; + if (randomLength < 1) { + throw new Error(`Nonce length should be greater than ${(nonceTimestampLength + nonceCounterLength) * 8} bits`); + } + nonce.set(await randomBytesImplementation(randomLength)); + const timestampArray = new Uint8Array(nonceTimestampLength); + let timestamp = Date.now(); + let timestampMsb = Math.floor(timestamp / 2 ** 32); + const lsbCount = Math.min(4, nonceTimestampLength); + for (let i = 0; i < lsbCount; i++) { + timestampArray[i] = timestamp & 0xFF; + timestamp >>>= 8; + } + // The bitwise operators in Typescript only take the 32 LSBs to calculate, so we need to extract the MSBs manually. + for (let i = 4; i < nonceTimestampLength; i++) { + timestampArray[i] = timestampMsb & 0xFF; + timestampMsb >>>= 8; + } + nonce.set(timestampArray, randomLength); + nonce.set(new Uint8Array(nonceCounterLength), randomLength + nonceTimestampLength); + return nonce; +}; + +export const increaseNonce = async (nonce: Uint8Array) => { + const carry = 1; + const end = nonce.length - nonceCounterLength; + let i = nonce.length; + while (i-- > end) { + nonce[i] += carry; + if (nonce[i] !== 0 || carry !== 1) { + break; + } + } + if (i < end) { + await generateNonce(nonce); + } + return nonce; +}; diff --git a/packages/lib/services/e2ee/cryptoTestUtils.ts b/packages/lib/services/e2ee/cryptoTestUtils.ts new file mode 100644 index 000000000..0c56f9311 --- /dev/null +++ b/packages/lib/services/e2ee/cryptoTestUtils.ts @@ -0,0 +1,285 @@ +import EncryptionService, { EncryptionMethod } from './EncryptionService'; +import BaseItem from '../../models/BaseItem'; +import Folder from '../../models/Folder'; +import MasterKey from '../../models/MasterKey'; +import Note from '../../models/Note'; +import Setting from '../../models/Setting'; +import shim from '../..//shim'; +import { getEncryptionEnabled, setEncryptionEnabled } from '../synchronizer/syncInfoUtils'; +import Logger from '@joplin/utils/Logger'; + +interface DecryptTestData { + method: EncryptionMethod; + password: string; + plaintext: string; + ciphertext: string; +} + +let serviceInstance: EncryptionService = null; + +const logger = Logger.create('Crypto Tests'); + +// This is convenient to quickly generate some data to verify for example that +// react-native-quick-crypto and node:crypto can decrypt the same data. +export async function createDecryptTestData() { + const method = EncryptionMethod.StringV1; + const password = 'just testing'; + const plaintext = 'δΈ­ζ–‡γ«γ£γ½γ‚“γ”ν•œκ΅­μ–΄πŸ˜€\uD83D\0\r\nenglish01234567890'; + const ciphertext = await serviceInstance.encrypt(method, password, plaintext); + + return { + method: method, + password: password, + plaintext: plaintext, + ciphertext: ciphertext, + }; +} + +interface CheckTestDataOptions { + throwOnError?: boolean; + silent?: boolean; + testLabel?: string; +} + +export async function checkDecryptTestData(data: DecryptTestData, options: CheckTestDataOptions = null) { + options = { + throwOnError: false, + silent: false, + ...options, + }; + + // Verify that the ciphertext decrypted on this device and producing the same plaintext. + const messages: string[] = []; + let hasError = false; + + try { + const decrypted = await EncryptionService.instance().decrypt(data.method, data.password, data.ciphertext); + if (decrypted !== data.plaintext) { + messages.push('Data could not be decrypted'); + messages.push('Expected:', data.plaintext); + messages.push('Got:', decrypted); + hasError = true; + } else { + messages.push('Data could be decrypted'); + } + } catch (error) { + hasError = true; + messages.push(`Failed to decrypt data: Error: ${error}`); + } + + if (hasError && options.throwOnError) { + const label = options.testLabel ? ` (test ${options.testLabel})` : ''; + throw new Error(`Testing Crypto failed${label}: \n${messages.join('\n')}`); + } else { + for (const msg of messages) { + if (hasError) { + logger.warn(msg); + } else { + if (!options.silent) logger.info(msg); + } + } + } +} + +export async function testStringPerformance(method: EncryptionMethod, dataSize: number, count: number, options: CheckTestDataOptions = null) { + options = { + throwOnError: false, + silent: false, + ...options, + }; + + // Verify that the ciphertext decrypted on this device and producing the same plaintext. + const messages: string[] = []; + let hasError = false; + + try { + serviceInstance.defaultEncryptionMethod_ = method; + let masterKey = await serviceInstance.generateMasterKey('123456'); + masterKey = await MasterKey.save(masterKey); + await serviceInstance.loadMasterKey(masterKey, '123456', true); + + const crypto = shim.crypto; + + const content = (await crypto.randomBytes(dataSize / 2)).toString('hex'); + const folder = await Folder.save({ title: 'folder' }); + const note = await Note.save({ title: 'encrypted note', body: content, parent_id: folder.id }); + + let encryptTime = 0.0; + let decryptTime = 0.0; + for (let i = 0; i < count; i++) { + const tick1 = performance.now(); + const serialized = await Note.serializeForSync(note); + const tick2 = performance.now(); + const deserialized = await Note.unserialize(serialized); + const decryptedNote = await Note.decrypt(deserialized); + const tick3 = performance.now(); + (decryptedNote.title === note.title); + encryptTime += tick2 - tick1; + decryptTime += tick3 - tick2; + } + + messages.push(`testStringPerformance(): method: ${method}, count: ${count}, dataSize: ${dataSize}, encryptTime: ${encryptTime}, decryptTime: ${decryptTime}, encryptTime/count: ${encryptTime / count}, decryptTime/count: ${decryptTime / count}.`); + + } catch (error) { + hasError = true; + messages.push(`testStringPerformance() failed. Error: ${error}`); + } + + if (hasError && options.throwOnError) { + const label = options.testLabel ? ` (test ${options.testLabel})` : ''; + throw new Error(`Testing Crypto failed${label}: \n${messages.join('\n')}`); + } else { + for (const msg of messages) { + if (hasError) { + logger.warn(msg); + } else { + if (!options.silent) logger.info(msg); + } + } + } +} + +export async function testFilePerformance(method: EncryptionMethod, dataSize: number, count: number, options: CheckTestDataOptions = null) { + options = { + throwOnError: false, + silent: false, + ...options, + }; + + // Verify that the ciphertext decrypted on this device and producing the same plaintext. + const messages: string[] = []; + let hasError = false; + + try { + serviceInstance.defaultFileEncryptionMethod_ = method; + let masterKey = await serviceInstance.generateMasterKey('123456'); + masterKey = await MasterKey.save(masterKey); + await serviceInstance.loadMasterKey(masterKey, '123456', true); + + const fsDriver = shim.fsDriver(); + const crypto = shim.crypto; + + const sourcePath = `${Setting.value('tempDir')}/testData.txt`; + const encryptedPath = `${Setting.value('tempDir')}/testData.crypted`; + const decryptedPath = `${Setting.value('tempDir')}/testData.decrypted`; + await fsDriver.writeFile(sourcePath, ''); + await fsDriver.appendFile(sourcePath, (await crypto.randomBytes(dataSize)).toString('base64'), 'base64'); + + let encryptTime = 0.0; + let decryptTime = 0.0; + for (let i = 0; i < count; i++) { + const tick1 = performance.now(); + await serviceInstance.encryptFile(sourcePath, encryptedPath); + const tick2 = performance.now(); + await serviceInstance.decryptFile(encryptedPath, decryptedPath); + const tick3 = performance.now(); + encryptTime += tick2 - tick1; + decryptTime += tick3 - tick2; + } + + messages.push(`testFilePerformance(): method: ${method}, count: ${count}, dataSize: ${dataSize}, encryptTime: ${encryptTime}, decryptTime: ${decryptTime}, encryptTime/count: ${encryptTime / count}, decryptTime/count: ${decryptTime / count}.`); + + } catch (error) { + hasError = true; + messages.push(`testFilePerformance() failed. Error: ${error}`); + } + + if (hasError && options.throwOnError) { + const label = options.testLabel ? ` (test ${options.testLabel})` : ''; + throw new Error(`Testing Crypto failed${label}: \n${messages.join('\n')}`); + } else { + for (const msg of messages) { + if (hasError) { + logger.warn(msg); + } else { + if (!options.silent) logger.info(msg); + } + } + } +} + +// cSpell:disable + +// Data generated on desktop, using node:crypto in packages/lib/services/e2ee/crypto.ts +const decryptTestData: Record = { + shortString: { + method: EncryptionMethod.StringV1, + password: '4BfJl8YbM,nXx.LVgs!AzkWWA]', + plaintext: 'δΈ­ζ–‡γ«γ£γ½γ‚“γ”ν•œκ΅­μ–΄πŸ˜€\uD83D\0\r\nenglish01234567890', + ciphertext: '{"salt":"6NKebMdcrFSGSzEWusbY8JUfG9rB98PxNZtk0QkYEFQ=","iv":"zHdXu8V5SYsO+vR/","ct":"4C3uyjOtRsNQZxCECvCeRaP+oPXMxjUMxsND67odnyiFg2A+fG6QW6O8axb6RHWU7QKHRG9/kHEs283DHL3hOAJbl4LS47R/dEDJbl8kWmGtLAsn"}', + }, + hexKey: { + method: EncryptionMethod.KeyV1, + password: '4BfJl8YbM,nXx.LVgs!AzkWWA]', + plaintext: 'ea5d8dd43823a812c9a64f4eb09b57e1aa2dbfcda50bf9e416dcd1f8c569a6462f7f61861c3ccda0c7c16bf4192ed9c7ecaf3e1248517bd6f397d1080d2632fc69e1ead59a398a07e7478c8d90142745c0dce39c2105b6d34117424697de03879caa3b6410325f14dc5755b69efa944eb83f06970d04444e83fe054576af20576c0f3a5dc23e0d6dcfa3e84ec05c21139070c0809bd2bdc86a7782368c9d99eb072c858c61ec8926136e6e50dfd57b7e8e0084ad08b2d984db0436f50433cab44b3c462ccd22a8567c8ff86675fff618b11526030f09f735646a9f189f54ba5485d069ee25ac468ec0a873c1223eed34962bd219972020cc147041d4b00a3ab8', + ciphertext: '{"salt":"iv5kKP+scMyXKO2jqzef3q9y9p/o/mj6zAoKVltbPx0=","iv":"Gz1PtbDzoaZzRx5m","ct":"YSS4ga1Q0MeVpFbMn2V45TaAbbuJM12vU/qYQ/VEGYPXhNQ4YchfRt7+LjhVxwpvfc/rD0znt6kpAh4ROGS2CLDy27/n5VICgTVY2VVff+YjPAma6odKn2pm4Z88fZlkoJVLi7QyU96Mvb6bYVbuNjQ16hOjFQ3iIIztLcafsHaW6v6gUFrDWYVQPi1/xovmmCe/GaM3JeMye5QQiFrmLQIxEJMNv8YiNyppMVf5b1YGFtDlOjm3XE2H9bb+wWNAd82mcwcAe+ZUedz3AH9PKlsRyBGHfGQ/rfFzeoFj+Wjm4fvPniPa1muRSMQDHU2Zw5YGWQwMNVUHt+y7lDoqPF2NQv4DvPmY1kLz2yohIzc="}', + }, + base64File: { + method: EncryptionMethod.FileV1, + password: '4BfJl8YbM,nXx.LVgs!AzkWWA]', + plaintext: '5Lit5paH44Gr44Gj44G944KT44GU7ZWc6rWt7Ja08J+YgO+/vQANCmVuZ2xpc2gwMTIzNDU2Nzg5MA==', + ciphertext: '{"salt":"19Zx/+hxpZ+Trc7MGBt837SsTOJjHe9aiY5UPnXP6Oo=","iv":"N4PTzsyh4wONNJWa","ct":"LxAibOnVox1q2WBtLAKxeZxIIxKOEd6xdD3NKAn5mgHhv4i60yPiyPbr8rS+MzHmeq7Z3BhHjR7540rtdeBugbmf1+b3tYuRudI="}', + }, +}; + +// cSpell:enable + +// This can be used to run integration tests directly on device. It will throw +// an error if something cannot be decrypted, or else print info messages. +export const runIntegrationTests = async (silent = false, testPerformance = false) => { + const log = (s: string) => { + if (silent) return; + logger.info(s); + }; + + log('Running integration tests...'); + const encryptionEnabled = getEncryptionEnabled(); + serviceInstance = EncryptionService.instance(); + BaseItem.encryptionService_ = EncryptionService.instance(); + setEncryptionEnabled(true); + + log('Decrypting using known data...'); + for (const testLabel in decryptTestData) { + log(`Running decrypt test data case ${testLabel}...`); + await checkDecryptTestData(decryptTestData[testLabel], { silent, testLabel, throwOnError: true }); + } + + log('Decrypting using local data...'); + const newData = await createDecryptTestData(); + await checkDecryptTestData(newData, { silent, throwOnError: true }); + + // The performance test is very slow so it is disabled by default. + if (testPerformance) { + log('Testing performance...'); + if (shim.mobilePlatform() === '') { + await testStringPerformance(EncryptionMethod.StringV1, 100, 1000); + await testStringPerformance(EncryptionMethod.StringV1, 1000000, 10); + await testStringPerformance(EncryptionMethod.StringV1, 5000000, 10); + await testStringPerformance(EncryptionMethod.SJCL1a, 100, 1000); + await testStringPerformance(EncryptionMethod.SJCL1a, 1000000, 10); + await testStringPerformance(EncryptionMethod.SJCL1a, 5000000, 10); + await testFilePerformance(EncryptionMethod.FileV1, 100, 1000); + await testFilePerformance(EncryptionMethod.FileV1, 1000000, 3); + await testFilePerformance(EncryptionMethod.FileV1, 5000000, 3); + await testFilePerformance(EncryptionMethod.SJCL1a, 100, 1000); + await testFilePerformance(EncryptionMethod.SJCL1a, 1000000, 3); + await testFilePerformance(EncryptionMethod.SJCL1a, 5000000, 3); + } else { + await testStringPerformance(EncryptionMethod.StringV1, 100, 100); + await testStringPerformance(EncryptionMethod.StringV1, 500000, 3); + await testStringPerformance(EncryptionMethod.StringV1, 1000000, 3); + await testStringPerformance(EncryptionMethod.SJCL1a, 100, 100); + await testStringPerformance(EncryptionMethod.SJCL1a, 500000, 3); + await testStringPerformance(EncryptionMethod.SJCL1a, 1000000, 3); + await testFilePerformance(EncryptionMethod.FileV1, 100, 100); + await testFilePerformance(EncryptionMethod.FileV1, 100000, 3); + await testFilePerformance(EncryptionMethod.FileV1, 500000, 3); + await testFilePerformance(EncryptionMethod.SJCL1a, 100, 100); + await testFilePerformance(EncryptionMethod.SJCL1a, 100000, 3); + await testFilePerformance(EncryptionMethod.SJCL1a, 500000, 3); + } + + } + + setEncryptionEnabled(encryptionEnabled); +}; diff --git a/packages/lib/services/e2ee/types.ts b/packages/lib/services/e2ee/types.ts index be7235e9d..5a5ed7f20 100644 --- a/packages/lib/services/e2ee/types.ts +++ b/packages/lib/services/e2ee/types.ts @@ -25,3 +25,46 @@ export interface RSA { publicKey(rsaKeyPair: RSAKeyPair): string; privateKey(rsaKeyPair: RSAKeyPair): string; } + +export interface Crypto { + randomBytes(size: number): Promise; + digest(algorithm: Digest, data: Uint8Array): Promise; + generateNonce(nonce: Uint8Array): Promise; + increaseNonce(nonce: Uint8Array): Promise; + encrypt(password: string, salt: CryptoBuffer, data: CryptoBuffer, options: EncryptionParameters): Promise; + decrypt(password: string, data: EncryptionResult, options: EncryptionParameters): Promise; + encryptString(password: string, salt: CryptoBuffer, data: string, encoding: BufferEncoding, options: EncryptionParameters): Promise; +} + +export interface CryptoBuffer extends Uint8Array { + toString(encoding?: BufferEncoding, start?: number, end?: number): string; +} + +// A subset of react-native-quick-crypto.HashAlgorithm, supported by Web Crypto API +export enum Digest { + sha1 = 'SHA-1', + sha256 = 'SHA-256', + sha384 = 'SHA-384', + sha512 = 'SHA-512', +} + +export enum CipherAlgorithm { + AES_128_GCM = 'aes-128-gcm', + AES_192_GCM = 'aes-192-gcm', + AES_256_GCM = 'aes-256-gcm', +} + +export interface EncryptionResult { + salt: string; // base64 encoded + iv: string; // base64 encoded + ct: string; // cipherText, base64 encoded +} + +export interface EncryptionParameters { + cipherAlgorithm: CipherAlgorithm; + authTagLength: number; // in bytes + digestAlgorithm: Digest; + keyLength: number; // in bytes + associatedData: Uint8Array; + iterationCount: number; +} diff --git a/packages/lib/services/synchronizer/Synchronizer.e2ee.test.ts b/packages/lib/services/synchronizer/Synchronizer.e2ee.test.ts index e50c0db93..3a878e1a5 100644 --- a/packages/lib/services/synchronizer/Synchronizer.e2ee.test.ts +++ b/packages/lib/services/synchronizer/Synchronizer.e2ee.test.ts @@ -12,6 +12,7 @@ import Synchronizer from '../../Synchronizer'; import { fetchSyncInfo, getEncryptionEnabled, localSyncInfo, setEncryptionEnabled } from '../synchronizer/syncInfoUtils'; import { loadMasterKeysFromSettings, setupAndDisableEncryption, setupAndEnableEncryption } from '../e2ee/utils'; import { remoteNotesAndFolders } from '../../testing/test-utils-synchronizer'; +import { EncryptionMethod } from '../e2ee/EncryptionService'; let insideBeforeEach = false; @@ -31,8 +32,12 @@ describe('Synchronizer.e2ee', () => { insideBeforeEach = false; }); - it('notes and folders should get encrypted when encryption is enabled', (async () => { + it.each([ + EncryptionMethod.SJCL1a, + EncryptionMethod.StringV1, + ])('notes and folders should get encrypted when encryption is enabled', (async (encryptionMethod) => { setEncryptionEnabled(true); + encryptionService().defaultEncryptionMethod_ = encryptionMethod; const masterKey = await loadEncryptionMasterKey(); const folder1 = await Folder.save({ title: 'folder1' }); let note1 = await Note.save({ title: 'un', body: 'to be encrypted', parent_id: folder1.id }); @@ -255,8 +260,13 @@ describe('Synchronizer.e2ee', () => { expect(allEncrypted).toBe(false); })); - it('should set the resource file size after decryption', (async () => { + it.each([ + [EncryptionMethod.SJCL1a, EncryptionMethod.SJCL1a], + [EncryptionMethod.StringV1, EncryptionMethod.FileV1], + ])('should set the resource file size after decryption', (async (stringEncryptionMethod, fileEncryptionMethod) => { setEncryptionEnabled(true); + encryptionService().defaultEncryptionMethod_ = stringEncryptionMethod; + encryptionService().defaultFileEncryptionMethod_ = fileEncryptionMethod; const masterKey = await loadEncryptionMasterKey(); const folder1 = await Folder.save({ title: 'folder1' }); @@ -282,9 +292,15 @@ describe('Synchronizer.e2ee', () => { expect(resource1_2.size).toBe(2720); })); - it('should encrypt remote resources after encryption has been enabled', (async () => { + it.each([ + [EncryptionMethod.SJCL1a, EncryptionMethod.SJCL1a], + [EncryptionMethod.StringV1, EncryptionMethod.FileV1], + ])('should encrypt remote resources after encryption has been enabled', (async (stringEncryptionMethod, fileEncryptionMethod) => { while (insideBeforeEach) await time.msleep(100); + encryptionService().defaultEncryptionMethod_ = stringEncryptionMethod; + encryptionService().defaultFileEncryptionMethod_ = fileEncryptionMethod; + const folder1 = await Folder.save({ title: 'folder1' }); const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id }); await shim.attachFileToNote(note1, `${supportDir}/photo.jpg`); @@ -350,9 +366,13 @@ describe('Synchronizer.e2ee', () => { expect(!!resource.encryption_blob_encrypted).toBe(false); })); - it('should stop trying to decrypt item after a few attempts', (async () => { + it.each([ + EncryptionMethod.SJCL1a, + EncryptionMethod.StringV1, + ])('should stop trying to decrypt item after a few attempts', (async (encryptionMethod) => { let hasThrown; + encryptionService().defaultEncryptionMethod_ = encryptionMethod; const note = await Note.save({ title: 'ma note' }); const masterKey = await loadEncryptionMasterKey(); await setupAndEnableEncryption(encryptionService(), masterKey, '123456'); diff --git a/packages/lib/shim-init-node.ts b/packages/lib/shim-init-node.ts index b3d80845f..4a4b5f8e9 100644 --- a/packages/lib/shim-init-node.ts +++ b/packages/lib/shim-init-node.ts @@ -12,6 +12,7 @@ import { ResourceEntity } from './services/database/types'; import { TextItem } from 'pdfjs-dist/types/src/display/api'; import replaceUnsupportedCharacters from './utils/replaceUnsupportedCharacters'; import { FetchBlobOptions } from './types'; +import crypto from './services/e2ee/crypto'; import FileApiDriverLocal from './file-api-driver-local'; import * as mimeUtils from './mime-utils'; @@ -135,6 +136,7 @@ function shimInit(options: ShimInitOptions = null) { shim.Geolocation = GeolocationNode; shim.FormData = require('form-data'); shim.sjclModule = require('./vendor/sjcl.js'); + shim.crypto = crypto; shim.electronBridge_ = options.electronBridge; shim.fsDriver = () => { diff --git a/packages/lib/shim.ts b/packages/lib/shim.ts index d90b89ea6..8cf494471 100644 --- a/packages/lib/shim.ts +++ b/packages/lib/shim.ts @@ -2,6 +2,7 @@ import * as React from 'react'; import { NoteEntity, ResourceEntity } from './services/database/types'; import type FsDriverBase from './fs-driver-base'; import type FileApiDriverLocal from './file-api-driver-local'; +import { Crypto } from './services/e2ee/types'; export interface CreateResourceFromPathOptions { resizeLargeImages?: 'always' | 'never' | 'ask'; @@ -292,6 +293,8 @@ const shim = { // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied sjclModule: null as any, + crypto: null as Crypto, + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied randomBytes: async (_count: number): Promise => { throw new Error('Not implemented: randomBytes'); diff --git a/packages/tools/cspell/dictionary4.txt b/packages/tools/cspell/dictionary4.txt index c29723112..a4ad8f084 100644 --- a/packages/tools/cspell/dictionary4.txt +++ b/packages/tools/cspell/dictionary4.txt @@ -119,7 +119,6 @@ Credentialless coepdegrade COEP Stormlikes -Stormlikes BYTV keyval traineddata @@ -130,6 +129,8 @@ Zocial agplv Famegear rcompare +rnqc +owasp tabindex Backblaze onnx diff --git a/yarn.lock b/yarn.lock index a4b774131..1d3079430 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5160,6 +5160,16 @@ __metadata: languageName: node linkType: hard +"@craftzdog/react-native-buffer@npm:^6.0.5": + version: 6.0.5 + resolution: "@craftzdog/react-native-buffer@npm:6.0.5" + dependencies: + ieee754: ^1.2.1 + react-native-quick-base64: ^2.0.5 + checksum: 921b8bc7f84778e355e81e475792399276d611a346a7e51b6266a45cf4aa82194beb3a8106af796ed143d958c8476070c59e3720c0eec0a3c31e368fbb08b350 + languageName: node + linkType: hard + "@cronvel/get-pixels@npm:^3.4.0": version: 3.4.0 resolution: "@cronvel/get-pixels@npm:3.4.0" @@ -7540,6 +7550,7 @@ __metadata: react-native-paper: 5.12.3 react-native-popup-menu: 0.16.1 react-native-quick-actions: 0.3.13 + react-native-quick-crypto: 0.7.1 react-native-rsa-native: 2.0.5 react-native-safe-area-context: 4.10.8 react-native-securerandom: 1.0.1 @@ -12228,7 +12239,7 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:18.19.42": +"@types/node@npm:18.19.42, @types/node@npm:^18.0.0": version: 18.19.42 resolution: "@types/node@npm:18.19.42" dependencies: @@ -12244,15 +12255,6 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^18.0.0": - version: 18.19.33 - resolution: "@types/node@npm:18.19.33" - dependencies: - undici-types: ~5.26.4 - checksum: b6db87d095bc541d64a410fa323a35c22c6113220b71b608bbe810b2397932d0f0a51c3c0f3ef90c20d8180a1502d950a7c5314b907e182d9cc10b36efd2a44e - languageName: node - linkType: hard - "@types/node@npm:^20.10.6": version: 20.11.16 resolution: "@types/node@npm:20.11.16" @@ -37900,6 +37902,31 @@ __metadata: languageName: node linkType: hard +"react-native-quick-base64@npm:^2.0.5": + version: 2.1.2 + resolution: "react-native-quick-base64@npm:2.1.2" + dependencies: + base64-js: ^1.5.1 + peerDependencies: + react: "*" + react-native: "*" + checksum: 46f3b26f48b26978686b0c043336220d681e6a02af5abcf3eb4ab7b9216251d1eb2fac5c559e984d963e93f54bd9f323651daac09762196815558abbd551729b + languageName: node + linkType: hard + +"react-native-quick-crypto@npm:0.7.1": + version: 0.7.1 + resolution: "react-native-quick-crypto@npm:0.7.1" + dependencies: + "@craftzdog/react-native-buffer": ^6.0.5 + events: ^3.3.0 + readable-stream: ^4.5.2 + string_decoder: ^1.3.0 + util: ^0.12.5 + checksum: 9a7a1fb1456410db30f062078f570bc566cac36fbc165e7d8ee8677bec09fcc96923de3cf7a0464804142af242a822bf66ea22460951399b9247e4a03fcfe059 + languageName: node + linkType: hard + "react-native-rsa-native@npm:2.0.5": version: 2.0.5 resolution: "react-native-rsa-native@npm:2.0.5" @@ -38654,6 +38681,19 @@ __metadata: languageName: node linkType: hard +"readable-stream@npm:^4.5.2": + version: 4.5.2 + resolution: "readable-stream@npm:4.5.2" + dependencies: + abort-controller: ^3.0.0 + buffer: ^6.0.3 + events: ^3.3.0 + process: ^0.11.10 + string_decoder: ^1.3.0 + checksum: c4030ccff010b83e4f33289c535f7830190773e274b3fcb6e2541475070bdfd69c98001c3b0cb78763fc00c8b62f514d96c2b10a8bd35d5ce45203a25fa1d33a + languageName: node + linkType: hard + "readable-stream@npm:~2.0.0": version: 2.0.6 resolution: "readable-stream@npm:2.0.6" @@ -45383,7 +45423,7 @@ __metadata: languageName: node linkType: hard -"util@npm:^0.12.4": +"util@npm:^0.12.4, util@npm:^0.12.5": version: 0.12.5 resolution: "util@npm:0.12.5" dependencies: