Android: Add support for offline speech to text (Beta - FR only) (#8115)

2024-12-21 09:38:01 +02:00 · 2023-05-03 12:19:43 +01:00 · 2023-05-03 12:19:43 +01:00 · 36c121523a
commit 36c121523a
parent 991c12025b
22 changed files with 388 additions and 7 deletions
--- a/.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch
+++ b/.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch
@ -0,0 +1,15 @@
+diff --git a/android/src/main/java/com/reactnativevosk/VoskModule.kt b/android/src/main/java/com/reactnativevosk/VoskModule.kt
+index 0e2b6595b1b2cf1ee01c6c64239c4b0ea37fce19..e0daf712a7f2d2c2dd22e623bac6daba7f6a4ac1 100644
+--- a/android/src/main/java/com/reactnativevosk/VoskModule.kt
+++ b/android/src/main/java/com/reactnativevosk/VoskModule.kt
+@@ -25,7 +25,9 @@ class VoskModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaMo
+ 
+     // Stop recording if data found
+     if (text != null && text.isNotEmpty()) {
+-      cleanRecognizer();
+      // Don't auto-stop the recogniser - we want to do that when the user
+      // presses on "stop" only.
+      // cleanRecognizer();
+       sendEvent("onResult", text)
+     }
+   }
--- a/package.json
+++ b/package.json
@ -96,6 +96,7 @@
  "packageManager": "yarn@3.3.1",
  "resolutions": {
    "react-native-camera@4.2.1": "patch:react-native-camera@npm%3A4.2.1#./.yarn/patches/react-native-camera-npm-4.2.1-24b2600a7e.patch",
-    "rn-fetch-blob@0.12.0": "patch:rn-fetch-blob@npm%3A0.12.0#./.yarn/patches/rn-fetch-blob-npm-0.12.0-cf02e3c544.patch"
+    "rn-fetch-blob@0.12.0": "patch:rn-fetch-blob@npm%3A0.12.0#./.yarn/patches/rn-fetch-blob-npm-0.12.0-cf02e3c544.patch",
+    "react-native-vosk@0.1.12": "patch:react-native-vosk@npm%3A0.1.12#./.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch"
  }
 }
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/README
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/README
@ -0,0 +1,7 @@
+French small model for Vosk
+
+WER
+
+%WER 23.95 [ 37203 / 155330, 5373 ins, 4427 del, 27403 sub ] exp/chain_a/tdnn/decode_test_cv/wer_12_0.0
+%WER 19.30 [ 2975 / 15412, 683 ins, 672 del, 1620 sub ] exp/chain_a/tdnn/decode_test_mtedx/wer_10_0.0
+%WER 27.25 [ 20208 / 74145, 2647 ins, 5852 del, 11709 sub ] exp/chain_a/tdnn/decode_test_podcast_reseg/wer_10_0.0
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/am/final.mdl
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/am/final.mdl
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/conf/mfcc.conf
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/conf/mfcc.conf
@ -0,0 +1,8 @@
+--use-energy=false
+--sample-frequency=16000
+--num-mel-bins=40
+--num-ceps=40
+--low-freq=40
+--high-freq=-200
+--allow-upsample=true
+--allow-downsample=true
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/conf/model.conf
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/conf/model.conf
@ -0,0 +1,10 @@
+--min-active=200
+--max-active=7000
+--beam=13.0
+--lattice-beam=4.0
+--acoustic-scale=1.0
+--frame-subsampling-factor=3
+--endpoint.silence-phones=1:2:3:4:5:6:7:8:9:10
+--endpoint.rule2.min-trailing-silence=0.5
+--endpoint.rule3.min-trailing-silence=1.0
+--endpoint.rule4.min-trailing-silence=2.0
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/Gr.fst
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/Gr.fst
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/HCLr.fst
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/HCLr.fst
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/disambig_tid.int
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/disambig_tid.int
@ -0,0 +1,76 @@
+9365
+9366
+9367
+9368
+9369
+9370
+9371
+9372
+9373
+9374
+9375
+9376
+9377
+9378
+9379
+9380
+9381
+9382
+9383
+9384
+9385
+9386
+9387
+9388
+9389
+9390
+9391
+9392
+9393
+9394
+9395
+9396
+9397
+9398
+9399
+9400
+9401
+9402
+9403
+9404
+9405
+9406
+9407
+9408
+9409
+9410
+9411
+9412
+9413
+9414
+9415
+9416
+9417
+9418
+9419
+9420
+9421
+9422
+9423
+9424
+9425
+9426
+9427
+9428
+9429
+9430
+9431
+9432
+9433
+9434
+9435
+9436
+9437
+9438
+9439
+9440
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/phones/word_boundary.int
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/phones/word_boundary.int
@ -0,0 +1,154 @@
+1 nonword
+2 begin
+3 end
+4 internal
+5 singleton
+6 nonword
+7 begin
+8 end
+9 internal
+10 singleton
+11 begin
+12 end
+13 internal
+14 singleton
+15 begin
+16 end
+17 internal
+18 singleton
+19 begin
+20 end
+21 internal
+22 singleton
+23 begin
+24 end
+25 internal
+26 singleton
+27 begin
+28 end
+29 internal
+30 singleton
+31 begin
+32 end
+33 internal
+34 singleton
+35 begin
+36 end
+37 internal
+38 singleton
+39 begin
+40 end
+41 internal
+42 singleton
+43 begin
+44 end
+45 internal
+46 singleton
+47 begin
+48 end
+49 internal
+50 singleton
+51 begin
+52 end
+53 internal
+54 singleton
+55 begin
+56 end
+57 internal
+58 singleton
+59 begin
+60 end
+61 internal
+62 singleton
+63 begin
+64 end
+65 internal
+66 singleton
+67 begin
+68 end
+69 internal
+70 singleton
+71 begin
+72 end
+73 internal
+74 singleton
+75 begin
+76 end
+77 internal
+78 singleton
+79 begin
+80 end
+81 internal
+82 singleton
+83 begin
+84 end
+85 internal
+86 singleton
+87 begin
+88 end
+89 internal
+90 singleton
+91 begin
+92 end
+93 internal
+94 singleton
+95 begin
+96 end
+97 internal
+98 singleton
+99 begin
+100 end
+101 internal
+102 singleton
+103 begin
+104 end
+105 internal
+106 singleton
+107 begin
+108 end
+109 internal
+110 singleton
+111 begin
+112 end
+113 internal
+114 singleton
+115 begin
+116 end
+117 internal
+118 singleton
+119 begin
+120 end
+121 internal
+122 singleton
+123 begin
+124 end
+125 internal
+126 singleton
+127 begin
+128 end
+129 internal
+130 singleton
+131 begin
+132 end
+133 internal
+134 singleton
+135 begin
+136 end
+137 internal
+138 singleton
+139 begin
+140 end
+141 internal
+142 singleton
+143 begin
+144 end
+145 internal
+146 singleton
+147 begin
+148 end
+149 internal
+150 singleton
+151 begin
+152 end
+153 internal
+154 singleton
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.dubm
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.dubm
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.ie
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.ie
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.mat
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.mat
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/global_cmvn.stats
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/global_cmvn.stats
@ -0,0 +1,3 @@
+ [
+  1.022245e+11 -6.33291e+09 -2.480997e+09 8.290258e+09 -9.084483e+09 -8.092173e+09 -1.4735e+10 -7.041795e+09 -1.171205e+10 -2.976464e+08 -1.009425e+10 -6765179 -7.821326e+09 1.449499e+09 -6.413975e+09 -5.303802e+08 -4.998635e+09 9.521598e+07 -3.073041e+09 1.56756e+08 -1.287956e+09 1.738752e+08 -2.382392e+08 -2.716675e+07 4.404485e+08 -1.913359e+08 7.780919e+08 -4.006922e+08 7.895809e+08 -5.401082e+08 5.17605e+08 -6.227134e+08 6.58271e+08 -6.204593e+07 5.187754e+08 -4.497048e+08 4.219366e+07 -2.78742e+08 -1.797385e+07 -3.604475e+07 1.053647e+09 
+  1.040194e+13 6.245521e+11 4.223293e+11 6.831219e+11 6.078478e+11 6.3425e+11 7.943839e+11 6.013323e+11 6.781652e+11 5.272091e+11 5.810814e+11 4.353831e+11 4.473305e+11 3.42063e+11 3.083377e+11 2.14257e+11 1.892057e+11 1.163827e+11 8.367058e+10 4.203224e+10 2.297476e+10 7.596307e+09 1.099877e+09 2.886651e+08 3.797438e+09 9.372847e+09 1.629059e+10 2.196351e+10 2.747149e+10 3.072878e+10 3.238528e+10 3.330232e+10 3.407238e+10 3.230687e+10 2.676914e+10 2.252055e+10 1.914305e+10 1.565974e+10 1.224627e+10 8.415393e+09 0 ]
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/online_cmvn.conf
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/online_cmvn.conf
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/splice.conf
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/splice.conf
@ -0,0 +1,2 @@
+--left-context=3
+--right-context=3
--- a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/uuid
+++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/uuid
@ -0,0 +1 @@
+1d689fce-7d11-4a95-a13e-19b64b5da0c0
--- a/packages/app-mobile/components/screens/Note.tsx
+++ b/packages/app-mobile/components/screens/Note.tsx
@ -44,6 +44,7 @@ import ShareExtension from '../../utils/ShareExtension.js';
 import CameraView from '../CameraView';
 import { NoteEntity } from '@joplin/lib/services/database/types';
 import Logger from '@joplin/lib/Logger';
+import Vosk from 'react-native-vosk';
 const urlUtils = require('@joplin/lib/urlUtils');

 const emptyArray: any[] = [];
@ -51,6 +52,10 @@ const emptyArray: any[] = [];
 const logger = Logger.create('screens/Note');

 class NoteScreenComponent extends BaseScreenComponent {
+
+	private vosk_: Vosk|null = null;
+	private voskResult_: string[] = [];
+
 	public static navigationOptions(): any {
 		return { header: null };
 	}
@ -754,6 +759,65 @@ class NoteScreenComponent extends BaseScreenComponent {
 		}
 	}

+	private async getVosk() {
+		if (this.vosk_) return this.vosk_;
+		this.vosk_ = new Vosk();
+		await this.vosk_.loadModel('model-fr-fr');
+		return this.vosk_;
+	}
+
+	private async voiceRecording_onPress() {
+		logger.info('Vosk: Getting instance...');
+
+		const vosk = await this.getVosk();
+
+		this.voskResult_ = [];
+
+		const eventHandlers: any[] = [];
+
+		eventHandlers.push(vosk.onResult(e => {
+			logger.info('Vosk: result', e.data);
+			this.voskResult_.push(e.data);
+		}));
+
+		eventHandlers.push(vosk.onError(e => {
+			logger.warn('Vosk: error', e.data);
+		}));
+
+		eventHandlers.push(vosk.onTimeout(e => {
+			logger.warn('Vosk: timeout', e.data);
+		}));
+
+		eventHandlers.push(vosk.onFinalResult(e => {
+			logger.info('Vosk: final result', e.data);
+		}));
+
+		logger.info('Vosk: Starting recording...');
+
+		void vosk.start();
+
+		const buttonId = await dialogs.pop(this, 'Voice recording in progress...', [
+			{ text: 'Stop recording', id: 'stop' },
+			{ text: _('Cancel'), id: 'cancel' },
+		]);
+
+		logger.info('Vosk: Stopping recording...');
+		vosk.stop();
+
+		for (const eventHandler of eventHandlers) {
+			eventHandler.remove();
+		}
+
+		logger.info('Vosk: Recording stopped:', this.voskResult_);
+
+		if (buttonId === 'cancel') return;
+
+		const newNote: NoteEntity = { ...this.state.note };
+		newNote.body = `${newNote.body} ${this.voskResult_.join(' ')}`;
+		this.setState({ note: newNote });
+		this.scheduleSave();
+	}
+
 	private toggleIsTodo_onPress() {
 		shared.toggleIsTodo_onPress(this);

@ -914,6 +978,16 @@ class NoteScreenComponent extends BaseScreenComponent {
 				void this.share_onPress();
 			},
 		});
+
+		if (shim.mobilePlatform() === 'android') {
+			output.push({
+				title: 'Voice recording (Beta - FR only)',
+				onPress: () => {
+					void this.voiceRecording_onPress();
+				},
+			});
+		}
+
 		if (isSaved) {
 			output.push({
 				title: _('Tags'),
--- a/packages/app-mobile/ios/Joplin.xcodeproj/project.pbxproj
+++ b/packages/app-mobile/ios/Joplin.xcodeproj/project.pbxproj
@ -422,6 +422,7 @@
 				"${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/SimpleLineIcons.ttf",
 				"${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/Zocial.ttf",
 				"${PODS_CONFIGURATION_BUILD_DIR}/React-Core/AccessibilityResources.bundle",
+				"${PODS_CONFIGURATION_BUILD_DIR}/react-native-vosk/Vosk.bundle",
 			);
 			name = "[CP] Copy Pods Resources";
 			outputPaths = (
@ -442,6 +443,7 @@
 				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/SimpleLineIcons.ttf",
 				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/Zocial.ttf",
 				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/AccessibilityResources.bundle",
+				"${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/Vosk.bundle",
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 			shellPath = /bin/sh;
--- a/packages/app-mobile/ios/Podfile.lock
+++ b/packages/app-mobile/ios/Podfile.lock
@ -316,7 +316,7 @@ PODS:
    - React-Core
  - react-native-camera/RN (4.2.1):
    - React-Core
-  - react-native-document-picker (8.1.4):
+  - react-native-document-picker (8.2.0):
    - React-Core
  - react-native-fingerprint-scanner (6.0.0):
    - React
@ -328,13 +328,13 @@ PODS:
    - React-Core
  - react-native-image-resizer (1.4.5):
    - React-Core
-  - react-native-netinfo (9.3.8):
+  - react-native-netinfo (9.3.9):
    - React-Core
  - react-native-rsa-native (2.0.5):
    - React
  - react-native-saf-x (2.11.0):
    - React-Core
-  - react-native-safe-area-context (4.5.0):
+  - react-native-safe-area-context (4.5.1):
    - RCT-Folly
    - RCTRequired
    - RCTTypeSafety
@ -346,6 +346,8 @@ PODS:
    - React-Core
  - react-native-version-info (1.1.1):
    - React-Core
+  - react-native-vosk (0.1.12):
+    - React-Core
  - react-native-webview (11.26.1):
    - React-Core
  - React-perflogger (0.70.6)
@ -505,6 +507,7 @@ DEPENDENCIES:
  - "react-native-slider (from `../node_modules/@react-native-community/slider`)"
  - react-native-sqlite-storage (from `../node_modules/react-native-sqlite-storage`)
  - react-native-version-info (from `../node_modules/react-native-version-info`)
+  - react-native-vosk (from `../node_modules/react-native-vosk`)
  - react-native-webview (from `../node_modules/react-native-webview`)
  - React-perflogger (from `../node_modules/react-native/ReactCommon/reactperflogger`)
  - React-RCTActionSheet (from `../node_modules/react-native/Libraries/ActionSheetIOS`)
@ -626,6 +629,8 @@ EXTERNAL SOURCES:
    :path: "../node_modules/react-native-sqlite-storage"
  react-native-version-info:
    :path: "../node_modules/react-native-version-info"
+  react-native-vosk:
+    :path: "../node_modules/react-native-vosk"
  react-native-webview:
    :path: "../node_modules/react-native-webview"
  React-perflogger:
@ -716,19 +721,20 @@ SPEC CHECKSUMS:
  React-logger: 1623c216abaa88974afce404dc8f479406bbc3a0
  react-native-alarm-notification: 26527410a6162d07a9dc57f4bbc62e94ff48e65d
  react-native-camera: 3eae183c1d111103963f3dd913b65d01aef8110f
-  react-native-document-picker: a9bd26996d1b2e4f412dd186041714c79af381d0
+  react-native-document-picker: 495c444c0c773c6e83a5d91165890ecb1c0a399a
  react-native-fingerprint-scanner: ac6656f18c8e45a7459302b84da41a44ad96dbbe
  react-native-geolocation: 69f4fd37650b8e7fee91816d395e62dd16f5ab8d
  react-native-get-random-values: a6ea6a8a65dc93e96e24a11105b1a9c8cfe1d72a
  react-native-image-picker: ec9b713e248760bfa0f879f0715391de4651a7cb
  react-native-image-resizer: d9fb629a867335bdc13230ac2a58702bb8c8828f
-  react-native-netinfo: fbc23bc2fe217155d85f2f7e0644b1654df8029b
+  react-native-netinfo: 22c082970cbd99071a4e5aa7a612ac20d66b08f0
  react-native-rsa-native: 12132eb627797529fdb1f0d22fd0f8f9678df64a
  react-native-saf-x: 9bd5238d3b43d76bbec64aa82c173ac20a4bce9f
-  react-native-safe-area-context: 39c2d8be3328df5d437ac1700f4f3a4f75716acc
+  react-native-safe-area-context: f5549f36508b1b7497434baa0cd97d7e470920d4
  react-native-slider: 33b8d190b59d4f67a541061bb91775d53d617d9d
  react-native-sqlite-storage: f6d515e1c446d1e6d026aa5352908a25d4de3261
  react-native-version-info: a106f23009ac0db4ee00de39574eb546682579b9
+  react-native-vosk: 33b8e82a46cc56f31bb4847a40efa2d160270e2e
  react-native-webview: 9f111dfbcfc826084d6c507f569e5e03342ee1c1
  React-perflogger: 8c79399b0500a30ee8152d0f9f11beae7fc36595
  React-RCTActionSheet: 7316773acabb374642b926c19aef1c115df5c466
--- a/packages/app-mobile/package.json
+++ b/packages/app-mobile/package.json
@ -66,6 +66,7 @@
    "react-native-url-polyfill": "1.3.0",
    "react-native-vector-icons": "9.2.0",
    "react-native-version-info": "1.1.1",
+    "react-native-vosk": "0.1.12",
    "react-native-webview": "11.26.1",
    "react-redux": "7.2.9",
    "redux": "4.2.1",
--- a/yarn.lock
+++ b/yarn.lock
@ -4983,6 +4983,7 @@ __metadata:
    react-native-url-polyfill: 1.3.0
    react-native-vector-icons: 9.2.0
    react-native-version-info: 1.1.1
+    react-native-vosk: 0.1.12
    react-native-webview: 11.26.1
    react-redux: 7.2.9
    redux: 4.2.1
@ -27550,6 +27551,26 @@ __metadata:
  languageName: node
  linkType: hard

+"react-native-vosk@npm:0.1.12":
+  version: 0.1.12
+  resolution: "react-native-vosk@npm:0.1.12"
+  peerDependencies:
+    react: "*"
+    react-native: "*"
+  checksum: 49dd234d0822d7f3deb9563a903260a8478bb78eb20367b50284df40e1e64e23dc52d632b329176883c048b8224182eee000fd7dbd3c42401a9a03bd0ce1ae10
+  languageName: node
+  linkType: hard
+
+"react-native-vosk@patch:react-native-vosk@npm%3A0.1.12#./.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch::locator=root%40workspace%3A.":
+  version: 0.1.12
+  resolution: "react-native-vosk@patch:react-native-vosk@npm%3A0.1.12#./.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch::version=0.1.12&hash=edde10&locator=root%40workspace%3A."
+  peerDependencies:
+    react: "*"
+    react-native: "*"
+  checksum: 53cd3db17e859466345dd80a485c9ceaa35498dbcf2159af95a70151e09b56aff8743b0323b3cb6f04653c2f24aed06e31045988a5af91331ad7bdb8c50be0b3
+  languageName: node
+  linkType: hard
+
 "react-native-webview@npm:11.26.1":
  version: 11.26.1
  resolution: "react-native-webview@npm:11.26.1"