diff --git a/.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch b/.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch new file mode 100644 index 000000000..c737954df --- /dev/null +++ b/.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch @@ -0,0 +1,15 @@ +diff --git a/android/src/main/java/com/reactnativevosk/VoskModule.kt b/android/src/main/java/com/reactnativevosk/VoskModule.kt +index 0e2b6595b1b2cf1ee01c6c64239c4b0ea37fce19..e0daf712a7f2d2c2dd22e623bac6daba7f6a4ac1 100644 +--- a/android/src/main/java/com/reactnativevosk/VoskModule.kt ++++ b/android/src/main/java/com/reactnativevosk/VoskModule.kt +@@ -25,7 +25,9 @@ class VoskModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaMo + + // Stop recording if data found + if (text != null && text.isNotEmpty()) { +- cleanRecognizer(); ++ // Don't auto-stop the recogniser - we want to do that when the user ++ // presses on "stop" only. ++ // cleanRecognizer(); + sendEvent("onResult", text) + } + } diff --git a/package.json b/package.json index a3b10492a..d4c231b56 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "packageManager": "yarn@3.3.1", "resolutions": { "react-native-camera@4.2.1": "patch:react-native-camera@npm%3A4.2.1#./.yarn/patches/react-native-camera-npm-4.2.1-24b2600a7e.patch", - "rn-fetch-blob@0.12.0": "patch:rn-fetch-blob@npm%3A0.12.0#./.yarn/patches/rn-fetch-blob-npm-0.12.0-cf02e3c544.patch" + "rn-fetch-blob@0.12.0": "patch:rn-fetch-blob@npm%3A0.12.0#./.yarn/patches/rn-fetch-blob-npm-0.12.0-cf02e3c544.patch", + "react-native-vosk@0.1.12": "patch:react-native-vosk@npm%3A0.1.12#./.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch" } } diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/README b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/README new file mode 100644 index 000000000..ce22e30d7 --- /dev/null +++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/README @@ -0,0 +1,7 @@ +French small model for Vosk + +WER + +%WER 23.95 [ 37203 / 155330, 5373 ins, 4427 del, 27403 sub ] exp/chain_a/tdnn/decode_test_cv/wer_12_0.0 +%WER 19.30 [ 2975 / 15412, 683 ins, 672 del, 1620 sub ] exp/chain_a/tdnn/decode_test_mtedx/wer_10_0.0 +%WER 27.25 [ 20208 / 74145, 2647 ins, 5852 del, 11709 sub ] exp/chain_a/tdnn/decode_test_podcast_reseg/wer_10_0.0 diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/am/final.mdl b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/am/final.mdl new file mode 100644 index 000000000..3ece48fb4 Binary files /dev/null and b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/am/final.mdl differ diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/conf/mfcc.conf b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/conf/mfcc.conf new file mode 100644 index 000000000..12fdad4ed --- /dev/null +++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/conf/mfcc.conf @@ -0,0 +1,8 @@ +--use-energy=false +--sample-frequency=16000 +--num-mel-bins=40 +--num-ceps=40 +--low-freq=40 +--high-freq=-200 +--allow-upsample=true +--allow-downsample=true diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/conf/model.conf b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/conf/model.conf new file mode 100644 index 000000000..e66bcae8f --- /dev/null +++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/conf/model.conf @@ -0,0 +1,10 @@ +--min-active=200 +--max-active=7000 +--beam=13.0 +--lattice-beam=4.0 +--acoustic-scale=1.0 +--frame-subsampling-factor=3 +--endpoint.silence-phones=1:2:3:4:5:6:7:8:9:10 +--endpoint.rule2.min-trailing-silence=0.5 +--endpoint.rule3.min-trailing-silence=1.0 +--endpoint.rule4.min-trailing-silence=2.0 diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/Gr.fst b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/Gr.fst new file mode 100644 index 000000000..bf81b42de Binary files /dev/null and b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/Gr.fst differ diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/HCLr.fst b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/HCLr.fst new file mode 100644 index 000000000..b8ad5671d Binary files /dev/null and b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/HCLr.fst differ diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/disambig_tid.int b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/disambig_tid.int new file mode 100644 index 000000000..07298933f --- /dev/null +++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/disambig_tid.int @@ -0,0 +1,76 @@ +9365 +9366 +9367 +9368 +9369 +9370 +9371 +9372 +9373 +9374 +9375 +9376 +9377 +9378 +9379 +9380 +9381 +9382 +9383 +9384 +9385 +9386 +9387 +9388 +9389 +9390 +9391 +9392 +9393 +9394 +9395 +9396 +9397 +9398 +9399 +9400 +9401 +9402 +9403 +9404 +9405 +9406 +9407 +9408 +9409 +9410 +9411 +9412 +9413 +9414 +9415 +9416 +9417 +9418 +9419 +9420 +9421 +9422 +9423 +9424 +9425 +9426 +9427 +9428 +9429 +9430 +9431 +9432 +9433 +9434 +9435 +9436 +9437 +9438 +9439 +9440 diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/phones/word_boundary.int b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/phones/word_boundary.int new file mode 100644 index 000000000..9a6c32ed5 --- /dev/null +++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/graph/phones/word_boundary.int @@ -0,0 +1,154 @@ +1 nonword +2 begin +3 end +4 internal +5 singleton +6 nonword +7 begin +8 end +9 internal +10 singleton +11 begin +12 end +13 internal +14 singleton +15 begin +16 end +17 internal +18 singleton +19 begin +20 end +21 internal +22 singleton +23 begin +24 end +25 internal +26 singleton +27 begin +28 end +29 internal +30 singleton +31 begin +32 end +33 internal +34 singleton +35 begin +36 end +37 internal +38 singleton +39 begin +40 end +41 internal +42 singleton +43 begin +44 end +45 internal +46 singleton +47 begin +48 end +49 internal +50 singleton +51 begin +52 end +53 internal +54 singleton +55 begin +56 end +57 internal +58 singleton +59 begin +60 end +61 internal +62 singleton +63 begin +64 end +65 internal +66 singleton +67 begin +68 end +69 internal +70 singleton +71 begin +72 end +73 internal +74 singleton +75 begin +76 end +77 internal +78 singleton +79 begin +80 end +81 internal +82 singleton +83 begin +84 end +85 internal +86 singleton +87 begin +88 end +89 internal +90 singleton +91 begin +92 end +93 internal +94 singleton +95 begin +96 end +97 internal +98 singleton +99 begin +100 end +101 internal +102 singleton +103 begin +104 end +105 internal +106 singleton +107 begin +108 end +109 internal +110 singleton +111 begin +112 end +113 internal +114 singleton +115 begin +116 end +117 internal +118 singleton +119 begin +120 end +121 internal +122 singleton +123 begin +124 end +125 internal +126 singleton +127 begin +128 end +129 internal +130 singleton +131 begin +132 end +133 internal +134 singleton +135 begin +136 end +137 internal +138 singleton +139 begin +140 end +141 internal +142 singleton +143 begin +144 end +145 internal +146 singleton +147 begin +148 end +149 internal +150 singleton +151 begin +152 end +153 internal +154 singleton diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.dubm b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.dubm new file mode 100644 index 000000000..cab561c5e Binary files /dev/null and b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.dubm differ diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.ie b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.ie new file mode 100644 index 000000000..335fabfb8 Binary files /dev/null and b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.ie differ diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.mat b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.mat new file mode 100644 index 000000000..2ba204b56 Binary files /dev/null and b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/final.mat differ diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/global_cmvn.stats b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/global_cmvn.stats new file mode 100644 index 000000000..23cafcb5f --- /dev/null +++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/global_cmvn.stats @@ -0,0 +1,3 @@ + [ + 1.022245e+11 -6.33291e+09 -2.480997e+09 8.290258e+09 -9.084483e+09 -8.092173e+09 -1.4735e+10 -7.041795e+09 -1.171205e+10 -2.976464e+08 -1.009425e+10 -6765179 -7.821326e+09 1.449499e+09 -6.413975e+09 -5.303802e+08 -4.998635e+09 9.521598e+07 -3.073041e+09 1.56756e+08 -1.287956e+09 1.738752e+08 -2.382392e+08 -2.716675e+07 4.404485e+08 -1.913359e+08 7.780919e+08 -4.006922e+08 7.895809e+08 -5.401082e+08 5.17605e+08 -6.227134e+08 6.58271e+08 -6.204593e+07 5.187754e+08 -4.497048e+08 4.219366e+07 -2.78742e+08 -1.797385e+07 -3.604475e+07 1.053647e+09 + 1.040194e+13 6.245521e+11 4.223293e+11 6.831219e+11 6.078478e+11 6.3425e+11 7.943839e+11 6.013323e+11 6.781652e+11 5.272091e+11 5.810814e+11 4.353831e+11 4.473305e+11 3.42063e+11 3.083377e+11 2.14257e+11 1.892057e+11 1.163827e+11 8.367058e+10 4.203224e+10 2.297476e+10 7.596307e+09 1.099877e+09 2.886651e+08 3.797438e+09 9.372847e+09 1.629059e+10 2.196351e+10 2.747149e+10 3.072878e+10 3.238528e+10 3.330232e+10 3.407238e+10 3.230687e+10 2.676914e+10 2.252055e+10 1.914305e+10 1.565974e+10 1.224627e+10 8.415393e+09 0 ] diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/online_cmvn.conf b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/online_cmvn.conf new file mode 100644 index 000000000..e69de29bb diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/splice.conf b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/splice.conf new file mode 100644 index 000000000..960cd2e4c --- /dev/null +++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/ivector/splice.conf @@ -0,0 +1,2 @@ +--left-context=3 +--right-context=3 diff --git a/packages/app-mobile/android/app/src/main/assets/model-fr-fr/uuid b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/uuid new file mode 100644 index 000000000..0a61b6aae --- /dev/null +++ b/packages/app-mobile/android/app/src/main/assets/model-fr-fr/uuid @@ -0,0 +1 @@ +1d689fce-7d11-4a95-a13e-19b64b5da0c0 \ No newline at end of file diff --git a/packages/app-mobile/components/screens/Note.tsx b/packages/app-mobile/components/screens/Note.tsx index 7babfd6f4..0dcfc620b 100644 --- a/packages/app-mobile/components/screens/Note.tsx +++ b/packages/app-mobile/components/screens/Note.tsx @@ -44,6 +44,7 @@ import ShareExtension from '../../utils/ShareExtension.js'; import CameraView from '../CameraView'; import { NoteEntity } from '@joplin/lib/services/database/types'; import Logger from '@joplin/lib/Logger'; +import Vosk from 'react-native-vosk'; const urlUtils = require('@joplin/lib/urlUtils'); const emptyArray: any[] = []; @@ -51,6 +52,10 @@ const emptyArray: any[] = []; const logger = Logger.create('screens/Note'); class NoteScreenComponent extends BaseScreenComponent { + + private vosk_: Vosk|null = null; + private voskResult_: string[] = []; + public static navigationOptions(): any { return { header: null }; } @@ -754,6 +759,65 @@ class NoteScreenComponent extends BaseScreenComponent { } } + private async getVosk() { + if (this.vosk_) return this.vosk_; + this.vosk_ = new Vosk(); + await this.vosk_.loadModel('model-fr-fr'); + return this.vosk_; + } + + private async voiceRecording_onPress() { + logger.info('Vosk: Getting instance...'); + + const vosk = await this.getVosk(); + + this.voskResult_ = []; + + const eventHandlers: any[] = []; + + eventHandlers.push(vosk.onResult(e => { + logger.info('Vosk: result', e.data); + this.voskResult_.push(e.data); + })); + + eventHandlers.push(vosk.onError(e => { + logger.warn('Vosk: error', e.data); + })); + + eventHandlers.push(vosk.onTimeout(e => { + logger.warn('Vosk: timeout', e.data); + })); + + eventHandlers.push(vosk.onFinalResult(e => { + logger.info('Vosk: final result', e.data); + })); + + logger.info('Vosk: Starting recording...'); + + void vosk.start(); + + const buttonId = await dialogs.pop(this, 'Voice recording in progress...', [ + { text: 'Stop recording', id: 'stop' }, + { text: _('Cancel'), id: 'cancel' }, + ]); + + logger.info('Vosk: Stopping recording...'); + vosk.stop(); + + for (const eventHandler of eventHandlers) { + eventHandler.remove(); + } + + logger.info('Vosk: Recording stopped:', this.voskResult_); + + if (buttonId === 'cancel') return; + + const newNote: NoteEntity = { ...this.state.note }; + newNote.body = `${newNote.body} ${this.voskResult_.join(' ')}`; + this.setState({ note: newNote }); + this.scheduleSave(); + } + private toggleIsTodo_onPress() { shared.toggleIsTodo_onPress(this); @@ -914,6 +978,16 @@ class NoteScreenComponent extends BaseScreenComponent { void this.share_onPress(); }, }); + + if (shim.mobilePlatform() === 'android') { + output.push({ + title: 'Voice recording (Beta - FR only)', + onPress: () => { + void this.voiceRecording_onPress(); + }, + }); + } + if (isSaved) { output.push({ title: _('Tags'), diff --git a/packages/app-mobile/ios/Joplin.xcodeproj/project.pbxproj b/packages/app-mobile/ios/Joplin.xcodeproj/project.pbxproj index eb4d8f326..c044486d8 100644 --- a/packages/app-mobile/ios/Joplin.xcodeproj/project.pbxproj +++ b/packages/app-mobile/ios/Joplin.xcodeproj/project.pbxproj @@ -422,6 +422,7 @@ "${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/SimpleLineIcons.ttf", "${PODS_ROOT}/../../node_modules/react-native-vector-icons/Fonts/Zocial.ttf", "${PODS_CONFIGURATION_BUILD_DIR}/React-Core/AccessibilityResources.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/react-native-vosk/Vosk.bundle", ); name = "[CP] Copy Pods Resources"; outputPaths = ( @@ -442,6 +443,7 @@ "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/SimpleLineIcons.ttf", "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/Zocial.ttf", "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/AccessibilityResources.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/Vosk.bundle", ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; diff --git a/packages/app-mobile/ios/Podfile.lock b/packages/app-mobile/ios/Podfile.lock index 8ac849607..528729f7a 100644 --- a/packages/app-mobile/ios/Podfile.lock +++ b/packages/app-mobile/ios/Podfile.lock @@ -316,7 +316,7 @@ PODS: - React-Core - react-native-camera/RN (4.2.1): - React-Core - - react-native-document-picker (8.1.4): + - react-native-document-picker (8.2.0): - React-Core - react-native-fingerprint-scanner (6.0.0): - React @@ -328,13 +328,13 @@ PODS: - React-Core - react-native-image-resizer (1.4.5): - React-Core - - react-native-netinfo (9.3.8): + - react-native-netinfo (9.3.9): - React-Core - react-native-rsa-native (2.0.5): - React - react-native-saf-x (2.11.0): - React-Core - - react-native-safe-area-context (4.5.0): + - react-native-safe-area-context (4.5.1): - RCT-Folly - RCTRequired - RCTTypeSafety @@ -346,6 +346,8 @@ PODS: - React-Core - react-native-version-info (1.1.1): - React-Core + - react-native-vosk (0.1.12): + - React-Core - react-native-webview (11.26.1): - React-Core - React-perflogger (0.70.6) @@ -505,6 +507,7 @@ DEPENDENCIES: - "react-native-slider (from `../node_modules/@react-native-community/slider`)" - react-native-sqlite-storage (from `../node_modules/react-native-sqlite-storage`) - react-native-version-info (from `../node_modules/react-native-version-info`) + - react-native-vosk (from `../node_modules/react-native-vosk`) - react-native-webview (from `../node_modules/react-native-webview`) - React-perflogger (from `../node_modules/react-native/ReactCommon/reactperflogger`) - React-RCTActionSheet (from `../node_modules/react-native/Libraries/ActionSheetIOS`) @@ -626,6 +629,8 @@ EXTERNAL SOURCES: :path: "../node_modules/react-native-sqlite-storage" react-native-version-info: :path: "../node_modules/react-native-version-info" + react-native-vosk: + :path: "../node_modules/react-native-vosk" react-native-webview: :path: "../node_modules/react-native-webview" React-perflogger: @@ -716,19 +721,20 @@ SPEC CHECKSUMS: React-logger: 1623c216abaa88974afce404dc8f479406bbc3a0 react-native-alarm-notification: 26527410a6162d07a9dc57f4bbc62e94ff48e65d react-native-camera: 3eae183c1d111103963f3dd913b65d01aef8110f - react-native-document-picker: a9bd26996d1b2e4f412dd186041714c79af381d0 + react-native-document-picker: 495c444c0c773c6e83a5d91165890ecb1c0a399a react-native-fingerprint-scanner: ac6656f18c8e45a7459302b84da41a44ad96dbbe react-native-geolocation: 69f4fd37650b8e7fee91816d395e62dd16f5ab8d react-native-get-random-values: a6ea6a8a65dc93e96e24a11105b1a9c8cfe1d72a react-native-image-picker: ec9b713e248760bfa0f879f0715391de4651a7cb react-native-image-resizer: d9fb629a867335bdc13230ac2a58702bb8c8828f - react-native-netinfo: fbc23bc2fe217155d85f2f7e0644b1654df8029b + react-native-netinfo: 22c082970cbd99071a4e5aa7a612ac20d66b08f0 react-native-rsa-native: 12132eb627797529fdb1f0d22fd0f8f9678df64a react-native-saf-x: 9bd5238d3b43d76bbec64aa82c173ac20a4bce9f - react-native-safe-area-context: 39c2d8be3328df5d437ac1700f4f3a4f75716acc + react-native-safe-area-context: f5549f36508b1b7497434baa0cd97d7e470920d4 react-native-slider: 33b8d190b59d4f67a541061bb91775d53d617d9d react-native-sqlite-storage: f6d515e1c446d1e6d026aa5352908a25d4de3261 react-native-version-info: a106f23009ac0db4ee00de39574eb546682579b9 + react-native-vosk: 33b8e82a46cc56f31bb4847a40efa2d160270e2e react-native-webview: 9f111dfbcfc826084d6c507f569e5e03342ee1c1 React-perflogger: 8c79399b0500a30ee8152d0f9f11beae7fc36595 React-RCTActionSheet: 7316773acabb374642b926c19aef1c115df5c466 diff --git a/packages/app-mobile/package.json b/packages/app-mobile/package.json index f7db6fb32..a255ddb66 100644 --- a/packages/app-mobile/package.json +++ b/packages/app-mobile/package.json @@ -66,6 +66,7 @@ "react-native-url-polyfill": "1.3.0", "react-native-vector-icons": "9.2.0", "react-native-version-info": "1.1.1", + "react-native-vosk": "0.1.12", "react-native-webview": "11.26.1", "react-redux": "7.2.9", "redux": "4.2.1", diff --git a/yarn.lock b/yarn.lock index a003e16e1..ed9d41fb8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4983,6 +4983,7 @@ __metadata: react-native-url-polyfill: 1.3.0 react-native-vector-icons: 9.2.0 react-native-version-info: 1.1.1 + react-native-vosk: 0.1.12 react-native-webview: 11.26.1 react-redux: 7.2.9 redux: 4.2.1 @@ -27550,6 +27551,26 @@ __metadata: languageName: node linkType: hard +"react-native-vosk@npm:0.1.12": + version: 0.1.12 + resolution: "react-native-vosk@npm:0.1.12" + peerDependencies: + react: "*" + react-native: "*" + checksum: 49dd234d0822d7f3deb9563a903260a8478bb78eb20367b50284df40e1e64e23dc52d632b329176883c048b8224182eee000fd7dbd3c42401a9a03bd0ce1ae10 + languageName: node + linkType: hard + +"react-native-vosk@patch:react-native-vosk@npm%3A0.1.12#./.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch::locator=root%40workspace%3A.": + version: 0.1.12 + resolution: "react-native-vosk@patch:react-native-vosk@npm%3A0.1.12#./.yarn/patches/react-native-vosk-npm-0.1.12-76b1caaae8.patch::version=0.1.12&hash=edde10&locator=root%40workspace%3A." + peerDependencies: + react: "*" + react-native: "*" + checksum: 53cd3db17e859466345dd80a485c9ceaa35498dbcf2159af95a70151e09b56aff8743b0323b3cb6f04653c2f24aed06e31045988a5af91331ad7bdb8c50be0b3 + languageName: node + linkType: hard + "react-native-webview@npm:11.26.1": version: 11.26.1 resolution: "react-native-webview@npm:11.26.1"