android: vosk integration testing screen

.eslintignore

@@ -410,6 +410,7 @@ packages/app-mobile/components/biometrics/biometricAuthenticate.js
 packages/app-mobile/components/biometrics/sensorInfo.js
 packages/app-mobile/components/getResponsiveValue.js
 packages/app-mobile/components/getResponsiveValue.test.js
+packages/app-mobile/components/screens/VoskScreen.js
 packages/app-mobile/components/screens/ConfigScreen.js
 packages/app-mobile/components/screens/Note.js
 packages/app-mobile/components/screens/Notes.js

.gitignore

@@ -397,6 +397,7 @@ packages/app-mobile/components/biometrics/biometricAuthenticate.js
 packages/app-mobile/components/biometrics/sensorInfo.js
 packages/app-mobile/components/getResponsiveValue.js
 packages/app-mobile/components/getResponsiveValue.test.js
+packages/app-mobile/components/screens/VoskScreen.js
 packages/app-mobile/components/screens/ConfigScreen.js
 packages/app-mobile/components/screens/Note.js
 packages/app-mobile/components/screens/Notes.js

packages/app-mobile/android/app/src/main/assets/vosk-model-small-fr-0.22/README

@@ -0,0 +1,7 @@
+French small model for Vosk
+
+WER
+
+%WER 23.95 [ 37203 / 155330, 5373 ins, 4427 del, 27403 sub ] exp/chain_a/tdnn/decode_test_cv/wer_12_0.0
+%WER 19.30 [ 2975 / 15412, 683 ins, 672 del, 1620 sub ] exp/chain_a/tdnn/decode_test_mtedx/wer_10_0.0
+%WER 27.25 [ 20208 / 74145, 2647 ins, 5852 del, 11709 sub ] exp/chain_a/tdnn/decode_test_podcast_reseg/wer_10_0.0

packages/app-mobile/android/app/src/main/assets/vosk-model-small-fr-0.22/conf/mfcc.conf

@@ -0,0 +1,8 @@
+--use-energy=false
+--sample-frequency=16000
+--num-mel-bins=40
+--num-ceps=40
+--low-freq=40
+--high-freq=-200
+--allow-upsample=true
+--allow-downsample=true

packages/app-mobile/android/app/src/main/assets/vosk-model-small-fr-0.22/conf/model.conf

@@ -0,0 +1,10 @@
+--min-active=200
+--max-active=7000
+--beam=13.0
+--lattice-beam=4.0
+--acoustic-scale=1.0
+--frame-subsampling-factor=3
+--endpoint.silence-phones=1:2:3:4:5:6:7:8:9:10
+--endpoint.rule2.min-trailing-silence=0.5
+--endpoint.rule3.min-trailing-silence=1.0
+--endpoint.rule4.min-trailing-silence=2.0

packages/app-mobile/android/app/src/main/assets/vosk-model-small-fr-0.22/graph/disambig_tid.int

@@ -0,0 +1,76 @@
+9365
+9366
+9367
+9368
+9369
+9370
+9371
+9372
+9373
+9374
+9375
+9376
+9377
+9378
+9379
+9380
+9381
+9382
+9383
+9384
+9385
+9386
+9387
+9388
+9389
+9390
+9391
+9392
+9393
+9394
+9395
+9396
+9397
+9398
+9399
+9400
+9401
+9402
+9403
+9404
+9405
+9406
+9407
+9408
+9409
+9410
+9411
+9412
+9413
+9414
+9415
+9416
+9417
+9418
+9419
+9420
+9421
+9422
+9423
+9424
+9425
+9426
+9427
+9428
+9429
+9430
+9431
+9432
+9433
+9434
+9435
+9436
+9437
+9438
+9439
+9440

packages/app-mobile/android/app/src/main/assets/vosk-model-small-fr-0.22/graph/phones/word_boundary.int

@@ -0,0 +1,154 @@
+1 nonword
+2 begin
+3 end
+4 internal
+5 singleton
+6 nonword
+7 begin
+8 end
+9 internal
+10 singleton
+11 begin
+12 end
+13 internal
+14 singleton
+15 begin
+16 end
+17 internal
+18 singleton
+19 begin
+20 end
+21 internal
+22 singleton
+23 begin
+24 end
+25 internal
+26 singleton
+27 begin
+28 end
+29 internal
+30 singleton
+31 begin
+32 end
+33 internal
+34 singleton
+35 begin
+36 end
+37 internal
+38 singleton
+39 begin
+40 end
+41 internal
+42 singleton
+43 begin
+44 end
+45 internal
+46 singleton
+47 begin
+48 end
+49 internal
+50 singleton
+51 begin
+52 end
+53 internal
+54 singleton
+55 begin
+56 end
+57 internal
+58 singleton
+59 begin
+60 end
+61 internal
+62 singleton
+63 begin
+64 end
+65 internal
+66 singleton
+67 begin
+68 end
+69 internal
+70 singleton
+71 begin
+72 end
+73 internal
+74 singleton
+75 begin
+76 end
+77 internal
+78 singleton
+79 begin
+80 end
+81 internal
+82 singleton
+83 begin
+84 end
+85 internal
+86 singleton
+87 begin
+88 end
+89 internal
+90 singleton
+91 begin
+92 end
+93 internal
+94 singleton
+95 begin
+96 end
+97 internal
+98 singleton
+99 begin
+100 end
+101 internal
+102 singleton
+103 begin
+104 end
+105 internal
+106 singleton
+107 begin
+108 end
+109 internal
+110 singleton
+111 begin
+112 end
+113 internal
+114 singleton
+115 begin
+116 end
+117 internal
+118 singleton
+119 begin
+120 end
+121 internal
+122 singleton
+123 begin
+124 end
+125 internal
+126 singleton
+127 begin
+128 end
+129 internal
+130 singleton
+131 begin
+132 end
+133 internal
+134 singleton
+135 begin
+136 end
+137 internal
+138 singleton
+139 begin
+140 end
+141 internal
+142 singleton
+143 begin
+144 end
+145 internal
+146 singleton
+147 begin
+148 end
+149 internal
+150 singleton
+151 begin
+152 end
+153 internal
+154 singleton

packages/app-mobile/android/app/src/main/assets/vosk-model-small-fr-0.22/ivector/global_cmvn.stats

@@ -0,0 +1,3 @@
+ [
+  1.022245e+11 -6.33291e+09 -2.480997e+09 8.290258e+09 -9.084483e+09 -8.092173e+09 -1.4735e+10 -7.041795e+09 -1.171205e+10 -2.976464e+08 -1.009425e+10 -6765179 -7.821326e+09 1.449499e+09 -6.413975e+09 -5.303802e+08 -4.998635e+09 9.521598e+07 -3.073041e+09 1.56756e+08 -1.287956e+09 1.738752e+08 -2.382392e+08 -2.716675e+07 4.404485e+08 -1.913359e+08 7.780919e+08 -4.006922e+08 7.895809e+08 -5.401082e+08 5.17605e+08 -6.227134e+08 6.58271e+08 -6.204593e+07 5.187754e+08 -4.497048e+08 4.219366e+07 -2.78742e+08 -1.797385e+07 -3.604475e+07 1.053647e+09 
+  1.040194e+13 6.245521e+11 4.223293e+11 6.831219e+11 6.078478e+11 6.3425e+11 7.943839e+11 6.013323e+11 6.781652e+11 5.272091e+11 5.810814e+11 4.353831e+11 4.473305e+11 3.42063e+11 3.083377e+11 2.14257e+11 1.892057e+11 1.163827e+11 8.367058e+10 4.203224e+10 2.297476e+10 7.596307e+09 1.099877e+09 2.886651e+08 3.797438e+09 9.372847e+09 1.629059e+10 2.196351e+10 2.747149e+10 3.072878e+10 3.238528e+10 3.330232e+10 3.407238e+10 3.230687e+10 2.676914e+10 2.252055e+10 1.914305e+10 1.565974e+10 1.224627e+10 8.415393e+09 0 ]

packages/app-mobile/android/app/src/main/assets/vosk-model-small-fr-0.22/ivector/splice.conf

@@ -0,0 +1,2 @@
+--left-context=3
+--right-context=3

packages/app-mobile/android/app/src/main/assets/vosk-model-small-fr-0.22/uuid

@@ -0,0 +1 @@
+b5f9c333-f2d1-4a54-9e87-628814fb747f

packages/app-mobile/components/screens/VoskScreen.tsx

@@ -0,0 +1,105 @@
+/* eslint-disable */
+const React = require('react')
+import { useState, useEffect, useRef, useCallback } from 'react';
+
+import { StyleSheet, View, Text, Button } from 'react-native';
+import Vosk from 'react-native-vosk';
+const { ScreenHeader } = require('../ScreenHeader');
+
+
+function VoskScreen() {
+	const [ready, setReady] = useState<Boolean>(false);
+	const [recognizing, setRecognizing] = useState<Boolean>(false);
+	const [result, setResult] = useState<String | undefined>();
+
+	const vosk = useRef(new Vosk()).current;
+
+	const load = useCallback(() => {
+		vosk
+			.loadModel('vosk-model-small-fr-0.22')
+		// .loadModel('model-en-us')
+			.then(() => setReady(true))
+			.catch((e: any) => console.log(e));
+	}, [vosk]);
+
+	const unload = useCallback(() => {
+		vosk.unload();
+		setReady(false);
+	}, [vosk]);
+
+	useEffect(() => {
+		const resultEvent = vosk.onResult((res: { data: String }) => {
+			console.log(res);
+
+			console.log(`A onResult event has been caught: ${res.data}`);
+		});
+
+		return () => {
+			resultEvent.remove();
+		};
+	}, [vosk]);
+
+	const grammar = ['gauche', 'droite', '[unk]'];
+	// const grammar = ['left', 'right', '[unk]'];
+
+	const record = () => {
+		if (!ready) return;
+		console.log('Starting recognition ...');
+
+		setRecognizing(true);
+
+		vosk
+			.start(grammar)
+			.then((res: String) => {
+				console.log(`Result is: ${res}`);
+				setResult(res);
+			})
+			.catch((e: any) => {
+				console.log(`Error: ${e}`);
+			})
+			.finally(() => {
+				setRecognizing(false);
+			});
+	};
+
+	return (
+		<View style={styles.container}>
+			<ScreenHeader title={"Vosk Test"} parentComponent={null} showSearchButton={false} />
+			<View style={{
+				display: 'flex',
+					alignItems: 'center',
+					justifyContent: 'center',
+					marginVertical: 34
+			}}>
+				<Button
+					onPress={ready ? unload : load}
+					title={ready ? 'Unload model' : 'Load model'}
+					color="blue"
+				/>
+				<Button
+					onPress={record}
+					title="Record"
+					disabled={ready === false || recognizing === true}
+					color="#841584"
+				/>
+				<Text>Recognized word:</Text>
+				<Text>{result}</Text>
+			</View>
+		</View>
+	);
+}
+
+const styles = StyleSheet.create({
+	container: {
+		flex: 1,
+		alignItems: 'center',
+		// justifyContent: 'center',
+	},
+	// box: {
+	// 	width: "100%",
+	// 	height: "100%",
+	// },
+});
+
+// export default VoskScreenWrapper
+export default VoskScreen

packages/app-mobile/components/side-menu-content.tsx

@@ -237,6 +237,15 @@ const SideMenuContentComponent = (props: Props) => {
 		});
 	};
 
+	const voskButton_press = () => {
+		props.dispatch({ type: 'SIDE_MENU_CLOSE' });
+
+		props.dispatch({
+			type: 'NAV_GO',
+			routeName: 'Vosk',
+		});
+	};
+
 	const performSync = useCallback(async () => {
 		const action = props.syncStarted ? 'cancel' : 'start';
 
@@ -412,6 +421,7 @@ const SideMenuContentComponent = (props: Props) => {
 		items.push(makeDivider('divider_1'));
 
 		items.push(renderSidebarButton('newFolder_button', _('New Notebook'), 'md-folder-open', newFolderButton_press));
+		items.push(renderSidebarButton('vosk_button', 'Vosk', 'md-folder-open', voskButton_press));
 
 		items.push(renderSidebarButton('tag_button', _('Tags'), 'md-pricetag', tagButton_press));
 

packages/app-mobile/package.json

@@ -66,6 +66,7 @@
     "react-native-url-polyfill": "1.3.0",
     "react-native-vector-icons": "9.2.0",
     "react-native-version-info": "1.1.1",
+    "react-native-vosk": "0.1.12",
     "react-native-webview": "11.26.1",
     "react-redux": "7.2.9",
     "redux": "4.2.1",
@@ -116,4 +117,4 @@
     "uglify-js": "3.17.4",
     "webpack": "5.74.0"
   }
-}
+}

packages/app-mobile/root.tsx

@@ -57,6 +57,7 @@ import RevisionService from '@joplin/lib/services/RevisionService';
 import JoplinDatabase from '@joplin/lib/JoplinDatabase';
 import Database from '@joplin/lib/database';
 import NotesScreen from './components/screens/Notes';
+import VoskScreen from './components/screens/VoskScreen';
 const { TagsScreen } = require('./components/screens/tags.js');
 import ConfigScreen from './components/screens/ConfigScreen';
 const { FolderScreen } = require('./components/screens/folder.js');
@@ -961,6 +962,7 @@ class AppComponent extends React.Component {
 		}
 
 		const appNavInit = {
+			Vosk: { screen: VoskScreen },
 			Notes: { screen: NotesScreen },
 			Note: { screen: NoteScreen },
 			Tags: { screen: TagsScreen },

yarn.lock

@@ -4990,6 +4990,7 @@ __metadata:
     react-native-url-polyfill: 1.3.0
     react-native-vector-icons: 9.2.0
     react-native-version-info: 1.1.1
+    react-native-vosk: ^0.1.12
     react-native-webview: 11.26.1
     react-redux: 7.2.9
     redux: 4.2.1
@@ -27557,6 +27558,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"react-native-vosk@npm:^0.1.12":
+  version: 0.1.12
+  resolution: "react-native-vosk@npm:0.1.12"
+  peerDependencies:
+    react: "*"
+    react-native: "*"
+  checksum: 49dd234d0822d7f3deb9563a903260a8478bb78eb20367b50284df40e1e64e23dc52d632b329176883c048b8224182eee000fd7dbd3c42401a9a03bd0ce1ae10
+  languageName: node
+  linkType: hard
+
 "react-native-webview@npm:11.26.1":
   version: 11.26.1
   resolution: "react-native-webview@npm:11.26.1"