import * as React from 'react'; import { useState, useEffect, useCallback, useRef, useMemo } from 'react'; import { Icon, ActivityIndicator, Text, Surface, Button } from 'react-native-paper'; import { _, languageName } from '@joplin/lib/locale'; import useAsyncEffect, { AsyncEffectEvent } from '@joplin/lib/hooks/useAsyncEffect'; import { IconSource } from 'react-native-paper/lib/typescript/components/Icon'; import VoiceTyping, { OnTextCallback, VoiceTypingSession } from '../../services/voiceTyping/VoiceTyping'; import whisper from '../../services/voiceTyping/whisper'; import vosk from '../../services/voiceTyping/vosk'; import { AppState } from '../../utils/types'; import { connect } from 'react-redux'; import { View, StyleSheet } from 'react-native'; import AccessibleView from '../accessibility/AccessibleView'; interface Props { locale: string; provider: string; onDismiss: ()=> void; onText: (text: string)=> void; } enum RecorderState { Loading = 1, Recording = 2, Processing = 3, Error = 4, Downloading = 5, } interface UseVoiceTypingProps { locale: string; provider: string; onSetPreview: OnTextCallback; onText: OnTextCallback; } const useWhisper = ({ locale, provider, onSetPreview, onText }: UseVoiceTypingProps): [Error | null, boolean, VoiceTypingSession|null] => { const [voiceTyping, setVoiceTyping] = useState(null); const [error, setError] = useState(null); const [mustDownloadModel, setMustDownloadModel] = useState(null); const onTextRef = useRef(onText); onTextRef.current = onText; const onSetPreviewRef = useRef(onSetPreview); onSetPreviewRef.current = onSetPreview; const voiceTypingRef = useRef(voiceTyping); voiceTypingRef.current = voiceTyping; const builder = useMemo(() => { return new VoiceTyping(locale, provider?.startsWith('whisper') ? [whisper] : [vosk]); }, [locale, provider]); useAsyncEffect(async (event: AsyncEffectEvent) => { try { await voiceTypingRef.current?.stop(); if (!await builder.isDownloaded()) { if (event.cancelled) return; await builder.download(); } if (event.cancelled) return; const voiceTyping = await builder.build({ onPreview: (text) => onSetPreviewRef.current(text), onFinalize: (text) => onTextRef.current(text), }); if (event.cancelled) return; setVoiceTyping(voiceTyping); } catch (error) { setError(error); } finally { setMustDownloadModel(false); } }, [builder]); useAsyncEffect(async (_event: AsyncEffectEvent) => { setMustDownloadModel(!(await builder.isDownloaded())); }, [builder]); useEffect(() => () => { void voiceTypingRef.current?.stop(); }, []); return [error, mustDownloadModel, voiceTyping]; }; const styles = StyleSheet.create({ container: { marginHorizontal: 1, width: '100%', maxWidth: 680, alignSelf: 'center', }, contentWrapper: { flexDirection: 'row', }, iconWrapper: { margin: 8, marginTop: 16, }, content: { marginTop: 16, marginHorizontal: 8, }, actionContainer: { flexDirection: 'row', justifyContent: 'flex-end', }, }); const VoiceTypingDialog: React.FC = props => { const [recorderState, setRecorderState] = useState(RecorderState.Loading); const [preview, setPreview] = useState(''); const [modelError, mustDownloadModel, voiceTyping] = useWhisper({ locale: props.locale, onSetPreview: setPreview, onText: props.onText, provider: props.provider, }); useEffect(() => { if (modelError) { setRecorderState(RecorderState.Error); } else if (voiceTyping) { setRecorderState(RecorderState.Recording); } }, [voiceTyping, modelError]); useEffect(() => { if (mustDownloadModel) { setRecorderState(RecorderState.Downloading); } }, [mustDownloadModel]); useEffect(() => { if (recorderState === RecorderState.Recording) { void voiceTyping.start(); } }, [recorderState, voiceTyping, props.onText]); const onDismiss = useCallback(() => { void voiceTyping?.stop(); props.onDismiss(); }, [voiceTyping, props.onDismiss]); const renderContent = () => { const components: Record string> = { [RecorderState.Loading]: () => _('Loading...'), [RecorderState.Recording]: () => _('Please record your voice...'), [RecorderState.Processing]: () => _('Converting speech to text...'), [RecorderState.Downloading]: () => _('Downloading %s language files...', languageName(props.locale)), [RecorderState.Error]: () => _('Error: %s', modelError.message), }; return components[recorderState](); }; const renderIcon = () => { const components: Record = { [RecorderState.Loading]: ({ size }: { size: number }) => , [RecorderState.Recording]: 'microphone', [RecorderState.Processing]: 'microphone', [RecorderState.Downloading]: ({ size }: { size: number }) => , [RecorderState.Error]: 'alert-circle-outline', }; return components[recorderState]; }; const renderPreview = () => { return {preview}; }; return ( {_('Voice typing...')} {renderContent()} {renderPreview()} ); }; export default connect((state: AppState) => ({ provider: state.settings['voiceTyping.preferredProvider'], }))(VoiceTypingDialog);