Mobile: Support attaching audio recordings (#11836)

.eslintignore

@@ -792,7 +792,10 @@ packages/app-mobile/components/screens/status.js
 packages/app-mobile/components/screens/tags.js
 packages/app-mobile/components/side-menu-content.js
 packages/app-mobile/components/testing/TestProviderStack.js
-packages/app-mobile/components/voiceTyping/VoiceTypingDialog.js
+packages/app-mobile/components/voiceTyping/AudioRecordingBanner.js
+packages/app-mobile/components/voiceTyping/RecordingControls.js
+packages/app-mobile/components/voiceTyping/SpeechToTextBanner.js
+packages/app-mobile/components/voiceTyping/types.js
 packages/app-mobile/gulpfile.js
 packages/app-mobile/index.web.js
 packages/app-mobile/root.js

.gitignore

@@ -767,7 +767,10 @@ packages/app-mobile/components/screens/status.js
 packages/app-mobile/components/screens/tags.js
 packages/app-mobile/components/side-menu-content.js
 packages/app-mobile/components/testing/TestProviderStack.js
-packages/app-mobile/components/voiceTyping/VoiceTypingDialog.js
+packages/app-mobile/components/voiceTyping/AudioRecordingBanner.js
+packages/app-mobile/components/voiceTyping/RecordingControls.js
+packages/app-mobile/components/voiceTyping/SpeechToTextBanner.js
+packages/app-mobile/components/voiceTyping/types.js
 packages/app-mobile/gulpfile.js
 packages/app-mobile/index.web.js
 packages/app-mobile/root.js

packages/app-mobile/components/EditorToolbar/EditorToolbar.tsx

@@ -6,7 +6,6 @@ import { ToolbarButtonInfo, ToolbarItem } from '@joplin/lib/services/commands/To
 import toolbarButtonsFromState from './utils/toolbarButtonsFromState';
 import { useCallback, useMemo, useRef, useState } from 'react';
 import { themeStyle } from '../global-style';
-import ToggleSpaceButton from '../ToggleSpaceButton';
 import ToolbarEditorDialog from './ToolbarEditorDialog';
 import { EditorState } from './types';
 import ToolbarButton from './ToolbarButton';
@@ -127,19 +126,17 @@ const EditorToolbar: React.FC<Props> = props => {
 	/>;
 
 	return <>
-		<ToggleSpaceButton themeId={props.themeId}>
-			<ScrollView
-				ref={scrollViewRef}
-				horizontal={true}
-				style={styles.content}
-				contentContainerStyle={styles.contentContainer}
-				onLayout={onContainerLayout}
-			>
-				{buttonInfos.map(renderButton)}
-				<View style={styles.spacer}/>
-				{settingsButton}
-			</ScrollView>
-		</ToggleSpaceButton>
+		<ScrollView
+			ref={scrollViewRef}
+			horizontal={true}
+			style={styles.content}
+			contentContainerStyle={styles.contentContainer}
+			onLayout={onContainerLayout}
+		>
+			{buttonInfos.map(renderButton)}
+			<View style={styles.spacer}/>
+			{settingsButton}
+		</ScrollView>
 		<ToolbarEditorDialog visible={settingsVisible} onDismiss={onDismissSettingsDialog} />
 	</>;
 };

packages/app-mobile/components/ToggleSpaceButton.tsx

@@ -12,7 +12,7 @@ import { themeStyle } from '@joplin/lib/theme';
 
 import * as React from 'react';
 import { ReactNode, useCallback, useState, useEffect } from 'react';
-import { Platform, View, ViewStyle } from 'react-native';
+import { Platform, useWindowDimensions, View, ViewStyle } from 'react-native';
 import IconButton from './IconButton';
 import useKeyboardVisible from '../utils/hooks/useKeyboardVisible';
 
@@ -77,7 +77,9 @@ const ToggleSpaceButton = (props: Props) => {
 	);
 
 	const { keyboardVisible } = useKeyboardVisible();
-	const spaceApplicable = keyboardVisible && Platform.OS === 'ios';
+	const windowSize = useWindowDimensions();
+	const isPortrait = windowSize.height > windowSize.width;
+	const spaceApplicable = keyboardVisible && Platform.OS === 'ios' && isPortrait;
 
 	const style: ViewStyle = {
 		marginBottom: spaceApplicable ? additionalSpace : 0,

packages/app-mobile/components/screens/Note/Note.tsx

@@ -40,8 +40,6 @@ import Logger from '@joplin/utils/Logger';
 import ImageEditor from '../../NoteEditor/ImageEditor/ImageEditor';
 import promptRestoreAutosave from '../../NoteEditor/ImageEditor/promptRestoreAutosave';
 import isEditableResource from '../../NoteEditor/ImageEditor/isEditableResource';
-import VoiceTypingDialog from '../../voiceTyping/VoiceTypingDialog';
-import { isSupportedLanguage } from '../../../services/voiceTyping/vosk';
 import { ChangeEvent as EditorChangeEvent, SelectionRangeChangeEvent, UndoRedoDepthChangeEvent } from '@joplin/editor/events';
 import { join } from 'path';
 import { Dispatch } from 'redux';
@@ -63,11 +61,14 @@ import { DialogContext, DialogControl } from '../../DialogManager';
 import { CommandRuntimeProps, EditorMode, PickerResponse } from './types';
 import commands from './commands';
 import { AttachFileAction, AttachFileOptions } from './commands/attachFile';
+import ToggleSpaceButton from '../../ToggleSpaceButton';
 import PluginService from '@joplin/lib/services/plugins/PluginService';
 import PluginUserWebView from '../../plugins/dialogs/PluginUserWebView';
 import getShownPluginEditorView from '@joplin/lib/services/plugins/utils/getShownPluginEditorView';
 import getActivePluginEditorView from '@joplin/lib/services/plugins/utils/getActivePluginEditorView';
 import EditorPluginHandler from '@joplin/lib/services/plugins/EditorPluginHandler';
+import AudioRecordingBanner from '../../voiceTyping/AudioRecordingBanner';
+import SpeechToTextBanner from '../../voiceTyping/SpeechToTextBanner';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
 const emptyArray: any[] = [];
@@ -126,6 +127,7 @@ interface State {
 	fromShare: boolean;
 	showCamera: boolean;
 	showImageEditor: boolean;
+	showAudioRecorder: boolean;
 	imageEditorResource: ResourceEntity;
 	imageEditorResourceFilepath: string;
 	noteResources: Record<string, ResourceInfo>;
@@ -137,7 +139,7 @@ interface State {
 		canRedo: boolean;
 	};
 
-	voiceTypingDialogShown: boolean;
+	showSpeechToTextDialog: boolean;
 }
 
 class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> implements BaseNoteScreenComponent {
@@ -195,6 +197,7 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 			fromShare: false,
 			showCamera: false,
 			showImageEditor: false,
+			showAudioRecorder: false,
 			imageEditorResource: null,
 			noteResources: {},
 			imageEditorResourceFilepath: null,
@@ -206,7 +209,7 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 				canRedo: false,
 			},
 
-			voiceTypingDialogShown: false,
+			showSpeechToTextDialog: false,
 		};
 
 		this.titleTextFieldRef = React.createRef();
@@ -323,8 +326,8 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 		this.screenHeader_redoButtonPress = this.screenHeader_redoButtonPress.bind(this);
 		this.onBodyViewerCheckboxChange = this.onBodyViewerCheckboxChange.bind(this);
 		this.onUndoRedoDepthChange = this.onUndoRedoDepthChange.bind(this);
-		this.voiceTypingDialog_onText = this.voiceTypingDialog_onText.bind(this);
-		this.voiceTypingDialog_onDismiss = this.voiceTypingDialog_onDismiss.bind(this);
+		this.speechToTextDialog_onText = this.speechToTextDialog_onText.bind(this);
+		this.audioRecorderDialog_onDismiss = this.audioRecorderDialog_onDismiss.bind(this);
 	}
 
 	private registerCommands() {
@@ -353,6 +356,9 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 
 					this.setState({ noteTagDialogShown: visible });
 				},
+				setAudioRecorderVisible: (visible) => {
+					this.setState({ showAudioRecorder: visible });
+				},
 				getMode: () => this.state.mode,
 				setMode: (mode: 'view'|'edit') => {
 					this.setState({ mode });
@@ -460,6 +466,9 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 			noteBodyViewer: {
 				flex: 1,
 			},
+			toggleSpaceButtonContent: {
+				flex: 1,
+			},
 			checkbox: {
 				color: theme.color,
 				paddingRight: 10,
@@ -1222,13 +1231,13 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 			});
 		}
 
-		// Voice typing is enabled only on Android for now
-		if (shim.mobilePlatform() === 'android' && isSupportedLanguage(currentLocale())) {
+		const voiceTypingSupported = Platform.OS === 'android';
+		if (voiceTypingSupported) {
 			output.push({
 				title: _('Voice typing...'),
 				onPress: () => {
 					// this.voiceRecording_onPress();
-					this.setState({ voiceTypingDialogShown: true });
+					this.setState({ showSpeechToTextDialog: true });
 				},
 				disabled: readOnly,
 			});
@@ -1416,7 +1425,7 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 		void this.saveOneProperty('body', newBody);
 	}
 
-	private voiceTypingDialog_onText(text: string) {
+	private speechToTextDialog_onText(text: string) {
 		if (this.state.mode === 'view') {
 			const newNote: NoteEntity = { ...this.state.note };
 			newNote.body = `${newNote.body} ${text}`;
@@ -1433,9 +1442,17 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 		}
 	}
 
-	private voiceTypingDialog_onDismiss() {
-		this.setState({ voiceTypingDialogShown: false });
-	}
+	private audioRecordingDialog_onFile = (file: PickerResponse) => {
+		return this.attachFile(file, 'audio');
+	};
+
+	private audioRecorderDialog_onDismiss = () => {
+		this.setState({ showSpeechToTextDialog: false, showAudioRecorder: false });
+	};
+
+	private speechToTextDialog_onDismiss = () => {
+		this.setState({ showSpeechToTextDialog: false });
+	};
 
 	private noteEditorVisible() {
 		return !this.state.showCamera && !this.state.showImageEditor;
@@ -1589,8 +1606,9 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 			}
 		}
 
+		const voiceTypingDialogShown = this.state.showSpeechToTextDialog || this.state.showAudioRecorder;
 		const renderActionButton = () => {
-			if (this.state.voiceTypingDialogShown) return null;
+			if (voiceTypingDialogShown) return null;
 			if (!this.state.note || !!this.state.note.deleted_time) return null;
 
 			const editButton = {
@@ -1637,9 +1655,37 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 
 		const noteTagDialog = !this.state.noteTagDialogShown ? null : <NoteTagsDialog onCloseRequested={this.noteTagDialog_closeRequested} />;
 
-		const renderVoiceTypingDialog = () => {
-			if (!this.state.voiceTypingDialogShown) return null;
-			return <VoiceTypingDialog locale={currentLocale()} onText={this.voiceTypingDialog_onText} onDismiss={this.voiceTypingDialog_onDismiss}/>;
+		const renderVoiceTypingDialogs = () => {
+			const result = [];
+			if (this.state.showAudioRecorder) {
+				result.push(<AudioRecordingBanner
+					key='audio-recorder'
+					onFileSaved={this.audioRecordingDialog_onFile}
+					onDismiss={this.audioRecorderDialog_onDismiss}
+				/>);
+			}
+			if (this.state.showSpeechToTextDialog) {
+				result.push(<SpeechToTextBanner
+					key='speech-to-text'
+					locale={currentLocale()}
+					onText={this.speechToTextDialog_onText}
+					onDismiss={this.speechToTextDialog_onDismiss}
+				/>);
+			}
+			return result;
+		};
+
+		const renderWrappedContent = () => {
+			const content = <>
+				{bodyComponent}
+				{renderVoiceTypingDialogs()}
+			</>;
+
+			return this.state.mode === 'edit' ? (
+				<ToggleSpaceButton themeId={this.props.themeId} style={this.styles().toggleSpaceButtonContent}>
+					{content}
+				</ToggleSpaceButton>
+			) : content;
 		};
 
 		const { editorPlugin: activeEditorPlugin } = getActivePluginEditorView(this.props.plugins);
@@ -1663,9 +1709,8 @@ class NoteScreenComponent extends BaseScreenComponent<ComponentProps, State> imp
 					title={getDisplayParentTitle(this.state.note, this.state.folder)}
 				/>
 				{titleComp}
-				{bodyComponent}
+				{renderWrappedContent()}
 				{renderActionButton()}
-				{renderVoiceTypingDialog()}
 
 				<SelectDateTimeDialog themeId={this.props.themeId} shown={this.state.alarmDialogShown} date={dueDate} onAccept={this.onAlarmDialogAccept} onReject={this.onAlarmDialogReject} />
 

packages/app-mobile/components/screens/Note/commands/attachFile.ts

@@ -14,6 +14,7 @@ export enum AttachFileAction {
 	AttachFile = 'attachFile',
 	AttachPhoto = 'attachPhoto',
 	AttachDrawing = 'attachDrawing',
+	RecordAudio = 'attachRecording',
 }
 
 export interface AttachFileOptions {
@@ -68,6 +69,10 @@ export const runtime = (props: CommandRuntimeProps): CommandRuntime => {
 		}
 	};
 
+	const recordAudio = async () => {
+		props.setAudioRecorderVisible(true);
+	};
+
 	const showAttachMenu = async (action: AttachFileAction = null) => {
 		props.hideKeyboard();
 
@@ -83,6 +88,7 @@ export const runtime = (props: CommandRuntimeProps): CommandRuntime => {
 			//
 			// On Android, it will depend on the phone, but usually it will allow browsing all files and photos.
 			buttons.push({ text: _('Attach file'), id: AttachFileAction.AttachFile });
+			buttons.push({ text: _('Record audio'), id: AttachFileAction.RecordAudio });
 
 			// Disabled on Android because it doesn't work due to permission issues, but enabled on iOS
 			// because that's only way to browse photos from the camera roll.
@@ -102,6 +108,7 @@ export const runtime = (props: CommandRuntimeProps): CommandRuntime => {
 		if (buttonId === AttachFileAction.TakePhoto) await takePhoto();
 		if (buttonId === AttachFileAction.AttachFile) await attachFile();
 		if (buttonId === AttachFileAction.AttachPhoto) await attachPhoto();
+		if (buttonId === AttachFileAction.RecordAudio) await recordAudio();
 	};
 
 	return {

packages/app-mobile/components/screens/Note/types.ts

@@ -18,5 +18,6 @@ export interface CommandRuntimeProps {
 	setMode(mode: EditorMode): void;
 	setCameraVisible(visible: boolean): void;
 	setTagDialogVisible(visible: boolean): void;
+	setAudioRecorderVisible(visible: boolean): void;
 	dialogs: DialogControl;
 }

packages/app-mobile/components/voiceTyping/AudioRecordingBanner.tsx

@@ -0,0 +1,235 @@
+import * as React from 'react';
+import { PrimaryButton, SecondaryButton } from '../buttons';
+import { _ } from '@joplin/lib/locale';
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { Audio, InterruptionModeIOS } from 'expo-av';
+import Logger from '@joplin/utils/Logger';
+import { OnFileSavedCallback, RecorderState } from './types';
+import { Platform } from 'react-native';
+import shim from '@joplin/lib/shim';
+import FsDriverWeb from '../../utils/fs-driver/fs-driver-rn.web';
+import uuid from '@joplin/lib/uuid';
+import RecordingControls from './RecordingControls';
+import { Text } from 'react-native-paper';
+import { AndroidAudioEncoder, AndroidOutputFormat, IOSAudioQuality, IOSOutputFormat, RecordingOptions } from 'expo-av/build/Audio';
+import time from '@joplin/lib/time';
+import { toFileExtension } from '@joplin/lib/mime-utils';
+import { formatMsToDurationLocal } from '@joplin/utils/time';
+
+const logger = Logger.create('AudioRecording');
+
+interface Props {
+	onFileSaved: OnFileSavedCallback;
+	onDismiss: ()=> void;
+}
+
+// Modified from the Expo default recording options to create
+// .m4a recordings on both Android and iOS (rather than .3gp on Android).
+const recordingOptions = (): RecordingOptions => ({
+	isMeteringEnabled: true,
+	android: {
+		extension: '.m4a',
+		outputFormat: AndroidOutputFormat.MPEG_4,
+		audioEncoder: AndroidAudioEncoder.AAC,
+		sampleRate: 44100,
+		numberOfChannels: 2,
+		bitRate: 64000,
+	},
+	ios: {
+		extension: '.m4a',
+		audioQuality: IOSAudioQuality.MIN,
+		outputFormat: IOSOutputFormat.MPEG4AAC,
+		sampleRate: 44100,
+		numberOfChannels: 2,
+		bitRate: 64000,
+		linearPCMBitDepth: 16,
+		linearPCMIsBigEndian: false,
+		linearPCMIsFloat: false,
+	},
+	web: Platform.OS === 'web' ? {
+		mimeType: [
+			// Different browsers support different audio formats.
+			// In most cases, prefer audio/ogg and audio/mp4 to audio/webm because
+			// Chrome and Firefox create .webm files without duration information.
+			// See https://issues.chromium.org/issues/40482588
+			'audio/ogg', 'audio/mp4', 'audio/webm',
+		].find(type => MediaRecorder.isTypeSupported(type)) ?? 'audio/webm',
+		bitsPerSecond: 128000,
+	} : {},
+});
+
+const getRecordingFileName = (extension: string) => {
+	return `recording-${time.formatDateToLocal(new Date())}${extension}`;
+};
+
+const recordingToSaveData = async (recording: Audio.Recording) => {
+	let uri = recording.getURI();
+	let type: string|undefined;
+	let fileName;
+
+	if (Platform.OS === 'web') {
+		// On web, we need to fetch the result (which is a blob URL) and save it in our
+		// virtual file system so that it can be processed elsewhere.
+		const fetchResult = await fetch(uri);
+		const blob = await fetchResult.blob();
+
+		type = recordingOptions().web.mimeType;
+		const extension = `.${toFileExtension(type)}`;
+		fileName = getRecordingFileName(extension);
+		const file = new File([blob], fileName);
+
+		const path = `/tmp/${uuid.create()}-${fileName}`;
+		await (shim.fsDriver() as FsDriverWeb).createReadOnlyVirtualFile(path, file);
+		uri = path;
+	} else {
+		const options = recordingOptions();
+		const extension = Platform.select({
+			android: options.android.extension,
+			ios: options.ios.extension,
+			default: '',
+		});
+		fileName = getRecordingFileName(extension);
+	}
+
+	return { uri, fileName, type };
+};
+
+const resetAudioMode = async () => {
+	await Audio.setAudioModeAsync({
+		// When enabled, iOS may use the small (phone call) speaker
+		// instead of the default one, so it's disabled when not recording:
+		allowsRecordingIOS: false,
+		playsInSilentModeIOS: false,
+	});
+};
+
+const useAudioRecorder = (onFileSaved: OnFileSavedCallback, onDismiss: ()=> void) => {
+	const [permissionResponse, requestPermissions] = Audio.usePermissions();
+	const [recordingState, setRecordingState] = useState<RecorderState>(RecorderState.Idle);
+	const [error, setError] = useState('');
+	const [duration, setDuration] = useState(0);
+
+	const recordingRef = useRef<Audio.Recording|null>();
+	const onStartRecording = useCallback(async () => {
+		try {
+			setRecordingState(RecorderState.Loading);
+
+			if (permissionResponse?.status !== 'granted') {
+				const response = await requestPermissions();
+				if (!response.granted) {
+					throw new Error(_('Missing permission to record audio.'));
+				}
+			}
+
+			await Audio.setAudioModeAsync({
+				allowsRecordingIOS: true,
+				playsInSilentModeIOS: true,
+				// Fixes an issue where opening a recording in the iOS audio player
+				// breaks creating new recordings.
+				// See https://github.com/expo/expo/issues/31152#issuecomment-2341811087
+				interruptionModeIOS: InterruptionModeIOS.DoNotMix,
+			});
+			setRecordingState(RecorderState.Recording);
+			const recording = new Audio.Recording();
+			await recording.prepareToRecordAsync(recordingOptions());
+			recording.setOnRecordingStatusUpdate(status => {
+				setDuration(status.durationMillis);
+			});
+			recordingRef.current = recording;
+			await recording.startAsync();
+		} catch (error) {
+			logger.error('Error starting recording:', error);
+			setError(`Recording error: ${error}`);
+			setRecordingState(RecorderState.Error);
+
+			void recordingRef.current?.stopAndUnloadAsync();
+			recordingRef.current = null;
+		}
+	}, [permissionResponse, requestPermissions]);
+
+	const onStopRecording = useCallback(async () => {
+		const recording = recordingRef.current;
+		recordingRef.current = null;
+
+		try {
+			setRecordingState(RecorderState.Processing);
+			await recording.stopAndUnloadAsync();
+			await resetAudioMode();
+
+			const saveEvent = await recordingToSaveData(recording);
+			onFileSaved(saveEvent);
+			onDismiss();
+		} catch (error) {
+			logger.error('Error saving recording:', error);
+			setError(`Save error: ${error}`);
+			setRecordingState(RecorderState.Error);
+		}
+	}, [onFileSaved, onDismiss]);
+
+	const onStartStopRecording = useCallback(async () => {
+		if (recordingState === RecorderState.Idle) {
+			await onStartRecording();
+		} else if (recordingState === RecorderState.Recording && recordingRef.current) {
+			await onStopRecording();
+		}
+	}, [recordingState, onStartRecording, onStopRecording]);
+
+	useEffect(() => () => {
+		if (recordingRef.current) {
+			void recordingRef.current?.stopAndUnloadAsync();
+			recordingRef.current = null;
+			void resetAudioMode();
+		}
+	}, []);
+
+	return { onStartStopRecording, error, duration, recordingState };
+};
+
+const AudioRecordingBanner: React.FC<Props> = props => {
+	const { recordingState, onStartStopRecording, duration, error } = useAudioRecorder(props.onFileSaved, props.onDismiss);
+
+	const onCancelPress = useCallback(async () => {
+		if (recordingState === RecorderState.Recording) {
+			const message = _('Cancelling will discard the recording. This cannot be undone. Are you sure you want to proceed?');
+			if (!await shim.showConfirmationDialog(message)) return;
+		}
+		props.onDismiss();
+	}, [recordingState, props.onDismiss]);
+
+	const startStopButtonLabel = recordingState === RecorderState.Idle ? _('Start recording') : _('Done');
+	const allowStartStop = recordingState === RecorderState.Idle || recordingState === RecorderState.Recording;
+	const actions = <>
+		<SecondaryButton onPress={onCancelPress}>{_('Cancel')}</SecondaryButton>
+		<PrimaryButton
+			disabled={!allowStartStop}
+			onPress={onStartStopRecording}
+			// Add additional accessibility information to make it clear that "Done" is
+			// associated with the voice recording banner:
+			accessibilityHint={recordingState === RecorderState.Recording ? _('Finishes recording') : undefined}
+		>{startStopButtonLabel}</PrimaryButton>
+	</>;
+
+	const renderDuration = () => {
+		if (recordingState !== RecorderState.Recording) return null;
+
+		const durationValue = formatMsToDurationLocal(duration);
+		return <Text
+			accessibilityLabel={_('Duration: %s', durationValue)}
+			accessibilityRole='timer'
+		>{durationValue}</Text>;
+	};
+
+	return <RecordingControls
+		recorderState={recordingState}
+		heading={recordingState === RecorderState.Recording ? _('Recording...') : _('Voice recorder')}
+		content={
+			recordingState === RecorderState.Idle
+				? _('Click "start" to attach a new voice memo to the note.')
+				: error
+		}
+		preview={renderDuration()}
+		actions={actions}
+	/>;
+};
+
+export default AudioRecordingBanner;

packages/app-mobile/components/voiceTyping/RecordingControls.tsx

@@ -0,0 +1,94 @@
+import * as React from 'react';
+import { View, StyleSheet } from 'react-native';
+import { ActivityIndicator, Icon, Surface, Text } from 'react-native-paper';
+import { IconSource } from 'react-native-paper/lib/typescript/components/Icon';
+import AccessibleView from '../accessibility/AccessibleView';
+import { RecorderState } from './types';
+
+interface Props {
+	recorderState: RecorderState;
+	heading: string;
+	content: React.ReactNode|string;
+	preview: React.ReactNode;
+	actions: React.ReactNode;
+}
+
+const styles = StyleSheet.create({
+	container: {
+		paddingHorizontal: 10,
+		width: 680,
+		flexShrink: 1,
+		maxWidth: '100%',
+		alignSelf: 'center',
+	},
+	contentWrapper: {
+		flexDirection: 'row',
+	},
+	iconWrapper: {
+		margin: 8,
+		marginTop: 16,
+	},
+	content: {
+		flexShrink: 1,
+		marginTop: 16,
+		marginHorizontal: 8,
+	},
+	actionContainer: {
+		flexDirection: 'row',
+		justifyContent: 'flex-end',
+		gap: 6,
+		marginBottom: 6,
+	},
+});
+
+const RecordingControls: React.FC<Props> = props => {
+	const renderIcon = () => {
+		const components: Record<RecorderState, IconSource> = {
+			[RecorderState.Loading]: ({ size }: { size: number }) => <ActivityIndicator animating={true} style={{ width: size, height: size }} />,
+			[RecorderState.Recording]: 'microphone',
+			[RecorderState.Idle]: 'microphone',
+			[RecorderState.Processing]: 'microphone',
+			[RecorderState.Downloading]: ({ size }: { size: number }) => <ActivityIndicator animating={true} style={{ width: size, height: size }} />,
+			[RecorderState.Error]: 'alert-circle-outline',
+		};
+
+		return components[props.recorderState];
+	};
+
+	return <Surface>
+		<View style={styles.container}>
+			<View style={styles.contentWrapper}>
+				<View style={styles.iconWrapper}>
+					<Icon source={renderIcon()} size={40}/>
+				</View>
+				<View style={styles.content}>
+					<AccessibleView
+						// Auto-focus
+						refocusCounter={1}
+						aria-live='polite'
+						role='heading'
+					>
+						<Text variant='bodyMedium'>
+							{props.heading}
+						</Text>
+					</AccessibleView>
+					<Text
+						variant='bodyMedium'
+						// role="status" might fit better here. However, react-native
+						// doesn't seem to support it.
+						role='alert'
+						// Although on web, role=alert should imply aria-live=polite,
+						// this does not seem to be the case for React Native:
+						accessibilityLiveRegion='polite'
+					>{props.content}</Text>
+					{props.preview}
+				</View>
+			</View>
+			<View style={styles.actionContainer}>
+				{props.actions}
+			</View>
+		</View>
+	</Surface>;
+};
+
+export default RecordingControls;

packages/app-mobile/components/voiceTyping/SpeechToTextBanner.tsx

@@ -1,17 +1,17 @@
 import * as React from 'react';
 import { useState, useEffect, useCallback, useRef, useMemo } from 'react';
-import { Icon, ActivityIndicator, Text, Surface, Button } from 'react-native-paper';
+import { Text, Button } from 'react-native-paper';
 import { _, languageName } from '@joplin/lib/locale';
 import useAsyncEffect, { AsyncEffectEvent } from '@joplin/lib/hooks/useAsyncEffect';
-import { IconSource } from 'react-native-paper/lib/typescript/components/Icon';
 import VoiceTyping, { OnTextCallback, VoiceTypingSession } from '../../services/voiceTyping/VoiceTyping';
 import whisper from '../../services/voiceTyping/whisper';
 import vosk from '../../services/voiceTyping/vosk';
 import { AppState } from '../../utils/types';
 import { connect } from 'react-redux';
-import { View, StyleSheet } from 'react-native';
-import AccessibleView from '../accessibility/AccessibleView';
 import Logger from '@joplin/utils/Logger';
+import { RecorderState } from './types';
+import RecordingControls from './RecordingControls';
+import { PrimaryButton } from '../buttons';
 
 const logger = Logger.create('VoiceTypingDialog');
 
@@ -22,14 +22,6 @@ interface Props {
 	onText: (text: string)=> void;
 }
 
-enum RecorderState {
-	Loading = 1,
-	Recording = 2,
-	Processing = 3,
-	Error = 4,
-	Downloading = 5,
-}
-
 interface UseVoiceTypingProps {
 	locale: string;
 	provider: string;
@@ -109,31 +101,7 @@ const useVoiceTyping = ({ locale, provider, onSetPreview, onText }: UseVoiceTypi
 	};
 };
 
-const styles = StyleSheet.create({
-	container: {
-		marginHorizontal: 1,
-		width: '100%',
-		maxWidth: 680,
-		alignSelf: 'center',
-	},
-	contentWrapper: {
-		flexDirection: 'row',
-	},
-	iconWrapper: {
-		margin: 8,
-		marginTop: 16,
-	},
-	content: {
-		marginTop: 16,
-		marginHorizontal: 8,
-	},
-	actionContainer: {
-		flexDirection: 'row',
-		justifyContent: 'flex-end',
-	},
-});
-
-const VoiceTypingDialog: React.FC<Props> = props => {
+const SpeechToTextComponent: React.FC<Props> = props => {
 	const [recorderState, setRecorderState] = useState<RecorderState>(RecorderState.Loading);
 	const [preview, setPreview] = useState<string>('');
 	const {
@@ -177,6 +145,7 @@ const VoiceTypingDialog: React.FC<Props> = props => {
 	const renderContent = () => {
 		const components: Record<RecorderState, ()=> string> = {
 			[RecorderState.Loading]: () => _('Loading...'),
+			[RecorderState.Idle]: () => 'Waiting...', // Not used for now
 			[RecorderState.Recording]: () => _('Please record your voice...'),
 			[RecorderState.Processing]: () => _('Converting speech to text...'),
 			[RecorderState.Downloading]: () => _('Downloading %s language files...', languageName(props.locale)),
@@ -186,18 +155,6 @@ const VoiceTypingDialog: React.FC<Props> = props => {
 		return components[recorderState]();
 	};
 
-	const renderIcon = () => {
-		const components: Record<RecorderState, IconSource> = {
-			[RecorderState.Loading]: ({ size }: { size: number }) => <ActivityIndicator animating={true} style={{ width: size, height: size }} />,
-			[RecorderState.Recording]: 'microphone',
-			[RecorderState.Processing]: 'microphone',
-			[RecorderState.Downloading]: ({ size }: { size: number }) => <ActivityIndicator animating={true} style={{ width: size, height: size }} />,
-			[RecorderState.Error]: 'alert-circle-outline',
-		};
-
-		return components[recorderState];
-	};
-
 	const renderPreview = () => {
 		return <Text variant='labelSmall'>{preview}</Text>;
 	};
@@ -207,48 +164,23 @@ const VoiceTypingDialog: React.FC<Props> = props => {
 	</Button>;
 	const allowReDownload = recorderState === RecorderState.Error || modelIsOutdated;
 
-	return (
-		<Surface>
-			<View style={styles.container}>
-				<View style={styles.contentWrapper}>
-					<View style={styles.iconWrapper}>
-						<Icon source={renderIcon()} size={40}/>
-					</View>
-					<View style={styles.content}>
-						<AccessibleView
-							// Auto-focus
-							refocusCounter={1}
-							aria-live='polite'
-							role='heading'
-						>
-							<Text variant='bodyMedium'>
-								{_('Voice typing...')}
-							</Text>
-						</AccessibleView>
-						<Text
-							variant='bodyMedium'
-							// role="status" might fit better here. However, react-native
-							// doesn't seem to support it.
-							role='alert'
-							// Although on web, role=alert should imply aria-live=polite,
-							// this does not seem to be the case for React Native:
-							accessibilityLiveRegion='polite'
-						>{renderContent()}</Text>
-						{renderPreview()}
-					</View>
-				</View>
-				<View style={styles.actionContainer}>
-					{allowReDownload ? reDownloadButton : null}
-					<Button
-						onPress={onDismiss}
-						accessibilityHint={_('Ends voice typing')}
-					>{_('Done')}</Button>
-				</View>
-			</View>
-		</Surface>
-	);
+	const actions = <>
+		{allowReDownload ? reDownloadButton : null}
+		<PrimaryButton
+			onPress={onDismiss}
+			accessibilityHint={_('Ends voice typing')}
+		>{_('Done')}</PrimaryButton>
+	</>;
+
+	return <RecordingControls
+		recorderState={recorderState}
+		heading={_('Voice typing...')}
+		content={renderContent()}
+		preview={renderPreview()}
+		actions={actions}
+	/>;
 };
 
 export default connect((state: AppState) => ({
 	provider: state.settings['voiceTyping.preferredProvider'],
-}))(VoiceTypingDialog);
+}))(SpeechToTextComponent);

packages/app-mobile/components/voiceTyping/types.ts

@@ -0,0 +1,16 @@
+export interface OnFileEvent {
+	uri: string;
+	fileName: string;
+	type: string|undefined;
+}
+
+export type OnFileSavedCallback = (event: OnFileEvent)=> void;
+
+export enum RecorderState {
+	Loading = 1,
+	Recording = 2,
+	Processing = 3,
+	Error = 4,
+	Downloading = 5,
+	Idle = 6,
+}

packages/app-mobile/ios/Joplin/Info.plist

@@ -72,6 +72,8 @@
 	<string>The images will be displayed on your notes.</string>
 	<key>NSPhotoLibraryUsageDescription</key>
 	<string>To allow attaching images to a note</string>
+	<key>NSMicrophoneUsageDescription</key>
+	<string>To allow attaching voice recordings to a note</string>
 	<key>UIAppFonts</key>
 	<array>
 		<string>AntDesign.ttf</string>

packages/app-mobile/jest.setup.js

@@ -69,6 +69,8 @@ const emptyMockPackages = [
 	'react-native-image-picker',
 	'react-native-document-picker',
 	'@joplin/react-native-saf-x',
+	'expo-av',
+	'expo-av/build/Audio',
 ];
 for (const packageName of emptyMockPackages) {
 	jest.doMock(packageName, () => {

packages/app-mobile/package.json

@@ -42,6 +42,7 @@
     "deprecated-react-native-prop-types": "5.0.0",
     "events": "3.3.0",
     "expo": "51.0.26",
+    "expo-av": "14.0.7",
     "expo-camera": "15.0.16",
     "lodash": "4.17.21",
     "md5": "2.3.0",

packages/app-mobile/web/public/index.html

@@ -10,7 +10,7 @@
 			http-equiv="Content-Security-Policy"
 			content="
 				default-src 'self' ;
-				connect-src 'self' * http://* https://* ;
+				connect-src 'self' * http://* https://* blob: ;
 				style-src 'unsafe-inline' 'self' blob: ;
 				child-src 'self' ;
 				script-src 'self' 'unsafe-eval' 'unsafe-inline' ;

packages/renderer/MdToHtml/renderMedia.ts

@@ -18,7 +18,9 @@ export interface Options {
 function resourceUrl(resourceFullPath: string): string {
 	if (
 		resourceFullPath.indexOf('http://') === 0 || resourceFullPath.indexOf('https://') === 0 || resourceFullPath.indexOf('joplin-content://') === 0 ||
-		resourceFullPath.indexOf('file://') === 0
+		resourceFullPath.indexOf('file://') === 0 ||
+		// On web, resources are loaded as blob URLs.
+		resourceFullPath.startsWith('blob:null/')
 	) {
 		return resourceFullPath;
 	}

packages/tools/cspell/dictionary4.txt

@@ -166,4 +166,6 @@ HMAC
 Siri
 cipherdecipher
 pmmmwh
+webm
+millis
 sideloading

packages/utils/time.test.ts

@@ -0,0 +1,12 @@
+import { formatMsToDurationLocal, Hour, Minute } from './time';
+
+describe('time', () => {
+	test.each([
+		[0, '0:00'],
+		[Minute * 3, '3:00'],
+		[Hour * 4 + Minute * 3, '4:03:00'],
+		[Hour * 25, '0000-00-01T01:00'],
+	])('should support formatting durations', (input, expected) => {
+		expect(formatMsToDurationLocal(input)).toBe(expected);
+	});
+});

packages/utils/time.ts

@@ -10,6 +10,8 @@ import * as dayjs from 'dayjs';
 // - import * as dayJsRelativeTimeType causes a runtime error.
 import type * as dayJsRelativeTimeType from 'dayjs/plugin/relativeTime';
 const dayJsRelativeTime: typeof dayJsRelativeTimeType = require('dayjs/plugin/relativeTime');
+import type * as dayJsDurationType from 'dayjs/plugin/duration';
+const dayJsDuration: typeof dayJsDurationType = require('dayjs/plugin/duration');
 
 const supportedLocales: Record<string, unknown> = {
 	'ar': require('dayjs/locale/ar'),
@@ -63,6 +65,7 @@ export const Week = 7 * Day;
 export const Month = 30 * Day;
 
 function initDayJs() {
+	dayjs.extend(dayJsDuration);
 	dayjs.extend(dayJsRelativeTime);
 }
 
@@ -157,3 +160,15 @@ export const isValidDate = (anything: string) => {
 export const formatDateTimeLocalToMs = (anything: string) => {
 	return dayjs(anything).unix() * 1000;
 };
+
+export const formatMsToDurationLocal = (ms: number) => {
+	let format;
+	if (ms < Hour) {
+		format = 'm:ss';
+	} else if (ms < Day) {
+		format = 'H:mm:ss';
+	} else {
+		format = 'YYYY-MM-DDTHH:mm';
+	}
+	return dayjs.duration(ms).format(format);
+};

renovate.json5

@@ -76,6 +76,7 @@
 		"browserify",
 		"codemirror",
 		"cspell",
+		"expo-av", // Must be updated with expo
 		"file-loader",
 		"gradle",
 		"html-webpack-plugin",

yarn.lock

@@ -8396,6 +8396,7 @@ __metadata:
     deprecated-react-native-prop-types: 5.0.0
     events: 3.3.0
     expo: 51.0.26
+    expo-av: 14.0.7
     expo-camera: 15.0.16
     fast-deep-equal: 3.1.3
     fs-extra: 11.2.0
@@ -24701,6 +24702,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"expo-av@npm:14.0.7":
+  version: 14.0.7
+  resolution: "expo-av@npm:14.0.7"
+  peerDependencies:
+    expo: "*"
+  checksum: 7518a8972b0d1b3b362d4dc9c4d21a39c7cf98b46abbdbb2349451115f4edf6e97fb6c723674fd3c62fa037fc77e00172955082d21b1c3fdf5273f33781ce78d
+  languageName: node
+  linkType: hard
+
 "expo-camera@npm:15.0.16":
   version: 15.0.16
   resolution: "expo-camera@npm:15.0.16"