diff --git a/app/android/app/src/main/AndroidManifest.xml b/app/android/app/src/main/AndroidManifest.xml index 201bde8d1..dee9d4875 100644 --- a/app/android/app/src/main/AndroidManifest.xml +++ b/app/android/app/src/main/AndroidManifest.xml @@ -2,6 +2,7 @@ + diff --git a/app/ios/Runner/Info.plist b/app/ios/Runner/Info.plist index 33e4ee169..c9102a703 100644 --- a/app/ios/Runner/Info.plist +++ b/app/ios/Runner/Info.plist @@ -43,6 +43,8 @@ NSAllowsArbitraryLoads + NSMicrophoneUsageDescription + Need microphone access for speech input and study recordings. NSCameraUsageDescription Need to access your camera to capture a photo for trial observations. UIApplicationSupportsIndirectInputEvents diff --git a/app/lib/l10n/app_de.arb b/app/lib/l10n/app_de.arb index b20f5c171..5c8413d2a 100644 --- a/app/lib/l10n/app_de.arb +++ b/app/lib/l10n/app_de.arb @@ -242,6 +242,7 @@ } } }, + "free_text_answer_hint": "Geben Sie Ihre Antwort hier ein...", "app_outdated_message": "Eine neue Version der StudyU App ist verfügbar. Bitte führen Sie eine Aktualisierung durch, um die neuesten Funktionen und Verbesserungen zu erhalten. Vielen Dank für Ihre Unterstützung!", "update_now": "Jetzt aktualisieren", @@ -455,5 +456,33 @@ "preview_mode_description": "Sie befinden sich derzeit im Vorschau-Modus. Dies ermöglicht Ihnen:\n\n• Schneller Vorlauf durch Studientage mit der \"Nächster Tag\" Schaltfläche\n• Mehrfaches Abschließen von Aufgaben ohne Einschränkungen\n• Erleben des vollständigen Studienablaufs ohne Beeinflussung echter Daten\n\nWichtig: Ergebnisse und Daten aus dem Vorschau-Modus werden nicht gespeichert oder mit tatsächlichen Teilnehmerergebnissen aus laufenden Studien vermischt.", "preview_mode_results_not_saved": "Ergebnisse werden zum Schutz der Studienintegrität nicht gespeichert.", "ok": "OK", - "submit" : "Absenden" + "submit" : "Absenden", + "speech_to_text_title": "Sprach-zu-Text", + "speech_to_text_description": "Diktiere Antworten mit der integrierten Spracherkennung deines Geräts. Die Verarbeitung übernimmt der Plattformdienst.", + "speech_to_text_enable_label": "Sprach­eingabe aktivieren", + "speech_to_text_enable_hint": "Tippen funktioniert weiterhin, falls das Mikrofon blockiert ist.", + "speech_to_text_language_label": "Bevorzugte Transkriptionssprache", + "speech_to_text_language_english": "Englisch", + "speech_to_text_language_german": "Deutsch", + "speech_to_text_not_enabled": "Die Sprach­eingabe bleibt deaktiviert, bis du sie einschaltest.", + "speech_to_text_not_enabled_brief": "Sprach­eingabe ist für dein Konto ausgeschaltet.", + "speech_to_text_go_to_settings": "In den Einstellungen verwalten", + "speech_to_text_unsupported": "Sprach­eingabe ist auf diesem Gerät nicht verfügbar.", + "speech_to_text_error_permission": "Bitte erlaube den Mikrofonzugriff, um diktieren zu können.", + "speech_to_text_error_general": "Sprach­eingabe fehlgeschlagen. Bitte erneut versuchen.", + "speech_to_text_error_connection": "Verbindung verloren. Bitte erneut versuchen.", + "speech_to_text_preparing": "Spracherkennung wird vorbereitet…", + "speech_to_text_live_caption_hint": "Live-Transkription", + "speech_to_text_listening": "Es wird zugehört…", + "speech_to_text_press_to_speak": "Zum Sprechen drücken", + "speech_to_text_idle": "Sprach­eingabe starten", + "speech_to_text_stop_listening": "Zuhören stoppen", + "speech_to_text_opt_out_action": "Sprach­eingabe deaktivieren", + "speech_to_text_enable_short_label": "Wieder aktivieren", + "speech_to_text_language_badge": "{language}", + "@speech_to_text_language_badge": { + "placeholders": { + "language": {} + } + } } diff --git a/app/lib/l10n/app_en.arb b/app/lib/l10n/app_en.arb index cb2e186a3..d161c5653 100644 --- a/app/lib/l10n/app_en.arb +++ b/app/lib/l10n/app_en.arb @@ -238,6 +238,7 @@ } } }, + "free_text_answer_hint": "Type your answer here...", "app_outdated_message": "A new version of the StudyU App is available. Please update to get the latest features and improvements. Thank you for your support!", "update_now": "Update now", @@ -451,5 +452,33 @@ "preview_mode_description": "You are currently in preview mode. This allows you to:\n\n• Fast-forward through study days using the \"Next Day\" button\n• Complete tasks multiple times without restrictions\n• Experience the full study flow without affecting real data\n\nImportant: Results and data from preview mode are not stored or mixed with actual participant results from running studies.", "preview_mode_results_not_saved": "Task completed in preview mode - results are not saved to protect study data integrity.", "ok": "OK", - "submit" : "Submit" + "submit" : "Submit", + "speech_to_text_title": "Speech to text", + "speech_to_text_description": "Dictate answers with your device's built-in speech recognition. Audio always stays with the platform service.", + "speech_to_text_enable_label": "Enable speech input", + "speech_to_text_enable_hint": "Typing continues to work even if the microphone is unavailable.", + "speech_to_text_language_label": "Preferred transcription language", + "speech_to_text_language_english": "English", + "speech_to_text_language_german": "German", + "speech_to_text_not_enabled": "Speech input stays disabled until you turn it on.", + "speech_to_text_not_enabled_brief": "Speech input is currently turned off for your account.", + "speech_to_text_go_to_settings": "Manage in Settings", + "speech_to_text_unsupported": "Speech input is unavailable on this device.", + "speech_to_text_error_permission": "Microphone permission is required for speech input.", + "speech_to_text_error_general": "Speech input failed. Please try again.", + "speech_to_text_error_connection": "Connection lost. Please try again.", + "speech_to_text_preparing": "Preparing speech recognizer…", + "speech_to_text_live_caption_hint": "Live transcription", + "speech_to_text_listening": "Listening…", + "speech_to_text_press_to_speak": "Press to speak", + "speech_to_text_idle": "Start speech input", + "speech_to_text_stop_listening": "Stop listening", + "speech_to_text_opt_out_action": "Disable speech input for now", + "speech_to_text_enable_short_label": "Enable again", + "speech_to_text_language_badge": "{language}", + "@speech_to_text_language_badge": { + "placeholders": { + "language": {} + } + } } diff --git a/app/lib/l10n/app_localizations.dart b/app/lib/l10n/app_localizations.dart index 2d8505c23..9f3aadeb4 100644 --- a/app/lib/l10n/app_localizations.dart +++ b/app/lib/l10n/app_localizations.dart @@ -1178,6 +1178,12 @@ abstract class AppLocalizations { /// **'Please enter only characters matching the pattern {pattern}'** String free_text_custom_error(String pattern); + /// No description provided for @free_text_answer_hint. + /// + /// In en, this message translates to: + /// **'Type your answer here...'** + String get free_text_answer_hint; + /// No description provided for @app_outdated_message. /// /// In en, this message translates to: @@ -2251,6 +2257,144 @@ abstract class AppLocalizations { /// In en, this message translates to: /// **'Submit'** String get submit; + + /// No description provided for @speech_to_text_title. + /// + /// In en, this message translates to: + /// **'Speech to text'** + String get speech_to_text_title; + + /// No description provided for @speech_to_text_description. + /// + /// In en, this message translates to: + /// **'Dictate answers with your device\'s built-in speech recognition. Audio always stays with the platform service.'** + String get speech_to_text_description; + + /// No description provided for @speech_to_text_enable_label. + /// + /// In en, this message translates to: + /// **'Enable speech input'** + String get speech_to_text_enable_label; + + /// No description provided for @speech_to_text_enable_hint. + /// + /// In en, this message translates to: + /// **'Typing continues to work even if the microphone is unavailable.'** + String get speech_to_text_enable_hint; + + /// No description provided for @speech_to_text_language_label. + /// + /// In en, this message translates to: + /// **'Preferred transcription language'** + String get speech_to_text_language_label; + + /// No description provided for @speech_to_text_language_english. + /// + /// In en, this message translates to: + /// **'English'** + String get speech_to_text_language_english; + + /// No description provided for @speech_to_text_language_german. + /// + /// In en, this message translates to: + /// **'German'** + String get speech_to_text_language_german; + + /// No description provided for @speech_to_text_not_enabled. + /// + /// In en, this message translates to: + /// **'Speech input stays disabled until you turn it on.'** + String get speech_to_text_not_enabled; + + /// No description provided for @speech_to_text_not_enabled_brief. + /// + /// In en, this message translates to: + /// **'Speech input is currently turned off for your account.'** + String get speech_to_text_not_enabled_brief; + + /// No description provided for @speech_to_text_go_to_settings. + /// + /// In en, this message translates to: + /// **'Manage in Settings'** + String get speech_to_text_go_to_settings; + + /// No description provided for @speech_to_text_unsupported. + /// + /// In en, this message translates to: + /// **'Speech input is unavailable on this device.'** + String get speech_to_text_unsupported; + + /// No description provided for @speech_to_text_error_permission. + /// + /// In en, this message translates to: + /// **'Microphone permission is required for speech input.'** + String get speech_to_text_error_permission; + + /// No description provided for @speech_to_text_error_general. + /// + /// In en, this message translates to: + /// **'Speech input failed. Please try again.'** + String get speech_to_text_error_general; + + /// No description provided for @speech_to_text_error_connection. + /// + /// In en, this message translates to: + /// **'Connection lost. Please try again.'** + String get speech_to_text_error_connection; + + /// No description provided for @speech_to_text_preparing. + /// + /// In en, this message translates to: + /// **'Preparing speech recognizer…'** + String get speech_to_text_preparing; + + /// No description provided for @speech_to_text_live_caption_hint. + /// + /// In en, this message translates to: + /// **'Live transcription'** + String get speech_to_text_live_caption_hint; + + /// No description provided for @speech_to_text_listening. + /// + /// In en, this message translates to: + /// **'Listening…'** + String get speech_to_text_listening; + + /// No description provided for @speech_to_text_press_to_speak. + /// + /// In en, this message translates to: + /// **'Press to speak'** + String get speech_to_text_press_to_speak; + + /// No description provided for @speech_to_text_idle. + /// + /// In en, this message translates to: + /// **'Start speech input'** + String get speech_to_text_idle; + + /// No description provided for @speech_to_text_stop_listening. + /// + /// In en, this message translates to: + /// **'Stop listening'** + String get speech_to_text_stop_listening; + + /// No description provided for @speech_to_text_opt_out_action. + /// + /// In en, this message translates to: + /// **'Disable speech input for now'** + String get speech_to_text_opt_out_action; + + /// No description provided for @speech_to_text_enable_short_label. + /// + /// In en, this message translates to: + /// **'Enable again'** + String get speech_to_text_enable_short_label; + + /// No description provided for @speech_to_text_language_badge. + /// + /// In en, this message translates to: + /// **'{language}'** + String speech_to_text_language_badge(Object language); } class _AppLocalizationsDelegate diff --git a/app/lib/l10n/app_localizations_de.dart b/app/lib/l10n/app_localizations_de.dart index 224338f8d..7a1f6376a 100644 --- a/app/lib/l10n/app_localizations_de.dart +++ b/app/lib/l10n/app_localizations_de.dart @@ -607,6 +607,9 @@ class AppLocalizationsDe extends AppLocalizations { return 'Bitte geben Sie nur Zeichen ein, die dem Muster $pattern entsprechen'; } + @override + String get free_text_answer_hint => 'Geben Sie Ihre Antwort hier ein...'; + @override String get app_outdated_message => 'Eine neue Version der StudyU App ist verfügbar. Bitte führen Sie eine Aktualisierung durch, um die neuesten Funktionen und Verbesserungen zu erhalten. Vielen Dank für Ihre Unterstützung!'; @@ -1166,4 +1169,84 @@ class AppLocalizationsDe extends AppLocalizations { @override String get submit => 'Absenden'; + + @override + String get speech_to_text_title => 'Sprach-zu-Text'; + + @override + String get speech_to_text_description => + 'Diktiere Antworten mit der integrierten Spracherkennung deines Geräts. Die Verarbeitung übernimmt der Plattformdienst.'; + + @override + String get speech_to_text_enable_label => 'Sprach­eingabe aktivieren'; + + @override + String get speech_to_text_enable_hint => + 'Tippen funktioniert weiterhin, falls das Mikrofon blockiert ist.'; + + @override + String get speech_to_text_language_label => + 'Bevorzugte Transkriptionssprache'; + + @override + String get speech_to_text_language_english => 'Englisch'; + + @override + String get speech_to_text_language_german => 'Deutsch'; + + @override + String get speech_to_text_not_enabled => + 'Die Sprach­eingabe bleibt deaktiviert, bis du sie einschaltest.'; + + @override + String get speech_to_text_not_enabled_brief => + 'Sprach­eingabe ist für dein Konto ausgeschaltet.'; + + @override + String get speech_to_text_go_to_settings => 'In den Einstellungen verwalten'; + + @override + String get speech_to_text_unsupported => + 'Sprach­eingabe ist auf diesem Gerät nicht verfügbar.'; + + @override + String get speech_to_text_error_permission => + 'Bitte erlaube den Mikrofonzugriff, um diktieren zu können.'; + + @override + String get speech_to_text_error_general => + 'Sprach­eingabe fehlgeschlagen. Bitte erneut versuchen.'; + + @override + String get speech_to_text_error_connection => + 'Verbindung verloren. Bitte erneut versuchen.'; + + @override + String get speech_to_text_preparing => 'Spracherkennung wird vorbereitet…'; + + @override + String get speech_to_text_live_caption_hint => 'Live-Transkription'; + + @override + String get speech_to_text_listening => 'Es wird zugehört…'; + + @override + String get speech_to_text_press_to_speak => 'Zum Sprechen drücken'; + + @override + String get speech_to_text_idle => 'Sprach­eingabe starten'; + + @override + String get speech_to_text_stop_listening => 'Zuhören stoppen'; + + @override + String get speech_to_text_opt_out_action => 'Sprach­eingabe deaktivieren'; + + @override + String get speech_to_text_enable_short_label => 'Wieder aktivieren'; + + @override + String speech_to_text_language_badge(Object language) { + return '$language'; + } } diff --git a/app/lib/l10n/app_localizations_en.dart b/app/lib/l10n/app_localizations_en.dart index 194435c68..b26635a09 100644 --- a/app/lib/l10n/app_localizations_en.dart +++ b/app/lib/l10n/app_localizations_en.dart @@ -600,6 +600,9 @@ class AppLocalizationsEn extends AppLocalizations { return 'Please enter only characters matching the pattern $pattern'; } + @override + String get free_text_answer_hint => 'Type your answer here...'; + @override String get app_outdated_message => 'A new version of the StudyU App is available. Please update to get the latest features and improvements. Thank you for your support!'; @@ -1156,4 +1159,84 @@ class AppLocalizationsEn extends AppLocalizations { @override String get submit => 'Submit'; + + @override + String get speech_to_text_title => 'Speech to text'; + + @override + String get speech_to_text_description => + 'Dictate answers with your device\'s built-in speech recognition. Audio always stays with the platform service.'; + + @override + String get speech_to_text_enable_label => 'Enable speech input'; + + @override + String get speech_to_text_enable_hint => + 'Typing continues to work even if the microphone is unavailable.'; + + @override + String get speech_to_text_language_label => + 'Preferred transcription language'; + + @override + String get speech_to_text_language_english => 'English'; + + @override + String get speech_to_text_language_german => 'German'; + + @override + String get speech_to_text_not_enabled => + 'Speech input stays disabled until you turn it on.'; + + @override + String get speech_to_text_not_enabled_brief => + 'Speech input is currently turned off for your account.'; + + @override + String get speech_to_text_go_to_settings => 'Manage in Settings'; + + @override + String get speech_to_text_unsupported => + 'Speech input is unavailable on this device.'; + + @override + String get speech_to_text_error_permission => + 'Microphone permission is required for speech input.'; + + @override + String get speech_to_text_error_general => + 'Speech input failed. Please try again.'; + + @override + String get speech_to_text_error_connection => + 'Connection lost. Please try again.'; + + @override + String get speech_to_text_preparing => 'Preparing speech recognizer…'; + + @override + String get speech_to_text_live_caption_hint => 'Live transcription'; + + @override + String get speech_to_text_listening => 'Listening…'; + + @override + String get speech_to_text_press_to_speak => 'Press to speak'; + + @override + String get speech_to_text_idle => 'Start speech input'; + + @override + String get speech_to_text_stop_listening => 'Stop listening'; + + @override + String get speech_to_text_opt_out_action => 'Disable speech input for now'; + + @override + String get speech_to_text_enable_short_label => 'Enable again'; + + @override + String speech_to_text_language_badge(Object language) { + return '$language'; + } } diff --git a/app/lib/screens/study/dashboard/settings.dart b/app/lib/screens/study/dashboard/settings.dart index 262552054..cc2a7b29c 100644 --- a/app/lib/screens/study/dashboard/settings.dart +++ b/app/lib/screens/study/dashboard/settings.dart @@ -96,50 +96,58 @@ class _SettingsState extends State { @override Widget build(BuildContext context) { final theme = Theme.of(context); + final loc = AppLocalizations.of(context)!; return Scaffold( - appBar: AppBar(title: Text(AppLocalizations.of(context)!.settings)), - body: Center( + appBar: AppBar(title: Text(loc.settings)), + body: SingleChildScrollView( + padding: const EdgeInsets.all(24), child: Column( - mainAxisSize: MainAxisSize.min, - mainAxisAlignment: MainAxisAlignment.spaceAround, - children: [ + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ getDropdownRow(context), - const SizedBox(height: 24), - Text( - '${AppLocalizations.of(context)!.study_current} ${subject!.study.title}', - style: theme.textTheme.titleLarge, - ), - const SizedBox(height: 8), - ElevatedButton.icon( - icon: Icon(MdiIcons.exitToApp), - label: Text(AppLocalizations.of(context)!.opt_out), - style: ElevatedButton.styleFrom( - backgroundColor: Colors.orange[800], - ), - onPressed: () { - showDialog( - context: context, - builder: (_) => OptOutAlertDialog(subject: subject), - ); - }, - ), - const SizedBox(height: 24), - ElevatedButton.icon( - icon: const Icon(Icons.delete), - label: Text(AppLocalizations.of(context)!.delete_data), - style: ElevatedButton.styleFrom(backgroundColor: Colors.red), - onPressed: () { - showDialog( - context: context, - builder: (_) => DeleteAlertDialog(subject: subject), - ); - }, - ), + const SizedBox(height: 32), + _buildStudySection(theme, loc), ], ), ), ); } + + Widget _buildStudySection(ThemeData theme, AppLocalizations loc) { + return Column( + children: [ + Text( + '${loc.study_current} ${subject!.study.title}', + style: theme.textTheme.titleLarge, + textAlign: TextAlign.center, + ), + const SizedBox(height: 8), + ElevatedButton.icon( + icon: Icon(MdiIcons.exitToApp), + label: Text(loc.opt_out), + style: ElevatedButton.styleFrom(backgroundColor: Colors.orange[800]), + onPressed: () { + showDialog( + context: context, + builder: (_) => OptOutAlertDialog(subject: subject), + ); + }, + ), + const SizedBox(height: 24), + ElevatedButton.icon( + icon: const Icon(Icons.delete), + label: Text(loc.delete_data), + style: ElevatedButton.styleFrom(backgroundColor: Colors.red), + onPressed: () { + showDialog( + context: context, + builder: (_) => DeleteAlertDialog(subject: subject), + ); + }, + ), + ], + ); + } } class OptOutAlertDialog extends StatelessWidget { diff --git a/app/lib/services/speech/speech_controller_models.dart b/app/lib/services/speech/speech_controller_models.dart new file mode 100644 index 000000000..3774aed39 --- /dev/null +++ b/app/lib/services/speech/speech_controller_models.dart @@ -0,0 +1,59 @@ +import 'package:flutter/foundation.dart'; + +typedef SpeechTranscriptCommit = void Function(String textSegment); + +enum SpeechLifecycleStatus { + unavailable, + idle, + preparing, + ready, + listening, + error, +} + +enum SpeechErrorType { microphonePermission, general } + +@immutable +class SpeechError { + const SpeechError(this.type, {this.details}); + + final SpeechErrorType type; + final String? details; +} + +@immutable +class SpeechControllerState { + const SpeechControllerState({ + required this.status, + this.error, + this.partialTranscript, + }); + + factory SpeechControllerState.initial({required bool supported}) { + return SpeechControllerState( + status: supported + ? SpeechLifecycleStatus.idle + : SpeechLifecycleStatus.unavailable, + ); + } + + final SpeechLifecycleStatus status; + final SpeechError? error; + final String? partialTranscript; + + SpeechControllerState copyWith({ + SpeechLifecycleStatus? status, + SpeechError? error, + bool clearError = false, + String? partialTranscript, + bool clearTranscript = false, + }) { + return SpeechControllerState( + status: status ?? this.status, + error: clearError ? null : error ?? this.error, + partialTranscript: clearTranscript + ? null + : partialTranscript ?? this.partialTranscript, + ); + } +} diff --git a/app/lib/services/speech/speech_to_text_controller.dart b/app/lib/services/speech/speech_to_text_controller.dart new file mode 100644 index 000000000..832dcdfe8 --- /dev/null +++ b/app/lib/services/speech/speech_to_text_controller.dart @@ -0,0 +1,3 @@ +export 'speech_controller_models.dart'; + +export 'websocket_speech_controller_io.dart'; diff --git a/app/lib/services/speech/websocket_speech_controller_io.dart b/app/lib/services/speech/websocket_speech_controller_io.dart new file mode 100644 index 000000000..3bf37c4a8 --- /dev/null +++ b/app/lib/services/speech/websocket_speech_controller_io.dart @@ -0,0 +1,383 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:record/record.dart'; +import 'package:studyu_app/services/speech/speech_controller_models.dart'; +import 'package:studyu_core/env.dart' as env; +import 'package:supabase_flutter/supabase_flutter.dart'; +import 'package:web_socket_channel/io.dart'; +import 'package:web_socket_channel/web_socket_channel.dart'; + +class SpeechToTextController extends ValueNotifier { + SpeechToTextController({ + required this.onFinalTranscription, + String? serverUrl, + }) : _serverUrl = serverUrl ?? env.sttWebSocketUrl ?? '', + _audioRecorder = AudioRecorder(), + super( + SpeechControllerState.initial( + supported: + _platformSupported && + (serverUrl ?? env.sttWebSocketUrl ?? '').isNotEmpty, + ), + ) { + _ensureInitialized(); + } + + final SpeechTranscriptCommit onFinalTranscription; + final AudioRecorder _audioRecorder; + final String _serverUrl; + + String? _latestTranscript; + bool _isDisposed = false; + Future? _initialization; + WebSocketChannel? _wsChannel; + StreamSubscription? _audioStreamSubscription; + bool _isListening = false; + bool _isReconnecting = false; + + static const int _sampleRate = 16000; // Vosk typically uses 16kHz + + /// Delay before attempting to reconnect after connection loss. + /// This prevents rapid reconnection attempts and gives the server/network + /// time to stabilize. 500ms provides a good balance between responsiveness + /// and avoiding connection spam. + static const int _reconnectionDelayMs = 500; + + /// Delay after sending EOF to allow server to process and send final results. + /// Vosk server needs time to finalize transcription after receiving EOF. + /// 300ms is typically sufficient for the server to send remaining data. + static const int _eofWaitDelayMs = 300; + + static bool get _platformSupported { + if (kIsWeb) return false; + return Platform.isAndroid || + Platform.isIOS || + Platform.isMacOS || + Platform.isWindows; + } + + static bool get isSupportedPlatform => _platformSupported; + + bool get isListening => value.status == SpeechLifecycleStatus.listening; + + Future _ensureInitialized() { + _initialization ??= _initializeRecorder(); + return _initialization!; + } + + Future _initializeRecorder() async { + if (!_platformSupported) return false; + try { + final hasPermission = await _audioRecorder.hasPermission(); + return hasPermission; + } catch (e) { + return false; + } + } + + Future startListening() async { + if (!_platformSupported) { + _emitError(SpeechErrorType.general); + return false; + } + + if (_serverUrl.isEmpty) { + return false; + } + if (value.status == SpeechLifecycleStatus.listening || + value.status == SpeechLifecycleStatus.preparing) { + return false; + } + + value = value.copyWith( + status: SpeechLifecycleStatus.preparing, + clearError: true, + clearTranscript: true, + ); + + try { + final available = await _ensureInitialized(); + if (!available) { + _emitError(SpeechErrorType.microphonePermission); + return false; + } + + _isListening = true; + await _startWebSocketConnection(); + await _startAudioRecording(); + + value = value.copyWith( + status: SpeechLifecycleStatus.listening, + clearError: true, + ); + + return true; + } catch (e, stack) { + _isListening = false; + _emitError( + SpeechErrorType.general, + details: 'Failed to start speech input: $e', + stackTrace: stack, + ); + return false; + } + } + + Future _startWebSocketConnection() async { + try { + final token = Supabase.instance.client.auth.currentSession?.accessToken; + + final ws = await WebSocket.connect( + _serverUrl, + headers: token != null ? {'Authorization': 'Bearer $token'} : null, + ); + + _wsChannel = IOWebSocketChannel(ws); + + final config = { + 'config': {'sample_rate': _sampleRate}, + }; + _wsChannel!.sink.add(jsonEncode(config)); + _wsChannel!.stream.listen( + (data) { + if (_isDisposed) return; + try { + final result = jsonDecode(data as String) as Map; + _handleTranscriptionResult(result); + } catch (e) { + debugPrint('Failed to parse WebSocket data: $e'); + } + }, + onError: (error) { + if (_isDisposed) return; + + _handleConnectionLoss(); + }, + onDone: () { + if (_isDisposed) return; + + _handleConnectionLoss(); + }, + cancelOnError: false, + ); + } catch (e) { + throw Exception('Failed to connect to WebSocket: $e'); + } + } + + void _handleConnectionLoss() { + // Atomic check-and-set to prevent race condition + if (_isListening && !_isReconnecting) { + _isReconnecting = true; + _reconnect(); + } + } + + Future _reconnect() async { + // Double-check in case of edge cases + if (_isDisposed) { + _isReconnecting = false; + return; + } + + try { + // Close existing connection if any + try { + await _wsChannel?.sink.close(); + } catch (e) { + debugPrint( + '[SpeechToText] Error closing WebSocket during reconnect: $e', + ); + } + _wsChannel = null; + + // Wait a bit before reconnecting + await Future.delayed(const Duration(milliseconds: _reconnectionDelayMs)); + + if (!_isListening) { + _isReconnecting = false; + return; + } + + await _startWebSocketConnection(); + } catch (e) { + _emitError( + SpeechErrorType.general, + details: 'Connection lost. Please try again.', + ); + } finally { + _isReconnecting = false; + } + } + + Future _startAudioRecording() async { + try { + final stream = await _audioRecorder.startStream( + const RecordConfig( + encoder: AudioEncoder.pcm16bits, + sampleRate: _sampleRate, + numChannels: 1, + autoGain: true, + echoCancel: true, + noiseSuppress: true, + ), + ); + + _audioStreamSubscription = stream.listen( + (audioData) { + if (_wsChannel != null && !_isDisposed && _isListening) { + try { + _wsChannel!.sink.add(audioData); + } catch (e) { + // If send fails, it might be because connection is closed + // The onDone/onError of WebSocket should handle this, but just in case + } + } + }, + onError: (error) { + if (_isDisposed) return; + _emitError( + SpeechErrorType.general, + details: 'Audio recording error: $error', + ); + }, + cancelOnError: false, + ); + } catch (e) { + throw Exception('Failed to start audio recording: $e'); + } + } + + void _handleTranscriptionResult(Map result) { + if (_isDisposed) return; + + // Vosk server returns partial results and final results + final text = result['text'] as String?; + final partial = result['partial'] as String?; + + if (partial != null && partial.isNotEmpty) { + // Show partial transcription (ongoing speech) + _latestTranscript = partial.trim(); + value = value.copyWith(partialTranscript: partial); + } else if (text != null) { + // Final transcription (silence detected or end of phrase) + final trimmed = text.trim(); + if (trimmed.isNotEmpty) { + // Commit the final result + _latestTranscript = trimmed; + _commitTranscript(); + // Keep listening - don't clear partial transcript immediately + // to avoid visual flicker between phrases + } else { + // Empty final result - just clear the partial transcript + value = value.copyWith(clearTranscript: true); + } + // Important: Reset to allow new partial results to show + _latestTranscript = null; + } + } + + Future stopListening() async { + if (!_isListening) return; + _isListening = false; + + try { + // Stop audio recording + await _audioStreamSubscription?.cancel(); + await _audioRecorder.stop(); + + // Send EOF to WebSocket + if (_wsChannel != null) { + try { + _wsChannel!.sink.add(jsonEncode({'eof': 1})); + // Give server a moment to send final results + await Future.delayed(const Duration(milliseconds: _eofWaitDelayMs)); + await _wsChannel!.sink.close(); + } catch (e) { + // Errors during closing are expected if connection is already dead + debugPrint( + '[SpeechToText] Error during graceful WebSocket close: $e', + ); + } + _wsChannel = null; + } + + value = value.copyWith( + status: SpeechLifecycleStatus.ready, + clearTranscript: true, + ); + } catch (e, stack) { + _emitError( + SpeechErrorType.general, + details: 'Failed to stop speech input: $e', + stackTrace: stack, + ); + } + } + + Future forceReset() async { + _isListening = false; + try { + await _audioStreamSubscription?.cancel(); + await _audioRecorder.stop(); + if (_wsChannel != null) { + await _wsChannel!.sink.close(); + _wsChannel = null; + } + } catch (e) { + debugPrint('[SpeechToText] Error during force reset cleanup: $e'); + } + value = value.copyWith( + status: SpeechLifecycleStatus.idle, + clearTranscript: true, + ); + } + + void _commitTranscript() { + final transcript = _latestTranscript; + if (transcript != null && transcript.isNotEmpty) { + onFinalTranscription(transcript); + } + } + + void _emitError( + SpeechErrorType type, { + String? details, + StackTrace? stackTrace, + }) { + _isListening = false; + _latestTranscript = null; + + try { + _audioStreamSubscription?.cancel(); + _audioRecorder.stop(); + } catch (e) { + debugPrint( + '[SpeechToText] Error stopping audio during error emission: $e', + ); + } + + value = value.copyWith( + status: SpeechLifecycleStatus.error, + error: SpeechError(type, details: details), + clearTranscript: true, + ); + } + + @override + Future dispose() async { + _isDisposed = true; + try { + await _audioStreamSubscription?.cancel(); + await _audioRecorder.stop(); + await _audioRecorder.dispose(); + await _wsChannel?.sink.close(); + } catch (e) { + debugPrint('[SpeechToText] Error during dispose cleanup: $e'); + } + super.dispose(); + } +} diff --git a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart index be16a4cc2..3622d2a93 100644 --- a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart +++ b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart @@ -3,7 +3,9 @@ import 'dart:async'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:studyu_app/l10n/app_localizations.dart'; +import 'package:studyu_app/services/speech/speech_to_text_controller.dart'; import 'package:studyu_app/widgets/questionnaire/questions/question_widget.dart'; + import 'package:studyu_core/core.dart'; class FreeTextQuestionWidget extends QuestionWidget { @@ -27,7 +29,13 @@ class _FreeTextQuestionWidgetState extends State { bool _hasInteracted = false; bool _hasSubmitted = false; Timer? _debounceTimer; + Timer? _listeningRestartTimer; AutovalidateMode _autovalidateMode = AutovalidateMode.disabled; + SpeechToTextController? _speechController; + bool _listeningRequested = false; + SpeechErrorType? _lastSpeechErrorType; + String _preSpeechText = ''; + int _insertionIndex = 0; @override void initState() { @@ -35,15 +43,221 @@ class _FreeTextQuestionWidgetState extends State { _focusNode.addListener(_onFocusChange); } + @override + void didChangeDependencies() { + super.didChangeDependencies(); + if (_speechController == null && + SpeechToTextController.isSupportedPlatform) { + _createSpeechController(); + } + } + @override void dispose() { _textFieldController.dispose(); _focusNode.removeListener(_onFocusChange); _focusNode.dispose(); _debounceTimer?.cancel(); + _listeningRestartTimer?.cancel(); + _speechController?.removeListener(_onSpeechStateChanged); + _speechController?.dispose(); super.dispose(); } + void _createSpeechController() { + _speechController?.removeListener(_onSpeechStateChanged); + _speechController?.dispose(); + _speechController = SpeechToTextController( + onFinalTranscription: _insertSpeechTranscript, + ); + _speechController!.addListener(_onSpeechStateChanged); + _listeningRequested = false; + setState(() {}); + } + + void _onSpeechStateChanged() { + final controller = _speechController; + if (controller == null) return; + final state = controller.value; + final currentError = controller.value.error; + if (currentError != null && currentError.type != _lastSpeechErrorType) { + final loc = AppLocalizations.of(context)!; + final message = _mapSpeechErrorToMessage(currentError, loc); + if (message != null) { + ScaffoldMessenger.of( + context, + ).showSnackBar(SnackBar(content: Text(message))); + } + } + _lastSpeechErrorType = currentError?.type; + if (!mounted) return; + + final hasError = state.error != null; + if (hasError && _listeningRequested) { + _setListeningRequested(false); + } + + // Handle partial transcript + if (state.status == SpeechLifecycleStatus.listening && + state.partialTranscript != null) { + _updateTextWithTranscript(state.partialTranscript!); + } + + WidgetsBinding.instance.addPostFrameCallback((_) { + if (!mounted) return; + setState(() {}); + }); + } + + void _updateTextWithTranscript(String transcript) { + if (transcript.isEmpty) return; + final controller = _textFieldController; + + // Ensure we have valid state to insert into + if (_insertionIndex > _preSpeechText.length) { + _insertionIndex = _preSpeechText.length; + } + + final previousChar = _insertionIndex > 0 + ? _preSpeechText.substring(_insertionIndex - 1, _insertionIndex) + : ''; + final needsSpace = previousChar.trim().isNotEmpty; + final insertion = '${needsSpace ? ' ' : ''}$transcript'; + + final newText = _preSpeechText.replaceRange( + _insertionIndex, + _insertionIndex, + insertion, + ); + + controller.value = controller.value.copyWith( + text: newText, + selection: TextSelection.collapsed( + offset: _insertionIndex + insertion.length, + ), + ); + _handleInteraction(); + _debouncedValidation(); + } + + void _insertSpeechTranscript(String transcript) { + final trimmed = transcript.trim(); + if (trimmed.isEmpty) return; + + _updateTextWithTranscript(trimmed); + + // Update state for next phrase (continuous listening) + _preSpeechText = _textFieldController.text; + _insertionIndex = _textFieldController.selection.isValid + ? _textFieldController.selection.extentOffset + : _textFieldController.text.length; + } + + String? _mapSpeechErrorToMessage(SpeechError error, AppLocalizations loc) { + final details = error.details?.toLowerCase(); + final isPermissionIssue = details?.contains('permission') == true; + final isConnectionIssue = details?.contains('connection') == true; + + if (error.type == SpeechErrorType.microphonePermission && + isPermissionIssue) { + return loc.speech_to_text_error_permission; + } + + if (isConnectionIssue) { + return loc.speech_to_text_error_connection; + } + + return loc.speech_to_text_error_general; + } + + void _toggleSpeechListening() { + if (_listeningRequested) { + _stopListeningIntent(); + } else { + _setListeningRequested(true); + _captureCurrentTextState(); + unawaited(_startListeningIfRequested()); + } + } + + void _captureCurrentTextState() { + final controller = _textFieldController; + _preSpeechText = controller.text; + _insertionIndex = controller.selection.isValid + ? controller.selection.extentOffset + : controller.text.length; + } + + void _stopListeningIntent() { + _setListeningRequested(false); + unawaited(_stopActiveListening()); + } + + void _setListeningRequested(bool requested) { + if (_listeningRequested == requested) return; + _listeningRequested = requested; + WidgetsBinding.instance.addPostFrameCallback((_) { + if (!mounted) return; + setState(() {}); + }); + } + + Future _startListeningIfRequested() async { + final controller = _speechController; + if (_listeningRequested == false || controller == null) { + return; + } + + var state = controller.value; + var status = state.status; + + if (status == SpeechLifecycleStatus.listening || + status == SpeechLifecycleStatus.preparing || + status == SpeechLifecycleStatus.unavailable) { + return; + } + + if (status == SpeechLifecycleStatus.error) { + await controller.forceReset(); + state = controller.value; + status = state.status; + } + + final canStart = + status == SpeechLifecycleStatus.idle || + status == SpeechLifecycleStatus.ready; + if (!canStart) { + return; + } + + await controller.startListening(); + } + + Future _stopActiveListening() async { + final controller = _speechController; + if (controller == null) return; + await controller.stopListening(); + } + + Widget _buildMicButton(ThemeData theme, bool isListening, bool isPreparing) { + final loc = AppLocalizations.of(context)!; + return IconButton.filledTonal( + onPressed: isPreparing ? null : _toggleSpeechListening, + icon: isListening + ? const Icon(Icons.stop_circle_outlined) + : const Icon(Icons.mic), + style: IconButton.styleFrom( + backgroundColor: isListening ? theme.colorScheme.errorContainer : null, + foregroundColor: isListening + ? theme.colorScheme.onErrorContainer + : null, + ), + tooltip: isListening + ? loc.speech_to_text_stop_listening + : loc.speech_to_text_idle, + ); + } + void _onFocusChange() { if (_focusNode.hasFocus) { _ensureTextFieldVisible(); @@ -137,6 +351,9 @@ class _FreeTextQuestionWidgetState extends State { @override Widget build(BuildContext context) { final question = widget.question; + final loc = AppLocalizations.of(context)!; + final theme = Theme.of(context); + return Column( crossAxisAlignment: CrossAxisAlignment.stretch, children: [ @@ -144,6 +361,7 @@ class _FreeTextQuestionWidgetState extends State { key: _formFieldKey, controller: _textFieldController, maxLines: null, + minLines: 4, focusNode: _focusNode, keyboardType: _getKeyboardType(), inputFormatters: _getInputFormatters(), @@ -208,15 +426,48 @@ class _FreeTextQuestionWidgetState extends State { } } }, + decoration: InputDecoration( + filled: true, + fillColor: theme.colorScheme.surfaceContainerHighest.withValues( + alpha: 0.3, + ), + hintText: loc.free_text_answer_hint, + border: OutlineInputBorder( + borderRadius: BorderRadius.circular(12), + borderSide: BorderSide.none, + ), + focusedBorder: OutlineInputBorder( + borderRadius: BorderRadius.circular(12), + borderSide: BorderSide( + color: theme.colorScheme.primary, + width: 2, + ), + ), + contentPadding: const EdgeInsets.all(16), + ), ), const SizedBox(height: 16), + Row( mainAxisAlignment: MainAxisAlignment.end, children: [ - OutlinedButton( - onPressed: _handleSubmit, - child: Text(AppLocalizations.of(context)!.submit), - ), + if (SpeechToTextController.isSupportedPlatform && + _speechController != null) + ValueListenableBuilder( + valueListenable: _speechController!, + builder: (context, state, _) { + if (state.status == SpeechLifecycleStatus.unavailable) { + return const SizedBox.shrink(); + } + return _buildMicButton( + theme, + state.status == SpeechLifecycleStatus.listening, + state.status == SpeechLifecycleStatus.preparing, + ); + }, + ), + const SizedBox(width: 8), + FilledButton(onPressed: _handleSubmit, child: Text(loc.submit)), ], ), ], diff --git a/app/pubspec.yaml b/app/pubspec.yaml index 8c1ec6980..056db0667 100644 --- a/app/pubspec.yaml +++ b/app/pubspec.yaml @@ -60,6 +60,7 @@ dependencies: url_launcher: ^6.3.2 uuid: ^4.5.2 wakelock_plus: ^1.4.0 + web_socket_channel: ^3.0.2 xml: ^6.6.1 dev_dependencies: diff --git a/core/lib/src/env/env.dart b/core/lib/src/env/env.dart index e9cee6fde..bd1a76da0 100644 --- a/core/lib/src/env/env.dart +++ b/core/lib/src/env/env.dart @@ -9,6 +9,7 @@ late final String? projectGeneratorUrl; late final String? playStoreUrl; late final String? appstoreUrl; late final String? developerEmail; +late final String? sttWebSocketUrl; void setEnv( String envSupabaseUrl, @@ -20,6 +21,7 @@ void setEnv( String? envPlayStoreUrl, String? envAppstoreUrl, String? envDeveloperEmail, + String? envSttWebSocketUrl, }) { supabaseUrl = envSupabaseUrl; supabaseAnonKey = envSupabaseAnonKey; @@ -47,4 +49,5 @@ void setEnv( playStoreUrl = envPlayStoreUrl; appstoreUrl = envAppstoreUrl; developerEmail = envDeveloperEmail; + sttWebSocketUrl = envSttWebSocketUrl; } diff --git a/designer_v2/lib/features/study/study_base_controller.dart b/designer_v2/lib/features/study/study_base_controller.dart index 633c5e210..786a10c02 100644 --- a/designer_v2/lib/features/study/study_base_controller.dart +++ b/designer_v2/lib/features/study/study_base_controller.dart @@ -16,7 +16,7 @@ part 'study_base_controller.g.dart'; @riverpod class StudyBaseController - extends _$StudyBaseController { + extends _$StudyBaseController { @override StudyControllerBaseState build(StudyID studyId) { state = StudyControllerBaseState( diff --git a/designer_v2/lib/features/study/study_base_controller.g.dart b/designer_v2/lib/features/study/study_base_controller.g.dart index 13dcb207c..368809384 100644 --- a/designer_v2/lib/features/study/study_base_controller.g.dart +++ b/designer_v2/lib/features/study/study_base_controller.g.dart @@ -68,7 +68,7 @@ final class StudyBaseControllerProvider } String _$studyBaseControllerHash() => - r'e2811f660e2b1086b07c702a0da8915d891e2004'; + r'2dd7b7b33e5705fd1c64fe268f023bbd9c87e1b5'; final class StudyBaseControllerFamily extends $Family { StudyBaseControllerFamily._() diff --git a/flutter_common/lib/envs/.env b/flutter_common/lib/envs/.env index 82f63798f..e9e4d5f0a 100644 --- a/flutter_common/lib/envs/.env +++ b/flutter_common/lib/envs/.env @@ -6,3 +6,4 @@ STUDYU_DESIGNER_URL=https://designer.studyu.health STUDYU_PLAY_STORE_URL=https://play.google.com/store/apps/details?id=health.studyu.app STUDYU_APPSTORE_URL=https://itunes.apple.com/app/id1571991198 STUDYU_DEVELOPER_EMAIL=stefan.konigorski@hpi.de +STUDYU_STT_WEBSOCKET_URL= diff --git a/flutter_common/lib/envs/.env.dev b/flutter_common/lib/envs/.env.dev index 3738415b9..5e265bc4d 100644 --- a/flutter_common/lib/envs/.env.dev +++ b/flutter_common/lib/envs/.env.dev @@ -3,6 +3,7 @@ STUDYU_SUPABASE_PUBLIC_ANON_KEY=eyJhbGciOiAiSFMyNTYiLCJ0eXAiOiAiSldUIn0.eyJyb2xl STUDYU_PROJECT_GENERATOR_URL= STUDYU_APP_URL=https://app.dev.studyu.health STUDYU_DESIGNER_URL=https://designer.dev.studyu.health +STUDYU_STT_WEBSOCKET_URL= STUDYU_PLAY_STORE_URL=https://play.google.com/store/apps/details?id=health.studyu.app STUDYU_APPSTORE_URL=https://itunes.apple.com/app/id1571991198 STUDYU_DEVELOPER_EMAIL=stefan.konigorski@hpi.de diff --git a/flutter_common/lib/envs/.env.local.example b/flutter_common/lib/envs/.env.local.example index 13b8253d8..999a02843 100644 --- a/flutter_common/lib/envs/.env.local.example +++ b/flutter_common/lib/envs/.env.local.example @@ -3,6 +3,7 @@ STUDYU_SUPABASE_PUBLIC_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJ STUDYU_PROJECT_GENERATOR_URL= STUDYU_APP_URL=http://localhost:8080 STUDYU_DESIGNER_URL=http://localhost:8081 +STUDYU_STT_WEBSOCKET_URL= STUDYU_PLAY_STORE_URL=https://play.google.com/store/apps/details?id=health.studyu.app STUDYU_APPSTORE_URL=https://itunes.apple.com/app/id1571991198 STUDYU_DEVELOPER_EMAIL=stefan.konigorski@hpi.de diff --git a/flutter_common/lib/src/utils/env_loader.dart b/flutter_common/lib/src/utils/env_loader.dart index 0226b6fdc..21432b75f 100644 --- a/flutter_common/lib/src/utils/env_loader.dart +++ b/flutter_common/lib/src/utils/env_loader.dart @@ -52,6 +52,7 @@ Future loadEnv() async { final envPlayStoreUrl = getEnv('STUDYU_PLAY_STORE_URL', optional: true); final envAppstoreUrl = getEnv('STUDYU_APPSTORE_URL', optional: true); final envDeveloperEmail = getEnv('STUDYU_DEVELOPER_EMAIL', optional: true); + final envSttWebSocketUrl = getEnv('STUDYU_STT_WEBSOCKET_URL', optional: true); // await SecureStorage.migrateSharedPreferencesToSecureStorage(); @@ -77,6 +78,7 @@ Future loadEnv() async { envPlayStoreUrl: envPlayStoreUrl, envAppstoreUrl: envAppstoreUrl, envDeveloperEmail: envDeveloperEmail, + envSttWebSocketUrl: envSttWebSocketUrl, supabaseClient: Supabase.instance.client, ); }