From 749b4a4932d31c15e107863bc08f97c7da6e9ec2 Mon Sep 17 00:00:00 2001 From: ibrahimozkan Date: Mon, 17 Nov 2025 13:57:53 +0100 Subject: [PATCH 01/16] feat: Add speech-to-text functionality with language preferences - Implemented speech-to-text feature allowing users to dictate answers using built-in speech recognition. - Added localization support for speech-to-text titles, descriptions, and error messages in English and German. - Introduced settings for managing speech input preferences, including language selection. - Integrated WebSocket communication for real-time speech recognition. - Enhanced UI components to support speech input and display live transcriptions. - Updated dependencies to include web_socket_channel for WebSocket communication. --- app/android/app/src/main/AndroidManifest.xml | 1 + app/ios/Runner/Info.plist | 16 +- app/lib/l10n/app_de.arb | 28 +- app/lib/l10n/app_en.arb | 28 +- app/lib/l10n/app_localizations.dart | 126 ++++++ app/lib/l10n/app_localizations_de.dart | 73 ++++ app/lib/l10n/app_localizations_en.dart | 73 ++++ app/lib/screens/study/dashboard/settings.dart | 161 ++++++-- .../speech/speech_controller_models.dart | 69 ++++ .../speech/speech_to_text_controller.dart | 3 + .../speech/speech_to_text_language.dart | 32 ++ .../speech/speech_to_text_preferences.dart | 25 ++ .../websocket_speech_controller_io.dart | 335 ++++++++++++++++ .../questions/free_text_question_widget.dart | 360 +++++++++++++++++- app/pubspec.yaml | 1 + 15 files changed, 1286 insertions(+), 45 deletions(-) create mode 100644 app/lib/services/speech/speech_controller_models.dart create mode 100644 app/lib/services/speech/speech_to_text_controller.dart create mode 100644 app/lib/services/speech/speech_to_text_language.dart create mode 100644 app/lib/services/speech/speech_to_text_preferences.dart create mode 100644 app/lib/services/speech/websocket_speech_controller_io.dart diff --git a/app/android/app/src/main/AndroidManifest.xml b/app/android/app/src/main/AndroidManifest.xml index 201bde8d1..dee9d4875 100644 --- a/app/android/app/src/main/AndroidManifest.xml +++ b/app/android/app/src/main/AndroidManifest.xml @@ -2,6 +2,7 @@ + diff --git a/app/ios/Runner/Info.plist b/app/ios/Runner/Info.plist index 33e4ee169..15ab68c54 100644 --- a/app/ios/Runner/Info.plist +++ b/app/ios/Runner/Info.plist @@ -38,13 +38,15 @@ LSSupportsOpeningDocumentsInPlace - NSAppTransportSecurity - - NSAllowsArbitraryLoads - - - NSCameraUsageDescription - Need to access your camera to capture a photo for trial observations. + NSAppTransportSecurity + + NSAllowsArbitraryLoads + + + NSMicrophoneUsageDescription + Need microphone access for speech input and study recordings. + NSCameraUsageDescription + Need to access your camera to capture a photo for trial observations. UIApplicationSupportsIndirectInputEvents UIBackgroundModes diff --git a/app/lib/l10n/app_de.arb b/app/lib/l10n/app_de.arb index b9aff165d..61e30c75c 100644 --- a/app/lib/l10n/app_de.arb +++ b/app/lib/l10n/app_de.arb @@ -455,5 +455,31 @@ "preview_mode_description": "Sie befinden sich derzeit im Vorschau-Modus. Dies ermöglicht Ihnen:\n\n• Schneller Vorlauf durch Studientage mit der \"Nächster Tag\" Schaltfläche\n• Mehrfaches Abschließen von Aufgaben ohne Einschränkungen\n• Erleben des vollständigen Studienablaufs ohne Beeinflussung echter Daten\n\nWichtig: Ergebnisse und Daten aus dem Vorschau-Modus werden nicht gespeichert oder mit tatsächlichen Teilnehmerergebnissen aus laufenden Studien vermischt.", "preview_mode_results_not_saved": "Ergebnisse werden zum Schutz der Studienintegrität nicht gespeichert.", "ok": "OK", - "submit" : "Absenden" + "submit" : "Absenden", + "speech_to_text_title": "Sprach-zu-Text", + "speech_to_text_description": "Diktiere Antworten mit der integrierten Spracherkennung deines Geräts. Die Verarbeitung übernimmt der Plattformdienst.", + "speech_to_text_enable_label": "Sprach­eingabe aktivieren", + "speech_to_text_enable_hint": "Tippen funktioniert weiterhin, falls das Mikrofon blockiert ist.", + "speech_to_text_language_label": "Bevorzugte Transkriptionssprache", + "speech_to_text_language_english": "Englisch", + "speech_to_text_language_german": "Deutsch", + "speech_to_text_not_enabled": "Die Sprach­eingabe bleibt deaktiviert, bis du sie einschaltest.", + "speech_to_text_not_enabled_brief": "Sprach­eingabe ist für dein Konto ausgeschaltet.", + "speech_to_text_go_to_settings": "In den Einstellungen verwalten", + "speech_to_text_unsupported": "Sprach­eingabe ist auf diesem Gerät nicht verfügbar.", + "speech_to_text_error_permission": "Bitte erlaube den Mikrofonzugriff, um diktieren zu können.", + "speech_to_text_error_general": "Sprach­eingabe fehlgeschlagen. Bitte erneut versuchen.", + "speech_to_text_preparing": "Spracherkennung wird vorbereitet…", + "speech_to_text_live_caption_hint": "Live-Transkription", + "speech_to_text_listening": "Es wird zugehört…", + "speech_to_text_press_to_speak": "Zum Sprechen drücken", + "speech_to_text_idle": "Sprach­eingabe starten", + "speech_to_text_opt_out_action": "Sprach­eingabe deaktivieren", + "speech_to_text_enable_short_label": "Wieder aktivieren", + "speech_to_text_language_badge": "{language}", + "@speech_to_text_language_badge": { + "placeholders": { + "language": {} + } + } } diff --git a/app/lib/l10n/app_en.arb b/app/lib/l10n/app_en.arb index cb2e186a3..236453ac2 100644 --- a/app/lib/l10n/app_en.arb +++ b/app/lib/l10n/app_en.arb @@ -451,5 +451,31 @@ "preview_mode_description": "You are currently in preview mode. This allows you to:\n\n• Fast-forward through study days using the \"Next Day\" button\n• Complete tasks multiple times without restrictions\n• Experience the full study flow without affecting real data\n\nImportant: Results and data from preview mode are not stored or mixed with actual participant results from running studies.", "preview_mode_results_not_saved": "Task completed in preview mode - results are not saved to protect study data integrity.", "ok": "OK", - "submit" : "Submit" + "submit" : "Submit", + "speech_to_text_title": "Speech to text", + "speech_to_text_description": "Dictate answers with your device's built-in speech recognition. Audio always stays with the platform service.", + "speech_to_text_enable_label": "Enable speech input", + "speech_to_text_enable_hint": "Typing continues to work even if the microphone is unavailable.", + "speech_to_text_language_label": "Preferred transcription language", + "speech_to_text_language_english": "English", + "speech_to_text_language_german": "German", + "speech_to_text_not_enabled": "Speech input stays disabled until you turn it on.", + "speech_to_text_not_enabled_brief": "Speech input is currently turned off for your account.", + "speech_to_text_go_to_settings": "Manage in Settings", + "speech_to_text_unsupported": "Speech input is unavailable on this device.", + "speech_to_text_error_permission": "Microphone permission is required for speech input.", + "speech_to_text_error_general": "Speech input failed. Please try again.", + "speech_to_text_preparing": "Preparing speech recognizer…", + "speech_to_text_live_caption_hint": "Live transcription", + "speech_to_text_listening": "Listening…", + "speech_to_text_press_to_speak": "Press to speak", + "speech_to_text_idle": "Start speech input", + "speech_to_text_opt_out_action": "Disable speech input for now", + "speech_to_text_enable_short_label": "Enable again", + "speech_to_text_language_badge": "{language}", + "@speech_to_text_language_badge": { + "placeholders": { + "language": {} + } + } } diff --git a/app/lib/l10n/app_localizations.dart b/app/lib/l10n/app_localizations.dart index 2d8505c23..7856d33b1 100644 --- a/app/lib/l10n/app_localizations.dart +++ b/app/lib/l10n/app_localizations.dart @@ -2251,6 +2251,132 @@ abstract class AppLocalizations { /// In en, this message translates to: /// **'Submit'** String get submit; + + /// No description provided for @speech_to_text_title. + /// + /// In en, this message translates to: + /// **'Speech to text'** + String get speech_to_text_title; + + /// No description provided for @speech_to_text_description. + /// + /// In en, this message translates to: + /// **'Dictate answers with your device\'s built-in speech recognition. Audio always stays with the platform service.'** + String get speech_to_text_description; + + /// No description provided for @speech_to_text_enable_label. + /// + /// In en, this message translates to: + /// **'Enable speech input'** + String get speech_to_text_enable_label; + + /// No description provided for @speech_to_text_enable_hint. + /// + /// In en, this message translates to: + /// **'Typing continues to work even if the microphone is unavailable.'** + String get speech_to_text_enable_hint; + + /// No description provided for @speech_to_text_language_label. + /// + /// In en, this message translates to: + /// **'Preferred transcription language'** + String get speech_to_text_language_label; + + /// No description provided for @speech_to_text_language_english. + /// + /// In en, this message translates to: + /// **'English'** + String get speech_to_text_language_english; + + /// No description provided for @speech_to_text_language_german. + /// + /// In en, this message translates to: + /// **'German'** + String get speech_to_text_language_german; + + /// No description provided for @speech_to_text_not_enabled. + /// + /// In en, this message translates to: + /// **'Speech input stays disabled until you turn it on.'** + String get speech_to_text_not_enabled; + + /// No description provided for @speech_to_text_not_enabled_brief. + /// + /// In en, this message translates to: + /// **'Speech input is currently turned off for your account.'** + String get speech_to_text_not_enabled_brief; + + /// No description provided for @speech_to_text_go_to_settings. + /// + /// In en, this message translates to: + /// **'Manage in Settings'** + String get speech_to_text_go_to_settings; + + /// No description provided for @speech_to_text_unsupported. + /// + /// In en, this message translates to: + /// **'Speech input is unavailable on this device.'** + String get speech_to_text_unsupported; + + /// No description provided for @speech_to_text_error_permission. + /// + /// In en, this message translates to: + /// **'Microphone permission is required for speech input.'** + String get speech_to_text_error_permission; + + /// No description provided for @speech_to_text_error_general. + /// + /// In en, this message translates to: + /// **'Speech input failed. Please try again.'** + String get speech_to_text_error_general; + + /// No description provided for @speech_to_text_preparing. + /// + /// In en, this message translates to: + /// **'Preparing speech recognizer…'** + String get speech_to_text_preparing; + + /// No description provided for @speech_to_text_live_caption_hint. + /// + /// In en, this message translates to: + /// **'Live transcription'** + String get speech_to_text_live_caption_hint; + + /// No description provided for @speech_to_text_listening. + /// + /// In en, this message translates to: + /// **'Listening…'** + String get speech_to_text_listening; + + /// No description provided for @speech_to_text_press_to_speak. + /// + /// In en, this message translates to: + /// **'Press to speak'** + String get speech_to_text_press_to_speak; + + /// No description provided for @speech_to_text_idle. + /// + /// In en, this message translates to: + /// **'Start speech input'** + String get speech_to_text_idle; + + /// No description provided for @speech_to_text_opt_out_action. + /// + /// In en, this message translates to: + /// **'Disable speech input for now'** + String get speech_to_text_opt_out_action; + + /// No description provided for @speech_to_text_enable_short_label. + /// + /// In en, this message translates to: + /// **'Enable again'** + String get speech_to_text_enable_short_label; + + /// No description provided for @speech_to_text_language_badge. + /// + /// In en, this message translates to: + /// **'{language}'** + String speech_to_text_language_badge(Object language); } class _AppLocalizationsDelegate diff --git a/app/lib/l10n/app_localizations_de.dart b/app/lib/l10n/app_localizations_de.dart index 29f2fa20a..c333503c2 100644 --- a/app/lib/l10n/app_localizations_de.dart +++ b/app/lib/l10n/app_localizations_de.dart @@ -1166,4 +1166,77 @@ class AppLocalizationsDe extends AppLocalizations { @override String get submit => 'Absenden'; + + @override + String get speech_to_text_title => 'Sprach-zu-Text'; + + @override + String get speech_to_text_description => + 'Diktiere Antworten mit der integrierten Spracherkennung deines Geräts. Die Verarbeitung übernimmt der Plattformdienst.'; + + @override + String get speech_to_text_enable_label => 'Sprach­eingabe aktivieren'; + + @override + String get speech_to_text_enable_hint => + 'Tippen funktioniert weiterhin, falls das Mikrofon blockiert ist.'; + + @override + String get speech_to_text_language_label => + 'Bevorzugte Transkriptionssprache'; + + @override + String get speech_to_text_language_english => 'Englisch'; + + @override + String get speech_to_text_language_german => 'Deutsch'; + + @override + String get speech_to_text_not_enabled => + 'Die Sprach­eingabe bleibt deaktiviert, bis du sie einschaltest.'; + + @override + String get speech_to_text_not_enabled_brief => + 'Sprach­eingabe ist für dein Konto ausgeschaltet.'; + + @override + String get speech_to_text_go_to_settings => 'In den Einstellungen verwalten'; + + @override + String get speech_to_text_unsupported => + 'Sprach­eingabe ist auf diesem Gerät nicht verfügbar.'; + + @override + String get speech_to_text_error_permission => + 'Bitte erlaube den Mikrofonzugriff, um diktieren zu können.'; + + @override + String get speech_to_text_error_general => + 'Sprach­eingabe fehlgeschlagen. Bitte erneut versuchen.'; + + @override + String get speech_to_text_preparing => 'Spracherkennung wird vorbereitet…'; + + @override + String get speech_to_text_live_caption_hint => 'Live-Transkription'; + + @override + String get speech_to_text_listening => 'Es wird zugehört…'; + + @override + String get speech_to_text_press_to_speak => 'Zum Sprechen drücken'; + + @override + String get speech_to_text_idle => 'Sprach­eingabe starten'; + + @override + String get speech_to_text_opt_out_action => 'Sprach­eingabe deaktivieren'; + + @override + String get speech_to_text_enable_short_label => 'Wieder aktivieren'; + + @override + String speech_to_text_language_badge(Object language) { + return '$language'; + } } diff --git a/app/lib/l10n/app_localizations_en.dart b/app/lib/l10n/app_localizations_en.dart index 194435c68..20a7543cd 100644 --- a/app/lib/l10n/app_localizations_en.dart +++ b/app/lib/l10n/app_localizations_en.dart @@ -1156,4 +1156,77 @@ class AppLocalizationsEn extends AppLocalizations { @override String get submit => 'Submit'; + + @override + String get speech_to_text_title => 'Speech to text'; + + @override + String get speech_to_text_description => + 'Dictate answers with your device\'s built-in speech recognition. Audio always stays with the platform service.'; + + @override + String get speech_to_text_enable_label => 'Enable speech input'; + + @override + String get speech_to_text_enable_hint => + 'Typing continues to work even if the microphone is unavailable.'; + + @override + String get speech_to_text_language_label => + 'Preferred transcription language'; + + @override + String get speech_to_text_language_english => 'English'; + + @override + String get speech_to_text_language_german => 'German'; + + @override + String get speech_to_text_not_enabled => + 'Speech input stays disabled until you turn it on.'; + + @override + String get speech_to_text_not_enabled_brief => + 'Speech input is currently turned off for your account.'; + + @override + String get speech_to_text_go_to_settings => 'Manage in Settings'; + + @override + String get speech_to_text_unsupported => + 'Speech input is unavailable on this device.'; + + @override + String get speech_to_text_error_permission => + 'Microphone permission is required for speech input.'; + + @override + String get speech_to_text_error_general => + 'Speech input failed. Please try again.'; + + @override + String get speech_to_text_preparing => 'Preparing speech recognizer…'; + + @override + String get speech_to_text_live_caption_hint => 'Live transcription'; + + @override + String get speech_to_text_listening => 'Listening…'; + + @override + String get speech_to_text_press_to_speak => 'Press to speak'; + + @override + String get speech_to_text_idle => 'Start speech input'; + + @override + String get speech_to_text_opt_out_action => 'Disable speech input for now'; + + @override + String get speech_to_text_enable_short_label => 'Enable again'; + + @override + String speech_to_text_language_badge(Object language) { + return '$language'; + } } diff --git a/app/lib/screens/study/dashboard/settings.dart b/app/lib/screens/study/dashboard/settings.dart index 262552054..495a7742b 100644 --- a/app/lib/screens/study/dashboard/settings.dart +++ b/app/lib/screens/study/dashboard/settings.dart @@ -6,6 +6,8 @@ import 'package:provider/provider.dart'; import 'package:studyu_app/l10n/app_localizations.dart'; import 'package:studyu_app/models/app_state.dart'; import 'package:studyu_app/routes.dart'; +import 'package:studyu_app/services/speech/speech_to_text_language.dart'; +import 'package:studyu_app/services/speech/speech_to_text_preferences.dart'; import 'package:studyu_app/util/app_analytics.dart'; import 'package:studyu_app/util/fitbit_handler.dart'; import 'package:studyu_app/util/localization.dart'; @@ -24,6 +26,9 @@ class _SettingsState extends State { Locale? _selectedValue; bool? _analyticsValue; StudySubject? subject; + bool _speechPrefsRequested = false; + bool _speechPrefsLoading = true; + SpeechRecognitionLanguage _speechLanguage = SpeechRecognitionLanguage.english; @override void initState() { @@ -33,6 +38,34 @@ class _SettingsState extends State { subject = context.read().activeSubject; } + @override + void didChangeDependencies() { + super.didChangeDependencies(); + if (!_speechPrefsRequested) { + _speechPrefsRequested = true; + _loadSpeechPreferences(); + } + } + + Future _loadSpeechPreferences() async { + final fallbackLocale = Localizations.maybeLocaleOf(context); + final language = await SpeechToTextPreferences.preferredLanguage( + fallbackLocale: fallbackLocale, + ); + if (!mounted) return; + setState(() { + _speechLanguage = language; + _speechPrefsLoading = false; + }); + } + + Future _changeSpeechLanguage(SpeechRecognitionLanguage language) async { + setState(() { + _speechLanguage = language; + }); + await SpeechToTextPreferences.setPreferredLanguage(language); + } + Widget getDropdownRow(BuildContext context) { final dropDownItems = >[]; @@ -96,50 +129,114 @@ class _SettingsState extends State { @override Widget build(BuildContext context) { final theme = Theme.of(context); + final loc = AppLocalizations.of(context)!; return Scaffold( - appBar: AppBar(title: Text(AppLocalizations.of(context)!.settings)), - body: Center( + appBar: AppBar(title: Text(loc.settings)), + body: SingleChildScrollView( + padding: const EdgeInsets.all(24), child: Column( - mainAxisSize: MainAxisSize.min, - mainAxisAlignment: MainAxisAlignment.spaceAround, - children: [ + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ getDropdownRow(context), const SizedBox(height: 24), + _buildSpeechSection(theme, loc), + const SizedBox(height: 32), + _buildStudySection(theme, loc), + ], + ), + ), + ); + } + + Widget _buildSpeechSection(ThemeData theme, AppLocalizations loc) { + return Card( + elevation: 0, + margin: EdgeInsets.zero, + child: Padding( + padding: const EdgeInsets.all(16), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text(loc.speech_to_text_title, style: theme.textTheme.titleMedium), + const SizedBox(height: 8), Text( - '${AppLocalizations.of(context)!.study_current} ${subject!.study.title}', - style: theme.textTheme.titleLarge, + loc.speech_to_text_description, + style: theme.textTheme.bodyMedium, ), - const SizedBox(height: 8), - ElevatedButton.icon( - icon: Icon(MdiIcons.exitToApp), - label: Text(AppLocalizations.of(context)!.opt_out), - style: ElevatedButton.styleFrom( - backgroundColor: Colors.orange[800], + const SizedBox(height: 12), + if (_speechPrefsLoading) + const LinearProgressIndicator() + else ...[ + DropdownMenu( + initialSelection: _speechLanguage, + label: Text(loc.speech_to_text_language_label), + onSelected: (lang) { + if (lang != null) _changeSpeechLanguage(lang); + }, + dropdownMenuEntries: SpeechRecognitionLanguage.values + .map( + (lang) => DropdownMenuEntry( + value: lang, + label: _languageLabel(lang, loc), + ), + ) + .toList(), ), - onPressed: () { - showDialog( - context: context, - builder: (_) => OptOutAlertDialog(subject: subject), - ); - }, - ), - const SizedBox(height: 24), - ElevatedButton.icon( - icon: const Icon(Icons.delete), - label: Text(AppLocalizations.of(context)!.delete_data), - style: ElevatedButton.styleFrom(backgroundColor: Colors.red), - onPressed: () { - showDialog( - context: context, - builder: (_) => DeleteAlertDialog(subject: subject), - ); - }, - ), + const SizedBox(height: 12), + ], ], ), ), ); } + + Widget _buildStudySection(ThemeData theme, AppLocalizations loc) { + return Column( + children: [ + Text( + '${loc.study_current} ${subject!.study.title}', + style: theme.textTheme.titleLarge, + textAlign: TextAlign.center, + ), + const SizedBox(height: 8), + ElevatedButton.icon( + icon: Icon(MdiIcons.exitToApp), + label: Text(loc.opt_out), + style: ElevatedButton.styleFrom(backgroundColor: Colors.orange[800]), + onPressed: () { + showDialog( + context: context, + builder: (_) => OptOutAlertDialog(subject: subject), + ); + }, + ), + const SizedBox(height: 24), + ElevatedButton.icon( + icon: const Icon(Icons.delete), + label: Text(loc.delete_data), + style: ElevatedButton.styleFrom(backgroundColor: Colors.red), + onPressed: () { + showDialog( + context: context, + builder: (_) => DeleteAlertDialog(subject: subject), + ); + }, + ), + ], + ); + } + + String _languageLabel( + SpeechRecognitionLanguage language, + AppLocalizations loc, + ) { + switch (language) { + case SpeechRecognitionLanguage.german: + return loc.speech_to_text_language_german; + case SpeechRecognitionLanguage.english: + return loc.speech_to_text_language_english; + } + } } class OptOutAlertDialog extends StatelessWidget { diff --git a/app/lib/services/speech/speech_controller_models.dart b/app/lib/services/speech/speech_controller_models.dart new file mode 100644 index 000000000..7b30b9241 --- /dev/null +++ b/app/lib/services/speech/speech_controller_models.dart @@ -0,0 +1,69 @@ +import 'package:flutter/foundation.dart'; + +import 'package:studyu_app/services/speech/speech_to_text_language.dart'; + +typedef SpeechTranscriptCommit = void Function(String textSegment); + +enum SpeechLifecycleStatus { + unavailable, + idle, + preparing, + ready, + listening, + error, +} + +enum SpeechErrorType { microphonePermission, general } + +@immutable +class SpeechError { + const SpeechError(this.type, {this.details}); + + final SpeechErrorType type; + final String? details; +} + +@immutable +class SpeechControllerState { + const SpeechControllerState({ + required this.status, + this.error, + this.partialTranscript, + required this.language, + }); + + factory SpeechControllerState.initial({ + required SpeechRecognitionLanguage language, + required bool supported, + }) { + return SpeechControllerState( + status: supported + ? SpeechLifecycleStatus.idle + : SpeechLifecycleStatus.unavailable, + language: language, + ); + } + + final SpeechLifecycleStatus status; + final SpeechError? error; + final String? partialTranscript; + final SpeechRecognitionLanguage language; + + SpeechControllerState copyWith({ + SpeechLifecycleStatus? status, + SpeechError? error, + bool clearError = false, + String? partialTranscript, + bool clearTranscript = false, + SpeechRecognitionLanguage? language, + }) { + return SpeechControllerState( + status: status ?? this.status, + error: clearError ? null : error ?? this.error, + partialTranscript: clearTranscript + ? null + : partialTranscript ?? this.partialTranscript, + language: language ?? this.language, + ); + } +} diff --git a/app/lib/services/speech/speech_to_text_controller.dart b/app/lib/services/speech/speech_to_text_controller.dart new file mode 100644 index 000000000..832dcdfe8 --- /dev/null +++ b/app/lib/services/speech/speech_to_text_controller.dart @@ -0,0 +1,3 @@ +export 'speech_controller_models.dart'; + +export 'websocket_speech_controller_io.dart'; diff --git a/app/lib/services/speech/speech_to_text_language.dart b/app/lib/services/speech/speech_to_text_language.dart new file mode 100644 index 000000000..2c7ce9f78 --- /dev/null +++ b/app/lib/services/speech/speech_to_text_language.dart @@ -0,0 +1,32 @@ +import 'package:flutter/material.dart'; + +enum SpeechRecognitionLanguage { + english(localeCode: 'en', localeId: 'en_US'), + german(localeCode: 'de', localeId: 'de_DE'); + + const SpeechRecognitionLanguage({ + required this.localeCode, + required this.localeId, + }); + + final String localeCode; + final String localeId; + + static SpeechRecognitionLanguage fromCode(String? code) { + if (code == null) return SpeechRecognitionLanguage.english; + return SpeechRecognitionLanguage.values.firstWhere( + (lang) => lang.localeCode == code, + orElse: () => SpeechRecognitionLanguage.english, + ); + } + + static SpeechRecognitionLanguage fromLocale(Locale? locale) { + if (locale == null) return SpeechRecognitionLanguage.english; + switch (locale.languageCode) { + case 'de': + return SpeechRecognitionLanguage.german; + default: + return SpeechRecognitionLanguage.english; + } + } +} diff --git a/app/lib/services/speech/speech_to_text_preferences.dart b/app/lib/services/speech/speech_to_text_preferences.dart new file mode 100644 index 000000000..8ac085012 --- /dev/null +++ b/app/lib/services/speech/speech_to_text_preferences.dart @@ -0,0 +1,25 @@ +import 'dart:ui'; + +import 'package:studyu_app/services/speech/speech_to_text_language.dart'; +import 'package:studyu_flutter_common/studyu_flutter_common.dart'; + +class SpeechToTextPreferences { + static const _languageKey = 'speech_to_text_language'; + + static Future preferredLanguage({ + Locale? fallbackLocale, + }) async { + final stored = await SecureStorage.read(_languageKey); + if (stored != null) return SpeechRecognitionLanguage.fromCode(stored); + if (fallbackLocale != null) { + return SpeechRecognitionLanguage.fromLocale(fallbackLocale); + } + return SpeechRecognitionLanguage.english; + } + + static Future setPreferredLanguage( + SpeechRecognitionLanguage language, + ) async { + await SecureStorage.write(_languageKey, language.localeCode); + } +} diff --git a/app/lib/services/speech/websocket_speech_controller_io.dart b/app/lib/services/speech/websocket_speech_controller_io.dart new file mode 100644 index 000000000..4d16bd756 --- /dev/null +++ b/app/lib/services/speech/websocket_speech_controller_io.dart @@ -0,0 +1,335 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/widgets.dart'; +import 'package:record/record.dart'; +import 'package:studyu_app/services/speech/speech_controller_models.dart'; +import 'package:studyu_app/services/speech/speech_to_text_language.dart'; +import 'package:studyu_app/services/speech/speech_to_text_preferences.dart'; +import 'package:web_socket_channel/io.dart'; +import 'package:web_socket_channel/web_socket_channel.dart'; + +class SpeechToTextController extends ValueNotifier { + SpeechToTextController({ + required this.onFinalTranscription, + required BuildContext context, + SpeechRecognitionLanguage initialLanguage = + SpeechRecognitionLanguage.english, + String? serverUrl, + }) : _language = initialLanguage, + _serverUrl = serverUrl ?? 'wss://stt.ibrahimozkan.dev', + // ignore: unused_field + _context = context, + _audioRecorder = AudioRecorder(), + super( + SpeechControllerState.initial( + language: initialLanguage, + supported: _platformSupported, + ), + ) { + _ensureInitialized(); + } + + final SpeechTranscriptCommit onFinalTranscription; + // ignore: unused_field + final BuildContext _context; + final AudioRecorder _audioRecorder; + final String _serverUrl; + + SpeechRecognitionLanguage _language; + String? _latestTranscript; + bool _isDisposed = false; + Future? _initialization; + WebSocketChannel? _wsChannel; + StreamSubscription? _audioStreamSubscription; + Timer? _silenceTimer; + bool _isListening = false; + + static const int _sampleRate = 16000; // Vosk typically uses 16kHz + + static bool get _platformSupported { + if (kIsWeb) return false; + return Platform.isAndroid || + Platform.isIOS || + Platform.isMacOS || + Platform.isWindows; + } + + static bool get isSupportedPlatform => _platformSupported; + + bool get isListening => value.status == SpeechLifecycleStatus.listening; + + SpeechRecognitionLanguage get language => _language; + + Future _ensureInitialized() { + _initialization ??= _initializeRecorder(); + return _initialization!; + } + + Future _initializeRecorder() async { + if (!_platformSupported) return false; + try { + final hasPermission = await _audioRecorder.hasPermission(); + return hasPermission; + } catch (e, stack) { + debugPrint('[WebSocketSpeech] init error: $e'); + debugPrint(stack.toString()); + return false; + } + } + + Future updateLanguage(SpeechRecognitionLanguage language) async { + if (_language == language) return; + _language = language; + value = value.copyWith(language: language); + await SpeechToTextPreferences.setPreferredLanguage(language); + } + + Future startListening() async { + if (!_platformSupported) { + _emitError(SpeechErrorType.general); + return false; + } + if (value.status == SpeechLifecycleStatus.listening || + value.status == SpeechLifecycleStatus.preparing) { + return false; + } + + value = value.copyWith( + status: SpeechLifecycleStatus.preparing, + clearError: true, + clearTranscript: true, + ); + + try { + final available = await _ensureInitialized(); + if (!available) { + _emitError(SpeechErrorType.microphonePermission); + return false; + } + + await _startWebSocketConnection(); + await _startAudioRecording(); + + _isListening = true; + value = value.copyWith( + status: SpeechLifecycleStatus.listening, + clearError: true, + ); + + return true; + } catch (e, stack) { + _emitError( + SpeechErrorType.general, + details: 'Failed to start speech input: $e', + stackTrace: stack, + ); + return false; + } + } + + Future _startWebSocketConnection() async { + try { + _wsChannel = IOWebSocketChannel.connect( + Uri.parse(_serverUrl), + ); + + // Send configuration with sample rate + final config = { + 'config': { + 'sample_rate': _sampleRate, + } + }; + _wsChannel!.sink.add(jsonEncode(config)); + + // Listen for transcription results + _wsChannel!.stream.listen( + (data) { + if (_isDisposed) return; + try { + final result = jsonDecode(data as String) as Map; + _handleTranscriptionResult(result); + } catch (e) { + debugPrint('[WebSocketSpeech] Error parsing response: $e'); + } + }, + onError: (error) { + if (_isDisposed) return; + _emitError( + SpeechErrorType.general, + details: 'WebSocket error: $error', + ); + }, + onDone: () { + if (_isDisposed) return; + }, + cancelOnError: false, + ); + } catch (e) { + throw Exception('Failed to connect to WebSocket: $e'); + } + } + + Future _startAudioRecording() async { + try { + final stream = await _audioRecorder.startStream( + const RecordConfig( + encoder: AudioEncoder.pcm16bits, + sampleRate: _sampleRate, + numChannels: 1, + autoGain: true, + echoCancel: true, + noiseSuppress: true, + ), + ); + + _audioStreamSubscription = stream.listen( + (audioData) { + if (_wsChannel != null && !_isDisposed && _isListening) { + _wsChannel!.sink.add(audioData); + _resetSilenceTimer(); + } + }, + onError: (error) { + if (_isDisposed) return; + _emitError( + SpeechErrorType.general, + details: 'Audio recording error: $error', + ); + }, + cancelOnError: false, + ); + } catch (e) { + throw Exception('Failed to start audio recording: $e'); + } + } + + void _resetSilenceTimer() { + _silenceTimer?.cancel(); + // No automatic stop on silence - user must manually stop + } + + void _handleTranscriptionResult(Map result) { + if (_isDisposed) return; + + // Vosk server returns partial results and final results + final text = result['text'] as String?; + final partial = result['partial'] as String?; + + if (partial != null && partial.isNotEmpty) { + // Show partial transcription (ongoing speech) + _latestTranscript = partial.trim(); + value = value.copyWith(partialTranscript: partial); + } else if (text != null) { + // Final transcription (silence detected or end of phrase) + final trimmed = text.trim(); + if (trimmed.isNotEmpty) { + // Commit the final result + _latestTranscript = trimmed; + _commitTranscript(); + // Keep listening - don't clear partial transcript immediately + // to avoid visual flicker between phrases + } else { + // Empty final result - just clear the partial transcript + value = value.copyWith(clearTranscript: true); + } + // Important: Reset to allow new partial results to show + _latestTranscript = null; + } + } + + Future stopListening() async { + if (!_isListening) return; + _isListening = false; + + try { + // Cancel silence timer + _silenceTimer?.cancel(); + + // Stop audio recording + await _audioStreamSubscription?.cancel(); + await _audioRecorder.stop(); + + // Send EOF to WebSocket + if (_wsChannel != null) { + _wsChannel!.sink.add(jsonEncode({'eof': 1})); + // Give server a moment to send final results + await Future.delayed(const Duration(milliseconds: 300)); + await _wsChannel!.sink.close(); + _wsChannel = null; + } + + value = value.copyWith( + status: SpeechLifecycleStatus.ready, + clearTranscript: true, + ); + } catch (e, stack) { + _emitError( + SpeechErrorType.general, + details: 'Failed to stop speech input: $e', + stackTrace: stack, + ); + } + } + + Future forceReset() async { + _isListening = false; + _silenceTimer?.cancel(); + try { + await _audioStreamSubscription?.cancel(); + await _audioRecorder.stop(); + if (_wsChannel != null) { + await _wsChannel!.sink.close(); + _wsChannel = null; + } + } catch (_) {} + value = value.copyWith( + status: SpeechLifecycleStatus.idle, + clearTranscript: true, + ); + } + + void _commitTranscript() { + final transcript = _latestTranscript; + if (transcript != null && transcript.isNotEmpty) { + onFinalTranscription(transcript); + // Don't clear immediately to avoid flicker - the next partial will replace it + // or we'll clear it when we get an empty final result + } + } + + void _emitError( + SpeechErrorType type, { + String? details, + StackTrace? stackTrace, + }) { + _isListening = false; + _latestTranscript = null; + debugPrint('[WebSocketSpeech] error=$type details=${details ?? ''}'); + if (stackTrace != null) { + debugPrint(stackTrace.toString()); + } + value = value.copyWith( + status: SpeechLifecycleStatus.error, + error: SpeechError(type, details: details), + clearTranscript: true, + ); + } + + @override + void dispose() { + _isDisposed = true; + _silenceTimer?.cancel(); + try { + unawaited(_audioStreamSubscription?.cancel()); + unawaited(_audioRecorder.stop()); + unawaited(_audioRecorder.dispose()); + unawaited(_wsChannel?.sink.close()); + } catch (_) { + // Ignore dispose errors + } + super.dispose(); + } +} diff --git a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart index be16a4cc2..76377418c 100644 --- a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart +++ b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart @@ -3,7 +3,11 @@ import 'dart:async'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:studyu_app/l10n/app_localizations.dart'; +import 'package:studyu_app/services/speech/speech_to_text_controller.dart'; +import 'package:studyu_app/services/speech/speech_to_text_language.dart'; +import 'package:studyu_app/services/speech/speech_to_text_preferences.dart'; import 'package:studyu_app/widgets/questionnaire/questions/question_widget.dart'; + import 'package:studyu_core/core.dart'; class FreeTextQuestionWidget extends QuestionWidget { @@ -27,7 +31,13 @@ class _FreeTextQuestionWidgetState extends State { bool _hasInteracted = false; bool _hasSubmitted = false; Timer? _debounceTimer; + Timer? _listeningRestartTimer; AutovalidateMode _autovalidateMode = AutovalidateMode.disabled; + SpeechToTextController? _speechController; + bool _speechPrefsRequested = false; + bool _speechPrefsInitialized = false; + bool _listeningRequested = false; + SpeechErrorType? _lastSpeechErrorType; @override void initState() { @@ -35,15 +45,350 @@ class _FreeTextQuestionWidgetState extends State { _focusNode.addListener(_onFocusChange); } + @override + void didChangeDependencies() { + super.didChangeDependencies(); + if (!_speechPrefsRequested && SpeechToTextController.isSupportedPlatform) { + _speechPrefsRequested = true; + _initSpeechPreferences(); + } + } + @override void dispose() { _textFieldController.dispose(); _focusNode.removeListener(_onFocusChange); _focusNode.dispose(); _debounceTimer?.cancel(); + _listeningRestartTimer?.cancel(); + _speechController?.removeListener(_onSpeechStateChanged); + _speechController?.dispose(); super.dispose(); } + Future _initSpeechPreferences() async { + final fallbackLocale = Localizations.maybeLocaleOf(context); + final language = await SpeechToTextPreferences.preferredLanguage( + fallbackLocale: fallbackLocale, + ); + if (!mounted) return; + setState(() { + _speechPrefsInitialized = true; + }); + _createSpeechController(language); + } + + void _createSpeechController(SpeechRecognitionLanguage language) { + _speechController?.removeListener(_onSpeechStateChanged); + _speechController?.dispose(); + _speechController = SpeechToTextController( + onFinalTranscription: _insertSpeechTranscript, + initialLanguage: language, + context: context, + ); + _speechController!.addListener(_onSpeechStateChanged); + _listeningRequested = false; + setState(() {}); + } + + void _onSpeechStateChanged() { + final controller = _speechController; + if (controller == null) return; + final state = controller.value; + final currentError = controller.value.error; + if (currentError != null && currentError.type != _lastSpeechErrorType) { + final loc = AppLocalizations.of(context)!; + final message = _mapSpeechErrorToMessage(currentError, loc); + if (message != null) { + ScaffoldMessenger.of( + context, + ).showSnackBar(SnackBar(content: Text(message))); + } + } + _lastSpeechErrorType = currentError?.type; + if (!mounted) return; + final isIdleOrReady = state.status == SpeechLifecycleStatus.idle || + state.status == SpeechLifecycleStatus.ready; + final hasError = state.error != null; + if (hasError && _listeningRequested) { + _setListeningRequested(false); + } + final shouldRestart = + _listeningRequested && isIdleOrReady && !hasError; + WidgetsBinding.instance.addPostFrameCallback((_) { + if (!mounted) return; + setState(() {}); + if (shouldRestart) { + _scheduleListeningRestart(); + } else if (state.error != null || !_listeningRequested) { + _listeningRestartTimer?.cancel(); + } + }); + } + + void _insertSpeechTranscript(String transcript) { + final trimmed = transcript.trim(); + if (trimmed.isEmpty) return; + final controller = _textFieldController; + final selection = controller.selection; + final baseOffset = selection.isValid + ? selection.extentOffset + : controller.text.length; + final currentText = controller.text; + final previousChar = baseOffset > 0 + ? currentText.substring(baseOffset - 1, baseOffset) + : ''; + final needsSpace = previousChar.trim().isNotEmpty; + final insertion = '${needsSpace ? ' ' : ''}$trimmed'; + final newText = currentText.replaceRange(baseOffset, baseOffset, insertion); + controller.value = controller.value.copyWith( + text: newText, + selection: TextSelection.collapsed(offset: baseOffset + insertion.length), + ); + _handleInteraction(); + _debouncedValidation(); + } + + String? _mapSpeechErrorToMessage(SpeechError error, AppLocalizations loc) { + final details = error.details?.toLowerCase(); + final isPermissionIssue = details?.contains('permission') == true; + if (error.type == SpeechErrorType.microphonePermission && + isPermissionIssue) { + return loc.speech_to_text_error_permission; + } + return loc.speech_to_text_error_general; + } + + void _toggleSpeechListening() { + if (_listeningRequested) { + _stopListeningIntent(); + } else { + _setListeningRequested(true); + unawaited(_startListeningIfRequested()); + } + } + + void _stopListeningIntent() { + _setListeningRequested(false); + _listeningRestartTimer?.cancel(); + unawaited(_stopActiveListening()); + } + + void _setListeningRequested(bool requested) { + if (_listeningRequested == requested) return; + _listeningRequested = requested; + WidgetsBinding.instance.addPostFrameCallback((_) { + if (!mounted) return; + setState(() {}); + }); + } + + void _scheduleListeningRestart() { + if (!_listeningRequested) return; + final controller = _speechController; + if (controller == null) return; + final state = controller.value; + final readyForRestart = state.error == null && + (state.status == SpeechLifecycleStatus.idle || + state.status == SpeechLifecycleStatus.ready); + if (!readyForRestart) { + return; + } + _listeningRestartTimer?.cancel(); + _listeningRestartTimer = Timer(const Duration(milliseconds: 250), () { + if (!_listeningRequested) return; + final restartController = _speechController; + if (restartController == null) return; + final restartState = restartController.value; + final canRestart = restartState.error == null && + (restartState.status == SpeechLifecycleStatus.idle || + restartState.status == SpeechLifecycleStatus.ready); + if (!canRestart) { + return; + } + unawaited(_startListeningIfRequested()); + }); + } + + Future _startListeningIfRequested() async { + final controller = _speechController; + if (_listeningRequested == false || controller == null) { + return; + } + + var state = controller.value; + var status = state.status; + + if (status == SpeechLifecycleStatus.listening || + status == SpeechLifecycleStatus.preparing || + status == SpeechLifecycleStatus.unavailable) { + return; + } + + if (status == SpeechLifecycleStatus.error) { + await controller.forceReset(); + state = controller.value; + status = state.status; + } + + final canStart = + status == SpeechLifecycleStatus.idle || status == SpeechLifecycleStatus.ready; + if (!canStart) { + return; + } + + await controller.startListening(); + } + + Future _stopActiveListening() async { + final controller = _speechController; + if (controller == null) return; + await controller.stopListening(); + } + + Widget _buildSpeechAssistant(ThemeData theme, AppLocalizations loc) { + if (!SpeechToTextController.isSupportedPlatform) { + return _speechInfoCard(loc.speech_to_text_unsupported, theme); + } + if (!_speechPrefsInitialized) { + return const Padding( + padding: EdgeInsets.symmetric(vertical: 12), + child: LinearProgressIndicator(), + ); + } + final controller = _speechController; + if (controller == null) return const SizedBox.shrink(); + + return ValueListenableBuilder( + valueListenable: controller, + builder: (context, state, _) { + final isListening = state.status == SpeechLifecycleStatus.listening; + final isPreparing = state.status == SpeechLifecycleStatus.preparing; + final statusLabel = isListening + ? loc.speech_to_text_listening + : loc.speech_to_text_press_to_speak; + return Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Center( + child: _buildSpeechToggleButton( + theme, + loc, + isListening, + isPreparing, + ), + ), + const SizedBox(height: 12), + Center( + child: Text( + statusLabel, + style: theme.textTheme.bodyMedium, + textAlign: TextAlign.center, + ), + ), + if (isPreparing) ...[ + const LinearProgressIndicator(), + const SizedBox(height: 4), + Text( + loc.speech_to_text_preparing, + style: theme.textTheme.bodySmall, + ), + ], + if ((state.partialTranscript ?? '').isNotEmpty) ...[ + const SizedBox(height: 8), + Container( + padding: const EdgeInsets.all(12), + decoration: BoxDecoration( + color: theme.colorScheme.surfaceContainerHighest, + borderRadius: BorderRadius.circular(8), + ), + child: Text( + '${loc.speech_to_text_live_caption_hint}: ${state.partialTranscript!}', + style: theme.textTheme.bodyMedium, + ), + ), + ], + if (state.status == SpeechLifecycleStatus.error && + state.error != null) + Padding( + padding: const EdgeInsets.only(top: 8), + child: Text( + _mapSpeechErrorToMessage(state.error!, loc) ?? + loc.speech_to_text_error_general, + style: theme.textTheme.bodySmall?.copyWith( + color: theme.colorScheme.error, + ), + ), + ), + ], + ); + }, + ); + } + + Widget _buildSpeechToggleButton( + ThemeData theme, + AppLocalizations loc, + bool isListening, + bool isPreparing, + ) { + final disabled = isPreparing; + final active = _listeningRequested && !disabled; + final circleColor = disabled + ? theme.colorScheme.surfaceContainerHighest + : active + ? theme.colorScheme.primary + : theme.colorScheme.surface; + final iconColor = disabled + ? theme.colorScheme.onSurface.withValues(alpha: 0.38) + : active + ? theme.colorScheme.onPrimary + : theme.colorScheme.primary; + final borderColor = active + ? theme.colorScheme.primary + : theme.colorScheme.outline; + final iconData = active || isListening ? Icons.pause : Icons.mic; + + return GestureDetector( + behavior: HitTestBehavior.opaque, + onTap: disabled ? null : _toggleSpeechListening, + child: AnimatedContainer( + duration: const Duration(milliseconds: 150), + width: 64, + height: 64, + decoration: BoxDecoration( + color: circleColor, + shape: BoxShape.circle, + border: Border.all(color: borderColor, width: 2), + boxShadow: active + ? [ + BoxShadow( + color: theme.colorScheme.primary.withValues(alpha: 0.35), + blurRadius: 12, + spreadRadius: 1, + ), + ] + : null, + ), + child: Icon( + iconData, + color: iconColor, + size: 30, + ), + ), + ); + } + + Widget _speechInfoCard(String text, ThemeData theme) { + return Card( + color: theme.colorScheme.surfaceContainerHighest, + child: Padding( + padding: const EdgeInsets.all(12), + child: Text(text, style: theme.textTheme.bodySmall), + ), + ); + } + void _onFocusChange() { if (_focusNode.hasFocus) { _ensureTextFieldVisible(); @@ -137,6 +482,9 @@ class _FreeTextQuestionWidgetState extends State { @override Widget build(BuildContext context) { final question = widget.question; + final loc = AppLocalizations.of(context)!; + final theme = Theme.of(context); + final speechSection = _buildSpeechAssistant(theme, loc); return Column( crossAxisAlignment: CrossAxisAlignment.stretch, children: [ @@ -210,13 +558,17 @@ class _FreeTextQuestionWidgetState extends State { }, ), const SizedBox(height: 16), + if (speechSection is SizedBox) + speechSection + else + Padding( + padding: const EdgeInsets.only(bottom: 16), + child: speechSection, + ), Row( mainAxisAlignment: MainAxisAlignment.end, children: [ - OutlinedButton( - onPressed: _handleSubmit, - child: Text(AppLocalizations.of(context)!.submit), - ), + OutlinedButton(onPressed: _handleSubmit, child: Text(loc.submit)), ], ), ], diff --git a/app/pubspec.yaml b/app/pubspec.yaml index 385bb3782..3f10bcfe5 100644 --- a/app/pubspec.yaml +++ b/app/pubspec.yaml @@ -61,6 +61,7 @@ dependencies: url_launcher: ^6.3.2 uuid: ^4.5.1 wakelock_plus: ^1.4.0 + web_socket_channel: ^3.0.2 xml: ^6.6.1 #dependency_overrides: From 7c1b7b73df62c462df267498b29970e2b577298d Mon Sep 17 00:00:00 2001 From: ibrahimozkan Date: Thu, 20 Nov 2025 12:17:19 +0300 Subject: [PATCH 02/16] feat: Implement speech-to-text WebSocket reconnection --- app/ios/Podfile.lock | 28 ++- .../websocket_speech_controller_io.dart | 141 ++++++++---- .../questions/free_text_question_widget.dart | 203 ++++++------------ pubspec.lock | 16 +- 4 files changed, 184 insertions(+), 204 deletions(-) diff --git a/app/ios/Podfile.lock b/app/ios/Podfile.lock index f4327b174..3aedfa588 100644 --- a/app/ios/Podfile.lock +++ b/app/ios/Podfile.lock @@ -21,6 +21,8 @@ PODS: - just_audio (0.0.1): - Flutter - FlutterMacOS + - objective_c (0.0.1): + - Flutter - package_info_plus (0.4.5): - Flutter - path_provider_foundation (0.0.1): @@ -30,11 +32,11 @@ PODS: - Flutter - record_ios (1.1.0): - Flutter - - Sentry/HybridSDK (8.52.1) - - sentry_flutter (9.6.0): + - Sentry/HybridSDK (8.56.2) + - sentry_flutter (9.7.0): - Flutter - FlutterMacOS - - Sentry/HybridSDK (= 8.52.1) + - Sentry/HybridSDK (= 8.56.2) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -64,6 +66,7 @@ DEPENDENCIES: - flutter_timezone (from `.symlinks/plugins/flutter_timezone/ios`) - flutter_web_auth_2 (from `.symlinks/plugins/flutter_web_auth_2/ios`) - just_audio (from `.symlinks/plugins/just_audio/darwin`) + - objective_c (from `.symlinks/plugins/objective_c/ios`) - package_info_plus (from `.symlinks/plugins/package_info_plus/ios`) - path_provider_foundation (from `.symlinks/plugins/path_provider_foundation/darwin`) - permission_handler_apple (from `.symlinks/plugins/permission_handler_apple/ios`) @@ -103,6 +106,8 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/flutter_web_auth_2/ios" just_audio: :path: ".symlinks/plugins/just_audio/darwin" + objective_c: + :path: ".symlinks/plugins/objective_c/ios" package_info_plus: :path: ".symlinks/plugins/package_info_plus/ios" path_provider_foundation: @@ -129,7 +134,7 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: app_links: 3dbc685f76b1693c66a6d9dd1e9ab6f73d97dc0a audio_session: 9bb7f6c970f21241b19f5a3658097ae459681ba0 - camera_avfoundation: be3be85408cd4126f250386828e9b1dfa40ab436 + camera_avfoundation: 5675ca25298b6f81fa0a325188e7df62cc217741 Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467 flutter_file_dialog: ca8d7fbd1772d4f0c2777b4ab20a7787ef4e7dd8 flutter_local_notifications: a5a732f069baa862e728d839dd2ebb904737effb @@ -138,18 +143,19 @@ SPEC CHECKSUMS: flutter_timezone: 7c838e17ffd4645d261e87037e5bebf6d38fe544 flutter_web_auth_2: 5c8d9dcd7848b5a9efb086d24e7a9adcae979c80 just_audio: 4e391f57b79cad2b0674030a00453ca5ce817eed + objective_c: 89e720c30d716b036faf9c9684022048eee1eee2 package_info_plus: af8e2ca6888548050f16fa2f1938db7b5a5df499 - path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 + path_provider_foundation: bb55f6dbba17d0dccd6737fe6f7f34fbd0376880 permission_handler_apple: 4ed2196e43d0651e8ff7ca3483a069d469701f2d record_ios: f75fa1d57f840012775c0e93a38a7f3ceea1a374 - Sentry: 2cbbe3592f30050c60e916c63c7f5a2fa584005e - sentry_flutter: 9b1e3e6934cb7040ffb71383c95a35f73523180c - shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 + Sentry: b53951377b78e21a734f5dc8318e333dbfc682d7 + sentry_flutter: 7f88e12d1882ec6e648644a7c0ff5b1466f07f99 + shared_preferences_foundation: 7036424c3d8ec98dfe75ff1667cb0cd531ec82bb sqflite_darwin: 20b2a3a3b70e43edae938624ce550a3cbf66a3d0 - url_launcher_ios: 694010445543906933d732453a59da0a173ae33d - video_player_avfoundation: 2cef49524dd1f16c5300b9cd6efd9611ce03639b + url_launcher_ios: 7a95fa5b60cc718a708b8f2966718e93db0cef1b + video_player_avfoundation: dd410b52df6d2466a42d28550e33e4146928280a wakelock_plus: e29112ab3ef0b318e58cfa5c32326458be66b556 - webview_flutter_wkwebview: 1821ceac936eba6f7984d89a9f3bcb4dea99ebb2 + webview_flutter_wkwebview: 8ebf4fded22593026f7dbff1fbff31ea98573c8d PODFILE CHECKSUM: b00a565d52284bcdd04d643f399531a35b02c2c4 diff --git a/app/lib/services/speech/websocket_speech_controller_io.dart b/app/lib/services/speech/websocket_speech_controller_io.dart index 4d16bd756..aaebb30f2 100644 --- a/app/lib/services/speech/websocket_speech_controller_io.dart +++ b/app/lib/services/speech/websocket_speech_controller_io.dart @@ -3,7 +3,6 @@ import 'dart:convert'; import 'dart:io'; import 'package:flutter/foundation.dart'; -import 'package:flutter/widgets.dart'; import 'package:record/record.dart'; import 'package:studyu_app/services/speech/speech_controller_models.dart'; import 'package:studyu_app/services/speech/speech_to_text_language.dart'; @@ -14,27 +13,22 @@ import 'package:web_socket_channel/web_socket_channel.dart'; class SpeechToTextController extends ValueNotifier { SpeechToTextController({ required this.onFinalTranscription, - required BuildContext context, SpeechRecognitionLanguage initialLanguage = SpeechRecognitionLanguage.english, String? serverUrl, - }) : _language = initialLanguage, - _serverUrl = serverUrl ?? 'wss://stt.ibrahimozkan.dev', - // ignore: unused_field - _context = context, - _audioRecorder = AudioRecorder(), - super( - SpeechControllerState.initial( - language: initialLanguage, - supported: _platformSupported, - ), - ) { + }) : _language = initialLanguage, + _serverUrl = serverUrl ?? 'wss://stt.ibrahimozkan.dev', + _audioRecorder = AudioRecorder(), + super( + SpeechControllerState.initial( + language: initialLanguage, + supported: _platformSupported, + ), + ) { _ensureInitialized(); } final SpeechTranscriptCommit onFinalTranscription; - // ignore: unused_field - final BuildContext _context; final AudioRecorder _audioRecorder; final String _serverUrl; @@ -44,8 +38,8 @@ class SpeechToTextController extends ValueNotifier { Future? _initialization; WebSocketChannel? _wsChannel; StreamSubscription? _audioStreamSubscription; - Timer? _silenceTimer; bool _isListening = false; + bool _isReconnecting = false; static const int _sampleRate = 16000; // Vosk typically uses 16kHz @@ -88,12 +82,14 @@ class SpeechToTextController extends ValueNotifier { } Future startListening() async { + debugPrint('[WebSocketSpeech] startListening requested'); if (!_platformSupported) { _emitError(SpeechErrorType.general); return false; } if (value.status == SpeechLifecycleStatus.listening || value.status == SpeechLifecycleStatus.preparing) { + debugPrint('[WebSocketSpeech] Already listening or preparing'); return false; } @@ -110,17 +106,19 @@ class SpeechToTextController extends ValueNotifier { return false; } + _isListening = true; await _startWebSocketConnection(); await _startAudioRecording(); - _isListening = true; value = value.copyWith( status: SpeechLifecycleStatus.listening, clearError: true, ); + debugPrint('[WebSocketSpeech] Listening started successfully'); return true; } catch (e, stack) { + _isListening = false; _emitError( SpeechErrorType.general, details: 'Failed to start speech input: $e', @@ -132,17 +130,15 @@ class SpeechToTextController extends ValueNotifier { Future _startWebSocketConnection() async { try { - _wsChannel = IOWebSocketChannel.connect( - Uri.parse(_serverUrl), - ); + debugPrint('[WebSocketSpeech] Connecting to $_serverUrl...'); + _wsChannel = IOWebSocketChannel.connect(Uri.parse(_serverUrl)); // Send configuration with sample rate final config = { - 'config': { - 'sample_rate': _sampleRate, - } + 'config': {'sample_rate': _sampleRate}, }; _wsChannel!.sink.add(jsonEncode(config)); + debugPrint('[WebSocketSpeech] WebSocket connected, config sent'); // Listen for transcription results _wsChannel!.stream.listen( @@ -157,13 +153,13 @@ class SpeechToTextController extends ValueNotifier { }, onError: (error) { if (_isDisposed) return; - _emitError( - SpeechErrorType.general, - details: 'WebSocket error: $error', - ); + debugPrint('[WebSocketSpeech] WebSocket error: $error'); + _handleConnectionLoss(); }, onDone: () { if (_isDisposed) return; + debugPrint('[WebSocketSpeech] WebSocket closed by server'); + _handleConnectionLoss(); }, cancelOnError: false, ); @@ -172,8 +168,54 @@ class SpeechToTextController extends ValueNotifier { } } + void _handleConnectionLoss() { + debugPrint( + '[WebSocketSpeech] Connection loss detected. Listening: $_isListening, Reconnecting: $_isReconnecting', + ); + if (_isListening && !_isReconnecting) { + _reconnect(); + } + } + + Future _reconnect() async { + if (_isReconnecting || _isDisposed) return; + _isReconnecting = true; + debugPrint('[WebSocketSpeech] Attempting to reconnect...'); + + try { + // Close existing connection if any + try { + await _wsChannel?.sink.close(); + } catch (_) {} + _wsChannel = null; + + // Wait a bit before reconnecting + await Future.delayed(const Duration(milliseconds: 500)); + + if (!_isListening) { + debugPrint('[WebSocketSpeech] Reconnect aborted: not listening'); + _isReconnecting = false; + return; + } + + await _startWebSocketConnection(); + debugPrint('[WebSocketSpeech] Reconnected successfully'); + } catch (e) { + debugPrint('[WebSocketSpeech] Reconnection failed: $e'); + // If reconnection fails, we might want to stop listening or try again later + // For now, let's stop to avoid infinite loops if server is down + _emitError( + SpeechErrorType.general, + details: 'Connection lost. Please try again.', + ); + } finally { + _isReconnecting = false; + } + } + Future _startAudioRecording() async { try { + debugPrint('[WebSocketSpeech] Starting audio recording...'); final stream = await _audioRecorder.startStream( const RecordConfig( encoder: AudioEncoder.pcm16bits, @@ -188,8 +230,13 @@ class SpeechToTextController extends ValueNotifier { _audioStreamSubscription = stream.listen( (audioData) { if (_wsChannel != null && !_isDisposed && _isListening) { - _wsChannel!.sink.add(audioData); - _resetSilenceTimer(); + try { + _wsChannel!.sink.add(audioData); + } catch (e) { + // If send fails, it might be because connection is closed + // The onDone/onError of WebSocket should handle this, but just in case + debugPrint('[WebSocketSpeech] Failed to send audio: $e'); + } } }, onError: (error) { @@ -201,16 +248,12 @@ class SpeechToTextController extends ValueNotifier { }, cancelOnError: false, ); + debugPrint('[WebSocketSpeech] Audio recording started'); } catch (e) { throw Exception('Failed to start audio recording: $e'); } } - void _resetSilenceTimer() { - _silenceTimer?.cancel(); - // No automatic stop on silence - user must manually stop - } - void _handleTranscriptionResult(Map result) { if (_isDisposed) return; @@ -222,10 +265,12 @@ class SpeechToTextController extends ValueNotifier { // Show partial transcription (ongoing speech) _latestTranscript = partial.trim(); value = value.copyWith(partialTranscript: partial); + // debugPrint('[WebSocketSpeech] Partial: $partial'); // Uncomment for very verbose logs } else if (text != null) { // Final transcription (silence detected or end of phrase) final trimmed = text.trim(); if (trimmed.isNotEmpty) { + debugPrint('[WebSocketSpeech] Final transcript: $trimmed'); // Commit the final result _latestTranscript = trimmed; _commitTranscript(); @@ -241,23 +286,27 @@ class SpeechToTextController extends ValueNotifier { } Future stopListening() async { + debugPrint('[WebSocketSpeech] stopListening requested'); if (!_isListening) return; _isListening = false; try { - // Cancel silence timer - _silenceTimer?.cancel(); - // Stop audio recording await _audioStreamSubscription?.cancel(); await _audioRecorder.stop(); + debugPrint('[WebSocketSpeech] Audio recording stopped'); // Send EOF to WebSocket if (_wsChannel != null) { - _wsChannel!.sink.add(jsonEncode({'eof': 1})); - // Give server a moment to send final results - await Future.delayed(const Duration(milliseconds: 300)); - await _wsChannel!.sink.close(); + try { + _wsChannel!.sink.add(jsonEncode({'eof': 1})); + // Give server a moment to send final results + await Future.delayed(const Duration(milliseconds: 300)); + await _wsChannel!.sink.close(); + debugPrint('[WebSocketSpeech] WebSocket closed cleanly'); + } catch (_) { + // Ignore errors during closing + } _wsChannel = null; } @@ -275,8 +324,8 @@ class SpeechToTextController extends ValueNotifier { } Future forceReset() async { + debugPrint('[WebSocketSpeech] forceReset requested'); _isListening = false; - _silenceTimer?.cancel(); try { await _audioStreamSubscription?.cancel(); await _audioRecorder.stop(); @@ -307,6 +356,13 @@ class SpeechToTextController extends ValueNotifier { }) { _isListening = false; _latestTranscript = null; + + // Ensure recorder is stopped on error + try { + _audioStreamSubscription?.cancel(); + _audioRecorder.stop(); + } catch (_) {} + debugPrint('[WebSocketSpeech] error=$type details=${details ?? ''}'); if (stackTrace != null) { debugPrint(stackTrace.toString()); @@ -321,7 +377,6 @@ class SpeechToTextController extends ValueNotifier { @override void dispose() { _isDisposed = true; - _silenceTimer?.cancel(); try { unawaited(_audioStreamSubscription?.cancel()); unawaited(_audioRecorder.stop()); diff --git a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart index 76377418c..63d97a1f9 100644 --- a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart +++ b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart @@ -84,7 +84,6 @@ class _FreeTextQuestionWidgetState extends State { _speechController = SpeechToTextController( onFinalTranscription: _insertSpeechTranscript, initialLanguage: language, - context: context, ); _speechController!.addListener(_onSpeechStateChanged); _listeningRequested = false; @@ -107,22 +106,14 @@ class _FreeTextQuestionWidgetState extends State { } _lastSpeechErrorType = currentError?.type; if (!mounted) return; - final isIdleOrReady = state.status == SpeechLifecycleStatus.idle || - state.status == SpeechLifecycleStatus.ready; + final hasError = state.error != null; if (hasError && _listeningRequested) { _setListeningRequested(false); } - final shouldRestart = - _listeningRequested && isIdleOrReady && !hasError; WidgetsBinding.instance.addPostFrameCallback((_) { if (!mounted) return; setState(() {}); - if (shouldRestart) { - _scheduleListeningRestart(); - } else if (state.error != null || !_listeningRequested) { - _listeningRestartTimer?.cancel(); - } }); } @@ -170,7 +161,6 @@ class _FreeTextQuestionWidgetState extends State { void _stopListeningIntent() { _setListeningRequested(false); - _listeningRestartTimer?.cancel(); unawaited(_stopActiveListening()); } @@ -183,33 +173,6 @@ class _FreeTextQuestionWidgetState extends State { }); } - void _scheduleListeningRestart() { - if (!_listeningRequested) return; - final controller = _speechController; - if (controller == null) return; - final state = controller.value; - final readyForRestart = state.error == null && - (state.status == SpeechLifecycleStatus.idle || - state.status == SpeechLifecycleStatus.ready); - if (!readyForRestart) { - return; - } - _listeningRestartTimer?.cancel(); - _listeningRestartTimer = Timer(const Duration(milliseconds: 250), () { - if (!_listeningRequested) return; - final restartController = _speechController; - if (restartController == null) return; - final restartState = restartController.value; - final canRestart = restartState.error == null && - (restartState.status == SpeechLifecycleStatus.idle || - restartState.status == SpeechLifecycleStatus.ready); - if (!canRestart) { - return; - } - unawaited(_startListeningIfRequested()); - }); - } - Future _startListeningIfRequested() async { final controller = _speechController; if (_listeningRequested == false || controller == null) { @@ -232,7 +195,8 @@ class _FreeTextQuestionWidgetState extends State { } final canStart = - status == SpeechLifecycleStatus.idle || status == SpeechLifecycleStatus.ready; + status == SpeechLifecycleStatus.idle || + status == SpeechLifecycleStatus.ready; if (!canStart) { return; } @@ -248,13 +212,10 @@ class _FreeTextQuestionWidgetState extends State { Widget _buildSpeechAssistant(ThemeData theme, AppLocalizations loc) { if (!SpeechToTextController.isSupportedPlatform) { - return _speechInfoCard(loc.speech_to_text_unsupported, theme); + return const SizedBox.shrink(); } if (!_speechPrefsInitialized) { - return const Padding( - padding: EdgeInsets.symmetric(vertical: 12), - child: LinearProgressIndicator(), - ); + return const SizedBox(height: 2, child: LinearProgressIndicator()); } final controller = _speechController; if (controller == null) return const SizedBox.shrink(); @@ -262,56 +223,44 @@ class _FreeTextQuestionWidgetState extends State { return ValueListenableBuilder( valueListenable: controller, builder: (context, state, _) { - final isListening = state.status == SpeechLifecycleStatus.listening; final isPreparing = state.status == SpeechLifecycleStatus.preparing; - final statusLabel = isListening - ? loc.speech_to_text_listening - : loc.speech_to_text_press_to_speak; + + final hasError = + state.status == SpeechLifecycleStatus.error && state.error != null; + return Column( crossAxisAlignment: CrossAxisAlignment.stretch, children: [ - Center( - child: _buildSpeechToggleButton( - theme, - loc, - isListening, - isPreparing, - ), - ), - const SizedBox(height: 12), - Center( - child: Text( - statusLabel, - style: theme.textTheme.bodyMedium, - textAlign: TextAlign.center, - ), - ), - if (isPreparing) ...[ - const LinearProgressIndicator(), - const SizedBox(height: 4), - Text( - loc.speech_to_text_preparing, - style: theme.textTheme.bodySmall, + if (isPreparing) + const Padding( + padding: EdgeInsets.only(bottom: 8), + child: LinearProgressIndicator(), ), - ], - if ((state.partialTranscript ?? '').isNotEmpty) ...[ - const SizedBox(height: 8), + if ((state.partialTranscript ?? '').isNotEmpty) Container( - padding: const EdgeInsets.all(12), + margin: const EdgeInsets.only(bottom: 8), + padding: const EdgeInsets.symmetric( + horizontal: 12, + vertical: 8, + ), decoration: BoxDecoration( - color: theme.colorScheme.surfaceContainerHighest, + color: theme.colorScheme.surfaceContainerHighest.withValues( + alpha: 0.5, + ), borderRadius: BorderRadius.circular(8), + border: Border.all(color: theme.colorScheme.outlineVariant), ), child: Text( - '${loc.speech_to_text_live_caption_hint}: ${state.partialTranscript!}', - style: theme.textTheme.bodyMedium, + state.partialTranscript!, + style: theme.textTheme.bodyMedium?.copyWith( + fontStyle: FontStyle.italic, + color: theme.colorScheme.onSurfaceVariant, + ), ), ), - ], - if (state.status == SpeechLifecycleStatus.error && - state.error != null) + if (hasError) Padding( - padding: const EdgeInsets.only(top: 8), + padding: const EdgeInsets.only(bottom: 8), child: Text( _mapSpeechErrorToMessage(state.error!, loc) ?? loc.speech_to_text_error_general, @@ -326,66 +275,20 @@ class _FreeTextQuestionWidgetState extends State { ); } - Widget _buildSpeechToggleButton( - ThemeData theme, - AppLocalizations loc, - bool isListening, - bool isPreparing, - ) { - final disabled = isPreparing; - final active = _listeningRequested && !disabled; - final circleColor = disabled - ? theme.colorScheme.surfaceContainerHighest - : active - ? theme.colorScheme.primary - : theme.colorScheme.surface; - final iconColor = disabled - ? theme.colorScheme.onSurface.withValues(alpha: 0.38) - : active - ? theme.colorScheme.onPrimary - : theme.colorScheme.primary; - final borderColor = active - ? theme.colorScheme.primary - : theme.colorScheme.outline; - final iconData = active || isListening ? Icons.pause : Icons.mic; - - return GestureDetector( - behavior: HitTestBehavior.opaque, - onTap: disabled ? null : _toggleSpeechListening, - child: AnimatedContainer( - duration: const Duration(milliseconds: 150), - width: 64, - height: 64, - decoration: BoxDecoration( - color: circleColor, - shape: BoxShape.circle, - border: Border.all(color: borderColor, width: 2), - boxShadow: active - ? [ - BoxShadow( - color: theme.colorScheme.primary.withValues(alpha: 0.35), - blurRadius: 12, - spreadRadius: 1, - ), - ] - : null, - ), - child: Icon( - iconData, - color: iconColor, - size: 30, - ), - ), - ); - } - - Widget _speechInfoCard(String text, ThemeData theme) { - return Card( - color: theme.colorScheme.surfaceContainerHighest, - child: Padding( - padding: const EdgeInsets.all(12), - child: Text(text, style: theme.textTheme.bodySmall), - ), + Widget _buildMicButton(ThemeData theme, bool isListening, bool isPreparing) { + final color = isListening + ? theme.colorScheme.error + : isPreparing + ? theme.colorScheme.outline + : theme.colorScheme.primary; + + return IconButton( + onPressed: isPreparing ? null : _toggleSpeechListening, + icon: isListening + ? const Icon(Icons.stop_circle_outlined) + : const Icon(Icons.mic), + color: color, + tooltip: isListening ? 'Stop listening' : 'Start listening', ); } @@ -556,13 +459,29 @@ class _FreeTextQuestionWidgetState extends State { } } }, + decoration: InputDecoration( + suffixIcon: + SpeechToTextController.isSupportedPlatform && + _speechPrefsInitialized + ? ValueListenableBuilder( + valueListenable: _speechController!, + builder: (context, state, _) { + return _buildMicButton( + theme, + state.status == SpeechLifecycleStatus.listening, + state.status == SpeechLifecycleStatus.preparing, + ); + }, + ) + : null, + ), ), const SizedBox(height: 16), if (speechSection is SizedBox) speechSection else Padding( - padding: const EdgeInsets.only(bottom: 16), + padding: const EdgeInsets.only(bottom: 8), child: speechSection, ), Row( diff --git a/pubspec.lock b/pubspec.lock index 7e535dbfe..35cff9aaf 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -1177,10 +1177,10 @@ packages: dependency: transitive description: name: meta - sha256: e3641ec5d63ebf0d9b41bd43201a66e3fc79a65db5f61fc181f04cd27aab950c + sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394" url: "https://pub.dev" source: hosted - version: "1.16.0" + version: "1.17.0" mime: dependency: transitive description: @@ -2038,26 +2038,26 @@ packages: dependency: transitive description: name: test - sha256: "65e29d831719be0591f7b3b1a32a3cda258ec98c58c7b25f7b84241bc31215bb" + sha256: "75906bf273541b676716d1ca7627a17e4c4070a3a16272b7a3dc7da3b9f3f6b7" url: "https://pub.dev" source: hosted - version: "1.26.2" + version: "1.26.3" test_api: dependency: transitive description: name: test_api - sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00" + sha256: ab2726c1a94d3176a45960b6234466ec367179b87dd74f1611adb1f3b5fb9d55 url: "https://pub.dev" source: hosted - version: "0.7.6" + version: "0.7.7" test_core: dependency: transitive description: name: test_core - sha256: "80bf5a02b60af04b09e14f6fe68b921aad119493e26e490deaca5993fef1b05a" + sha256: "0cc24b5ff94b38d2ae73e1eb43cc302b77964fbf67abad1e296025b78deb53d0" url: "https://pub.dev" source: hosted - version: "0.6.11" + version: "0.6.12" timeline_tile: dependency: transitive description: From ea0facd09acdcf5025d477257c291f604d4824a3 Mon Sep 17 00:00:00 2001 From: ibrahimozkn <51259479+ibrahimozkn@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:21:03 +0000 Subject: [PATCH 03/16] chore: Apply static analysis changes --- pubspec.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pubspec.lock b/pubspec.lock index 35cff9aaf..7e535dbfe 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -1177,10 +1177,10 @@ packages: dependency: transitive description: name: meta - sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394" + sha256: e3641ec5d63ebf0d9b41bd43201a66e3fc79a65db5f61fc181f04cd27aab950c url: "https://pub.dev" source: hosted - version: "1.17.0" + version: "1.16.0" mime: dependency: transitive description: @@ -2038,26 +2038,26 @@ packages: dependency: transitive description: name: test - sha256: "75906bf273541b676716d1ca7627a17e4c4070a3a16272b7a3dc7da3b9f3f6b7" + sha256: "65e29d831719be0591f7b3b1a32a3cda258ec98c58c7b25f7b84241bc31215bb" url: "https://pub.dev" source: hosted - version: "1.26.3" + version: "1.26.2" test_api: dependency: transitive description: name: test_api - sha256: ab2726c1a94d3176a45960b6234466ec367179b87dd74f1611adb1f3b5fb9d55 + sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00" url: "https://pub.dev" source: hosted - version: "0.7.7" + version: "0.7.6" test_core: dependency: transitive description: name: test_core - sha256: "0cc24b5ff94b38d2ae73e1eb43cc302b77964fbf67abad1e296025b78deb53d0" + sha256: "80bf5a02b60af04b09e14f6fe68b921aad119493e26e490deaca5993fef1b05a" url: "https://pub.dev" source: hosted - version: "0.6.12" + version: "0.6.11" timeline_tile: dependency: transitive description: From 444d804a21c45f74ea2b5ccc066ddcb424fc9618 Mon Sep 17 00:00:00 2001 From: ibrahimozkan Date: Thu, 20 Nov 2025 14:14:39 +0300 Subject: [PATCH 04/16] refactor: remove speech-to-text language selection and persistence functionality --- app/lib/screens/study/dashboard/settings.dart | 89 ------------------- .../speech/speech_controller_models.dart | 12 +-- .../speech/speech_to_text_language.dart | 32 ------- .../speech/speech_to_text_preferences.dart | 25 ------ .../websocket_speech_controller_io.dart | 24 +---- .../questions/free_text_question_widget.dart | 33 ++----- 6 files changed, 10 insertions(+), 205 deletions(-) delete mode 100644 app/lib/services/speech/speech_to_text_language.dart delete mode 100644 app/lib/services/speech/speech_to_text_preferences.dart diff --git a/app/lib/screens/study/dashboard/settings.dart b/app/lib/screens/study/dashboard/settings.dart index 495a7742b..cc2a7b29c 100644 --- a/app/lib/screens/study/dashboard/settings.dart +++ b/app/lib/screens/study/dashboard/settings.dart @@ -6,8 +6,6 @@ import 'package:provider/provider.dart'; import 'package:studyu_app/l10n/app_localizations.dart'; import 'package:studyu_app/models/app_state.dart'; import 'package:studyu_app/routes.dart'; -import 'package:studyu_app/services/speech/speech_to_text_language.dart'; -import 'package:studyu_app/services/speech/speech_to_text_preferences.dart'; import 'package:studyu_app/util/app_analytics.dart'; import 'package:studyu_app/util/fitbit_handler.dart'; import 'package:studyu_app/util/localization.dart'; @@ -26,9 +24,6 @@ class _SettingsState extends State { Locale? _selectedValue; bool? _analyticsValue; StudySubject? subject; - bool _speechPrefsRequested = false; - bool _speechPrefsLoading = true; - SpeechRecognitionLanguage _speechLanguage = SpeechRecognitionLanguage.english; @override void initState() { @@ -38,34 +33,6 @@ class _SettingsState extends State { subject = context.read().activeSubject; } - @override - void didChangeDependencies() { - super.didChangeDependencies(); - if (!_speechPrefsRequested) { - _speechPrefsRequested = true; - _loadSpeechPreferences(); - } - } - - Future _loadSpeechPreferences() async { - final fallbackLocale = Localizations.maybeLocaleOf(context); - final language = await SpeechToTextPreferences.preferredLanguage( - fallbackLocale: fallbackLocale, - ); - if (!mounted) return; - setState(() { - _speechLanguage = language; - _speechPrefsLoading = false; - }); - } - - Future _changeSpeechLanguage(SpeechRecognitionLanguage language) async { - setState(() { - _speechLanguage = language; - }); - await SpeechToTextPreferences.setPreferredLanguage(language); - } - Widget getDropdownRow(BuildContext context) { final dropDownItems = >[]; @@ -138,8 +105,6 @@ class _SettingsState extends State { crossAxisAlignment: CrossAxisAlignment.stretch, children: [ getDropdownRow(context), - const SizedBox(height: 24), - _buildSpeechSection(theme, loc), const SizedBox(height: 32), _buildStudySection(theme, loc), ], @@ -148,48 +113,6 @@ class _SettingsState extends State { ); } - Widget _buildSpeechSection(ThemeData theme, AppLocalizations loc) { - return Card( - elevation: 0, - margin: EdgeInsets.zero, - child: Padding( - padding: const EdgeInsets.all(16), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Text(loc.speech_to_text_title, style: theme.textTheme.titleMedium), - const SizedBox(height: 8), - Text( - loc.speech_to_text_description, - style: theme.textTheme.bodyMedium, - ), - const SizedBox(height: 12), - if (_speechPrefsLoading) - const LinearProgressIndicator() - else ...[ - DropdownMenu( - initialSelection: _speechLanguage, - label: Text(loc.speech_to_text_language_label), - onSelected: (lang) { - if (lang != null) _changeSpeechLanguage(lang); - }, - dropdownMenuEntries: SpeechRecognitionLanguage.values - .map( - (lang) => DropdownMenuEntry( - value: lang, - label: _languageLabel(lang, loc), - ), - ) - .toList(), - ), - const SizedBox(height: 12), - ], - ], - ), - ), - ); - } - Widget _buildStudySection(ThemeData theme, AppLocalizations loc) { return Column( children: [ @@ -225,18 +148,6 @@ class _SettingsState extends State { ], ); } - - String _languageLabel( - SpeechRecognitionLanguage language, - AppLocalizations loc, - ) { - switch (language) { - case SpeechRecognitionLanguage.german: - return loc.speech_to_text_language_german; - case SpeechRecognitionLanguage.english: - return loc.speech_to_text_language_english; - } - } } class OptOutAlertDialog extends StatelessWidget { diff --git a/app/lib/services/speech/speech_controller_models.dart b/app/lib/services/speech/speech_controller_models.dart index 7b30b9241..3774aed39 100644 --- a/app/lib/services/speech/speech_controller_models.dart +++ b/app/lib/services/speech/speech_controller_models.dart @@ -1,7 +1,5 @@ import 'package:flutter/foundation.dart'; -import 'package:studyu_app/services/speech/speech_to_text_language.dart'; - typedef SpeechTranscriptCommit = void Function(String textSegment); enum SpeechLifecycleStatus { @@ -29,25 +27,19 @@ class SpeechControllerState { required this.status, this.error, this.partialTranscript, - required this.language, }); - factory SpeechControllerState.initial({ - required SpeechRecognitionLanguage language, - required bool supported, - }) { + factory SpeechControllerState.initial({required bool supported}) { return SpeechControllerState( status: supported ? SpeechLifecycleStatus.idle : SpeechLifecycleStatus.unavailable, - language: language, ); } final SpeechLifecycleStatus status; final SpeechError? error; final String? partialTranscript; - final SpeechRecognitionLanguage language; SpeechControllerState copyWith({ SpeechLifecycleStatus? status, @@ -55,7 +47,6 @@ class SpeechControllerState { bool clearError = false, String? partialTranscript, bool clearTranscript = false, - SpeechRecognitionLanguage? language, }) { return SpeechControllerState( status: status ?? this.status, @@ -63,7 +54,6 @@ class SpeechControllerState { partialTranscript: clearTranscript ? null : partialTranscript ?? this.partialTranscript, - language: language ?? this.language, ); } } diff --git a/app/lib/services/speech/speech_to_text_language.dart b/app/lib/services/speech/speech_to_text_language.dart deleted file mode 100644 index 2c7ce9f78..000000000 --- a/app/lib/services/speech/speech_to_text_language.dart +++ /dev/null @@ -1,32 +0,0 @@ -import 'package:flutter/material.dart'; - -enum SpeechRecognitionLanguage { - english(localeCode: 'en', localeId: 'en_US'), - german(localeCode: 'de', localeId: 'de_DE'); - - const SpeechRecognitionLanguage({ - required this.localeCode, - required this.localeId, - }); - - final String localeCode; - final String localeId; - - static SpeechRecognitionLanguage fromCode(String? code) { - if (code == null) return SpeechRecognitionLanguage.english; - return SpeechRecognitionLanguage.values.firstWhere( - (lang) => lang.localeCode == code, - orElse: () => SpeechRecognitionLanguage.english, - ); - } - - static SpeechRecognitionLanguage fromLocale(Locale? locale) { - if (locale == null) return SpeechRecognitionLanguage.english; - switch (locale.languageCode) { - case 'de': - return SpeechRecognitionLanguage.german; - default: - return SpeechRecognitionLanguage.english; - } - } -} diff --git a/app/lib/services/speech/speech_to_text_preferences.dart b/app/lib/services/speech/speech_to_text_preferences.dart deleted file mode 100644 index 8ac085012..000000000 --- a/app/lib/services/speech/speech_to_text_preferences.dart +++ /dev/null @@ -1,25 +0,0 @@ -import 'dart:ui'; - -import 'package:studyu_app/services/speech/speech_to_text_language.dart'; -import 'package:studyu_flutter_common/studyu_flutter_common.dart'; - -class SpeechToTextPreferences { - static const _languageKey = 'speech_to_text_language'; - - static Future preferredLanguage({ - Locale? fallbackLocale, - }) async { - final stored = await SecureStorage.read(_languageKey); - if (stored != null) return SpeechRecognitionLanguage.fromCode(stored); - if (fallbackLocale != null) { - return SpeechRecognitionLanguage.fromLocale(fallbackLocale); - } - return SpeechRecognitionLanguage.english; - } - - static Future setPreferredLanguage( - SpeechRecognitionLanguage language, - ) async { - await SecureStorage.write(_languageKey, language.localeCode); - } -} diff --git a/app/lib/services/speech/websocket_speech_controller_io.dart b/app/lib/services/speech/websocket_speech_controller_io.dart index aaebb30f2..18ee2de6f 100644 --- a/app/lib/services/speech/websocket_speech_controller_io.dart +++ b/app/lib/services/speech/websocket_speech_controller_io.dart @@ -5,26 +5,16 @@ import 'dart:io'; import 'package:flutter/foundation.dart'; import 'package:record/record.dart'; import 'package:studyu_app/services/speech/speech_controller_models.dart'; -import 'package:studyu_app/services/speech/speech_to_text_language.dart'; -import 'package:studyu_app/services/speech/speech_to_text_preferences.dart'; import 'package:web_socket_channel/io.dart'; import 'package:web_socket_channel/web_socket_channel.dart'; class SpeechToTextController extends ValueNotifier { SpeechToTextController({ required this.onFinalTranscription, - SpeechRecognitionLanguage initialLanguage = - SpeechRecognitionLanguage.english, String? serverUrl, - }) : _language = initialLanguage, - _serverUrl = serverUrl ?? 'wss://stt.ibrahimozkan.dev', + }) : _serverUrl = serverUrl ?? 'wss://stt.ibrahimozkan.dev', _audioRecorder = AudioRecorder(), - super( - SpeechControllerState.initial( - language: initialLanguage, - supported: _platformSupported, - ), - ) { + super(SpeechControllerState.initial(supported: _platformSupported)) { _ensureInitialized(); } @@ -32,7 +22,6 @@ class SpeechToTextController extends ValueNotifier { final AudioRecorder _audioRecorder; final String _serverUrl; - SpeechRecognitionLanguage _language; String? _latestTranscript; bool _isDisposed = false; Future? _initialization; @@ -55,8 +44,6 @@ class SpeechToTextController extends ValueNotifier { bool get isListening => value.status == SpeechLifecycleStatus.listening; - SpeechRecognitionLanguage get language => _language; - Future _ensureInitialized() { _initialization ??= _initializeRecorder(); return _initialization!; @@ -74,13 +61,6 @@ class SpeechToTextController extends ValueNotifier { } } - Future updateLanguage(SpeechRecognitionLanguage language) async { - if (_language == language) return; - _language = language; - value = value.copyWith(language: language); - await SpeechToTextPreferences.setPreferredLanguage(language); - } - Future startListening() async { debugPrint('[WebSocketSpeech] startListening requested'); if (!_platformSupported) { diff --git a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart index 63d97a1f9..86efefb67 100644 --- a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart +++ b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart @@ -4,8 +4,6 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:studyu_app/l10n/app_localizations.dart'; import 'package:studyu_app/services/speech/speech_to_text_controller.dart'; -import 'package:studyu_app/services/speech/speech_to_text_language.dart'; -import 'package:studyu_app/services/speech/speech_to_text_preferences.dart'; import 'package:studyu_app/widgets/questionnaire/questions/question_widget.dart'; import 'package:studyu_core/core.dart'; @@ -34,8 +32,6 @@ class _FreeTextQuestionWidgetState extends State { Timer? _listeningRestartTimer; AutovalidateMode _autovalidateMode = AutovalidateMode.disabled; SpeechToTextController? _speechController; - bool _speechPrefsRequested = false; - bool _speechPrefsInitialized = false; bool _listeningRequested = false; SpeechErrorType? _lastSpeechErrorType; @@ -48,9 +44,9 @@ class _FreeTextQuestionWidgetState extends State { @override void didChangeDependencies() { super.didChangeDependencies(); - if (!_speechPrefsRequested && SpeechToTextController.isSupportedPlatform) { - _speechPrefsRequested = true; - _initSpeechPreferences(); + if (_speechController == null && + SpeechToTextController.isSupportedPlatform) { + _createSpeechController(); } } @@ -66,24 +62,11 @@ class _FreeTextQuestionWidgetState extends State { super.dispose(); } - Future _initSpeechPreferences() async { - final fallbackLocale = Localizations.maybeLocaleOf(context); - final language = await SpeechToTextPreferences.preferredLanguage( - fallbackLocale: fallbackLocale, - ); - if (!mounted) return; - setState(() { - _speechPrefsInitialized = true; - }); - _createSpeechController(language); - } - - void _createSpeechController(SpeechRecognitionLanguage language) { + void _createSpeechController() { _speechController?.removeListener(_onSpeechStateChanged); _speechController?.dispose(); _speechController = SpeechToTextController( onFinalTranscription: _insertSpeechTranscript, - initialLanguage: language, ); _speechController!.addListener(_onSpeechStateChanged); _listeningRequested = false; @@ -214,8 +197,8 @@ class _FreeTextQuestionWidgetState extends State { if (!SpeechToTextController.isSupportedPlatform) { return const SizedBox.shrink(); } - if (!_speechPrefsInitialized) { - return const SizedBox(height: 2, child: LinearProgressIndicator()); + if (!SpeechToTextController.isSupportedPlatform) { + return const SizedBox.shrink(); } final controller = _speechController; if (controller == null) return const SizedBox.shrink(); @@ -460,9 +443,7 @@ class _FreeTextQuestionWidgetState extends State { } }, decoration: InputDecoration( - suffixIcon: - SpeechToTextController.isSupportedPlatform && - _speechPrefsInitialized + suffixIcon: SpeechToTextController.isSupportedPlatform ? ValueListenableBuilder( valueListenable: _speechController!, builder: (context, state, _) { From d6ace175e760da83ecd326a55cb24c89cf8a13dd Mon Sep 17 00:00:00 2001 From: ibrahimozkan Date: Thu, 20 Nov 2025 23:05:03 +0300 Subject: [PATCH 05/16] feat: Add `STUDYU_STT_WEBSOCKET_URL` environment variable for Speech-to-Text service configuration. --- app/lib/services/speech/websocket_speech_controller_io.dart | 3 ++- core/lib/src/env/env.dart | 3 +++ flutter_common/lib/envs/.env | 1 + flutter_common/lib/envs/.env.dev | 1 + flutter_common/lib/envs/.env.local.example | 1 + flutter_common/lib/src/utils/env_loader.dart | 2 ++ 6 files changed, 10 insertions(+), 1 deletion(-) diff --git a/app/lib/services/speech/websocket_speech_controller_io.dart b/app/lib/services/speech/websocket_speech_controller_io.dart index 18ee2de6f..2a9ea6e83 100644 --- a/app/lib/services/speech/websocket_speech_controller_io.dart +++ b/app/lib/services/speech/websocket_speech_controller_io.dart @@ -5,6 +5,7 @@ import 'dart:io'; import 'package:flutter/foundation.dart'; import 'package:record/record.dart'; import 'package:studyu_app/services/speech/speech_controller_models.dart'; +import 'package:studyu_core/env.dart' as env; import 'package:web_socket_channel/io.dart'; import 'package:web_socket_channel/web_socket_channel.dart'; @@ -12,7 +13,7 @@ class SpeechToTextController extends ValueNotifier { SpeechToTextController({ required this.onFinalTranscription, String? serverUrl, - }) : _serverUrl = serverUrl ?? 'wss://stt.ibrahimozkan.dev', + }) : _serverUrl = serverUrl ?? env.sttWebSocketUrl ?? '', _audioRecorder = AudioRecorder(), super(SpeechControllerState.initial(supported: _platformSupported)) { _ensureInitialized(); diff --git a/core/lib/src/env/env.dart b/core/lib/src/env/env.dart index e9cee6fde..bd1a76da0 100644 --- a/core/lib/src/env/env.dart +++ b/core/lib/src/env/env.dart @@ -9,6 +9,7 @@ late final String? projectGeneratorUrl; late final String? playStoreUrl; late final String? appstoreUrl; late final String? developerEmail; +late final String? sttWebSocketUrl; void setEnv( String envSupabaseUrl, @@ -20,6 +21,7 @@ void setEnv( String? envPlayStoreUrl, String? envAppstoreUrl, String? envDeveloperEmail, + String? envSttWebSocketUrl, }) { supabaseUrl = envSupabaseUrl; supabaseAnonKey = envSupabaseAnonKey; @@ -47,4 +49,5 @@ void setEnv( playStoreUrl = envPlayStoreUrl; appstoreUrl = envAppstoreUrl; developerEmail = envDeveloperEmail; + sttWebSocketUrl = envSttWebSocketUrl; } diff --git a/flutter_common/lib/envs/.env b/flutter_common/lib/envs/.env index 82f63798f..e9e4d5f0a 100644 --- a/flutter_common/lib/envs/.env +++ b/flutter_common/lib/envs/.env @@ -6,3 +6,4 @@ STUDYU_DESIGNER_URL=https://designer.studyu.health STUDYU_PLAY_STORE_URL=https://play.google.com/store/apps/details?id=health.studyu.app STUDYU_APPSTORE_URL=https://itunes.apple.com/app/id1571991198 STUDYU_DEVELOPER_EMAIL=stefan.konigorski@hpi.de +STUDYU_STT_WEBSOCKET_URL= diff --git a/flutter_common/lib/envs/.env.dev b/flutter_common/lib/envs/.env.dev index a734966f7..086edf47c 100644 --- a/flutter_common/lib/envs/.env.dev +++ b/flutter_common/lib/envs/.env.dev @@ -3,3 +3,4 @@ STUDYU_SUPABASE_PUBLIC_ANON_KEY=eyJhbGciOiAiSFMyNTYiLCJ0eXAiOiAiSldUIn0.eyJyb2xl STUDYU_PROJECT_GENERATOR_URL= STUDYU_APP_URL=https://app.dev.studyu.health STUDYU_DESIGNER_URL=https://designer.dev.studyu.health +STUDYU_STT_WEBSOCKET_URL= \ No newline at end of file diff --git a/flutter_common/lib/envs/.env.local.example b/flutter_common/lib/envs/.env.local.example index c797de031..a0c64e012 100644 --- a/flutter_common/lib/envs/.env.local.example +++ b/flutter_common/lib/envs/.env.local.example @@ -3,3 +3,4 @@ STUDYU_SUPABASE_PUBLIC_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJ STUDYU_PROJECT_GENERATOR_URL= STUDYU_APP_URL=http://localhost:8080 STUDYU_DESIGNER_URL=http://localhost:8081 +STUDYU_STT_WEBSOCKET_URL= \ No newline at end of file diff --git a/flutter_common/lib/src/utils/env_loader.dart b/flutter_common/lib/src/utils/env_loader.dart index 0226b6fdc..21432b75f 100644 --- a/flutter_common/lib/src/utils/env_loader.dart +++ b/flutter_common/lib/src/utils/env_loader.dart @@ -52,6 +52,7 @@ Future loadEnv() async { final envPlayStoreUrl = getEnv('STUDYU_PLAY_STORE_URL', optional: true); final envAppstoreUrl = getEnv('STUDYU_APPSTORE_URL', optional: true); final envDeveloperEmail = getEnv('STUDYU_DEVELOPER_EMAIL', optional: true); + final envSttWebSocketUrl = getEnv('STUDYU_STT_WEBSOCKET_URL', optional: true); // await SecureStorage.migrateSharedPreferencesToSecureStorage(); @@ -77,6 +78,7 @@ Future loadEnv() async { envPlayStoreUrl: envPlayStoreUrl, envAppstoreUrl: envAppstoreUrl, envDeveloperEmail: envDeveloperEmail, + envSttWebSocketUrl: envSttWebSocketUrl, supabaseClient: Supabase.instance.client, ); } From a45f4a18f9873deae620b7a525fcc6b8ce2d35b5 Mon Sep 17 00:00:00 2001 From: ibrahimozkn <51259479+ibrahimozkn@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:05:36 +0000 Subject: [PATCH 06/16] lint: Apply final newline changes --- flutter_common/lib/envs/.env.dev | 2 +- flutter_common/lib/envs/.env.local.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flutter_common/lib/envs/.env.dev b/flutter_common/lib/envs/.env.dev index 086edf47c..0bf785c53 100644 --- a/flutter_common/lib/envs/.env.dev +++ b/flutter_common/lib/envs/.env.dev @@ -3,4 +3,4 @@ STUDYU_SUPABASE_PUBLIC_ANON_KEY=eyJhbGciOiAiSFMyNTYiLCJ0eXAiOiAiSldUIn0.eyJyb2xl STUDYU_PROJECT_GENERATOR_URL= STUDYU_APP_URL=https://app.dev.studyu.health STUDYU_DESIGNER_URL=https://designer.dev.studyu.health -STUDYU_STT_WEBSOCKET_URL= \ No newline at end of file +STUDYU_STT_WEBSOCKET_URL= diff --git a/flutter_common/lib/envs/.env.local.example b/flutter_common/lib/envs/.env.local.example index a0c64e012..80f8c31d0 100644 --- a/flutter_common/lib/envs/.env.local.example +++ b/flutter_common/lib/envs/.env.local.example @@ -3,4 +3,4 @@ STUDYU_SUPABASE_PUBLIC_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJ STUDYU_PROJECT_GENERATOR_URL= STUDYU_APP_URL=http://localhost:8080 STUDYU_DESIGNER_URL=http://localhost:8081 -STUDYU_STT_WEBSOCKET_URL= \ No newline at end of file +STUDYU_STT_WEBSOCKET_URL= From d5e69d072e47f3aa1f50dcbd06ff27fb82436fd2 Mon Sep 17 00:00:00 2001 From: ibrahimozkan Date: Fri, 21 Nov 2025 15:18:14 +0300 Subject: [PATCH 07/16] refactor: Add WebSocket STT server URL validation and remove redundant comments. --- .../websocket_speech_controller_io.dart | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/app/lib/services/speech/websocket_speech_controller_io.dart b/app/lib/services/speech/websocket_speech_controller_io.dart index 2a9ea6e83..bfceeb352 100644 --- a/app/lib/services/speech/websocket_speech_controller_io.dart +++ b/app/lib/services/speech/websocket_speech_controller_io.dart @@ -15,7 +15,13 @@ class SpeechToTextController extends ValueNotifier { String? serverUrl, }) : _serverUrl = serverUrl ?? env.sttWebSocketUrl ?? '', _audioRecorder = AudioRecorder(), - super(SpeechControllerState.initial(supported: _platformSupported)) { + super( + SpeechControllerState.initial( + supported: + _platformSupported && + (serverUrl ?? env.sttWebSocketUrl ?? '').isNotEmpty, + ), + ) { _ensureInitialized(); } @@ -68,6 +74,13 @@ class SpeechToTextController extends ValueNotifier { _emitError(SpeechErrorType.general); return false; } + + if (_serverUrl.isEmpty) { + debugPrint( + '[WebSocketSpeech] WebSocket URL not configured, STT unavailable', + ); + return false; + } if (value.status == SpeechLifecycleStatus.listening || value.status == SpeechLifecycleStatus.preparing) { debugPrint('[WebSocketSpeech] Already listening or preparing'); @@ -113,15 +126,11 @@ class SpeechToTextController extends ValueNotifier { try { debugPrint('[WebSocketSpeech] Connecting to $_serverUrl...'); _wsChannel = IOWebSocketChannel.connect(Uri.parse(_serverUrl)); - - // Send configuration with sample rate final config = { 'config': {'sample_rate': _sampleRate}, }; _wsChannel!.sink.add(jsonEncode(config)); debugPrint('[WebSocketSpeech] WebSocket connected, config sent'); - - // Listen for transcription results _wsChannel!.stream.listen( (data) { if (_isDisposed) return; @@ -183,8 +192,6 @@ class SpeechToTextController extends ValueNotifier { debugPrint('[WebSocketSpeech] Reconnected successfully'); } catch (e) { debugPrint('[WebSocketSpeech] Reconnection failed: $e'); - // If reconnection fails, we might want to stop listening or try again later - // For now, let's stop to avoid infinite loops if server is down _emitError( SpeechErrorType.general, details: 'Connection lost. Please try again.', @@ -325,8 +332,6 @@ class SpeechToTextController extends ValueNotifier { final transcript = _latestTranscript; if (transcript != null && transcript.isNotEmpty) { onFinalTranscription(transcript); - // Don't clear immediately to avoid flicker - the next partial will replace it - // or we'll clear it when we get an empty final result } } @@ -338,7 +343,6 @@ class SpeechToTextController extends ValueNotifier { _isListening = false; _latestTranscript = null; - // Ensure recorder is stopped on error try { _audioStreamSubscription?.cancel(); _audioRecorder.stop(); @@ -363,9 +367,7 @@ class SpeechToTextController extends ValueNotifier { unawaited(_audioRecorder.stop()); unawaited(_audioRecorder.dispose()); unawaited(_wsChannel?.sink.close()); - } catch (_) { - // Ignore dispose errors - } + } catch (_) {} super.dispose(); } } From ab07ca7d1916929d8dbb9223c814ff0f296b4c8d Mon Sep 17 00:00:00 2001 From: ibrahimozkan Date: Fri, 28 Nov 2025 13:03:16 +0100 Subject: [PATCH 08/16] feat: Enhance free text question UI with continuous speech-to-text input, new styling, and hint text. --- app/lib/l10n/app_de.arb | 1 + app/lib/l10n/app_en.arb | 1 + app/lib/l10n/app_localizations.dart | 6 + app/lib/l10n/app_localizations_de.dart | 4 +- app/lib/l10n/app_localizations_en.dart | 3 + .../websocket_speech_controller_io.dart | 51 ++--- .../questions/free_text_question_widget.dart | 203 +++++++++--------- .../features/study/study_base_controller.dart | 2 +- 8 files changed, 125 insertions(+), 146 deletions(-) diff --git a/app/lib/l10n/app_de.arb b/app/lib/l10n/app_de.arb index 504522db5..0adbfa6cf 100644 --- a/app/lib/l10n/app_de.arb +++ b/app/lib/l10n/app_de.arb @@ -242,6 +242,7 @@ } } }, + "free_text_answer_hint": "Geben Sie Ihre Antwort hier ein...", "app_outdated_message": "Eine neue Version der StudyU App ist verfügbar. Bitte führen Sie eine Aktualisierung durch, um die neuesten Funktionen und Verbesserungen zu erhalten. Vielen Dank für Ihre Unterstützung!", "update_now": "Jetzt aktualisieren", diff --git a/app/lib/l10n/app_en.arb b/app/lib/l10n/app_en.arb index 236453ac2..5036ae0b7 100644 --- a/app/lib/l10n/app_en.arb +++ b/app/lib/l10n/app_en.arb @@ -238,6 +238,7 @@ } } }, + "free_text_answer_hint": "Type your answer here...", "app_outdated_message": "A new version of the StudyU App is available. Please update to get the latest features and improvements. Thank you for your support!", "update_now": "Update now", diff --git a/app/lib/l10n/app_localizations.dart b/app/lib/l10n/app_localizations.dart index 7856d33b1..6b15cf33b 100644 --- a/app/lib/l10n/app_localizations.dart +++ b/app/lib/l10n/app_localizations.dart @@ -1178,6 +1178,12 @@ abstract class AppLocalizations { /// **'Please enter only characters matching the pattern {pattern}'** String free_text_custom_error(String pattern); + /// No description provided for @free_text_answer_hint. + /// + /// In en, this message translates to: + /// **'Type your answer here...'** + String get free_text_answer_hint; + /// No description provided for @app_outdated_message. /// /// In en, this message translates to: diff --git a/app/lib/l10n/app_localizations_de.dart b/app/lib/l10n/app_localizations_de.dart index 38d24fed4..b9018ce49 100644 --- a/app/lib/l10n/app_localizations_de.dart +++ b/app/lib/l10n/app_localizations_de.dart @@ -1,6 +1,5 @@ // ignore: unused_import import 'package:intl/intl.dart' as intl; - import 'app_localizations.dart'; // ignore_for_file: type=lint @@ -608,6 +607,9 @@ class AppLocalizationsDe extends AppLocalizations { return 'Bitte geben Sie nur Zeichen ein, die dem Muster $pattern entsprechen'; } + @override + String get free_text_answer_hint => 'Geben Sie Ihre Antwort hier ein...'; + @override String get app_outdated_message => 'Eine neue Version der StudyU App ist verfügbar. Bitte führen Sie eine Aktualisierung durch, um die neuesten Funktionen und Verbesserungen zu erhalten. Vielen Dank für Ihre Unterstützung!'; diff --git a/app/lib/l10n/app_localizations_en.dart b/app/lib/l10n/app_localizations_en.dart index 20a7543cd..5b23e0040 100644 --- a/app/lib/l10n/app_localizations_en.dart +++ b/app/lib/l10n/app_localizations_en.dart @@ -600,6 +600,9 @@ class AppLocalizationsEn extends AppLocalizations { return 'Please enter only characters matching the pattern $pattern'; } + @override + String get free_text_answer_hint => 'Type your answer here...'; + @override String get app_outdated_message => 'A new version of the StudyU App is available. Please update to get the latest features and improvements. Thank you for your support!'; diff --git a/app/lib/services/speech/websocket_speech_controller_io.dart b/app/lib/services/speech/websocket_speech_controller_io.dart index bfceeb352..d1f931044 100644 --- a/app/lib/services/speech/websocket_speech_controller_io.dart +++ b/app/lib/services/speech/websocket_speech_controller_io.dart @@ -6,6 +6,7 @@ import 'package:flutter/foundation.dart'; import 'package:record/record.dart'; import 'package:studyu_app/services/speech/speech_controller_models.dart'; import 'package:studyu_core/env.dart' as env; +import 'package:supabase_flutter/supabase_flutter.dart'; import 'package:web_socket_channel/io.dart'; import 'package:web_socket_channel/web_socket_channel.dart'; @@ -61,29 +62,22 @@ class SpeechToTextController extends ValueNotifier { try { final hasPermission = await _audioRecorder.hasPermission(); return hasPermission; - } catch (e, stack) { - debugPrint('[WebSocketSpeech] init error: $e'); - debugPrint(stack.toString()); + } catch (e) { return false; } } Future startListening() async { - debugPrint('[WebSocketSpeech] startListening requested'); if (!_platformSupported) { _emitError(SpeechErrorType.general); return false; } if (_serverUrl.isEmpty) { - debugPrint( - '[WebSocketSpeech] WebSocket URL not configured, STT unavailable', - ); return false; } if (value.status == SpeechLifecycleStatus.listening || value.status == SpeechLifecycleStatus.preparing) { - debugPrint('[WebSocketSpeech] Already listening or preparing'); return false; } @@ -108,7 +102,6 @@ class SpeechToTextController extends ValueNotifier { status: SpeechLifecycleStatus.listening, clearError: true, ); - debugPrint('[WebSocketSpeech] Listening started successfully'); return true; } catch (e, stack) { @@ -124,31 +117,35 @@ class SpeechToTextController extends ValueNotifier { Future _startWebSocketConnection() async { try { - debugPrint('[WebSocketSpeech] Connecting to $_serverUrl...'); - _wsChannel = IOWebSocketChannel.connect(Uri.parse(_serverUrl)); + final token = Supabase.instance.client.auth.currentSession?.accessToken; + + final ws = await WebSocket.connect( + _serverUrl, + headers: token != null ? {'Authorization': 'Bearer $token'} : null, + ); + + _wsChannel = IOWebSocketChannel(ws); + final config = { 'config': {'sample_rate': _sampleRate}, }; _wsChannel!.sink.add(jsonEncode(config)); - debugPrint('[WebSocketSpeech] WebSocket connected, config sent'); _wsChannel!.stream.listen( (data) { if (_isDisposed) return; try { final result = jsonDecode(data as String) as Map; _handleTranscriptionResult(result); - } catch (e) { - debugPrint('[WebSocketSpeech] Error parsing response: $e'); - } + } catch (e) {} }, onError: (error) { if (_isDisposed) return; - debugPrint('[WebSocketSpeech] WebSocket error: $error'); + _handleConnectionLoss(); }, onDone: () { if (_isDisposed) return; - debugPrint('[WebSocketSpeech] WebSocket closed by server'); + _handleConnectionLoss(); }, cancelOnError: false, @@ -159,9 +156,6 @@ class SpeechToTextController extends ValueNotifier { } void _handleConnectionLoss() { - debugPrint( - '[WebSocketSpeech] Connection loss detected. Listening: $_isListening, Reconnecting: $_isReconnecting', - ); if (_isListening && !_isReconnecting) { _reconnect(); } @@ -170,7 +164,6 @@ class SpeechToTextController extends ValueNotifier { Future _reconnect() async { if (_isReconnecting || _isDisposed) return; _isReconnecting = true; - debugPrint('[WebSocketSpeech] Attempting to reconnect...'); try { // Close existing connection if any @@ -183,15 +176,12 @@ class SpeechToTextController extends ValueNotifier { await Future.delayed(const Duration(milliseconds: 500)); if (!_isListening) { - debugPrint('[WebSocketSpeech] Reconnect aborted: not listening'); _isReconnecting = false; return; } await _startWebSocketConnection(); - debugPrint('[WebSocketSpeech] Reconnected successfully'); } catch (e) { - debugPrint('[WebSocketSpeech] Reconnection failed: $e'); _emitError( SpeechErrorType.general, details: 'Connection lost. Please try again.', @@ -203,7 +193,6 @@ class SpeechToTextController extends ValueNotifier { Future _startAudioRecording() async { try { - debugPrint('[WebSocketSpeech] Starting audio recording...'); final stream = await _audioRecorder.startStream( const RecordConfig( encoder: AudioEncoder.pcm16bits, @@ -223,7 +212,6 @@ class SpeechToTextController extends ValueNotifier { } catch (e) { // If send fails, it might be because connection is closed // The onDone/onError of WebSocket should handle this, but just in case - debugPrint('[WebSocketSpeech] Failed to send audio: $e'); } } }, @@ -236,7 +224,6 @@ class SpeechToTextController extends ValueNotifier { }, cancelOnError: false, ); - debugPrint('[WebSocketSpeech] Audio recording started'); } catch (e) { throw Exception('Failed to start audio recording: $e'); } @@ -253,12 +240,10 @@ class SpeechToTextController extends ValueNotifier { // Show partial transcription (ongoing speech) _latestTranscript = partial.trim(); value = value.copyWith(partialTranscript: partial); - // debugPrint('[WebSocketSpeech] Partial: $partial'); // Uncomment for very verbose logs } else if (text != null) { // Final transcription (silence detected or end of phrase) final trimmed = text.trim(); if (trimmed.isNotEmpty) { - debugPrint('[WebSocketSpeech] Final transcript: $trimmed'); // Commit the final result _latestTranscript = trimmed; _commitTranscript(); @@ -274,7 +259,6 @@ class SpeechToTextController extends ValueNotifier { } Future stopListening() async { - debugPrint('[WebSocketSpeech] stopListening requested'); if (!_isListening) return; _isListening = false; @@ -282,7 +266,6 @@ class SpeechToTextController extends ValueNotifier { // Stop audio recording await _audioStreamSubscription?.cancel(); await _audioRecorder.stop(); - debugPrint('[WebSocketSpeech] Audio recording stopped'); // Send EOF to WebSocket if (_wsChannel != null) { @@ -291,7 +274,6 @@ class SpeechToTextController extends ValueNotifier { // Give server a moment to send final results await Future.delayed(const Duration(milliseconds: 300)); await _wsChannel!.sink.close(); - debugPrint('[WebSocketSpeech] WebSocket closed cleanly'); } catch (_) { // Ignore errors during closing } @@ -312,7 +294,6 @@ class SpeechToTextController extends ValueNotifier { } Future forceReset() async { - debugPrint('[WebSocketSpeech] forceReset requested'); _isListening = false; try { await _audioStreamSubscription?.cancel(); @@ -348,10 +329,6 @@ class SpeechToTextController extends ValueNotifier { _audioRecorder.stop(); } catch (_) {} - debugPrint('[WebSocketSpeech] error=$type details=${details ?? ''}'); - if (stackTrace != null) { - debugPrint(stackTrace.toString()); - } value = value.copyWith( status: SpeechLifecycleStatus.error, error: SpeechError(type, details: details), diff --git a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart index 86efefb67..ba4bb2eb0 100644 --- a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart +++ b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart @@ -34,6 +34,8 @@ class _FreeTextQuestionWidgetState extends State { SpeechToTextController? _speechController; bool _listeningRequested = false; SpeechErrorType? _lastSpeechErrorType; + String _preSpeechText = ''; + int _insertionIndex = 0; @override void initState() { @@ -94,35 +96,63 @@ class _FreeTextQuestionWidgetState extends State { if (hasError && _listeningRequested) { _setListeningRequested(false); } + + // Handle partial transcript + if (state.status == SpeechLifecycleStatus.listening && + state.partialTranscript != null) { + _updateTextWithTranscript(state.partialTranscript!); + } + WidgetsBinding.instance.addPostFrameCallback((_) { if (!mounted) return; setState(() {}); }); } - void _insertSpeechTranscript(String transcript) { - final trimmed = transcript.trim(); - if (trimmed.isEmpty) return; + void _updateTextWithTranscript(String transcript) { + if (transcript.isEmpty) return; final controller = _textFieldController; - final selection = controller.selection; - final baseOffset = selection.isValid - ? selection.extentOffset - : controller.text.length; - final currentText = controller.text; - final previousChar = baseOffset > 0 - ? currentText.substring(baseOffset - 1, baseOffset) + + // Ensure we have valid state to insert into + if (_insertionIndex > _preSpeechText.length) { + _insertionIndex = _preSpeechText.length; + } + + final previousChar = _insertionIndex > 0 + ? _preSpeechText.substring(_insertionIndex - 1, _insertionIndex) : ''; final needsSpace = previousChar.trim().isNotEmpty; - final insertion = '${needsSpace ? ' ' : ''}$trimmed'; - final newText = currentText.replaceRange(baseOffset, baseOffset, insertion); + final insertion = '${needsSpace ? ' ' : ''}$transcript'; + + final newText = _preSpeechText.replaceRange( + _insertionIndex, + _insertionIndex, + insertion, + ); + controller.value = controller.value.copyWith( text: newText, - selection: TextSelection.collapsed(offset: baseOffset + insertion.length), + selection: TextSelection.collapsed( + offset: _insertionIndex + insertion.length, + ), ); _handleInteraction(); _debouncedValidation(); } + void _insertSpeechTranscript(String transcript) { + final trimmed = transcript.trim(); + if (trimmed.isEmpty) return; + + _updateTextWithTranscript(trimmed); + + // Update state for next phrase (continuous listening) + _preSpeechText = _textFieldController.text; + _insertionIndex = _textFieldController.selection.isValid + ? _textFieldController.selection.extentOffset + : _textFieldController.text.length; + } + String? _mapSpeechErrorToMessage(SpeechError error, AppLocalizations loc) { final details = error.details?.toLowerCase(); final isPermissionIssue = details?.contains('permission') == true; @@ -138,10 +168,19 @@ class _FreeTextQuestionWidgetState extends State { _stopListeningIntent(); } else { _setListeningRequested(true); + _captureCurrentTextState(); unawaited(_startListeningIfRequested()); } } + void _captureCurrentTextState() { + final controller = _textFieldController; + _preSpeechText = controller.text; + _insertionIndex = controller.selection.isValid + ? controller.selection.extentOffset + : controller.text.length; + } + void _stopListeningIntent() { _setListeningRequested(false); unawaited(_stopActiveListening()); @@ -193,84 +232,18 @@ class _FreeTextQuestionWidgetState extends State { await controller.stopListening(); } - Widget _buildSpeechAssistant(ThemeData theme, AppLocalizations loc) { - if (!SpeechToTextController.isSupportedPlatform) { - return const SizedBox.shrink(); - } - if (!SpeechToTextController.isSupportedPlatform) { - return const SizedBox.shrink(); - } - final controller = _speechController; - if (controller == null) return const SizedBox.shrink(); - - return ValueListenableBuilder( - valueListenable: controller, - builder: (context, state, _) { - final isPreparing = state.status == SpeechLifecycleStatus.preparing; - - final hasError = - state.status == SpeechLifecycleStatus.error && state.error != null; - - return Column( - crossAxisAlignment: CrossAxisAlignment.stretch, - children: [ - if (isPreparing) - const Padding( - padding: EdgeInsets.only(bottom: 8), - child: LinearProgressIndicator(), - ), - if ((state.partialTranscript ?? '').isNotEmpty) - Container( - margin: const EdgeInsets.only(bottom: 8), - padding: const EdgeInsets.symmetric( - horizontal: 12, - vertical: 8, - ), - decoration: BoxDecoration( - color: theme.colorScheme.surfaceContainerHighest.withValues( - alpha: 0.5, - ), - borderRadius: BorderRadius.circular(8), - border: Border.all(color: theme.colorScheme.outlineVariant), - ), - child: Text( - state.partialTranscript!, - style: theme.textTheme.bodyMedium?.copyWith( - fontStyle: FontStyle.italic, - color: theme.colorScheme.onSurfaceVariant, - ), - ), - ), - if (hasError) - Padding( - padding: const EdgeInsets.only(bottom: 8), - child: Text( - _mapSpeechErrorToMessage(state.error!, loc) ?? - loc.speech_to_text_error_general, - style: theme.textTheme.bodySmall?.copyWith( - color: theme.colorScheme.error, - ), - ), - ), - ], - ); - }, - ); - } - Widget _buildMicButton(ThemeData theme, bool isListening, bool isPreparing) { - final color = isListening - ? theme.colorScheme.error - : isPreparing - ? theme.colorScheme.outline - : theme.colorScheme.primary; - - return IconButton( + return IconButton.filledTonal( onPressed: isPreparing ? null : _toggleSpeechListening, icon: isListening ? const Icon(Icons.stop_circle_outlined) : const Icon(Icons.mic), - color: color, + style: IconButton.styleFrom( + backgroundColor: isListening ? theme.colorScheme.errorContainer : null, + foregroundColor: isListening + ? theme.colorScheme.onErrorContainer + : null, + ), tooltip: isListening ? 'Stop listening' : 'Start listening', ); } @@ -370,7 +343,7 @@ class _FreeTextQuestionWidgetState extends State { final question = widget.question; final loc = AppLocalizations.of(context)!; final theme = Theme.of(context); - final speechSection = _buildSpeechAssistant(theme, loc); + return Column( crossAxisAlignment: CrossAxisAlignment.stretch, children: [ @@ -378,6 +351,7 @@ class _FreeTextQuestionWidgetState extends State { key: _formFieldKey, controller: _textFieldController, maxLines: null, + minLines: 4, focusNode: _focusNode, keyboardType: _getKeyboardType(), inputFormatters: _getInputFormatters(), @@ -443,32 +417,47 @@ class _FreeTextQuestionWidgetState extends State { } }, decoration: InputDecoration( - suffixIcon: SpeechToTextController.isSupportedPlatform - ? ValueListenableBuilder( - valueListenable: _speechController!, - builder: (context, state, _) { - return _buildMicButton( - theme, - state.status == SpeechLifecycleStatus.listening, - state.status == SpeechLifecycleStatus.preparing, - ); - }, - ) - : null, + filled: true, + fillColor: theme.colorScheme.surfaceContainerHighest.withOpacity( + 0.3, + ), + hintText: loc.free_text_answer_hint, + border: OutlineInputBorder( + borderRadius: BorderRadius.circular(12), + borderSide: BorderSide.none, + ), + focusedBorder: OutlineInputBorder( + borderRadius: BorderRadius.circular(12), + borderSide: BorderSide( + color: theme.colorScheme.primary, + width: 2, + ), + ), + contentPadding: const EdgeInsets.all(16), ), ), const SizedBox(height: 16), - if (speechSection is SizedBox) - speechSection - else - Padding( - padding: const EdgeInsets.only(bottom: 8), - child: speechSection, - ), + Row( mainAxisAlignment: MainAxisAlignment.end, children: [ - OutlinedButton(onPressed: _handleSubmit, child: Text(loc.submit)), + if (SpeechToTextController.isSupportedPlatform && + _speechController != null) + ValueListenableBuilder( + valueListenable: _speechController!, + builder: (context, state, _) { + if (state.status == SpeechLifecycleStatus.unavailable) { + return const SizedBox.shrink(); + } + return _buildMicButton( + theme, + state.status == SpeechLifecycleStatus.listening, + state.status == SpeechLifecycleStatus.preparing, + ); + }, + ), + const SizedBox(width: 8), + FilledButton(onPressed: _handleSubmit, child: Text(loc.submit)), ], ), ], diff --git a/designer_v2/lib/features/study/study_base_controller.dart b/designer_v2/lib/features/study/study_base_controller.dart index 633c5e210..786a10c02 100644 --- a/designer_v2/lib/features/study/study_base_controller.dart +++ b/designer_v2/lib/features/study/study_base_controller.dart @@ -16,7 +16,7 @@ part 'study_base_controller.g.dart'; @riverpod class StudyBaseController - extends _$StudyBaseController { + extends _$StudyBaseController { @override StudyControllerBaseState build(StudyID studyId) { state = StudyControllerBaseState( From d4c751b68f344dacac7e6734a06cc54669fb550b Mon Sep 17 00:00:00 2001 From: ibrahimozkan Date: Fri, 28 Nov 2025 13:08:15 +0100 Subject: [PATCH 09/16] refactor: improve WebSocket error logging and update InputDecoration fillColor alpha usage --- app/lib/services/speech/websocket_speech_controller_io.dart | 4 +++- .../questionnaire/questions/free_text_question_widget.dart | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/lib/services/speech/websocket_speech_controller_io.dart b/app/lib/services/speech/websocket_speech_controller_io.dart index d1f931044..a7100aada 100644 --- a/app/lib/services/speech/websocket_speech_controller_io.dart +++ b/app/lib/services/speech/websocket_speech_controller_io.dart @@ -136,7 +136,9 @@ class SpeechToTextController extends ValueNotifier { try { final result = jsonDecode(data as String) as Map; _handleTranscriptionResult(result); - } catch (e) {} + } catch (e) { + debugPrint('Failed to parse WebSocket data: $e'); + } }, onError: (error) { if (_isDisposed) return; diff --git a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart index ba4bb2eb0..232e65b4a 100644 --- a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart +++ b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart @@ -418,8 +418,8 @@ class _FreeTextQuestionWidgetState extends State { }, decoration: InputDecoration( filled: true, - fillColor: theme.colorScheme.surfaceContainerHighest.withOpacity( - 0.3, + fillColor: theme.colorScheme.surfaceContainerHighest.withValues( + alpha: 0.3, ), hintText: loc.free_text_answer_hint, border: OutlineInputBorder( From 10e21bac0b4dd55652e92a5b008244b884b29441 Mon Sep 17 00:00:00 2001 From: ibrahimozkn <51259479+ibrahimozkn@users.noreply.github.com> Date: Fri, 28 Nov 2025 12:11:49 +0000 Subject: [PATCH 10/16] chore: Apply static analysis changes --- designer_v2/lib/features/study/study_base_controller.g.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/designer_v2/lib/features/study/study_base_controller.g.dart b/designer_v2/lib/features/study/study_base_controller.g.dart index 9acb16716..4e29eda40 100644 --- a/designer_v2/lib/features/study/study_base_controller.g.dart +++ b/designer_v2/lib/features/study/study_base_controller.g.dart @@ -68,7 +68,7 @@ final class StudyBaseControllerProvider } String _$studyBaseControllerHash() => - r'e2811f660e2b1086b07c702a0da8915d891e2004'; + r'2dd7b7b33e5705fd1c64fe268f023bbd9c87e1b5'; final class StudyBaseControllerFamily extends $Family { const StudyBaseControllerFamily._() From ef579dd63095d41dea650a2ee7cb605f35c605f5 Mon Sep 17 00:00:00 2001 From: Ibrahim Ozkan <51259479+ibrahimozkn@users.noreply.github.com> Date: Fri, 28 Nov 2025 13:20:30 +0100 Subject: [PATCH 11/16] Update app/lib/services/speech/websocket_speech_controller_io.dart Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../speech/websocket_speech_controller_io.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/lib/services/speech/websocket_speech_controller_io.dart b/app/lib/services/speech/websocket_speech_controller_io.dart index a7100aada..d3466a475 100644 --- a/app/lib/services/speech/websocket_speech_controller_io.dart +++ b/app/lib/services/speech/websocket_speech_controller_io.dart @@ -339,13 +339,13 @@ class SpeechToTextController extends ValueNotifier { } @override - void dispose() { + Future dispose() async { _isDisposed = true; try { - unawaited(_audioStreamSubscription?.cancel()); - unawaited(_audioRecorder.stop()); - unawaited(_audioRecorder.dispose()); - unawaited(_wsChannel?.sink.close()); + await _audioStreamSubscription?.cancel(); + await _audioRecorder.stop(); + await _audioRecorder.dispose(); + await _wsChannel?.sink.close(); } catch (_) {} super.dispose(); } From bec2a5d839c1bd4c53c6ab6eb9f8037c5da28bd8 Mon Sep 17 00:00:00 2001 From: Ibrahim Ozkan <51259479+ibrahimozkn@users.noreply.github.com> Date: Fri, 28 Nov 2025 13:20:46 +0100 Subject: [PATCH 12/16] Update app/lib/widgets/questionnaire/questions/free_text_question_widget.dart Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../questionnaire/questions/free_text_question_widget.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart index 232e65b4a..499e907c5 100644 --- a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart +++ b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart @@ -154,8 +154,8 @@ class _FreeTextQuestionWidgetState extends State { } String? _mapSpeechErrorToMessage(SpeechError error, AppLocalizations loc) { - final details = error.details?.toLowerCase(); - final isPermissionIssue = details?.contains('permission') == true; + final details = error.details; + final isPermissionIssue = details?.toLowerCase().contains('permission') == true; if (error.type == SpeechErrorType.microphonePermission && isPermissionIssue) { return loc.speech_to_text_error_permission; From 84c8724160daf83caf3bbaf038fc2cc5936ce320 Mon Sep 17 00:00:00 2001 From: Ibrahim Ozkan <51259479+ibrahimozkn@users.noreply.github.com> Date: Fri, 28 Nov 2025 13:21:09 +0100 Subject: [PATCH 13/16] Update app/ios/Runner/Info.plist Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- app/ios/Runner/Info.plist | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/ios/Runner/Info.plist b/app/ios/Runner/Info.plist index 15ab68c54..c9102a703 100644 --- a/app/ios/Runner/Info.plist +++ b/app/ios/Runner/Info.plist @@ -38,15 +38,15 @@ LSSupportsOpeningDocumentsInPlace - NSAppTransportSecurity - - NSAllowsArbitraryLoads - - - NSMicrophoneUsageDescription - Need microphone access for speech input and study recordings. - NSCameraUsageDescription - Need to access your camera to capture a photo for trial observations. + NSAppTransportSecurity + + NSAllowsArbitraryLoads + + + NSMicrophoneUsageDescription + Need microphone access for speech input and study recordings. + NSCameraUsageDescription + Need to access your camera to capture a photo for trial observations. UIApplicationSupportsIndirectInputEvents UIBackgroundModes From 007651980a6968738a41f5bbf5389268c793bc69 Mon Sep 17 00:00:00 2001 From: ibrahimozkn <51259479+ibrahimozkn@users.noreply.github.com> Date: Fri, 28 Nov 2025 12:24:46 +0000 Subject: [PATCH 14/16] chore: Apply static analysis changes --- .../questionnaire/questions/free_text_question_widget.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart index 499e907c5..16f060631 100644 --- a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart +++ b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart @@ -155,7 +155,8 @@ class _FreeTextQuestionWidgetState extends State { String? _mapSpeechErrorToMessage(SpeechError error, AppLocalizations loc) { final details = error.details; - final isPermissionIssue = details?.toLowerCase().contains('permission') == true; + final isPermissionIssue = + details?.toLowerCase().contains('permission') == true; if (error.type == SpeechErrorType.microphonePermission && isPermissionIssue) { return loc.speech_to_text_error_permission; From 1dd6e6f3177dc92d0313b4f4776573cb8d62e389 Mon Sep 17 00:00:00 2001 From: ibrahimozkan Date: Fri, 28 Nov 2025 14:06:34 +0100 Subject: [PATCH 15/16] feat: Add localization for speech-to-text connection errors and stop listening label, and implement connection error handling and localized microphone button tooltips. --- app/lib/l10n/app_de.arb | 2 ++ app/lib/l10n/app_en.arb | 2 ++ app/lib/l10n/app_localizations.dart | 12 ++++++++++++ app/lib/l10n/app_localizations_de.dart | 7 +++++++ app/lib/l10n/app_localizations_en.dart | 7 +++++++ .../questions/free_text_question_widget.dart | 17 +++++++++++++---- 6 files changed, 43 insertions(+), 4 deletions(-) diff --git a/app/lib/l10n/app_de.arb b/app/lib/l10n/app_de.arb index 0adbfa6cf..5c8413d2a 100644 --- a/app/lib/l10n/app_de.arb +++ b/app/lib/l10n/app_de.arb @@ -470,11 +470,13 @@ "speech_to_text_unsupported": "Sprach­eingabe ist auf diesem Gerät nicht verfügbar.", "speech_to_text_error_permission": "Bitte erlaube den Mikrofonzugriff, um diktieren zu können.", "speech_to_text_error_general": "Sprach­eingabe fehlgeschlagen. Bitte erneut versuchen.", + "speech_to_text_error_connection": "Verbindung verloren. Bitte erneut versuchen.", "speech_to_text_preparing": "Spracherkennung wird vorbereitet…", "speech_to_text_live_caption_hint": "Live-Transkription", "speech_to_text_listening": "Es wird zugehört…", "speech_to_text_press_to_speak": "Zum Sprechen drücken", "speech_to_text_idle": "Sprach­eingabe starten", + "speech_to_text_stop_listening": "Zuhören stoppen", "speech_to_text_opt_out_action": "Sprach­eingabe deaktivieren", "speech_to_text_enable_short_label": "Wieder aktivieren", "speech_to_text_language_badge": "{language}", diff --git a/app/lib/l10n/app_en.arb b/app/lib/l10n/app_en.arb index 5036ae0b7..d161c5653 100644 --- a/app/lib/l10n/app_en.arb +++ b/app/lib/l10n/app_en.arb @@ -466,11 +466,13 @@ "speech_to_text_unsupported": "Speech input is unavailable on this device.", "speech_to_text_error_permission": "Microphone permission is required for speech input.", "speech_to_text_error_general": "Speech input failed. Please try again.", + "speech_to_text_error_connection": "Connection lost. Please try again.", "speech_to_text_preparing": "Preparing speech recognizer…", "speech_to_text_live_caption_hint": "Live transcription", "speech_to_text_listening": "Listening…", "speech_to_text_press_to_speak": "Press to speak", "speech_to_text_idle": "Start speech input", + "speech_to_text_stop_listening": "Stop listening", "speech_to_text_opt_out_action": "Disable speech input for now", "speech_to_text_enable_short_label": "Enable again", "speech_to_text_language_badge": "{language}", diff --git a/app/lib/l10n/app_localizations.dart b/app/lib/l10n/app_localizations.dart index 6b15cf33b..9f3aadeb4 100644 --- a/app/lib/l10n/app_localizations.dart +++ b/app/lib/l10n/app_localizations.dart @@ -2336,6 +2336,12 @@ abstract class AppLocalizations { /// **'Speech input failed. Please try again.'** String get speech_to_text_error_general; + /// No description provided for @speech_to_text_error_connection. + /// + /// In en, this message translates to: + /// **'Connection lost. Please try again.'** + String get speech_to_text_error_connection; + /// No description provided for @speech_to_text_preparing. /// /// In en, this message translates to: @@ -2366,6 +2372,12 @@ abstract class AppLocalizations { /// **'Start speech input'** String get speech_to_text_idle; + /// No description provided for @speech_to_text_stop_listening. + /// + /// In en, this message translates to: + /// **'Stop listening'** + String get speech_to_text_stop_listening; + /// No description provided for @speech_to_text_opt_out_action. /// /// In en, this message translates to: diff --git a/app/lib/l10n/app_localizations_de.dart b/app/lib/l10n/app_localizations_de.dart index b9018ce49..7a1f6376a 100644 --- a/app/lib/l10n/app_localizations_de.dart +++ b/app/lib/l10n/app_localizations_de.dart @@ -1217,6 +1217,10 @@ class AppLocalizationsDe extends AppLocalizations { String get speech_to_text_error_general => 'Sprach­eingabe fehlgeschlagen. Bitte erneut versuchen.'; + @override + String get speech_to_text_error_connection => + 'Verbindung verloren. Bitte erneut versuchen.'; + @override String get speech_to_text_preparing => 'Spracherkennung wird vorbereitet…'; @@ -1232,6 +1236,9 @@ class AppLocalizationsDe extends AppLocalizations { @override String get speech_to_text_idle => 'Sprach­eingabe starten'; + @override + String get speech_to_text_stop_listening => 'Zuhören stoppen'; + @override String get speech_to_text_opt_out_action => 'Sprach­eingabe deaktivieren'; diff --git a/app/lib/l10n/app_localizations_en.dart b/app/lib/l10n/app_localizations_en.dart index 5b23e0040..b26635a09 100644 --- a/app/lib/l10n/app_localizations_en.dart +++ b/app/lib/l10n/app_localizations_en.dart @@ -1207,6 +1207,10 @@ class AppLocalizationsEn extends AppLocalizations { String get speech_to_text_error_general => 'Speech input failed. Please try again.'; + @override + String get speech_to_text_error_connection => + 'Connection lost. Please try again.'; + @override String get speech_to_text_preparing => 'Preparing speech recognizer…'; @@ -1222,6 +1226,9 @@ class AppLocalizationsEn extends AppLocalizations { @override String get speech_to_text_idle => 'Start speech input'; + @override + String get speech_to_text_stop_listening => 'Stop listening'; + @override String get speech_to_text_opt_out_action => 'Disable speech input for now'; diff --git a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart index 16f060631..3622d2a93 100644 --- a/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart +++ b/app/lib/widgets/questionnaire/questions/free_text_question_widget.dart @@ -154,13 +154,19 @@ class _FreeTextQuestionWidgetState extends State { } String? _mapSpeechErrorToMessage(SpeechError error, AppLocalizations loc) { - final details = error.details; - final isPermissionIssue = - details?.toLowerCase().contains('permission') == true; + final details = error.details?.toLowerCase(); + final isPermissionIssue = details?.contains('permission') == true; + final isConnectionIssue = details?.contains('connection') == true; + if (error.type == SpeechErrorType.microphonePermission && isPermissionIssue) { return loc.speech_to_text_error_permission; } + + if (isConnectionIssue) { + return loc.speech_to_text_error_connection; + } + return loc.speech_to_text_error_general; } @@ -234,6 +240,7 @@ class _FreeTextQuestionWidgetState extends State { } Widget _buildMicButton(ThemeData theme, bool isListening, bool isPreparing) { + final loc = AppLocalizations.of(context)!; return IconButton.filledTonal( onPressed: isPreparing ? null : _toggleSpeechListening, icon: isListening @@ -245,7 +252,9 @@ class _FreeTextQuestionWidgetState extends State { ? theme.colorScheme.onErrorContainer : null, ), - tooltip: isListening ? 'Stop listening' : 'Start listening', + tooltip: isListening + ? loc.speech_to_text_stop_listening + : loc.speech_to_text_idle, ); } From fe60bd2be5b9981d28093489d54194a3664eb085 Mon Sep 17 00:00:00 2001 From: ibrahimozkan Date: Fri, 28 Nov 2025 15:02:20 +0100 Subject: [PATCH 16/16] refactor: Introduce constants for reconnection and EOF delays and add detailed error logging for cleanup operations. --- .../websocket_speech_controller_io.dart | 51 +++++++++++++++---- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/app/lib/services/speech/websocket_speech_controller_io.dart b/app/lib/services/speech/websocket_speech_controller_io.dart index d3466a475..3bf37c4a8 100644 --- a/app/lib/services/speech/websocket_speech_controller_io.dart +++ b/app/lib/services/speech/websocket_speech_controller_io.dart @@ -40,6 +40,17 @@ class SpeechToTextController extends ValueNotifier { static const int _sampleRate = 16000; // Vosk typically uses 16kHz + /// Delay before attempting to reconnect after connection loss. + /// This prevents rapid reconnection attempts and gives the server/network + /// time to stabilize. 500ms provides a good balance between responsiveness + /// and avoiding connection spam. + static const int _reconnectionDelayMs = 500; + + /// Delay after sending EOF to allow server to process and send final results. + /// Vosk server needs time to finalize transcription after receiving EOF. + /// 300ms is typically sufficient for the server to send remaining data. + static const int _eofWaitDelayMs = 300; + static bool get _platformSupported { if (kIsWeb) return false; return Platform.isAndroid || @@ -158,24 +169,33 @@ class SpeechToTextController extends ValueNotifier { } void _handleConnectionLoss() { + // Atomic check-and-set to prevent race condition if (_isListening && !_isReconnecting) { + _isReconnecting = true; _reconnect(); } } Future _reconnect() async { - if (_isReconnecting || _isDisposed) return; - _isReconnecting = true; + // Double-check in case of edge cases + if (_isDisposed) { + _isReconnecting = false; + return; + } try { // Close existing connection if any try { await _wsChannel?.sink.close(); - } catch (_) {} + } catch (e) { + debugPrint( + '[SpeechToText] Error closing WebSocket during reconnect: $e', + ); + } _wsChannel = null; // Wait a bit before reconnecting - await Future.delayed(const Duration(milliseconds: 500)); + await Future.delayed(const Duration(milliseconds: _reconnectionDelayMs)); if (!_isListening) { _isReconnecting = false; @@ -274,10 +294,13 @@ class SpeechToTextController extends ValueNotifier { try { _wsChannel!.sink.add(jsonEncode({'eof': 1})); // Give server a moment to send final results - await Future.delayed(const Duration(milliseconds: 300)); + await Future.delayed(const Duration(milliseconds: _eofWaitDelayMs)); await _wsChannel!.sink.close(); - } catch (_) { - // Ignore errors during closing + } catch (e) { + // Errors during closing are expected if connection is already dead + debugPrint( + '[SpeechToText] Error during graceful WebSocket close: $e', + ); } _wsChannel = null; } @@ -304,7 +327,9 @@ class SpeechToTextController extends ValueNotifier { await _wsChannel!.sink.close(); _wsChannel = null; } - } catch (_) {} + } catch (e) { + debugPrint('[SpeechToText] Error during force reset cleanup: $e'); + } value = value.copyWith( status: SpeechLifecycleStatus.idle, clearTranscript: true, @@ -329,7 +354,11 @@ class SpeechToTextController extends ValueNotifier { try { _audioStreamSubscription?.cancel(); _audioRecorder.stop(); - } catch (_) {} + } catch (e) { + debugPrint( + '[SpeechToText] Error stopping audio during error emission: $e', + ); + } value = value.copyWith( status: SpeechLifecycleStatus.error, @@ -346,7 +375,9 @@ class SpeechToTextController extends ValueNotifier { await _audioRecorder.stop(); await _audioRecorder.dispose(); await _wsChannel?.sink.close(); - } catch (_) {} + } catch (e) { + debugPrint('[SpeechToText] Error during dispose cleanup: $e'); + } super.dispose(); } }