diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistActivity.kt b/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistActivity.kt index 3778168ad1d..3207d898aa8 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistActivity.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistActivity.kt @@ -48,11 +48,14 @@ class AssistActivity : BaseActivity() { private var contextIsLocked = true companion object { - private const val EXTRA_SERVER = "server" - private const val EXTRA_PIPELINE = "pipeline" - private const val EXTRA_START_LISTENING = "start_listening" - private const val EXTRA_FROM_FRONTEND = "from_frontend" - private const val EXTRA_FROM_WAKE_WORD_PHRASE = "from_wake_word_phrase" + const val EXTRA_SERVER = "server" + const val EXTRA_PIPELINE = "pipeline" + const val EXTRA_START_LISTENING = "start_listening" + const val EXTRA_FROM_FRONTEND = "from_frontend" + const val EXTRA_FROM_WAKE_WORD_PHRASE = "from_wake_word_phrase" + const val EXTRA_TRIGGER_SOURCE = "trigger_source" + const val TRIGGER_SOURCE_ASSIST = "assist" + const val ACTION_TRIGGER_AUTOMOTIVE_ASSIST = "ACTION_TRIGGER_AUTOMOTIVE_ASSIST" fun newInstance( context: Context, @@ -61,6 +64,7 @@ class AssistActivity : BaseActivity() { startListening: Boolean = true, fromFrontend: Boolean = true, wakeWordPhrase: String? = null, + triggerSource: String? = null, ): Intent { return Intent(context, AssistActivity::class.java).apply { putExtra(EXTRA_SERVER, serverId) @@ -68,6 +72,7 @@ class AssistActivity : BaseActivity() { putExtra(EXTRA_START_LISTENING, startListening) putExtra(EXTRA_FROM_FRONTEND, fromFrontend) putExtra(EXTRA_FROM_WAKE_WORD_PHRASE, wakeWordPhrase) + putExtra(EXTRA_TRIGGER_SOURCE, triggerSource) } } } @@ -113,6 +118,27 @@ class AssistActivity : BaseActivity() { } val fromFrontend = intent.getBooleanExtra(EXTRA_FROM_FRONTEND, false) + val triggerSource = intent.getStringExtra(EXTRA_TRIGGER_SOURCE) + + if (triggerSource == TRIGGER_SOURCE_ASSIST) { + if (io.homeassistant.companion.android.vehicle.HaCarAppService.carInfo != null) { + val automotiveIntent = Intent( + io.homeassistant.companion.android.vehicle.HaCarAppService.ACTION_NAVIGATE_TO_AUTOMOTIVE_ASSIST, + ).apply { + putExtra( + io.homeassistant.companion.android.vehicle.HaCarAppService.EXTRA_SERVER, + if (intent.hasExtra(EXTRA_SERVER)) { + intent.getIntExtra(EXTRA_SERVER, ServerManager.SERVER_ID_ACTIVE) + } else { + ServerManager.SERVER_ID_ACTIVE + }, + ) + } + sendBroadcast(automotiveIntent) + finish() + return + } + } setContent { if (viewModel.shouldFinish) { @@ -166,6 +192,9 @@ class AssistActivity : BaseActivity() { override fun onPause() { super.onPause() + // The error says onPause() is protected in AssistViewModel. + // We need to call it, but it's protected. + // Let's see if we can change the visibility in AssistViewModel. viewModel.onPause() } diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistAudioStrategyFactory.kt b/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistAudioStrategyFactory.kt index 329d7b4e106..2e737115b7a 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistAudioStrategyFactory.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistAudioStrategyFactory.kt @@ -1,5 +1,4 @@ package io.homeassistant.companion.android.assist - import android.content.Context import androidx.core.content.getSystemService import io.homeassistant.companion.android.assist.service.AssistVoiceInteractionService diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistViewModel.kt b/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistViewModel.kt index 6581cc57ac1..a978cbb1256 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistViewModel.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/assist/AssistViewModel.kt @@ -506,13 +506,13 @@ class AssistViewModel @AssistedInject constructor( if (!proactive) requestSilently = false } - fun onPause() { + override fun onPause() { requestPermission = null inactivityTimerJob?.cancel() stopRecording() } - fun onDestroy() { + override fun onDestroy() { requestPermission = null inactivityTimerJob?.cancel() stopRecording() diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/assist/AutomotiveAssistViewModel.kt b/app/src/main/kotlin/io/homeassistant/companion/android/assist/AutomotiveAssistViewModel.kt new file mode 100644 index 00000000000..61242585436 --- /dev/null +++ b/app/src/main/kotlin/io/homeassistant/companion/android/assist/AutomotiveAssistViewModel.kt @@ -0,0 +1,338 @@ +package io.homeassistant.companion.android.assist + +import android.app.Application +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.setValue +import androidx.lifecycle.viewModelScope +import dagger.assisted.Assisted +import dagger.assisted.AssistedFactory +import dagger.assisted.AssistedInject +import io.homeassistant.companion.android.assist.ui.AssistMessage +import io.homeassistant.companion.android.common.assist.AssistAudioStrategy +import io.homeassistant.companion.android.common.assist.AssistEvent +import io.homeassistant.companion.android.common.assist.AssistViewModelBase +import io.homeassistant.companion.android.common.data.servers.ServerManager +import io.homeassistant.companion.android.common.util.AudioUrlPlayer +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import timber.log.Timber + +/** A ViewModel for the automotive Assist UI. It provides a simplified state + * compared to [AssistViewModel], focusing on voice-only interaction. + */ +class AutomotiveAssistViewModel @AssistedInject constructor( + @Assisted override val serverManager: ServerManager, + @Assisted override val audioStrategy: AssistAudioStrategy, + @Assisted private val audioUrlPlayer: AudioUrlPlayer, + @Assisted private val application: Application, +) : AssistViewModelBase(serverManager, audioStrategy, audioUrlPlayer, application) { + + val isAudioPlaying: StateFlow = isPlayingAudioState + + private val _processingState = MutableStateFlow(false) + val processingState: StateFlow = _processingState + + private var pipelineId: String? = null + private var pipelineJob: Job? = null + private var activeUserMessage: AssistMessage? = null + private var activeHaMessage: AssistMessage? = null + private var isContinuationTurn = false + + var isProcessing by mutableStateOf(false) + private set + + @AssistedFactory + interface Factory { + fun create( + serverManager: ServerManager, + audioStrategy: AssistAudioStrategy, + audioUrlPlayer: AudioUrlPlayer, + application: Application, + ): AutomotiveAssistViewModel + } + + private val _conversation = MutableStateFlow>(emptyList()) + val conversation: StateFlow> = _conversation.asStateFlow() + + var inputMode by mutableStateOf(null) + private set + + var shouldFinish by mutableStateOf(false) + private set + + var recorderAutoStart by mutableStateOf(false) + private set + + override fun getInput(): AssistInputMode? = inputMode + + override fun setInput(inputMode: AssistInputMode) { + this.inputMode = inputMode + } + + init { + viewModelScope.launch { + audioStrategy.wakeWordDetected.collect { detectedPhrase -> + if (inputMode != AssistInputMode.VOICE_ACTIVE) { + onMicrophoneInput(clearConversation = false) + } + } + } + } + + fun onCreate(hasPermission: Boolean, serverId: Int?, pipelineId: String?, startListening: Boolean?) { + viewModelScope.launch { + this@AutomotiveAssistViewModel.hasPermission = hasPermission + serverId?.let { + selectedServerId = it + } + startListening?.let { recorderAutoStart = it } + + if (!serverManager.isRegistered()) { + inputMode = AssistInputMode.BLOCKED + _conversation.value = listOf( + AssistMessage( + app.getString(io.homeassistant.companion.android.common.R.string.not_registered), + isInput = false, + ), + ) + return@launch + } + + if (pipelineId != null) { + setPipeline(pipelineId) + } else { + val lastPipelineId = serverManager.integrationRepository(selectedServerId).getLastUsedPipelineId() + Timber.tag("[AA-Assist]").d("onCreate: lastPipelineId=%s", lastPipelineId) + if (lastPipelineId != null) { + setPipeline(lastPipelineId) + } else { + val allPipelines = try { + serverManager.webSocketRepository(selectedServerId).getAssistPipelines() + } catch (e: Exception) { + Timber.e(e, "Failed to get assist pipelines") + null + } + Timber.tag("[AA-Assist]").d("onCreate: allPipelines=%s", allPipelines?.pipelines?.map { it.id }) + val ttsPipeline = allPipelines?.pipelines?.firstOrNull { it.ttsEngine != null } + if (ttsPipeline != null) { + Timber.tag("[AA-Assist]").d("onCreate: using TTS pipeline=%s", ttsPipeline.id) + setPipeline(ttsPipeline.id) + } else { + inputMode = AssistInputMode.BLOCKED + _conversation.value = listOf( + AssistMessage( + app.getString(io.homeassistant.companion.android.common.R.string.assist_error), + isInput = false, + ), + ) + } + } + } + + if (hasPermission && recorderAutoStart) { + onMicrophoneInput(proactive = true, clearConversation = true) + } + } + } + + private suspend fun setPipeline(id: String?) { + pipelineId = id + Timber.tag("[AA-Assist]").d("setPipeline: id=%s", id) + val pipeline = try { + serverManager.webSocketRepository(selectedServerId).getAssistPipeline(id) + } catch (e: Exception) { + Timber.e(e, "Failed to get assist pipeline") + null + } + + Timber.tag("[AA-Assist]").d( + "setPipeline: pipeline=%s, ttsEngine=%s, sttEngine=%s", + pipeline?.id, + pipeline?.ttsEngine, + pipeline?.sttEngine, + ) + if (pipeline != null) { + _conversation.value = emptyList() + activeUserMessage = null + activeHaMessage = null + inputMode = if (pipeline.sttEngine != null) AssistInputMode.VOICE_INACTIVE else AssistInputMode.TEXT_ONLY + } else { + inputMode = AssistInputMode.BLOCKED + } + } + + fun onMicrophoneInput( + proactive: Boolean = false, + isContinuation: Boolean = false, + clearConversation: Boolean = false, + ) { + Timber.d( + "onMicrophoneInput called " + + "(proactive=$proactive, isContinuation=$isContinuation, clearConversation=$clearConversation)", + ) + if (!hasPermission) { + Timber.w("onMicrophoneInput aborted: no permission") + return + } + + if (clearConversation) { + _conversation.value = emptyList() + activeUserMessage = null + activeHaMessage = null + pipelineJob?.cancel() + } + + stopPlayback() + setupRecorder(onError = { + stopRecording() + _conversation.value = _conversation.value + AssistMessage( + app.getString(io.homeassistant.companion.android.common.R.string.assist_error), + isInput = false, + isError = true, + ) + Timber.e(it, "Recorder setup failed") + }) + if (!isContinuation) { + inputMode = AssistInputMode.VOICE_ACTIVE + } + + if (proactive) { + if (isContinuation) { + // Just add user placeholder, pipeline already running + activeUserMessage = AssistMessage.placeholder(isInput = true) + _conversation.value = _conversation.value + activeUserMessage!! + activeHaMessage = AssistMessage.placeholder(isInput = false) + } else { + // New pipeline, add placeholders and start pipeline + activeUserMessage = AssistMessage.placeholder(isInput = true) + activeHaMessage = AssistMessage.placeholder(isInput = false) + _conversation.value = _conversation.value + activeUserMessage!! + activeHaMessage!! + runAssistPipeline(null) + } + } + } + + private fun runAssistPipeline(text: String?, skipStopPlayback: Boolean = false) { + Timber.tag("[AA-Assist]").d("runAssistPipeline: text=%s, isVoice=%s", text, text == null) + if (!skipStopPlayback) { + stopPlayback() + } + + pipelineJob = viewModelScope.launch { + val pipeline = try { + val id = pipelineId ?: serverManager.integrationRepository(selectedServerId).getLastUsedPipelineId() + Timber.tag("[AA-Assist]").d( + "runAssistPipeline: pipelineId=%s, lastPipelineId=%s", + pipelineId, + serverManager.integrationRepository(selectedServerId).getLastUsedPipelineId(), + ) + id?.let { + serverManager.webSocketRepository(selectedServerId).getAssistPipeline(it) + } + } catch (e: Exception) { + Timber.e(e, "Failed to get assist pipeline") + null + } + + Timber.tag("[AA-Assist]").d( + "runAssistPipeline: pipeline=%s, ttsEngine=%s, sttEngine=%s", + pipeline?.id, + pipeline?.ttsEngine, + pipeline?.sttEngine, + ) + isProcessing = true + runAssistPipelineInternal( + text = text, + pipeline = pipeline, + wakeWordPhrase = null, + ) { event -> + when (event) { + is AssistEvent.Message -> { + val currentList = _conversation.value.toMutableList() + if (event is AssistEvent.Message.Error) { + if (activeHaMessage != null) { + val haIndex = currentList.indexOf(activeHaMessage) + if (haIndex != -1) { + currentList[haIndex] = activeHaMessage!!.copy( + message = event.message.trim(), + isError = true, + ) + _conversation.value = currentList + } + } + } else if (event is AssistEvent.Message.Input) { + if (activeUserMessage != null) { + val userIndex = currentList.indexOf(activeUserMessage) + if (userIndex != -1) { + currentList[userIndex] = activeUserMessage!!.copy( + message = event.message.trim(), + isError = false, + ) + // Add assistant placeholder for the response if not already in list + if (currentList.indexOf(activeHaMessage) == -1) { + activeHaMessage = AssistMessage.placeholder(isInput = false) + currentList.add(activeHaMessage!!) + } + _conversation.value = currentList + } + } + } else if (event is AssistEvent.Message.Output) { + if (activeHaMessage != null) { + val haIndex = currentList.indexOf(activeHaMessage) + if (haIndex != -1) { + currentList[haIndex] = activeHaMessage!!.copy( + message = event.message.trim(), + isError = false, + ) + _conversation.value = currentList + } + } + } + } + + is AssistEvent.MessageChunk -> { + val currentList = _conversation.value.toMutableList() + if (activeHaMessage != null) { + val haIndex = currentList.indexOf(activeHaMessage) + if (haIndex != -1) { + activeHaMessage = activeHaMessage!!.copy( + message = activeHaMessage!!.message + event.chunk, + ) + currentList[haIndex] = activeHaMessage!! + _conversation.value = currentList + } + } + } + + is AssistEvent.Dismiss -> { + isProcessing = false + _processingState.value = false + shouldFinish = true + } + + is AssistEvent.ContinueConversation -> { + onMicrophoneInput(proactive = true, isContinuation = false) + isContinuationTurn = true + runAssistPipeline(null, skipStopPlayback = true) + } + + is AssistEvent.PipelineEnded -> { + isProcessing = false + _processingState.value = false + if (!isContinuationTurn) { + activeUserMessage = null + activeHaMessage = null + } + isContinuationTurn = false + } + + else -> {} + } + } + } + } +} diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/AutomotiveAssistScreen.kt b/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/AutomotiveAssistScreen.kt new file mode 100644 index 00000000000..ca65fcc2a47 --- /dev/null +++ b/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/AutomotiveAssistScreen.kt @@ -0,0 +1,144 @@ +package io.homeassistant.companion.android.vehicle + +import android.app.Application +import androidx.annotation.RequiresApi +import androidx.car.app.CarContext +import androidx.car.app.Screen +import androidx.car.app.model.Action +import androidx.car.app.model.CarColor +import androidx.car.app.model.CarIcon +import androidx.car.app.model.CarText +import androidx.car.app.model.ItemList +import androidx.car.app.model.ListTemplate +import androidx.car.app.model.Row +import androidx.car.app.model.Template +import androidx.compose.runtime.snapshotFlow +import com.mikepenz.iconics.IconicsDrawable +import com.mikepenz.iconics.typeface.library.community.material.CommunityMaterial +import com.mikepenz.iconics.utils.sizeDp +import com.mikepenz.iconics.utils.toAndroidIconCompat +import dagger.assisted.Assisted +import dagger.assisted.AssistedFactory +import dagger.assisted.AssistedInject +import io.homeassistant.companion.android.assist.AssistAudioStrategyFactory +import io.homeassistant.companion.android.assist.AutomotiveAssistViewModel +import io.homeassistant.companion.android.common.R as commonR +import io.homeassistant.companion.android.common.assist.AssistViewModelBase +import io.homeassistant.companion.android.common.data.servers.ServerManager +import io.homeassistant.companion.android.common.util.AudioUrlPlayer +import io.homeassistant.companion.android.util.vehicle.getHeaderBuilder +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import timber.log.Timber + +/** A CarApp Screen that displays the automotive Assist UI. + * It uses a simplified interface suitable for vehicle use. + */ +@RequiresApi +class AutomotiveAssistScreen @AssistedInject constructor( + @Assisted private val carContext: CarContext, + @Assisted private val serverManager: ServerManager, + @Assisted private val serverId: Int, + @Assisted private val audioStrategyFactory: AssistAudioStrategyFactory, + @Assisted private val audioUrlPlayer: AudioUrlPlayer, + @Assisted private val application: Application, + @Assisted private val viewModel: AutomotiveAssistViewModel, + @Assisted private val scope: CoroutineScope, +) : Screen(carContext) { + + init { + scope.launch { + viewModel.conversation.collect { + invalidate() + } + } + scope.launch { + viewModel.processingState.collect { + invalidate() + } + } + scope.launch { + viewModel.isAudioPlaying.collect { + invalidate() + } + } + scope.launch { + snapshotFlow { viewModel.inputMode }.collect { + invalidate() + } + } + } + + @AssistedFactory + interface Factory { + fun create( + carContext: CarContext, + serverManager: ServerManager, + serverId: Int, + audioStrategyFactory: AssistAudioStrategyFactory, + audioUrlPlayer: AudioUrlPlayer, + application: Application, + viewModel: AutomotiveAssistViewModel, + scope: CoroutineScope, + ): AutomotiveAssistScreen + } + + override fun onGetTemplate(): Template { + Timber.d("onGetTemplate called") + val conversation = viewModel.conversation.value + val isPlayingAudio = viewModel.isAudioPlaying.value + val isProcessing = viewModel.isProcessing + val inputMode = viewModel.getInput() + + val icon = when { + inputMode == AssistViewModelBase.AssistInputMode.VOICE_ACTIVE -> { + CommunityMaterial.Icon3.cmd_microphone + } + isPlayingAudio -> { + CommunityMaterial.Icon3.cmd_volume_high + } + isProcessing -> { + CommunityMaterial.Icon3.cmd_sync + } + else -> { + CommunityMaterial.Icon3.cmd_microphone_outline + } + } + + val header = carContext.getHeaderBuilder(commonR.string.assist_how_can_i_assist).apply { + addEndHeaderAction( + Action.Builder() + .setIcon( + CarIcon.Builder( + IconicsDrawable( + carContext, + icon, + ).apply { + sizeDp = 32 + }.toAndroidIconCompat(), + ).setTint(CarColor.DEFAULT).build(), + ) + .setOnClickListener { + Timber.d("Assist button clicked") + viewModel.onMicrophoneInput(proactive = true, clearConversation = true) + } + .build(), + ) + }.build() + + val itemListBuilder = ItemList.Builder() + conversation.forEach { msg -> + itemListBuilder.addItem( + Row.Builder() + .setTitle(CarText.create(if (msg.isInput) "You" else "Assistant")) + .addText(CarText.create(msg.message)) + .build(), + ) + } + + return ListTemplate.Builder() + .setHeader(header) + .setSingleList(itemListBuilder.build()) + .build() + } +} diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/HaCarAppService.kt b/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/HaCarAppService.kt index 02773c14142..555e3766ace 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/HaCarAppService.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/HaCarAppService.kt @@ -1,7 +1,13 @@ package io.homeassistant.companion.android.vehicle +import android.app.Application +import android.content.BroadcastReceiver +import android.content.Context import android.content.Intent +import android.content.IntentFilter import android.content.pm.ApplicationInfo +import android.content.pm.PackageManager +import android.media.AudioManager import android.os.Build import androidx.annotation.RequiresApi import androidx.car.app.CarAppService @@ -12,12 +18,18 @@ import androidx.car.app.SessionInfo import androidx.car.app.hardware.CarHardwareManager import androidx.car.app.hardware.info.CarInfo import androidx.car.app.validation.HostValidator +import androidx.core.content.ContextCompat import androidx.lifecycle.lifecycleScope import dagger.hilt.android.AndroidEntryPoint import io.homeassistant.companion.android.R +import io.homeassistant.companion.android.assist.AssistActivity +import io.homeassistant.companion.android.assist.AssistAudioStrategyFactory +import io.homeassistant.companion.android.assist.AutomotiveAssistViewModel import io.homeassistant.companion.android.common.data.integration.Entity import io.homeassistant.companion.android.common.data.prefs.PrefsRepository import io.homeassistant.companion.android.common.data.servers.ServerManager +import io.homeassistant.companion.android.common.util.AudioUrlPlayer +import io.homeassistant.companion.android.vehicle.AutomotiveAssistScreen.Factory as AutomotiveAssistScreenFactory import java.util.Collections import javax.inject.Inject import kotlinx.coroutines.CancellationException @@ -37,6 +49,36 @@ class HaCarAppService : CarAppService() { companion object { var carInfo: CarInfo? = null private set + + const val ACTION_NAVIGATE_TO_AUTOMOTIVE_ASSIST = + "io.homeassistant.companion.android.vehicle.ACTION_NAVIGATE_TO_AUTOMOTIVE_ASSIST" + const val EXTRA_SERVER = "server" + } + + private var currentSession: MySession? = null + + private val navigationReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context?, intent: Intent?) { + if (intent?.action == ACTION_NAVIGATE_TO_AUTOMOTIVE_ASSIST) { + val serverId = intent.getIntExtra(EXTRA_SERVER, ServerManager.SERVER_ID_ACTIVE) + currentSession?.navigateToAssist(serverId) + } + } + } + + @RequiresApi(Build.VERSION_CODES.O) + override fun onCreate() { + super.onCreate() + val filter = IntentFilter(ACTION_NAVIGATE_TO_AUTOMOTIVE_ASSIST) + registerReceiver(navigationReceiver, filter) + } + + @RequiresApi(Build.VERSION_CODES.O) + override fun onDestroy() { + super.onDestroy() + unregisterReceiver(navigationReceiver) + currentSession = null + carInfo = null } @Inject @@ -45,6 +87,15 @@ class HaCarAppService : CarAppService() { @Inject lateinit var prefsRepository: PrefsRepository + @Inject + lateinit var audioStrategyFactory: AssistAudioStrategyFactory + + @Inject + lateinit var automotiveAssistViewModelFactory: AutomotiveAssistViewModel.Factory + + @Inject + lateinit var automotiveAssistScreenFactory: AutomotiveAssistScreenFactory + private val serverId = MutableStateFlow(0) private val allEntities = MutableStateFlow>(emptyMap()) private var allEntitiesJob: Job? = null @@ -60,75 +111,133 @@ class HaCarAppService : CarAppService() { } override fun onCreateSession(sessionInfo: SessionInfo): Session { - return object : Session() { - init { - lifecycleScope.launch { - serverManager.getServer()?.let { - loadEntities(this, it.id) - } + val session = MySession() + currentSession = session + return session + } + + inner class MySession : Session() { + init { + lifecycleScope.launch { + serverManager.getServer()?.let { + loadEntities(this, it.id) } } + } - val serverIdFlow = serverId.asStateFlow() - val entityFlow = allEntities.shareIn( - lifecycleScope, - SharingStarted.WhileSubscribed(10_000), - 1, - ) + val serverIdFlow = serverId.asStateFlow() + val entityFlow = allEntities.shareIn( + lifecycleScope, + SharingStarted.WhileSubscribed(10_000), + 1, + ) - override fun onCreateScreen(intent: Intent): Screen { - carInfo = carContext.getCarService(CarHardwareManager::class.java).carInfo - - if (intent.getBooleanExtra("TRANSITION_LAUNCH", false)) { - carContext - .getCarService(ScreenManager::class.java).run { - push( - MainVehicleScreen( - carContext, - serverManager, - serverIdFlow, - entityFlow, - prefsRepository, - { loadEntities(lifecycleScope, it) }, - { loadEntities(lifecycleScope, serverId.value) }, - ), - ) - - push( - LoginScreen( - carContext, - serverManager, - ), - ) - } - return SwitchToDrivingOptimizedScreen(carContext) - } else { - carContext - .getCarService(ScreenManager::class.java).run { - push( - MainVehicleScreen( - carContext, - serverManager, - serverIdFlow, - entityFlow, - prefsRepository, - { loadEntities(lifecycleScope, it) }, - { loadEntities(lifecycleScope, serverId.value) }, - ), - ) - } - return LoginScreen( - carContext, - serverManager, - ) - } + override fun onCreateScreen(intent: Intent): Screen { + carInfo = carContext.getCarService(CarHardwareManager::class.java).carInfo + + if (intent.action == AssistActivity.ACTION_TRIGGER_AUTOMOTIVE_ASSIST) { + val serverId = intent.getIntExtra(AssistActivity.EXTRA_SERVER, ServerManager.SERVER_ID_ACTIVE) + + val audioManager = applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + + val audioUrlPlayerInstance = AudioUrlPlayer( + audioManager, + { player -> + val exoPlayer = androidx.media3.exoplayer.ExoPlayer.Builder(applicationContext).build() + exoPlayer.apply(player) + exoPlayer + }, + ) + + val automotiveAssistViewModel = automotiveAssistViewModelFactory.create( + serverManager, + audioStrategyFactory.create(applicationContext, null), + audioUrlPlayerInstance, + application as Application, + ) + automotiveAssistViewModel.onCreate( + hasPermission = ContextCompat.checkSelfPermission( + applicationContext, + android.Manifest.permission.RECORD_AUDIO, + ) == PackageManager.PERMISSION_GRANTED, + serverId = serverId, + pipelineId = null, + startListening = true, + ) + + return automotiveAssistScreenFactory.create( + carContext, + serverManager, + serverId, + audioStrategyFactory, + audioUrlPlayerInstance, + application as Application, + automotiveAssistViewModel, + lifecycleScope, + ) + } else { + carContext + .getCarService(ScreenManager::class.java).run { + push( + MainVehicleScreen( + carContext, + serverManager, + serverIdFlow, + entityFlow, + prefsRepository, + { loadEntities(lifecycleScope, it) }, + { loadEntities(lifecycleScope, serverId.value) }, + ), + ) + } + return LoginScreen( + carContext, + serverManager, + ) } } - } - override fun onDestroy() { - super.onDestroy() - carInfo = null + fun navigateToAssist(serverId: Int) { + val audioManager = applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + + val audioUrlPlayerInstance = AudioUrlPlayer( + audioManager, + { player -> + val exoPlayer = androidx.media3.exoplayer.ExoPlayer.Builder(applicationContext).build() + exoPlayer.apply(player) + exoPlayer + }, + ) + + val automotiveAssistViewModel = automotiveAssistViewModelFactory.create( + serverManager, + audioStrategyFactory.create(applicationContext, null), + audioUrlPlayerInstance, + application as Application, + ) + automotiveAssistViewModel.onCreate( + hasPermission = ContextCompat.checkSelfPermission( + applicationContext, + android.Manifest.permission.RECORD_AUDIO, + ) == PackageManager.PERMISSION_GRANTED, + serverId = serverId, + pipelineId = null, + startListening = true, + ) + + carContext.getCarService(ScreenManager::class.java).push( + automotiveAssistScreenFactory.create( + carContext, + serverManager, + serverId, + audioStrategyFactory, + audioUrlPlayerInstance, + application as Application, + automotiveAssistViewModel, + lifecycleScope, + ), + ) + } } private fun loadEntities(scope: CoroutineScope, id: Int) { @@ -151,11 +260,11 @@ class HaCarAppService : CarAppService() { null } if (entities != null) { - allEntities.emit(entities.toImmutableMap()) + allEntities.value = entities.toImmutableMap() try { serverManager.integrationRepository(id).getEntityUpdates()?.collect { entity -> entities[entity.entityId] = entity - allEntities.emit(entities.toImmutableMap()) + allEntities.value = entities.toImmutableMap() } } catch (e: CancellationException) { throw e @@ -164,7 +273,7 @@ class HaCarAppService : CarAppService() { } } else { Timber.w("No entities found?") - allEntities.emit(emptyMap()) + allEntities.value = emptyMap() } } } diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/MainVehicleScreen.kt b/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/MainVehicleScreen.kt index c1d2d760cc0..bab9137c12f 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/MainVehicleScreen.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/vehicle/MainVehicleScreen.kt @@ -1,11 +1,13 @@ package io.homeassistant.companion.android.vehicle +import android.content.Intent import android.os.Build import androidx.annotation.RequiresApi import androidx.car.app.CarContext import androidx.car.app.model.Action import androidx.car.app.model.CarColor import androidx.car.app.model.CarIcon +import androidx.car.app.model.GridItem import androidx.car.app.model.GridTemplate import androidx.car.app.model.ItemList import androidx.car.app.model.Template @@ -202,6 +204,35 @@ class MainVehicleScreen( ) { onChangeServer(it) }.build(), ) } + + listBuilder.addItem( + GridItem.Builder() + .setLoading(false) + .setTitle(carContext.getString(commonR.string.assist)) + .setImage( + CarIcon.Builder( + IconicsDrawable(carContext, CommunityMaterial.Icon3.cmd_microphone).apply { + sizeDp = 64 + }.toAndroidIconCompat(), + ) + .setTint(CarColor.DEFAULT) + .build(), + ) + .setOnClickListener { + val intent = Intent( + io.homeassistant.companion.android.vehicle.HaCarAppService.ACTION_NAVIGATE_TO_AUTOMOTIVE_ASSIST, + ).apply { + if (serverId.value != 0) { + putExtra( + io.homeassistant.companion.android.vehicle.HaCarAppService.EXTRA_SERVER, + serverId.value, + ) + } + } + carContext.sendBroadcast(intent) + } + .build(), + ) val refreshAction = Action.Builder() .setIcon( CarIcon.Builder( diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/widgets/assist/AssistShortcutActivity.kt b/app/src/main/kotlin/io/homeassistant/companion/android/widgets/assist/AssistShortcutActivity.kt index 3048c180887..60ff86fc8a7 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/widgets/assist/AssistShortcutActivity.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/widgets/assist/AssistShortcutActivity.kt @@ -50,6 +50,7 @@ class AssistShortcutActivity : BaseActivity() { fromFrontend = false, ).apply { action = Intent.ACTION_VIEW + putExtra(AssistActivity.EXTRA_TRIGGER_SOURCE, AssistActivity.TRIGGER_SOURCE_ASSIST) } val shortcutInfo = ShortcutInfoCompat.Builder(this, "$SHORTCUT_PREFIX${UUID.randomUUID()}") .setIntent(assistIntent) diff --git a/common/src/main/kotlin/io/homeassistant/companion/android/common/assist/AssistViewModelBase.kt b/common/src/main/kotlin/io/homeassistant/companion/android/common/assist/AssistViewModelBase.kt index 8b108542e2e..e019507344d 100644 --- a/common/src/main/kotlin/io/homeassistant/companion/android/common/assist/AssistViewModelBase.kt +++ b/common/src/main/kotlin/io/homeassistant/companion/android/common/assist/AssistViewModelBase.kt @@ -30,6 +30,8 @@ import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.Job import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.catch import kotlinx.coroutines.flow.emptyFlow import kotlinx.coroutines.flow.first @@ -59,12 +61,20 @@ sealed interface AssistEvent { } abstract class AssistViewModelBase( - protected val serverManager: ServerManager, - protected val audioStrategy: AssistAudioStrategy, + protected open val serverManager: ServerManager, + protected open val audioStrategy: AssistAudioStrategy, private val audioUrlPlayer: AudioUrlPlayer, application: Application, ) : AndroidViewModel(application) { + open fun onPause() {} + open fun onDestroy() {} + + override fun onCleared() { + super.onCleared() + onDestroy() + } + companion object { const val PIPELINE_PREFERRED = "preferred" const val PIPELINE_LAST_USED = "last_used" @@ -128,6 +138,15 @@ abstract class AssistViewModelBase( /** Whether TTS audio is currently being played back. Updated by playback handlers. */ protected var isPlayingAudio = false + private set + + private val _isPlayingAudioFlow = MutableStateFlow(false) + val isPlayingAudioState: StateFlow = _isPlayingAudioFlow + + protected fun setIsPlayingAudio(value: Boolean) { + isPlayingAudio = value + _isPlayingAudioFlow.value = value + } /** * @param text input to run an intent pipeline with, or `null` to run a STT pipeline (check if @@ -145,10 +164,13 @@ abstract class AssistViewModelBase( var job: Job? = null job = viewModelScope.launch { val flow = try { + val outputTts = pipeline?.ttsEngine?.isNotBlank() == true + Timber.tag("[AA-Assist]").d("runAssistPipelineInternal: isVoice=%s, pipelineId=%s, ttsEngine=%s, outputTts=%s", + isVoice, pipeline?.id, pipeline?.ttsEngine, outputTts) if (isVoice) { serverManager.webSocketRepository(selectedServerId).runAssistPipelineForVoice( sampleRate = VOICE_SAMPLE_RATE, - outputTts = pipeline?.ttsEngine?.isNotBlank() == true, + outputTts = outputTts, pipelineId = pipeline?.id, conversationId = conversationId, wakeWordPhrase = wakeWordPhrase, @@ -217,29 +239,36 @@ abstract class AssistViewModelBase( } private fun handleRunStart(data: AssistPipelineRunStart?, isVoice: Boolean, onEvent: (AssistEvent) -> Unit) { + Timber.tag("[AA-Assist]").d("handleRunStart: isVoice=%s, hasTtsOutput=%s", isVoice, data?.ttsOutput != null) if (!isVoice) return data?.ttsOutput?.let { ttsOutput -> val audioPath = ttsOutput.url + Timber.tag("[AA-Assist]").d("handleRunStart: audioPath=%s, currentPath=%s", audioPath, currentPathBeingPlayed) val shouldPlay = currentPathBeingPlayed != audioPath || currentPlayAudioJob?.isActive != true + Timber.tag("[AA-Assist]").d("handleRunStart: shouldPlay=%s, audioPathNotBlank=%s", shouldPlay, audioPath.isNotBlank()) if (audioPath.isNotBlank() && shouldPlay) { currentPathBeingPlayed = audioPath stopPlayback() currentPlayAudioJob = viewModelScope.launch { try { + Timber.tag("[AA-Assist]").d("handleRunStart: starting audio playback for %s", audioPath) playAudio(audioPath).collect { state -> + Timber.tag("[AA-Assist]").d("handleRunStart: playback state=%s", state) when (state) { - PlaybackState.PLAYING -> isPlayingAudio = true + PlaybackState.PLAYING -> setIsPlayingAudio(true) PlaybackState.STOP_PLAYING -> { - isPlayingAudio = false + setIsPlayingAudio(false) onEvent(AssistEvent.PlaybackFinished) notifyContinueConversationIfNeeded(onEvent) } PlaybackState.READY -> { /* No op */ } } } + Timber.tag("[AA-Assist]").d("handleRunStart: audio playback flow completed") } finally { - isPlayingAudio = false + Timber.tag("[AA-Assist]").d("handleRunStart: finally block - setting isPlayingAudio=false") + setIsPlayingAudio(false) } } } @@ -268,9 +297,12 @@ abstract class AssistViewModelBase( } private fun handleIntentEnd(data: AssistPipelineIntentEnd?, onEvent: (AssistEvent) -> Unit) { + Timber.tag("[AA-Assist]").d("handleIntentEnd: intentOutput=%s", data?.intentOutput != null) val intentOutput = data?.intentOutput ?: return conversationId = intentOutput.conversationId continueConversation.set(intentOutput.continueConversation) + Timber.tag("[AA-Assist]").d("handleIntentEnd: continueConversation=%s, speech=%s", + intentOutput.continueConversation, intentOutput.response.speech?.plain?.get("speech")) intentOutput.response.speech?.plain?.get("speech")?.let { speech -> onEvent(AssistEvent.Message.Output(speech)) } @@ -286,12 +318,12 @@ abstract class AssistViewModelBase( currentPlayAudioJob = viewModelScope.launch { val audioPath = data?.ttsOutput?.url - if (!audioPath.isNullOrBlank()) { - isPlayingAudio = true - try { + if (!audioPath.isNullOrBlank()) { + setIsPlayingAudio(true) + try { playAudio(audioPath).first { state -> state == PlaybackState.STOP_PLAYING } } finally { - isPlayingAudio = false + setIsPlayingAudio(false) } onEvent(AssistEvent.PlaybackFinished) } @@ -369,15 +401,24 @@ abstract class AssistViewModelBase( @OptIn(ExperimentalCoroutinesApi::class) private suspend fun playAudio(path: String): Flow { + Timber.tag("[AA-Assist]").d("playAudio: resolving path=%s", path) return serverManager.connectionStateProvider(selectedServerId).urlFlow().flatMapLatest { urlState -> val baseUrl = if (urlState is UrlState.HasUrl) { + Timber.tag("[AA-Assist]").d("playAudio: urlState=HasUrl, baseUrl=%s", urlState.url) urlState.url } else { + Timber.tag("[AA-Assist]").d("playAudio: urlState not HasUrl, baseUrl=null, stateType=%s", urlState::class.simpleName) null } - UrlUtil.handle(baseUrl, path)?.let { + val resolvedUrl = UrlUtil.handle(baseUrl, path) + Timber.tag("[AA-Assist]").d("playAudio: resolvedUrl=%s", resolvedUrl) + resolvedUrl?.let { + Timber.tag("[AA-Assist]").d("playAudio: calling audioUrlPlayer.playAudio(%s)", it) audioUrlPlayer.playAudio(it.toString()) - } ?: emptyFlow() + } ?: run { + Timber.tag("[AA-Assist]").e("playAudio: URL resolution failed, returning emptyFlow") + emptyFlow() + } } } diff --git a/common/src/main/kotlin/io/homeassistant/companion/android/common/util/AudioUrlPlayer.kt b/common/src/main/kotlin/io/homeassistant/companion/android/common/util/AudioUrlPlayer.kt index 4c40ef08fbc..89ed552ebb5 100644 --- a/common/src/main/kotlin/io/homeassistant/companion/android/common/util/AudioUrlPlayer.kt +++ b/common/src/main/kotlin/io/homeassistant/companion/android/common/util/AudioUrlPlayer.kt @@ -71,10 +71,13 @@ class AudioUrlPlayer @VisibleForTesting constructor( */ @OptIn(UnstableApi::class) fun playAudio(url: String, isAssistant: Boolean = true): Flow = callbackFlow { + Timber.tag("[AA-Assist]").d("AudioUrlPlayer.playAudio: url=%s, isAssistant=%s", url, isAssistant) if (!canPlayMusic()) { + Timber.tag("[AA-Assist]").d("AudioUrlPlayer.playAudio: canPlayMusic=false, closing flow") close() return@callbackFlow } + Timber.tag("[AA-Assist]").d("AudioUrlPlayer.playAudio: canPlayMusic=true, creating player") var request: AudioFocusRequestCompat? = null val player = playerCreator { @@ -88,10 +91,14 @@ class AudioUrlPlayer @VisibleForTesting constructor( addListener( object : Player.Listener { override fun onPlaybackStateChanged(playbackState: Int) { + Timber.tag("[AA-Assist]").d("AudioUrlPlayer: onPlaybackStateChanged=%s, hasStartedPlayback=%s", + playbackState, hasStartedPlayback) when (playbackState) { Player.STATE_READY -> { + Timber.tag("[AA-Assist]").d("AudioUrlPlayer: STATE_READY reached") if (!hasStartedPlayback) { hasStartedPlayback = true + Timber.tag("[AA-Assist]").d("AudioUrlPlayer: requesting focus and playing") trySend(PlaybackState.READY) request = requestFocus(isAssistant) play() @@ -142,7 +149,10 @@ class AudioUrlPlayer @VisibleForTesting constructor( private fun canPlayMusic(): Boolean { return try { - audioManager != null && audioManager.getStreamVolume(STREAM_MUSIC) != 0 + val canPlay = audioManager != null && audioManager.getStreamVolume(STREAM_MUSIC) != 0 + Timber.tag("[AA-Assist]").d("AudioUrlPlayer.canPlayMusic: audioManager=%s, volume=%s, canPlay=%s", + audioManager != null, audioManager?.getStreamVolume(STREAM_MUSIC), canPlay) + canPlay } catch (e: RuntimeException) { Timber.e(e, "Couldn't get stream volume") true