diff --git a/ai_agents/agents/integration_tests/tts_guarder/tests/bin/start b/ai_agents/agents/integration_tests/tts_guarder/tests/bin/start index 4afef8466f..83642cc985 100755 --- a/ai_agents/agents/integration_tests/tts_guarder/tests/bin/start +++ b/ai_agents/agents/integration_tests/tts_guarder/tests/bin/start @@ -9,7 +9,7 @@ EXTENSION_NAME=${EXT_NAME:-elevenlabs_tts_python} export PYTHONPATH=.:ten_packages/system/ten_runtime_python/lib:ten_packages/system/ten_runtime_python/interface:ten_packages/system/ten_ai_base/interface:ten_packages/extension/${EXTENSION_NAME}:$PYTHONPATH # some tts extension does not support sample rate comparison, so we need to set the enable_sample_rate parameter -if [[ "${EXTENSION_NAME}" == "humeai_tts_python" || "${EXTENSION_NAME}" == "openai_tts_python" || "${EXTENSION_NAME}" == "openai_tts2_python" || "${EXTENSION_NAME}" == "mistral_tts_python" || "${EXTENSION_NAME}" == "ezai_tw_tts_python" ]]; then +if [[ "${EXTENSION_NAME}" == "humeai_tts_python" || "${EXTENSION_NAME}" == "openai_tts_python" || "${EXTENSION_NAME}" == "openai_tts2_python" || "${EXTENSION_NAME}" == "mistral_tts_python" || "${EXTENSION_NAME}" == "ezai_tw_tts_python" || "${EXTENSION_NAME}" == "typecast_tts_python" ]]; then export ENABLE_SAMPLE_RATE="False" else export ENABLE_SAMPLE_RATE="True" diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/README.md b/ai_agents/agents/ten_packages/extension/typecast_tts_python/README.md new file mode 100644 index 0000000000..ff512e4e31 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/README.md @@ -0,0 +1,35 @@ +# Typecast TTS Extension + +Typecast text-to-speech extension for TEN Framework. + +The extension uses the official `typecast-python` SDK and Typecast's +`/v1/text-to-speech/stream` endpoint. Typecast streams WAV audio at 32 kHz, +16-bit, mono. The extension strips the initial WAV header and forwards PCM16 +mono audio to TEN. + +## Configuration + +```json +{ + "params": { + "api_key": "", + "url": "https://api.typecast.ai", + "voice_id": "tc_60e5426de8b95f1d3000d7b5", + "model": "ssfm-v30", + "output": { + "audio_format": "wav" + } + }, + "chunk_size": 8192 +} +``` + +`output.audio_format` is forced to `wav` because TEN consumes PCM audio frames. +MP3 streaming would require an additional decoder and is intentionally not used. + +## Validation + +```bash +tman -y install --standalone +./tests/bin/start +``` diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/__init__.py b/ai_agents/agents/ten_packages/extension/typecast_tts_python/__init__.py new file mode 100644 index 0000000000..72593ab225 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/__init__.py @@ -0,0 +1,6 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from . import addon diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/addon.py b/ai_agents/agents/ten_packages/extension/typecast_tts_python/addon.py new file mode 100644 index 0000000000..85d8d5980e --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/addon.py @@ -0,0 +1,15 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten_runtime import Addon, TenEnv, register_addon_as_extension + +from .extension import TypecastTTSExtension + + +@register_addon_as_extension("typecast_tts_python") +class TypecastTTSExtensionAddon(Addon): + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + ten_env.log_info("TypecastTTSExtensionAddon on_create_instance") + ten_env.on_create_instance_done(TypecastTTSExtension(name), context) diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/config.py b/ai_agents/agents/ten_packages/extension/typecast_tts_python/config.py new file mode 100644 index 0000000000..259aa496ac --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/config.py @@ -0,0 +1,71 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from pathlib import Path +from typing import Any +import copy + +from pydantic import Field +from ten_ai_base import utils +from ten_ai_base.tts2_http import AsyncTTS2HttpConfig + + +class TypecastTTSConfig(AsyncTTS2HttpConfig): + """Typecast TTS config.""" + + dump: bool = Field(default=False, description="Typecast TTS dump") + dump_path: str = Field( + default_factory=lambda: str( + Path(__file__).parent / "typecast_tts_in.pcm" + ), + description="Typecast TTS dump path", + ) + url: str = Field( + default="https://api.typecast.ai", + description="Typecast API URL", + ) + params: dict[str, Any] = Field( + default_factory=dict, + description="Typecast TTS params", + ) + chunk_size: int = Field( + default=8192, + ge=1, + description="Maximum bytes read from each Typecast streaming chunk", + ) + + def update_params(self) -> None: + if "url" in self.params: + self.url = str( + self.params.pop("url") # pylint: disable=no-member + ).rstrip("/") + else: + self.url = self.url.rstrip("/") + + if "model" not in self.params: + self.params["model"] = "ssfm-v30" + + output = dict( + self.params.get("output") or {} # pylint: disable=no-member + ) + output["audio_format"] = "wav" + self.params["output"] = output + + def validate(self) -> None: + if not self.params.get("api_key"): # pylint: disable=no-member + raise ValueError("api_key is required for Typecast TTS") + if not self.params.get("voice_id"): # pylint: disable=no-member + raise ValueError("voice_id is required for Typecast TTS") + if not self.params.get("model"): # pylint: disable=no-member + raise ValueError("model is required for Typecast TTS") + + def to_str(self, sensitive_handling: bool = True) -> str: + if not sensitive_handling: + return f"{self}" + + config = copy.deepcopy(self) + if config.params and "api_key" in config.params: + config.params["api_key"] = utils.encrypt(config.params["api_key"]) + return f"{config}" diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/extension.py b/ai_agents/agents/ten_packages/extension/typecast_tts_python/extension.py new file mode 100644 index 0000000000..f052eb4bf8 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/extension.py @@ -0,0 +1,56 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten_ai_base.tts2_http import ( + AsyncTTS2HttpClient, + AsyncTTS2HttpConfig, + AsyncTTS2HttpExtension, +) +from ten_ai_base.message import TTSAudioEndReason +from ten_runtime import AsyncTenEnv + +from .config import TypecastTTSConfig +from .typecast_tts import TYPECAST_STREAM_SAMPLE_RATE, TypecastTTSClient + + +class TypecastTTSExtension(AsyncTTS2HttpExtension): + """Typecast TTS extension using Typecast streaming TTS.""" + + def __init__(self, name: str) -> None: + super().__init__(name) + self.config: TypecastTTSConfig = None + self.client: TypecastTTSClient = None + + async def create_config(self, config_json_str: str) -> AsyncTTS2HttpConfig: + return TypecastTTSConfig.model_validate_json(config_json_str) + + async def create_client( + self, config: AsyncTTS2HttpConfig, ten_env: AsyncTenEnv + ) -> AsyncTTS2HttpClient: + return TypecastTTSClient(config=config, ten_env=ten_env) + + def vendor(self) -> str: + return "typecast" + + def synthesize_audio_sample_rate(self) -> int: + return TYPECAST_STREAM_SAMPLE_RATE + + async def _send_audio_end_and_finish( + self, + request_id: str, + reason: TTSAudioEndReason, + log_message: str | None = None, + ) -> None: + # The base PCMWriter may retain tail bytes while a write is in flight. + # Its cleanup flush then writes those bytes on this second pass. + recorder = self.recorder_map.get(request_id) + if recorder: + await recorder.flush() + + await super()._send_audio_end_and_finish( + request_id=request_id, + reason=reason, + log_message=log_message, + ) diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/manifest.json b/ai_agents/agents/ten_packages/extension/typecast_tts_python/manifest.json new file mode 100644 index 0000000000..5362033ff2 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/manifest.json @@ -0,0 +1,83 @@ +{ + "type": "extension", + "name": "typecast_tts_python", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.11" + }, + { + "type": "system", + "name": "ten_ai_base", + "version": "0.7" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "BUILD.gn", + "**.tent", + "**.py", + "README.md", + "pyproject.toml", + "requirements.txt" + ] + }, + "api": { + "interface": [ + { + "import_uri": "../../system/ten_ai_base/api/tts-interface.json" + } + ], + "property": { + "properties": { + "params": { + "type": "object", + "properties": { + "api_key": { + "type": "string" + }, + "url": { + "type": "string" + }, + "voice_id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "language": { + "type": "string" + }, + "output": { + "type": "object", + "properties": { + "audio_pitch": { + "type": "int64" + }, + "audio_tempo": { + "type": "float64" + }, + "audio_format": { + "type": "string" + }, + "target_lufs": { + "type": "float64" + } + } + }, + "seed": { + "type": "int64" + } + } + }, + "chunk_size": { + "type": "int64" + } + } + } + } +} diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/pcm.py b/ai_agents/agents/ten_packages/extension/typecast_tts_python/pcm.py new file mode 100644 index 0000000000..13f674c6cb --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/pcm.py @@ -0,0 +1,41 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +BYTES_PER_SAMPLE = 2 +NUMBER_OF_CHANNELS = 1 +WAV_HEADER_BYTES = 44 + + +class StreamingWavToPcm16: + """Strip the first WAV header and return PCM16-aligned chunks.""" + + def __init__(self) -> None: + self._header = bytearray() + self._remainder = bytearray() + self._header_stripped = False + + def feed(self, chunk: bytes) -> bytes: + if not chunk: + return b"" + + if not self._header_stripped: + needed = WAV_HEADER_BYTES - len(self._header) + self._header.extend(chunk[:needed]) + chunk = chunk[needed:] + if len(self._header) < WAV_HEADER_BYTES: + return b"" + self._header_stripped = True + + if self._remainder: + chunk = bytes(self._remainder) + chunk + self._remainder.clear() + + frame_size = BYTES_PER_SAMPLE * NUMBER_OF_CHANNELS + left_size = len(chunk) % frame_size + if left_size: + self._remainder.extend(chunk[-left_size:]) + chunk = chunk[:-left_size] + + return bytes(chunk) diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/property.json b/ai_agents/agents/ten_packages/extension/typecast_tts_python/property.json new file mode 100644 index 0000000000..818a04b078 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/property.json @@ -0,0 +1,13 @@ +{ + "params": { + "api_key": "", + "voice_id": "tc_60e5426de8b95f1d3000d7b5", + "model": "ssfm-v30", + "output": { + "audio_format": "wav" + } + }, + "chunk_size": 8192, + "dump": false, + "dump_path": "/tmp/typecast_tts_audio_dump" +} diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/pyproject.toml b/ai_agents/agents/ten_packages/extension/typecast_tts_python/pyproject.toml new file mode 100644 index 0000000000..929922693c --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "typecast-tts-python" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "typecast-python==0.3.9", +] diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/requirements.txt b/ai_agents/agents/ten_packages/extension/typecast_tts_python/requirements.txt new file mode 100644 index 0000000000..8bfdac59ff --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/requirements.txt @@ -0,0 +1 @@ +typecast-python==0.3.9 diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/bin/start b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/bin/start new file mode 100755 index 0000000000..8e78210572 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/bin/start @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." + +export PYTHONPATH=.ten/app:.ten/app/ten_packages/system/ten_runtime_python/lib:.ten/app/ten_packages/system/ten_runtime_python/interface:.ten/app/ten_packages/system/ten_ai_base/interface:$PYTHONPATH + +pytest -s tests/ "$@" diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_basic_audio_setting1.json b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_basic_audio_setting1.json new file mode 100644 index 0000000000..b71144d433 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_basic_audio_setting1.json @@ -0,0 +1,9 @@ +{ + "dump": false, + "dump_path": "./tests/keep_dump_output/", + "params": { + "api_key": "${env:TYPECAST_API_KEY}", + "voice_id": "tc_60e5426de8b95f1d3000d7b5", + "model": "ssfm-v30" + } +} diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_basic_audio_setting2.json b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_basic_audio_setting2.json new file mode 100644 index 0000000000..b71144d433 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_basic_audio_setting2.json @@ -0,0 +1,9 @@ +{ + "dump": false, + "dump_path": "./tests/keep_dump_output/", + "params": { + "api_key": "${env:TYPECAST_API_KEY}", + "voice_id": "tc_60e5426de8b95f1d3000d7b5", + "model": "ssfm-v30" + } +} diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_dump.json b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_dump.json new file mode 100644 index 0000000000..dfda288a3a --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_dump.json @@ -0,0 +1,9 @@ +{ + "dump": true, + "dump_path": "./dump/", + "params": { + "api_key": "${env:TYPECAST_API_KEY}", + "voice_id": "tc_60e5426de8b95f1d3000d7b5", + "model": "ssfm-v30" + } +} diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_invalid.json b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_invalid.json new file mode 100644 index 0000000000..2dfc3ec4b9 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_invalid.json @@ -0,0 +1,7 @@ +{ + "params": { + "api_key": "invalid", + "voice_id": "tc_60e5426de8b95f1d3000d7b5", + "model": "ssfm-v30" + } +} diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_miss_required.json b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_miss_required.json new file mode 100644 index 0000000000..f2d17f03de --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/configs/property_miss_required.json @@ -0,0 +1,7 @@ +{ + "params": { + "api_key": "", + "voice_id": "tc_60e5426de8b95f1d3000d7b5", + "model": "ssfm-v30" + } +} diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/test_typecast_tts.py b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/test_typecast_tts.py new file mode 100644 index 0000000000..e467971342 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/tests/test_typecast_tts.py @@ -0,0 +1,267 @@ +import sys +from pathlib import Path +import asyncio +import json +import threading +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from ten_runtime import AsyncExtensionTester, AsyncTenEnvTester, Data, TenError +from ten_runtime import TenErrorCode +from ten_ai_base.struct import TTS2HttpResponseEventType, TTSTextInput +from ten_ai_base.helper import PCMWriter +from ten_ai_base.helper import write_pcm_to_file +from ten_ai_base.message import TTSAudioEndReason +from ten_ai_base.tts2_http import AsyncTTS2HttpExtension + +from pcm import StreamingWavToPcm16 +from typecast_tts_python.config import TypecastTTSConfig +from typecast_tts_python.extension import TypecastTTSExtension +from typecast_tts_python.typecast_tts import TypecastTTSClient +from typecast import TypecastError, UnauthorizedError + + +def test_config_defaults_and_forces_wav(): + config = TypecastTTSConfig( + params={ + "api_key": "key", + "voice_id": "voice", + "url": "https://example.com/", + "output": {"audio_format": "mp3", "audio_tempo": 1.1}, + } + ) + + config.update_params() + config.validate() + + assert config.url == "https://example.com" + assert "url" not in config.params + assert config.params["model"] == "ssfm-v30" + assert config.params["output"] == { + "audio_format": "wav", + "audio_tempo": 1.1, + } + + +@pytest.mark.parametrize("missing", ["api_key", "voice_id"]) +def test_config_requires_credentials_and_voice(missing): + params = {"api_key": "key", "voice_id": "voice"} + params[missing] = "" + config = TypecastTTSConfig(params=params) + config.update_params() + + with pytest.raises(ValueError, match=missing): + config.validate() + + +def test_streaming_wav_to_pcm16_strips_header_across_chunks(): + converter = StreamingWavToPcm16() + header = b"h" * 44 + + assert converter.feed(header[:20]) == b"" + assert converter.feed(header[20:] + b"\x01\x02\x03") == b"\x01\x02" + assert converter.feed(b"\x04\x05") == b"\x03\x04" + assert converter.feed(b"\x06") == b"\x05\x06" + + +def test_streaming_wav_to_pcm16_strips_header_in_single_chunk(): + converter = StreamingWavToPcm16() + header = b"h" * 44 + + assert converter.feed(header + b"\x01\x02\x03\x04") == b"\x01\x02\x03\x04" + + +def test_extension_flushes_dump_tail_added_during_write(tmp_path): + output = tmp_path / "dump.pcm" + first_write_started = threading.Event() + release_first_write = threading.Event() + write_count = 0 + + def delayed_write(buffer, file_name): + nonlocal write_count + write_count += 1 + if write_count == 1: + first_write_started.set() + release_first_write.wait() + write_pcm_to_file(buffer, file_name) + + async def run(): + writer = PCMWriter(str(output), buffer_size=4) + extension = TypecastTTSExtension("test") + extension.recorder_map = {"request-id": writer} + + async def base_finish(self, request_id, reason, log_message=None): + await self.recorder_map[request_id].flush() + + with patch( + "ten_ai_base.helper.write_pcm_to_file", + side_effect=delayed_write, + ), patch.object( + AsyncTTS2HttpExtension, + "_send_audio_end_and_finish", + base_finish, + ): + await writer.write(b"head") + while not first_write_started.is_set(): + await asyncio.sleep(0) + await writer.write(b"tail") + flush_task = asyncio.create_task( + extension._send_audio_end_and_finish( + request_id="request-id", + reason=TTSAudioEndReason.REQUEST_END, + ) + ) + await asyncio.sleep(0) + release_first_write.set() + await flush_task + + asyncio.run(run()) + + assert output.read_bytes() == b"headtail" + + +def test_client_empty_text_ends_without_request(): + config = TypecastTTSConfig(params={"api_key": "key", "voice_id": "voice"}) + config.update_params() + client = TypecastTTSClient(config, MagicMock()) + + async def collect(): + return [event async for event in client.get(" ", "request-id")] + + assert asyncio.run(collect()) == [(None, TTS2HttpResponseEventType.END)] + + +@pytest.mark.parametrize( + ("error", "expected_event"), + [ + ( + UnauthorizedError("invalid key"), + TTS2HttpResponseEventType.INVALID_KEY_ERROR, + ), + (TypecastError("rate limited", 429), TTS2HttpResponseEventType.ERROR), + ], +) +def test_client_maps_vendor_errors(error, expected_event): + config = TypecastTTSConfig(params={"api_key": "key", "voice_id": "voice"}) + config.update_params() + client = TypecastTTSClient(config, MagicMock()) + + async def failing_stream(request, chunk_size): + raise error + yield b"" # pragma: no cover + + mock_sdk = MagicMock() + mock_sdk.__aenter__ = AsyncMock(return_value=mock_sdk) + mock_sdk.__aexit__ = AsyncMock(return_value=None) + mock_sdk.text_to_speech_stream = failing_stream + + async def collect(): + with patch( + "typecast_tts_python.typecast_tts.AsyncTypecast", + return_value=mock_sdk, + ): + return [event async for event in client.get("hello", "request-id")] + + events = asyncio.run(collect()) + assert events == [(str(error).encode(), expected_event)] + + +class TypecastTTSExtensionTester(AsyncExtensionTester): + def __init__(self): + super().__init__() + self.audio_end_received = False + self.received_audio_chunks = [] + + async def on_start(self, ten_env: AsyncTenEnvTester) -> None: + tts_input = TTSTextInput( + request_id="tts_request_1", + text="hello typecast", + text_input_end=True, + ) + data = Data.create("tts_text_input") + data.set_property_from_json(None, tts_input.model_dump_json()) + await ten_env.send_data(data) + asyncio.create_task(self._stop_on_timeout(ten_env)) + + async def on_data(self, ten_env: AsyncTenEnvTester, data: Data) -> None: + if data.get_name() == "tts_audio_end": + data_json, _ = data.get_property_to_json() + data_dict = json.loads(data_json) + assert data_dict["request_id"] == "tts_request_1" + self.audio_end_received = True + ten_env.stop_test() + + async def on_audio_frame(self, ten_env: AsyncTenEnvTester, audio_frame): + buf = audio_frame.lock_buf() + try: + self.received_audio_chunks.append(bytes(buf)) + finally: + audio_frame.unlock_buf(buf) + + async def _stop_on_timeout(self, ten_env: AsyncTenEnvTester) -> None: + await asyncio.sleep(10) + ten_env.stop_test( + TenError.create( + error_code=TenErrorCode.ErrorCodeGeneric, + error_message="test timeout", + ) + ) + + +def test_typecast_tts_extension_success(): + wav_header = b"h" * 44 + audio_chunk_1 = b"\x01\x02\x03\x04" + audio_chunk_2 = b"\x05\x06\x07\x08" + + async def mock_text_to_speech_stream(request, chunk_size): + assert request.text == "hello typecast" + assert request.output.audio_format == "wav" + assert chunk_size == 8192 + yield wav_header[:20] + yield wav_header[20:] + audio_chunk_1 + yield audio_chunk_2 + + with patch("typecast_tts_python.typecast_tts.AsyncTypecast") as mock_cls: + mock_client = MagicMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + mock_client.text_to_speech_stream = mock_text_to_speech_stream + mock_cls.return_value = mock_client + + property_json = { + "params": { + "api_key": "test_api_key", + "voice_id": "test_voice_id", + "model": "ssfm-v30", + } + } + + tester = TypecastTTSExtensionTester() + tester.set_test_mode_single( + "typecast_tts_python", json.dumps(property_json) + ) + + err = tester.run() + + assert err is None, ( + "test_typecast_tts_extension_success err: " + f"{err.error_message() if err else 'None'}" + ) + assert tester.audio_end_received + assert b"".join(tester.received_audio_chunks) == ( + audio_chunk_1 + audio_chunk_2 + ) + mock_cls.assert_called_once_with( + host="https://api.typecast.ai", + api_key="test_api_key", + ) + mock_client.__aexit__.assert_awaited_once_with(None, None, None) + + +if __name__ == "__main__": + test_streaming_wav_to_pcm16_strips_header_across_chunks() + test_streaming_wav_to_pcm16_strips_header_in_single_chunk() diff --git a/ai_agents/agents/ten_packages/extension/typecast_tts_python/typecast_tts.py b/ai_agents/agents/ten_packages/extension/typecast_tts_python/typecast_tts.py new file mode 100644 index 0000000000..a4383b4095 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/typecast_tts_python/typecast_tts.py @@ -0,0 +1,121 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from typing import Any, AsyncIterator, Tuple + +from ten_ai_base.const import LOG_CATEGORY_VENDOR +from ten_ai_base.struct import TTS2HttpResponseEventType +from ten_ai_base.tts2_http import AsyncTTS2HttpClient +from ten_runtime import AsyncTenEnv +from typecast import AsyncTypecast, TypecastError +from typecast.models import TTSRequestStream + +from .config import TypecastTTSConfig +from .pcm import StreamingWavToPcm16 + +TYPECAST_STREAM_SAMPLE_RATE = 32000 + + +class TypecastTTSClient(AsyncTTS2HttpClient): + """Typecast TTS client backed by the official Typecast Python SDK.""" + + def __init__(self, config: TypecastTTSConfig, ten_env: AsyncTenEnv): + super().__init__() + self.config = config + self.ten_env = ten_env + self._is_cancelled = False + self._client: AsyncTypecast | None = None + + ten_env.log_info(f"TypecastTTS initialized with URL: {self.config.url}") + + async def _ensure_client(self) -> AsyncTypecast: + if self._client is None: + self._client = AsyncTypecast( + host=self.config.url, + api_key=self.config.params["api_key"], + ) + await self._client.__aenter__() + return self._client + + async def cancel(self): + self.ten_env.log_debug("TypecastTTS: cancel() called.") + self._is_cancelled = True + + async def get( + self, text: str, request_id: str + ) -> AsyncIterator[Tuple[bytes | None, TTS2HttpResponseEventType]]: + self._is_cancelled = False + + if len(text.strip()) == 0: + self.ten_env.log_warn( + f"TypecastTTS: empty text for request_id: {request_id}.", + category=LOG_CATEGORY_VENDOR, + ) + yield None, TTS2HttpResponseEventType.END + return + + try: + payload = {**self.config.params} + payload.pop("api_key", None) + + output = dict(payload.get("output") or {}) + output["audio_format"] = "wav" + payload["output"] = output + payload["text"] = text + + request = TTSRequestStream.model_validate(payload) + converter = StreamingWavToPcm16() + client = await self._ensure_client() + + self.ten_env.log_debug( + f"TypecastTTS: sending request for request_id: {request_id}" + ) + + async for chunk in client.text_to_speech_stream( + request, chunk_size=self.config.chunk_size + ): + if self._is_cancelled: + self.ten_env.log_debug( + f"Cancellation detected, flushing TTS stream for request_id: {request_id}" + ) + yield None, TTS2HttpResponseEventType.FLUSH + return + + pcm = converter.feed(chunk) + if pcm: + yield pcm, TTS2HttpResponseEventType.RESPONSE + + if not self._is_cancelled: + self.ten_env.log_debug( + f"TypecastTTS: sending END event for request_id: {request_id}" + ) + yield None, TTS2HttpResponseEventType.END + + except Exception as e: + error_message = str(e) + self.ten_env.log_error( + f"vendor_error: {error_message} for request_id: {request_id}", + category=LOG_CATEGORY_VENDOR, + ) + event = ( + TTS2HttpResponseEventType.INVALID_KEY_ERROR + if isinstance(e, TypecastError) + and getattr(e, "status_code", None) in (401, 403) + else TTS2HttpResponseEventType.ERROR + ) + yield error_message.encode("utf-8"), event + + async def clean(self): + self.ten_env.log_debug("TypecastTTS: clean() called.") + if self._client is not None: + await self._client.__aexit__(None, None, None) + self._client = None + + def get_extra_metadata(self) -> dict[str, Any]: + return { + "vendor": "typecast", + "model": self.config.params.get("model", ""), + "voice_id": self.config.params.get("voice_id", ""), + }