diff --git a/ai_agents/.env.example b/ai_agents/.env.example index 3fcf728744..0a0b7c28d6 100644 --- a/ai_agents/.env.example +++ b/ai_agents/.env.example @@ -104,6 +104,14 @@ DEEPGRAM_API_KEY= # xAI Voice API key XAI_API_KEY= +# Extension: blaze_stt_python, blaze_tts_python +# Blaze Speech API (https://api.blaze.vn) +BLAZE_API_KEY= +BLAZE_STT_API_KEY= +BLAZE_TTS_API_KEY= +BLAZE_STT_API_URL=https://api.blaze.vn +BLAZE_TTS_API_URL=https://api.blaze.vn + # Azure ASR AZURE_ASR_API_KEY= AZURE_ASR_REGION= diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json b/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json index 56dbf3c5f3..6eada221ee 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/manifest-lock.json @@ -44,8 +44,8 @@ { "type": "system", "name": "ten_ai_base", - "version": "0.7.41", - "hash": "ad64eef476fc583a5021559ede87f2071440128180c63d43f03acb7c46245895", + "version": "0.7.54", + "hash": "61bebb7b08d87373188f5e47a02d8cb1aeea52d4e629016019e23c637c0b204d", "dependencies": [ { "type": "system", @@ -56,8 +56,8 @@ { "type": "extension", "name": "streamid_adapter", - "version": "0.2.0", - "hash": "6df9a1b48a4007339ccbda84ff96a4ed4eb5df498b8fef021ca2302be7c55051", + "version": "0.2.1", + "hash": "9bd513b0d1032485ada19b7288c77e21ba01d621af91dd671c3bb2fbed36e670", "dependencies": [ { "type": "system", @@ -69,8 +69,8 @@ { "type": "extension", "name": "aliyun_asr", - "version": "0.2.0", - "hash": "e8c6f1b5c78d17d3ba3dbeb3c7d6ff4a2d293a04ebd5e1f45e7ddbec47bf8b79", + "version": "0.2.1", + "hash": "68cba364f1fe7a785d55720f95bfb049c8833df03f4ce408b3618a872f381f5e", "dependencies": [ { "type": "system", @@ -86,8 +86,8 @@ { "type": "extension", "name": "aliyun_asr_bigmodel_python", - "version": "0.2.5", - "hash": "d5659201fc4ea8568b094b06fb486ade2c81ad732af58f7fa1fc1fe51659c127", + "version": "0.2.7", + "hash": "bcf8edb94d1a47b356b34d85402a906c3bfcf65adf607fae70db2b0528a1fef5", "dependencies": [ { "type": "system", @@ -103,8 +103,8 @@ { "type": "extension", "name": "assemblyai_asr_python", - "version": "0.2.0", - "hash": "af62c5f66f7f908631aa8593f5df5ab471fe4c022a40e2f5e9d9be3b2b7369a0", + "version": "0.2.2", + "hash": "e0547ae9ae3b09bbfff5c86014cfe081eb8b38d567b91bec934afbd49ca530b0", "dependencies": [ { "type": "system", @@ -120,8 +120,8 @@ { "type": "extension", "name": "aws_asr_python", - "version": "0.2.2", - "hash": "dae7c06862b141ea5f6e1265d67c37a6a560ab86c47861ff66894965ba824282", + "version": "0.2.3", + "hash": "320207a0f5b382e2e35f174e83c527bed1e39cc6af21011131c93f0fc49e2274", "dependencies": [ { "type": "system", @@ -137,8 +137,8 @@ { "type": "extension", "name": "azure_asr_python", - "version": "0.2.11", - "hash": "5a557adf46971fd1b78faf5bf6ca8ab20507f9864191a1d04427ff295a7a0f37", + "version": "0.3.0", + "hash": "73f77093444a17533f95dfa1fa92b09cff9c740f990ad3ab0092619a5e031013", "dependencies": [ { "type": "system", @@ -154,8 +154,8 @@ { "type": "extension", "name": "bytedance_asr", - "version": "0.2.4", - "hash": "082b28fda013d4f029324a9203863b3599813a15a589f2de6500a77680d1e5d5", + "version": "0.2.6", + "hash": "a42d6bed1d611f84395797951d861bcbfdf14d59162d24251b385a8457aaff6d", "dependencies": [ { "type": "system", @@ -171,8 +171,8 @@ { "type": "extension", "name": "bytedance_llm_based_asr", - "version": "0.4.3", - "hash": "b87db0ab66de621642cfbd877492a3783806a57089af00e895a99f61bf0d49ba", + "version": "0.5.1", + "hash": "79262c21d87a542868665beee4261025d516839d48b9610ae31d598e1fa636d5", "dependencies": [ { "type": "system", @@ -188,8 +188,8 @@ { "type": "extension", "name": "deepgram_asr_python", - "version": "0.4.1", - "hash": "55db49e9724d7375047455d3f6eb3820228b13560bb1b69ee3e817b71e30699b", + "version": "0.5.0", + "hash": "d4e0ed7c594ae50b255a05d5c81ee490b24dd418d367a4b22d04ca7f9e7aae55", "dependencies": [ { "type": "system", @@ -205,8 +205,8 @@ { "type": "extension", "name": "gladia_asr_python", - "version": "0.2.0", - "hash": "9bf62bc647b25d4c28b3418c6a3893e4610e29af4b0225f219c1527f1c0ddde2", + "version": "0.2.2", + "hash": "078e782ad5b9d76288c0074e397fc4b6a39c58b1ede5adae6949271ae7c32790", "dependencies": [ { "type": "system", @@ -222,8 +222,8 @@ { "type": "extension", "name": "google_asr_python", - "version": "0.2.3", - "hash": "9bb857f2af29cadef2af76ee35877cd702dc3025a93dc0ee6cf90160a3dbba7d", + "version": "0.2.4", + "hash": "2f0ef954e67dfb662a47c99e342cbac05824dc0bfc80eaec46638d506481a9f4", "dependencies": [ { "type": "system", @@ -239,8 +239,8 @@ { "type": "extension", "name": "openai_asr_python", - "version": "0.3.1", - "hash": "e396e4fcc0c3056d1b3c8ddf21758238afbad8bb6f1eecc1982ebdc08e49da29", + "version": "0.3.3", + "hash": "a46a67dc1c4a16612248e6aff4518555955e1b9690f7867541d563fe1b9a63fc", "dependencies": [ { "type": "system", @@ -256,8 +256,8 @@ { "type": "extension", "name": "sarvam_asr_python", - "version": "0.1.2", - "hash": "e821256e20456fa01534b140ce3c5176bcdc83abd00c22eef1e5d191bae160d2", + "version": "0.1.4", + "hash": "ce5df60dd755e3822d861bf67b1d24d0389ef28bbb84fcec4acbca2baf4fe5e9", "dependencies": [ { "type": "system", @@ -273,8 +273,8 @@ { "type": "extension", "name": "soniox_asr_python", - "version": "0.4.5", - "hash": "2e66d1b6e8ca7fb1563ce1f86cadc2b8b6eda0e7bde6ea35bb8d5cbdc8aae901", + "version": "0.6.0", + "hash": "944b983cb1383dfc990a6e0a0c914115042d1ced9403015e08fb91a17a184c88", "dependencies": [ { "type": "system", @@ -290,8 +290,8 @@ { "type": "extension", "name": "speechmatics_asr_python", - "version": "0.2.5", - "hash": "5c9b91322f73d0ebe4b5d1e4379650b7657092e0f7c5411d061f16cf4963bf59", + "version": "0.2.6", + "hash": "99f04fdf2996e2a9d4e7fceec3646713266b67e7a0f95295ef67f8a3f69030a0", "dependencies": [ { "type": "system", @@ -307,8 +307,8 @@ { "type": "extension", "name": "tencent_asr_python", - "version": "0.2.12", - "hash": "172ed04f00eb25e3c8c4b5b1870d161d9d79cfdfefa6ab0704466299070de58d", + "version": "0.3.0", + "hash": "75431c0c7e6d5772ac90838219a94e4e5fbebc2cec81bfaf98a87d7d372e629a", "dependencies": [ { "type": "system", @@ -324,8 +324,8 @@ { "type": "extension", "name": "xai_asr_python", - "version": "0.4.0", - "hash": "b93659f14987bad6708a8b65f8bc6d39922320c76d2208949105a76e8d51f832", + "version": "0.4.1", + "hash": "a628d8a73894d00e17526e5925f3e9e2d2407d8fe9de6078271a200d620ca5b4", "dependencies": [ { "type": "system", @@ -341,8 +341,8 @@ { "type": "extension", "name": "xfyun_asr_bigmodel_python", - "version": "0.2.3", - "hash": "27e32d41eb23ea049fd8b5410fbaf86708aa8e69afdc19673d3e713d1f158373", + "version": "0.2.4", + "hash": "aa13a686b4b49a5e42c472d91d6eacd1e9227a44a36f7d38466201b2c563fa5f", "dependencies": [ { "type": "system", @@ -358,8 +358,8 @@ { "type": "extension", "name": "xfyun_asr_dialect_python", - "version": "0.2.4", - "hash": "6de98594ceb7f5deddf590517d668d868d98f4036d17a8604b4cb5a26a83ab29", + "version": "0.2.5", + "hash": "29b87b083f6c77d3e0d1fb83173b1f15dbebc81a12bdd83ceabd4dc49c0d3cf1", "dependencies": [ { "type": "system", @@ -375,8 +375,8 @@ { "type": "extension", "name": "xfyun_asr_python", - "version": "0.2.3", - "hash": "c09124358d343cc962bc0cbd4f8031447571bc7f79a2ab2086d1092722dbc1a8", + "version": "0.2.4", + "hash": "77d5d595fa7b27d2f8a03379cd9a4f8a07c5e04d04a44a8df809ee77245ea14b", "dependencies": [ { "type": "system", @@ -392,8 +392,8 @@ { "type": "extension", "name": "coze_llm2_python", - "version": "0.2.0", - "hash": "6056efcc889c5cdb85b104d19d97f01f6cd33dc831634d2249b6729fa4340325", + "version": "0.2.1", + "hash": "75166c983014a6e0143b16e6280ea1a144f0277caf33955497c06d14cf6d630f", "dependencies": [ { "type": "system", @@ -409,8 +409,8 @@ { "type": "extension", "name": "dify_llm2_python", - "version": "0.2.0", - "hash": "2e4e13f3a99d855ffd6715cc830fd65dcfa39bf5a2a8b2e7ffbac938f82d870b", + "version": "0.2.1", + "hash": "5def073aa3c294c4433f982b45356664c3fab269c1b6e834b1abeb848ebf8a34", "dependencies": [ { "type": "system", @@ -426,8 +426,8 @@ { "type": "extension", "name": "openai_llm2_python", - "version": "0.2.1", - "hash": "99afd20bed116d440fd60c9cbff768668342a46a13394390abf244dd9eb186bc", + "version": "0.2.3", + "hash": "83d1c9cb728eeff525ec85cd308f09f07072d7aeb298c78c9aacd1dbb20a54e7", "dependencies": [ { "type": "system", @@ -443,8 +443,8 @@ { "type": "extension", "name": "azure_tts_python", - "version": "0.3.0", - "hash": "2cb8ff4f0927e283c9e9a78cef7f241792f058fbbc7ac5e83e2bf679e9192c07", + "version": "0.3.1", + "hash": "56ffc058cd410e926d351943890ad47ab1a0990893424fd33be7a1872a84fb6a", "dependencies": [ { "type": "system", @@ -460,8 +460,8 @@ { "type": "extension", "name": "bytedance_tts_duplex", - "version": "0.4.0", - "hash": "9c15517b311464a960ebdbb130b9f5fe0fc9c66cfabf58304f0fc47fbb902dd9", + "version": "0.4.2", + "hash": "7d0454a6a3a371c33be19e8379e9d92381ee1922f47f78cfb696f47d79389ac2", "dependencies": [ { "type": "system", @@ -477,8 +477,8 @@ { "type": "extension", "name": "cartesia_tts", - "version": "0.8.1", - "hash": "1a703a7b66b2aebf204610a5f56b4baa16ae5d37745e8dc7b056f7c797af083f", + "version": "0.8.2", + "hash": "04cb5f35c509dd863fb141353f141318cbd2159e85d54c5e80acd34b77a3aafd", "dependencies": [ { "type": "system", @@ -494,8 +494,8 @@ { "type": "extension", "name": "cosy_tts_python", - "version": "0.4.0", - "hash": "eaff006ccecab4af9b96589eace754b1f693641b8eb146efb9466b03b8b9b6e5", + "version": "0.4.1", + "hash": "b31f2f08e790cf290a40c5fb97eb1738bce8b68a9603f7956d78abe1862d9c7e", "dependencies": [ { "type": "system", @@ -511,8 +511,8 @@ { "type": "extension", "name": "dubverse_tts", - "version": "0.2.0", - "hash": "92bc225d5a46d9b09b06c72b93ae5def0105b1915a91f6b99b88babf1527d8eb", + "version": "0.2.1", + "hash": "1148a245a7664b375c2f39c4a8e5b64146a131539122a7d0866724b426bf3d85", "dependencies": [ { "type": "system", @@ -524,8 +524,8 @@ { "type": "extension", "name": "elevenlabs_tts2_python", - "version": "0.4.2", - "hash": "0127c79ea84fee1d139996b63261c64fd61dbcbd71ae7af0dbff30bb30d4707c", + "version": "0.4.3", + "hash": "92b449d50d04855ff522042bcf33fe8821c4e3d677913af8cd84b45a3318a4cc", "dependencies": [ { "type": "system", @@ -541,8 +541,8 @@ { "type": "extension", "name": "fish_audio_tts_python", - "version": "0.4.0", - "hash": "84c3f56a11127d0fe609c5f904dae8ca5022cbd6e9f20735e1738d05518b0ff5", + "version": "0.4.1", + "hash": "0000de3ee859e7e38a1de68ae3d46e7a291b45c444a3510d2f2cdee58a286ae0", "dependencies": [ { "type": "system", @@ -558,8 +558,8 @@ { "type": "extension", "name": "google_tts_python", - "version": "0.4.2", - "hash": "f7f23fed062d4400180efc9c0a1e081d9bdc6f1a1e4f37a84dd1b359dca845a3", + "version": "0.4.3", + "hash": "57f18cd9319317200ea23580fa7400d078c675b298bba391f7c81c122cbb54f4", "dependencies": [ { "type": "system", @@ -575,8 +575,8 @@ { "type": "extension", "name": "groq_tts_python", - "version": "0.5.1", - "hash": "c5672974d99662be9d256a2a9333427414c6e14b759b898cf730e4d6e2afa299", + "version": "0.5.2", + "hash": "b85dcd5f620949402443b7c7bb0077eb44b8b78872e29cbf9570ca0fa91dae44", "dependencies": [ { "type": "system", @@ -592,8 +592,8 @@ { "type": "extension", "name": "humeai_tts_python", - "version": "0.5.1", - "hash": "062279a3e65c207838d408c7752034e8b043492de11952ef2ea5a82ac33769c4", + "version": "0.5.2", + "hash": "56bcddb8e55cb093be08c441aafbf9811558c63c028014a2d4a84e7f49b4e9c5", "dependencies": [ { "type": "system", @@ -609,8 +609,8 @@ { "type": "extension", "name": "minimax_tts_websocket_python", - "version": "0.4.0", - "hash": "8bc3ae28854a5fdcc6a64bde759e80d33b20e0be2df7aa56fa75381b77b38521", + "version": "0.4.2", + "hash": "4f7b5f857385a6446d5f0ded434098ae73c012f323e9fff1321c64395b418dc1", "dependencies": [ { "type": "system", @@ -626,8 +626,8 @@ { "type": "extension", "name": "neuphonic_tts", - "version": "0.2.0", - "hash": "d37d2f4c394eac42c4fb18b9763176312be7ead3d604119293191956aea1a94e", + "version": "0.2.2", + "hash": "ebcb5740968b85ed65b58575b7cf5ae385546bd873a4500eb62290334bd387d8", "dependencies": [ { "type": "system", @@ -639,8 +639,8 @@ { "type": "extension", "name": "openai_tts2_python", - "version": "0.6.5", - "hash": "2da958bdf55c7f1aac93b484773ae81deb750af849e827207b3895f0e9ee5d5e", + "version": "0.6.7", + "hash": "aa3c8cd6f15bc5541a55e573987fce6e82849716842a718c2e9e08a344483c89", "dependencies": [ { "type": "system", @@ -656,8 +656,8 @@ { "type": "extension", "name": "polly_tts", - "version": "0.5.4", - "hash": "e2550055690c134b3ca60c752abc2d83b6ddf26d9e3ae67e31c40ef9c4efacd3", + "version": "0.5.5", + "hash": "2f3ee6e4131c394f190b5e59b5205cbb2f35b67dae86caaa9baa456edd6c71e3", "dependencies": [ { "type": "system", @@ -673,8 +673,8 @@ { "type": "extension", "name": "rime_http_tts", - "version": "0.5.1", - "hash": "e5eb307ea24023796d05d520c6b1d34795283918747cfb78286dce0b0bd58105", + "version": "0.5.2", + "hash": "0e434e5c04d56cd6a419f429997fc6ac1b0bbfc9c931676808b303c83630647c", "dependencies": [ { "type": "system", @@ -690,8 +690,8 @@ { "type": "extension", "name": "rime_tts", - "version": "0.4.5", - "hash": "eb0ec7f19e57562472deab3d5639991b81c56554de9de555cd6a4949f2624faa", + "version": "0.4.10", + "hash": "202317613e495d1eb99c0741c08020672f83cb0c6fcab340d37dbdf9beadb912", "dependencies": [ { "type": "system", @@ -707,8 +707,8 @@ { "type": "extension", "name": "sarvam_http_tts", - "version": "0.5.1", - "hash": "6d94f49739d745b59b0f9d9f1dc4fe8c1efd9e6084b4ad583b22e89a67c17092", + "version": "0.5.2", + "hash": "91de052497c81cb8f32733f534c20d66dc5add51b335ae6048cd7ae89443e851", "dependencies": [ { "type": "system", @@ -724,8 +724,8 @@ { "type": "extension", "name": "stepfun_tts_python", - "version": "0.4.2", - "hash": "0dafe791d3fed62233926fd98f9b4fcee7ed446664eacd47bd25b69007e7a8e4", + "version": "0.4.4", + "hash": "7485f7d5c553380e39df046b3e4462246c7acb0d61ccb34291dd627da23da90e", "dependencies": [ { "type": "system", @@ -741,8 +741,8 @@ { "type": "extension", "name": "tencent_tts_python", - "version": "0.4.5", - "hash": "a42d673e186d325bcae6086d88a05a8442486b3ecce499ea79ad291c12b28658", + "version": "0.4.6", + "hash": "6ae54c9e1c4d4e73e44a3598b351e30178a84e749c2b0c3a97a4265edef99b0d", "dependencies": [ { "type": "system", @@ -758,8 +758,8 @@ { "type": "extension", "name": "xai_tts_python", - "version": "0.1.1", - "hash": "cf22cea83e7cd4dc7b39120e827ce36b80a99ec83b261abd2e309d6320694568", + "version": "0.1.2", + "hash": "1e686da1ab7ac68dc48237a3bc6803b22a300e566c1f7839ceea99c4d5a1fb43", "dependencies": [ { "type": "system", @@ -775,8 +775,8 @@ { "type": "extension", "name": "message_collector2", - "version": "0.2.0", - "hash": "b4cdc75c61ed572952e2f14bdb5c440f993d3a5bf2da6fe5f5cbb1b4d772a837", + "version": "0.2.1", + "hash": "41071d407402c084aa504675199ad2940672fc135400e92e596ac20b6f75dfbd", "dependencies": [ { "type": "system", @@ -788,8 +788,8 @@ { "type": "extension", "name": "weatherapi_tool_python", - "version": "0.1.0", - "hash": "372a4ffffd03992ee2e7a50b0fe83d534b4ed49c0d15099fb0c9cf5ea5cc926c", + "version": "0.1.1", + "hash": "284202626f01d235dab90545413075d91801a24bc7d2b375cb3408ecfc6b7c5d", "dependencies": [ { "type": "system", @@ -801,8 +801,8 @@ { "type": "extension", "name": "oracle_asr_python", - "version": "0.1.2", - "hash": "b5aa785e0fb263c709c3b448f5301c08e44347b992becb266913896524b95ec1", + "version": "0.1.3", + "hash": "5652fa198d32f1487c01bea02d5fc9fb04f9be8109165d17913bd55d14a02003", "dependencies": [ { "type": "system", @@ -818,8 +818,8 @@ { "type": "extension", "name": "oracle_tts_python", - "version": "0.1.2", - "hash": "df3fb21a40d1c5ea7b756f97d1558dcbeeb680abc2a66c48677fe8b2ed5416ec", + "version": "0.1.3", + "hash": "fe25c1e525677dbfeef8d7bc2e0848bdd67cea30703b83a87b004718b90d7d63", "dependencies": [ { "type": "system", @@ -835,8 +835,8 @@ { "type": "extension", "name": "deepgram_tts", - "version": "0.1.2", - "hash": "f4c885fa40f482262c863519193bed5d9d3a57967d377a7c45faa002e5d622ac", + "version": "0.1.3", + "hash": "7a94eed2d71bf877f339ebcc46206d65b17eac1fc32abea9823d019dc8bb0903", "dependencies": [ { "type": "system", @@ -852,8 +852,8 @@ { "type": "extension", "name": "gradium_tts_python", - "version": "0.1.1", - "hash": "e83cf112d80598e184d7215c3beac8f3064819f875757773a2cf0f508f7b586b", + "version": "0.1.3", + "hash": "bd0606608e372ad3a2cbfbe8849bbb2939f1a52f0f5048688a05da34bc913ca0", "dependencies": [ { "type": "system", @@ -869,8 +869,8 @@ { "type": "extension", "name": "murf_tts_python", - "version": "0.1.1", - "hash": "b7dccdadc9bbb1d80894981c59416e5c409d990bce0355e43c69fa7420ad5a9a", + "version": "0.1.3", + "hash": "e8b6a21881036c0f2503f75aed4d03a9f668a4c6a705501d4420313e5bd06bd2", "dependencies": [ { "type": "system", @@ -883,6 +883,40 @@ ], "path": "../../../ten_packages/extension/murf_tts_python" }, + { + "type": "extension", + "name": "blaze_stt_python", + "version": "1.0.1", + "hash": "86b6295019101d33477b835c2f0b1cd7dd0368121b54dafda59711d08d8b54f0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python" + }, + { + "type": "system", + "name": "ten_ai_base" + } + ], + "path": "../../../ten_packages/extension/blaze_stt_python" + }, + { + "type": "extension", + "name": "blaze_tts_python", + "version": "1.0.1", + "hash": "fac55efe4566587ed5c207437b38b7824ce572a14ac683707ed81e860fa5173c", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python" + }, + { + "type": "system", + "name": "ten_ai_base" + } + ], + "path": "../../../ten_packages/extension/blaze_tts_python" + }, { "type": "system", "name": "ten_runtime_python", diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json b/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json index bc24bd14d1..251ec1c938 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json @@ -173,9 +173,15 @@ }, { "path": "../../../ten_packages/extension/smallest_tts_python" + }, + { + "path": "../../../ten_packages/extension/blaze_stt_python" + }, + { + "path": "../../../ten_packages/extension/blaze_tts_python" } ], "scripts": { "start": "scripts/start.sh" } -} \ No newline at end of file +} diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/property.json b/ai_agents/agents/examples/voice-assistant/tenapp/property.json index b9cc5b8ee1..d311f09c8c 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/property.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/property.json @@ -1578,6 +1578,567 @@ } ] } + }, + { + "name": "voice_assistant_blaze_stt", + "auto_start": false, + "graph": { + "nodes": [ + { + "type": "extension", + "name": "agora_rtc", + "addon": "agora_rtc", + "extension_group": "default", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "app_certificate": "${env:AGORA_APP_CERTIFICATE|}", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": false + } + }, + { + "type": "extension", + "name": "stt", + "addon": "blaze_stt_python", + "extension_group": "stt", + "property": { + "dump": false, + "dump_path": "./", + "params": { + "api_key": "${env:BLAZE_STT_API_KEY}", + "api_url": "${env:BLAZE_STT_API_URL|https://api.blaze.vn}", + "language": "vi", + "sample_rate": 16000, + "model": "stt-stream-1.5" + } + } + }, + { + "type": "extension", + "name": "llm", + "addon": "openai_llm2_python", + "extension_group": "chatgpt", + "property": { + "base_url": "https://api.openai.com/v1", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "${env:OPENAI_MODEL}", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL|}", + "greeting": "Xin chào, TEN Agent với Blaze đã kết nối. Tôi có thể giúp gì cho bạn?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "name": "tts", + "addon": "elevenlabs_tts2_python", + "extension_group": "tts", + "property": { + "dump": false, + "dump_path": "./", + "params": { + "key": "${env:ELEVENLABS_TTS_KEY}", + "model_id": "eleven_multilingual_v2", + "voice_id": "pNInz6obpgDQGcFmaJgB", + "output_format": "pcm_16000" + } + } + }, + { + "type": "extension", + "name": "main_control", + "addon": "main_python", + "extension_group": "control", + "property": { + "greeting": "Xin chào, TEN Agent với Blaze đã kết nối. Tôi có thể giúp gì cho bạn?" + } + }, + { + "type": "extension", + "name": "message_collector", + "addon": "message_collector2", + "extension_group": "transcriber", + "property": {} + }, + { + "type": "extension", + "name": "streamid_adapter", + "addon": "streamid_adapter", + "property": {} + } + ], + "connections": [ + { + "extension": "main_control", + "cmd": [ + { + "names": [ + "on_user_joined", + "on_user_left" + ], + "source": [ + { + "extension": "agora_rtc" + } + ] + } + ], + "data": [ + { + "name": "asr_result", + "source": [ + { + "extension": "stt" + } + ] + }, + { + "name": "tts_audio_start", + "source": [ + { + "extension": "tts" + } + ] + }, + { + "name": "tts_audio_end", + "source": [ + { + "extension": "tts" + } + ] + } + ] + }, + { + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "streamid_adapter" + } + ] + }, + { + "name": "pcm_frame", + "source": [ + { + "extension": "tts" + } + ] + } + ], + "data": [ + { + "name": "data", + "source": [ + { + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension": "streamid_adapter", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "stt" + } + ] + } + ] + } + ] + } + }, + { + "name": "voice_assistant_blaze_tts", + "auto_start": false, + "graph": { + "nodes": [ + { + "type": "extension", + "name": "agora_rtc", + "addon": "agora_rtc", + "extension_group": "default", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "app_certificate": "${env:AGORA_APP_CERTIFICATE|}", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": false + } + }, + { + "type": "extension", + "name": "stt", + "addon": "deepgram_asr_python", + "extension_group": "stt", + "property": { + "params": { + "api_key": "${env:DEEPGRAM_API_KEY}", + "language": "en-US", + "model": "nova-3" + } + } + }, + { + "type": "extension", + "name": "llm", + "addon": "openai_llm2_python", + "extension_group": "chatgpt", + "property": { + "base_url": "https://api.openai.com/v1", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "${env:OPENAI_MODEL}", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL|}", + "greeting": "Xin chào, TEN Agent với Blaze đã kết nối. Tôi có thể giúp gì cho bạn?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "name": "tts", + "addon": "blaze_tts_python", + "extension_group": "tts", + "property": { + "dump": false, + "dump_path": "./", + "params": { + "api_key": "${env:BLAZE_TTS_API_KEY}", + "api_url": "${env:BLAZE_TTS_API_URL|https://api.blaze.vn}", + "language": "vi", + "speaker_id": "HN-Nu-CSKH-HuongGiang", + "audio_speed": 1.0, + "audio_quality": 64, + "sample_rate": 24000, + "model": "2.0-realtime", + "audio_format": "pcm" + } + } + }, + { + "type": "extension", + "name": "main_control", + "addon": "main_python", + "extension_group": "control", + "property": { + "greeting": "Xin chào, TEN Agent với Blaze đã kết nối. Tôi có thể giúp gì cho bạn?" + } + }, + { + "type": "extension", + "name": "message_collector", + "addon": "message_collector2", + "extension_group": "transcriber", + "property": {} + }, + { + "type": "extension", + "name": "streamid_adapter", + "addon": "streamid_adapter", + "property": {} + } + ], + "connections": [ + { + "extension": "main_control", + "cmd": [ + { + "names": [ + "on_user_joined", + "on_user_left" + ], + "source": [ + { + "extension": "agora_rtc" + } + ] + } + ], + "data": [ + { + "name": "asr_result", + "source": [ + { + "extension": "stt" + } + ] + }, + { + "name": "tts_audio_start", + "source": [ + { + "extension": "tts" + } + ] + }, + { + "name": "tts_audio_end", + "source": [ + { + "extension": "tts" + } + ] + } + ] + }, + { + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "streamid_adapter" + } + ] + }, + { + "name": "pcm_frame", + "source": [ + { + "extension": "tts" + } + ] + } + ], + "data": [ + { + "name": "data", + "source": [ + { + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension": "streamid_adapter", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "stt" + } + ] + } + ] + } + ] + } + }, + { + "name": "voice_assistant_blaze_full", + "auto_start": false, + "graph": { + "nodes": [ + { + "type": "extension", + "name": "agora_rtc", + "addon": "agora_rtc", + "extension_group": "default", + "property": { + "app_id": "${env:AGORA_APP_ID}", + "app_certificate": "${env:AGORA_APP_CERTIFICATE|}", + "channel": "ten_agent_test", + "stream_id": 1234, + "remote_stream_id": 123, + "subscribe_audio": true, + "publish_audio": true, + "publish_data": true, + "enable_agora_asr": false + } + }, + { + "type": "extension", + "name": "stt", + "addon": "blaze_stt_python", + "extension_group": "stt", + "property": { + "dump": false, + "dump_path": "./", + "params": { + "api_key": "${env:BLAZE_STT_API_KEY}", + "api_url": "${env:BLAZE_STT_API_URL|https://api.blaze.vn}", + "language": "vi", + "sample_rate": 16000, + "model": "stt-stream-1.5" + } + } + }, + { + "type": "extension", + "name": "llm", + "addon": "openai_llm2_python", + "extension_group": "chatgpt", + "property": { + "base_url": "https://api.openai.com/v1", + "api_key": "${env:OPENAI_API_KEY}", + "frequency_penalty": 0.9, + "model": "${env:OPENAI_MODEL}", + "max_tokens": 512, + "prompt": "", + "proxy_url": "${env:OPENAI_PROXY_URL|}", + "greeting": "Xin chào, TEN Agent với Blaze đã kết nối. Tôi có thể giúp gì cho bạn?", + "max_memory_length": 10 + } + }, + { + "type": "extension", + "name": "tts", + "addon": "blaze_tts_python", + "extension_group": "tts", + "property": { + "dump": false, + "dump_path": "./", + "params": { + "api_key": "${env:BLAZE_TTS_API_KEY}", + "api_url": "${env:BLAZE_TTS_API_URL|https://api.blaze.vn}", + "language": "vi", + "speaker_id": "HN-Nu-CSKH-HuongGiang", + "audio_speed": 1.0, + "audio_quality": 64, + "sample_rate": 24000, + "model": "2.0-realtime", + "audio_format": "pcm" + } + } + }, + { + "type": "extension", + "name": "main_control", + "addon": "main_python", + "extension_group": "control", + "property": { + "greeting": "Xin chào, TEN Agent với Blaze đã kết nối. Tôi có thể giúp gì cho bạn?" + } + }, + { + "type": "extension", + "name": "message_collector", + "addon": "message_collector2", + "extension_group": "transcriber", + "property": {} + }, + { + "type": "extension", + "name": "streamid_adapter", + "addon": "streamid_adapter", + "property": {} + } + ], + "connections": [ + { + "extension": "main_control", + "cmd": [ + { + "names": [ + "on_user_joined", + "on_user_left" + ], + "source": [ + { + "extension": "agora_rtc" + } + ] + } + ], + "data": [ + { + "name": "asr_result", + "source": [ + { + "extension": "stt" + } + ] + }, + { + "name": "tts_audio_start", + "source": [ + { + "extension": "tts" + } + ] + }, + { + "name": "tts_audio_end", + "source": [ + { + "extension": "tts" + } + ] + } + ] + }, + { + "extension": "agora_rtc", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "streamid_adapter" + } + ] + }, + { + "name": "pcm_frame", + "source": [ + { + "extension": "tts" + } + ] + } + ], + "data": [ + { + "name": "data", + "source": [ + { + "extension": "message_collector" + } + ] + } + ] + }, + { + "extension": "streamid_adapter", + "audio_frame": [ + { + "name": "pcm_frame", + "dest": [ + { + "extension": "stt" + } + ] + } + ] + } + ] + } } ], "log": { diff --git a/ai_agents/agents/ten_packages/extension/blaze_stt_python/README.md b/ai_agents/agents/ten_packages/extension/blaze_stt_python/README.md index ac790fd340..e89ff68959 100644 --- a/ai_agents/agents/ten_packages/extension/blaze_stt_python/README.md +++ b/ai_agents/agents/ten_packages/extension/blaze_stt_python/README.md @@ -18,29 +18,46 @@ pip install httpx pydantic ### Environment Variables -Set the following environment variables: - ```bash -export BLAZE_STT_API_URL="http://localhost:8000" -export BLAZE_STT_API_KEY="your-api-key-here" # Optional +export BLAZE_API_KEY="your-api-key-here" +export BLAZE_STT_API_KEY="$BLAZE_API_KEY" +export BLAZE_STT_API_URL="https://api.blaze.vn" ``` ### Property.json (TEN Framework) -The extension includes a `property.json` file with default configuration that TEN framework can use: - ```json { + "dump": false, + "dump_path": "./", "params": { - "api_url": "${env:BLAZE_STT_API_URL}", + "api_url": "${env:BLAZE_STT_API_URL|https://api.blaze.vn}", "api_key": "${env:BLAZE_STT_API_KEY}", "language": "vi", + "sample_rate": 16000, "timeout": 3600 } } ``` -TEN framework will automatically read this file and use environment variables for configuration. +### Voice assistant graphs + +In `agents/examples/voice-assistant`: + +| Graph name | STT | TTS | +| ---------- | --- | --- | +| `voice_assistant_blaze_stt` | Blaze | ElevenLabs | +| `voice_assistant_blaze_full` | Blaze | Blaze | + +**Note:** Blaze STT is batch HTTP (`/v1/stt/execute`). The TEN extension buffers +PCM and submits a WAV on finalize (turn-based), not true streaming ASR. + +### Live smoke test + +```bash +python ai_agents/scripts/smoke_blaze.py +python ai_agents/scripts/smoke_blaze.py --skip-tts --audio /path/to/sample.wav +``` ## Usage diff --git a/ai_agents/agents/ten_packages/extension/blaze_stt_python/__init__.py b/ai_agents/agents/ten_packages/extension/blaze_stt_python/__init__.py index 123f767d29..9c9fb60f08 100644 --- a/ai_agents/agents/ten_packages/extension/blaze_stt_python/__init__.py +++ b/ai_agents/agents/ten_packages/extension/blaze_stt_python/__init__.py @@ -1,11 +1,15 @@ -""" -Blaze STT Extension for TEN Framework +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +"""Blaze STT package.""" -This extension provides Speech-to-Text (STT) functionality using Blaze API. -Implements TEN framework extension interface. -""" +try: + from . import addon as addon # noqa: F401 +except Exception: # pragma: no cover - host unit tests without ten_runtime + addon = None -from .blaze_stt import BlazeSTTExtension, BlazeSTTConfig +from .blaze_stt import BlazeSTTConfig, BlazeSTTExtension __all__ = ["BlazeSTTExtension", "BlazeSTTConfig"] -__version__ = "1.0.0" diff --git a/ai_agents/agents/ten_packages/extension/blaze_stt_python/addon.py b/ai_agents/agents/ten_packages/extension/blaze_stt_python/addon.py new file mode 100644 index 0000000000..2735431a80 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/blaze_stt_python/addon.py @@ -0,0 +1,19 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten_runtime import ( + Addon, + register_addon_as_extension, + TenEnv, +) + +from .extension import BlazeSTTExtension + + +@register_addon_as_extension("blaze_stt_python") +class BlazeSTTExtensionAddon(Addon): + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + ten_env.log_info("BlazeSTTExtensionAddon on_create_instance") + ten_env.on_create_instance_done(BlazeSTTExtension(name), context) diff --git a/ai_agents/agents/ten_packages/extension/blaze_stt_python/config.py b/ai_agents/agents/ten_packages/extension/blaze_stt_python/config.py new file mode 100644 index 0000000000..dc3c7217c7 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/blaze_stt_python/config.py @@ -0,0 +1,73 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +"""TEN framework config for Blaze realtime STT (WebSocket).""" + +from __future__ import annotations + +import copy +from pathlib import Path + +from pydantic import BaseModel, Field +from ten_ai_base import utils + +DEFAULT_API_URL = "https://api.blaze.vn" +DEFAULT_LANGUAGE = "vi" +DEFAULT_SAMPLE_RATE = 16000 +DEFAULT_MODEL = "stt-stream-1.5" + + +class BlazeASRParams(BaseModel): + """Params nested under property.params.""" + + api_key: str = Field(default="", description="Blaze API key (Bearer token)") + api_url: str = Field(default=DEFAULT_API_URL, description="Blaze base URL") + language: str = Field( + default=DEFAULT_LANGUAGE, description="Language code (e.g. vi, en)" + ) + sample_rate: int = Field( + default=DEFAULT_SAMPLE_RATE, + description="Input PCM sample rate from RTC (must be 16000 for stream)", + ) + model: str = Field( + default=DEFAULT_MODEL, + description="Realtime STT model (stt-stream-1.5)", + ) + # Optional domain adaptation (stt-stream-1.5 only) + topic: str = Field(default="", description="Optional topic hint") + context: str = Field(default="", description="Optional free-form context") + + +class BlazeASRConfig(BaseModel): + """Top-level extension property.""" + + dump: bool = Field(default=False, description="Dump input audio") + dump_path: str = Field( + default_factory=lambda: str(Path(__file__).parent / "blaze_stt_in.pcm"), + description="Dump path for input audio", + ) + params: BlazeASRParams = Field(default_factory=BlazeASRParams) + + def validate_config(self) -> None: + if not self.params.api_key: + raise ValueError("API key is required for Blaze STT") + if not self.params.api_url: + raise ValueError("api_url is required for Blaze STT") + + def to_str(self, sensitive_handling: bool = True) -> str: + if not sensitive_handling: + return f"{self}" + config = copy.deepcopy(self) + if config.params.api_key: + config.params.api_key = utils.encrypt(config.params.api_key) + return f"{config}" + + def ws_url(self) -> str: + base = self.params.api_url.rstrip("/") + if base.startswith("https://"): + base = "wss://" + base[len("https://") :] + elif base.startswith("http://"): + base = "ws://" + base[len("http://") :] + return f"{base}/v1/stt/realtime" diff --git a/ai_agents/agents/ten_packages/extension/blaze_stt_python/extension.py b/ai_agents/agents/ten_packages/extension/blaze_stt_python/extension.py new file mode 100644 index 0000000000..5c99472268 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/blaze_stt_python/extension.py @@ -0,0 +1,280 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +""" +Blaze realtime STT (WebSocket /v1/stt/realtime, model stt-stream-1.5). + +Protocol: + 1. Connect WS, send JSON {token, language, model, ...} + 2. Stream binary PCM s16le mono @ 16kHz + 3. Receive JSON {type: partial|final, text: ...} +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import websockets +from typing_extensions import override +from websockets.exceptions import ConnectionClosed + +from ten_ai_base.asr import ( + ASRBufferConfig, + ASRBufferConfigModeDiscard, + AsyncASRBaseExtension, +) +from ten_ai_base.const import LOG_CATEGORY_KEY_POINT, LOG_CATEGORY_VENDOR +from ten_ai_base.dumper import Dumper +from ten_ai_base.message import ( + ModuleError, + ModuleErrorCode, + ModuleErrorVendorInfo, + ModuleType, +) +from ten_ai_base.struct import ASRResult +from ten_runtime import AsyncTenEnv, AudioFrame + +from .config import BlazeASRConfig, DEFAULT_SAMPLE_RATE + + +class BlazeSTTExtension(AsyncASRBaseExtension): + """Realtime streaming Speech-to-Text via Blaze WebSocket.""" + + def __init__(self, name: str): + super().__init__(name) + self.config: BlazeASRConfig | None = None + self.session_id: str | None = None + self._ws = None + self._recv_task: asyncio.Task | None = None + self._connected = False + self.audio_dumper: Dumper | None = None + self._utterance_start_ms: int = 0 + + @override + def vendor(self) -> str: + return "blaze" + + @override + async def on_init(self, ten_env: AsyncTenEnv) -> None: + await super().on_init(ten_env) + config_json, _ = await ten_env.get_property_to_json("") + try: + self.config = BlazeASRConfig.model_validate_json(config_json) + self.config.validate_config() + ten_env.log_info( + f"config: {self.config.to_str(sensitive_handling=True)}", + category=LOG_CATEGORY_KEY_POINT, + ) + if self.config.dump: + dump_path = Path(self.config.dump_path) + if dump_path.suffix != ".pcm": + dump_path = dump_path / "blaze_stt_in.pcm" + dump_path.parent.mkdir(parents=True, exist_ok=True) + self.audio_dumper = Dumper(str(dump_path)) + await self.audio_dumper.start() + except Exception as exc: # pylint: disable=broad-exception-caught + ten_env.log_error( + f"invalid property: {exc}", category=LOG_CATEGORY_KEY_POINT + ) + self.config = None + await self.send_asr_error( + ModuleError( + module=ModuleType.ASR, + code=ModuleErrorCode.FATAL_ERROR.value, + message=str(exc), + ), + ) + + @override + async def start_connection(self) -> None: + if self.config is None: + return + await self.stop_connection() + + url = self.config.ws_url() + self.ten_env.log_info( + f"vendor_status_changed: connecting Blaze STT {url} " + f"model={self.config.params.model}", + category=LOG_CATEGORY_VENDOR, + ) + try: + self._ws = await websockets.connect( + url, + max_size=8 * 1024 * 1024, + ping_interval=20, + ping_timeout=20, + ) + init_msg = { + "token": self.config.params.api_key, + "language": self.config.params.language, + "model": self.config.params.model, + } + if self.config.params.topic: + init_msg["topic"] = self.config.params.topic + if self.config.params.context: + init_msg["context"] = self.config.params.context + + await self._ws.send(json.dumps(init_msg)) + self._connected = True + self._recv_task = asyncio.create_task(self._recv_loop()) + self.ten_env.log_info( + "vendor_status_changed: Blaze STT realtime ready " + f"model={self.config.params.model}", + category=LOG_CATEGORY_VENDOR, + ) + except Exception as exc: # pylint: disable=broad-exception-caught + self._connected = False + self._ws = None + self.ten_env.log_error( + f"vendor_error: STT connect failed: {exc}", + category=LOG_CATEGORY_VENDOR, + ) + await self.send_asr_error( + ModuleError( + module=ModuleType.ASR, + code=ModuleErrorCode.NON_FATAL_ERROR.value, + message=str(exc), + ), + ModuleErrorVendorInfo( + vendor="blaze", code="", message=str(exc) + ), + ) + + async def _recv_loop(self) -> None: + assert self._ws is not None + try: + async for raw in self._ws: + if isinstance(raw, bytes): + continue + try: + msg = json.loads(raw) + except json.JSONDecodeError: + continue + await self._handle_message(msg) + except ConnectionClosed: + self.ten_env.log_info( + "vendor_status_changed: Blaze STT WS closed", + category=LOG_CATEGORY_VENDOR, + ) + except asyncio.CancelledError: + raise + except Exception as exc: # pylint: disable=broad-exception-caught + self.ten_env.log_error( + f"vendor_error: STT recv loop: {exc}", + category=LOG_CATEGORY_VENDOR, + ) + finally: + self._connected = False + + async def _handle_message(self, msg: dict) -> None: + if self.config is None: + return + msg_type = msg.get("type") + if msg_type == "error": + text = msg.get("text") or str(msg) + self.ten_env.log_error( + f"vendor_error: {text}", + category=LOG_CATEGORY_VENDOR, + ) + await self.send_asr_error( + ModuleError( + module=ModuleType.ASR, + code=ModuleErrorCode.NON_FATAL_ERROR.value, + message=text, + ), + ModuleErrorVendorInfo(vendor="blaze", code="", message=text), + ) + return + + if msg_type not in ("partial", "final"): + return + + text = (msg.get("text") or "").strip() + is_final = msg_type == "final" + if not text and not is_final: + return + + language = msg.get("language") or self.config.params.language + asr_result = ASRResult( + text=text, + final=is_final, + start_ms=0, + duration_ms=0, + language=language, + words=[], + ) + self.ten_env.log_debug( + f"Blaze STT {msg_type}: {text[:120]!r}", + category=LOG_CATEGORY_VENDOR, + ) + await self.send_asr_result(asr_result) + if is_final: + await self.send_asr_finalize_end() + + @override + async def stop_connection(self) -> None: + self._connected = False + if self._recv_task is not None: + self._recv_task.cancel() + try: + await self._recv_task + except (asyncio.CancelledError, Exception): + pass + self._recv_task = None + if self._ws is not None: + try: + await self._ws.close() + except Exception: # pylint: disable=broad-exception-caught + pass + self._ws = None + self.ten_env.log_info("Blaze STT connection stopped") + + @override + def is_connected(self) -> bool: + return self._connected and self._ws is not None + + @override + def input_audio_sample_rate(self) -> int: + if self.config is None: + return DEFAULT_SAMPLE_RATE + return int(self.config.params.sample_rate) + + @override + def buffer_strategy(self) -> ASRBufferConfig: + return ASRBufferConfigModeDiscard() + + @override + async def send_audio( + self, frame: AudioFrame, session_id: str | None + ) -> bool: + if not self.is_connected() or self._ws is None: + return False + self.session_id = session_id + try: + chunk = bytes(frame.get_buf()) + if not chunk: + return True + if self.audio_dumper: + await self.audio_dumper.push_bytes(chunk) + await self._ws.send(chunk) + return True + except Exception as exc: # pylint: disable=broad-exception-caught + self.ten_env.log_error( + f"vendor_error: send_audio failed: {exc}", + category=LOG_CATEGORY_VENDOR, + ) + self._connected = False + return False + + @override + async def finalize(self, session_id: str | None) -> None: + # Server-side endpoint detection emits finals; finalize is a no-op flush + # signal so the framework lifecycle stays happy. + self.session_id = session_id or self.session_id + self.ten_env.log_debug( + "Blaze STT finalize (endpoint detection is server-side)" + ) diff --git a/ai_agents/agents/ten_packages/extension/blaze_stt_python/manifest.json b/ai_agents/agents/ten_packages/extension/blaze_stt_python/manifest.json index a1ba9de831..2dba5d916a 100644 --- a/ai_agents/agents/ten_packages/extension/blaze_stt_python/manifest.json +++ b/ai_agents/agents/ten_packages/extension/blaze_stt_python/manifest.json @@ -18,7 +18,7 @@ "content": "Blaze Speech-to-Text extension for TEN Framework" }, "vi-VN": { - "content": "Extension chuyển đổi giọng nói thành văn bản cho TEN Framework" + "content": "Extension chuy\u1ec3n \u0111\u1ed5i gi\u1ecdng n\u00f3i th\u00e0nh v\u0103n b\u1ea3n cho TEN Framework" } } }, @@ -38,7 +38,6 @@ "include": [ "manifest.json", "property.json", - "BUILD.gn", "**.tent", "**.py", "README.md", @@ -49,23 +48,32 @@ "api": { "interface": [ { - "import_uri": "../../system/ten_ai_base/api/stt-interface.json" + "import_uri": "../../system/ten_ai_base/api/asr-interface.json" } ], "property": { "properties": { + "dump": { + "type": "bool" + }, + "dump_path": { + "type": "string" + }, "params": { "type": "object", "properties": { - "api_url": { + "api_key": { "type": "string" }, - "api_key": { + "api_url": { "type": "string" }, "language": { "type": "string" }, + "sample_rate": { + "type": "int32" + }, "timeout": { "type": "int32" } diff --git a/ai_agents/agents/ten_packages/extension/blaze_stt_python/property.json b/ai_agents/agents/ten_packages/extension/blaze_stt_python/property.json index 84ac80ecaf..6106133b67 100644 --- a/ai_agents/agents/ten_packages/extension/blaze_stt_python/property.json +++ b/ai_agents/agents/ten_packages/extension/blaze_stt_python/property.json @@ -1,8 +1,11 @@ { + "dump": false, + "dump_path": "./", "params": { - "api_url": "${env:BLAZE_STT_API_URL}", "api_key": "${env:BLAZE_STT_API_KEY}", + "api_url": "${env:BLAZE_STT_API_URL|https://api.blaze.vn}", "language": "vi", - "timeout": 3600 + "sample_rate": 16000, + "model": "stt-stream-1.5" } } diff --git a/ai_agents/agents/ten_packages/extension/blaze_stt_python/requirements.txt b/ai_agents/agents/ten_packages/extension/blaze_stt_python/requirements.txt index ad010f96fe..e623313a67 100644 --- a/ai_agents/agents/ten_packages/extension/blaze_stt_python/requirements.txt +++ b/ai_agents/agents/ten_packages/extension/blaze_stt_python/requirements.txt @@ -1,2 +1,3 @@ httpx>=0.24.0 pydantic>=2.0.0 +websockets>=12.0 diff --git a/ai_agents/agents/ten_packages/extension/blaze_tts_python/README.md b/ai_agents/agents/ten_packages/extension/blaze_tts_python/README.md index b44fae58a3..a8573d5acd 100644 --- a/ai_agents/agents/ten_packages/extension/blaze_tts_python/README.md +++ b/ai_agents/agents/ten_packages/extension/blaze_tts_python/README.md @@ -18,32 +18,50 @@ pip install httpx pydantic ### Environment Variables -Set the following environment variables: - ```bash -export BLAZE_TTS_API_URL="http://localhost:8000" -export BLAZE_TTS_API_KEY="your-api-key-here" # Optional +export BLAZE_API_KEY="your-api-key-here" +export BLAZE_TTS_API_KEY="$BLAZE_API_KEY" +export BLAZE_TTS_API_URL="https://api.blaze.vn" ``` ### Property.json (TEN Framework) -The extension includes a `property.json` file with default configuration that TEN framework can use: - ```json { + "dump": false, + "dump_path": "./", "params": { - "api_url": "${env:BLAZE_TTS_API_URL}", + "api_url": "${env:BLAZE_TTS_API_URL|https://api.blaze.vn}", "api_key": "${env:BLAZE_TTS_API_KEY}", "language": "vi", - "speaker_id": null, + "speaker_id": "HN-Nam-2-BL", "audio_speed": 1.0, "audio_quality": 64, + "sample_rate": 24000, + "model": "v1.5_pro", "timeout": 3600 } } ``` -TEN framework will automatically read this file and use environment variables for configuration. +Output is WAV PCM16 mono at **24 kHz**. The TEN HTTP client posts `/v1/tts`, +polls `/v1/tts/{id}/download` (handles HTTP 425), then strips the WAV header. + +### Voice assistant graphs + +In `agents/examples/voice-assistant`: + +| Graph name | STT | TTS | +| ---------- | --- | --- | +| `voice_assistant_blaze_tts` | Deepgram | Blaze | +| `voice_assistant_blaze_full` | Blaze | Blaze | + +### Live smoke test + +```bash +python ai_agents/scripts/smoke_blaze.py +python ai_agents/scripts/smoke_blaze.py --skip-stt --text "Xin chào" +``` ## Usage diff --git a/ai_agents/agents/ten_packages/extension/blaze_tts_python/__init__.py b/ai_agents/agents/ten_packages/extension/blaze_tts_python/__init__.py index 0659904987..3b26d9cc64 100644 --- a/ai_agents/agents/ten_packages/extension/blaze_tts_python/__init__.py +++ b/ai_agents/agents/ten_packages/extension/blaze_tts_python/__init__.py @@ -1,11 +1,15 @@ -""" -Blaze TTS Extension for TEN Framework +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +"""Blaze TTS package.""" -This extension provides Text-to-Speech (TTS) functionality using Blaze API. -Implements TEN framework extension interface. -""" +try: + from . import addon as addon # noqa: F401 +except Exception: # pragma: no cover - host unit tests without ten_runtime + addon = None -from .blaze_tts import BlazeTTSExtension, BlazeTTSConfig +from .blaze_tts import BlazeTTSConfig, BlazeTTSExtension __all__ = ["BlazeTTSExtension", "BlazeTTSConfig"] -__version__ = "1.0.0" diff --git a/ai_agents/agents/ten_packages/extension/blaze_tts_python/addon.py b/ai_agents/agents/ten_packages/extension/blaze_tts_python/addon.py new file mode 100644 index 0000000000..6f0e8b2362 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/blaze_tts_python/addon.py @@ -0,0 +1,19 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +from ten_runtime import ( + Addon, + register_addon_as_extension, + TenEnv, +) + +from .extension import BlazeTTSExtension + + +@register_addon_as_extension("blaze_tts_python") +class BlazeTTSExtensionAddon(Addon): + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + ten_env.log_info("BlazeTTSExtensionAddon on_create_instance") + ten_env.on_create_instance_done(BlazeTTSExtension(name), context) diff --git a/ai_agents/agents/ten_packages/extension/blaze_tts_python/config.py b/ai_agents/agents/ten_packages/extension/blaze_tts_python/config.py new file mode 100644 index 0000000000..3f7dfd5a19 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/blaze_tts_python/config.py @@ -0,0 +1,95 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +"""TEN framework config for Blaze realtime TTS (WebSocket).""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any + +from pydantic import Field +from ten_ai_base import utils +from ten_ai_base.tts2_http import AsyncTTS2HttpConfig + +DEFAULT_API_URL = "https://api.blaze.vn" +DEFAULT_SPEAKER_ID = "HN-Nu-CSKH-HuongGiang" +DEFAULT_SAMPLE_RATE = 24000 +DEFAULT_LANGUAGE = "vi" +# Public realtime alias; HTTP client maps this to v2.0_flash backend +DEFAULT_MODEL = "2.0-realtime" + + +class BlazeTTSConfig(AsyncTTS2HttpConfig): + """Blaze realtime TTS configuration.""" + + dump: bool = Field(default=False, description="Dump synthesized audio") + dump_path: str = Field( + default_factory=lambda: str(Path(__file__).parent / "blaze_tts_in.pcm"), + description="Dump path for synthesized audio", + ) + params: dict[str, Any] = Field( + default_factory=dict, description="Blaze TTS params" + ) + + def update_params(self) -> None: + self.params.setdefault("api_url", DEFAULT_API_URL) + self.params.setdefault("language", DEFAULT_LANGUAGE) + self.params.setdefault("speaker_id", DEFAULT_SPEAKER_ID) + self.params.setdefault("audio_speed", 1.0) + self.params.setdefault("audio_quality", 64) + self.params.setdefault("model", DEFAULT_MODEL) + self.params.setdefault("sample_rate", DEFAULT_SAMPLE_RATE) + # pcm for TEN pcm_frame (docs also allow mp3/wav/opus) + self.params.setdefault("audio_format", "pcm") + # Official API value is "request" (not internal name request_base) + self.params.setdefault("strategy", "request") + self.params.setdefault("normalization", "basic") + # docs use string speed + if "audio_speed" in self.params: + self.params["audio_speed"] = str(self.params["audio_speed"]) + + if self.params.get("speaker_id") == "": + self.params["speaker_id"] = DEFAULT_SPEAKER_ID + + def to_str(self, sensitive_handling: bool = True) -> str: + if not sensitive_handling: + return f"{self}" + config = copy.deepcopy(self) + if config.params and "api_key" in config.params: + config.params["api_key"] = utils.encrypt(config.params["api_key"]) + return f"{config}" + + def validate(self) -> None: + if not self.params.get("api_key"): + raise ValueError("API key is required for Blaze TTS") + if not self.params.get("speaker_id"): + raise ValueError("speaker_id is required for Blaze TTS") + if not self.params.get("api_url"): + raise ValueError("api_url is required for Blaze TTS") + + def ws_url(self) -> str: + base = str(self.params.get("api_url", DEFAULT_API_URL)).rstrip("/") + if base.startswith("https://"): + base = "wss://" + base[len("https://") :] + elif base.startswith("http://"): + base = "ws://" + base[len("http://") :] + + # Session defaults as query params + from urllib.parse import urlencode + + q = { + "speaker_id": self.params.get("speaker_id", DEFAULT_SPEAKER_ID), + "language": self.params.get("language", DEFAULT_LANGUAGE), + "model": self.params.get("model", DEFAULT_MODEL), + "sample_rate": str( + self.params.get("sample_rate", DEFAULT_SAMPLE_RATE) + ), + "audio_format": self.params.get("audio_format", "pcm"), + "audio_speed": str(self.params.get("audio_speed", 1.0)), + "audio_quality": str(self.params.get("audio_quality", 64)), + } + return f"{base}/v1/tts/realtime?{urlencode(q)}" diff --git a/ai_agents/agents/ten_packages/extension/blaze_tts_python/extension.py b/ai_agents/agents/ten_packages/extension/blaze_tts_python/extension.py new file mode 100644 index 0000000000..b81e459ac6 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/blaze_tts_python/extension.py @@ -0,0 +1,41 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +"""Blaze TTS TEN extension (HTTP job + download polling).""" + +from ten_ai_base.tts2_http import ( + AsyncTTS2HttpClient, + AsyncTTS2HttpConfig, + AsyncTTS2HttpExtension, +) +from ten_runtime import AsyncTenEnv + +from .config import BlazeTTSConfig, DEFAULT_SAMPLE_RATE +from .ten_tts_client import BlazeTTSHttpClient + + +class BlazeTTSExtension(AsyncTTS2HttpExtension): + """Text-to-speech via Blaze (2.0-realtime alias -> v2.0_flash HTTP).""" + + def __init__(self, name: str) -> None: + super().__init__(name) + self.config: BlazeTTSConfig | None = None + self.client: BlazeTTSHttpClient | None = None + + async def create_config(self, config_json_str: str) -> AsyncTTS2HttpConfig: + return BlazeTTSConfig.model_validate_json(config_json_str) + + async def create_client( + self, config: AsyncTTS2HttpConfig, ten_env: AsyncTenEnv + ) -> AsyncTTS2HttpClient: + return BlazeTTSHttpClient(config=config, ten_env=ten_env) + + def vendor(self) -> str: + return "blaze" + + def synthesize_audio_sample_rate(self) -> int: + if self.config is None: + return DEFAULT_SAMPLE_RATE + return int(self.config.params.get("sample_rate", DEFAULT_SAMPLE_RATE)) diff --git a/ai_agents/agents/ten_packages/extension/blaze_tts_python/manifest.json b/ai_agents/agents/ten_packages/extension/blaze_tts_python/manifest.json index eb05eaf662..898553099e 100644 --- a/ai_agents/agents/ten_packages/extension/blaze_tts_python/manifest.json +++ b/ai_agents/agents/ten_packages/extension/blaze_tts_python/manifest.json @@ -18,7 +18,7 @@ "content": "Blaze Text-to-Speech extension for TEN Framework" }, "vi-VN": { - "content": "Extension chuyển đổi văn bản thành giọng nói cho TEN Framework" + "content": "Extension chuy\u1ec3n \u0111\u1ed5i v\u0103n b\u1ea3n th\u00e0nh gi\u1ecdng n\u00f3i cho TEN Framework" } } }, @@ -38,7 +38,6 @@ "include": [ "manifest.json", "property.json", - "BUILD.gn", "**.tent", "**.py", "README.md", @@ -54,13 +53,19 @@ ], "property": { "properties": { + "dump": { + "type": "bool" + }, + "dump_path": { + "type": "string" + }, "params": { "type": "object", "properties": { - "api_url": { + "api_key": { "type": "string" }, - "api_key": { + "api_url": { "type": "string" }, "language": { @@ -73,7 +78,13 @@ "type": "float64" }, "audio_quality": { - "type": "float64" + "type": "int32" + }, + "sample_rate": { + "type": "int32" + }, + "model": { + "type": "string" }, "timeout": { "type": "int32" diff --git a/ai_agents/agents/ten_packages/extension/blaze_tts_python/property.json b/ai_agents/agents/ten_packages/extension/blaze_tts_python/property.json index 25ca345113..21342b0b04 100644 --- a/ai_agents/agents/ten_packages/extension/blaze_tts_python/property.json +++ b/ai_agents/agents/ten_packages/extension/blaze_tts_python/property.json @@ -1,12 +1,15 @@ { + "dump": false, + "dump_path": "./", "params": { - "api_url": "${env:BLAZE_TTS_API_URL}", "api_key": "${env:BLAZE_TTS_API_KEY}", + "api_url": "${env:BLAZE_TTS_API_URL|https://api.blaze.vn}", "language": "vi", - "speaker_id": null, + "speaker_id": "HN-Nu-CSKH-HuongGiang", "audio_speed": 1.0, "audio_quality": 64, - "timeout": 3600 + "sample_rate": 24000, + "model": "2.0-realtime", + "audio_format": "pcm" } } - diff --git a/ai_agents/agents/ten_packages/extension/blaze_tts_python/requirements.txt b/ai_agents/agents/ten_packages/extension/blaze_tts_python/requirements.txt index ad010f96fe..e623313a67 100644 --- a/ai_agents/agents/ten_packages/extension/blaze_tts_python/requirements.txt +++ b/ai_agents/agents/ten_packages/extension/blaze_tts_python/requirements.txt @@ -1,2 +1,3 @@ httpx>=0.24.0 pydantic>=2.0.0 +websockets>=12.0 diff --git a/ai_agents/agents/ten_packages/extension/blaze_tts_python/ten_tts_client.py b/ai_agents/agents/ten_packages/extension/blaze_tts_python/ten_tts_client.py new file mode 100644 index 0000000000..72ee5fb1de --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/blaze_tts_python/ten_tts_client.py @@ -0,0 +1,325 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +""" +Blaze realtime TTS client — matches official docs: +https://app.blaze.vn/api/documentation/realTimeTts + +Protocol: + 1. Connect wss://api.blaze.vn/v1/tts/realtime + 2. Recv {"type": "successful-connection"} + 3. Send {"token": "...", "strategy": "request"} + 4. Recv {"type": "successful-authentication"} + 5. Send TTS query JSON (query, language, audio_format, ...) + 6. Recv processing-request → started-byte-stream → binary chunks + → finished-byte-stream +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any, AsyncIterator, Tuple +from urllib.parse import urlencode + +import websockets +from websockets.exceptions import ConnectionClosed + +from ten_ai_base.const import LOG_CATEGORY_VENDOR +from ten_ai_base.struct import TTS2HttpResponseEventType +from ten_ai_base.tts2_http import AsyncTTS2HttpClient +from ten_runtime import AsyncTenEnv + +from .config import BlazeTTSConfig, DEFAULT_API_URL + + +class BlazeTTSHttpClient(AsyncTTS2HttpClient): + """Realtime TTS over WebSocket (class name kept for extension wiring).""" + + def __init__(self, config: BlazeTTSConfig, ten_env: AsyncTenEnv): + super().__init__() + self.config = config + self.ten_env = ten_env + self._is_cancelled = False + self._ws = None + self._ready = False + self._lock = asyncio.Lock() + ten_env.log_info( + f"BlazeTTS realtime client model=" + f"{config.params.get('model', '2.0-realtime')} " + f"speaker={config.params.get('speaker_id')}" + ) + + def _ws_url(self) -> str: + base = str(self.config.params.get("api_url", DEFAULT_API_URL)).rstrip( + "/" + ) + if base.startswith("https://"): + base = "wss://" + base[len("https://") :] + elif base.startswith("http://"): + base = "ws://" + base[len("http://") :] + # Optional session defaults as query params (docs allow bare URL too) + q = { + "speaker_id": self.config.params.get("speaker_id", ""), + "language": self.config.params.get("language", "vi"), + "model": self.config.params.get("model", "2.0-realtime"), + "sample_rate": str( + int(self.config.params.get("sample_rate", 24000)) + ), + "audio_format": self.config.params.get("audio_format", "pcm"), + } + return f"{base}/v1/tts/realtime?{urlencode(q)}" + + async def cancel(self): + self.ten_env.log_debug("BlazeTTS: cancel() called.") + self._is_cancelled = True + + async def _recv_json(self, timeout: float = 30.0) -> dict: + assert self._ws is not None + raw = await asyncio.wait_for(self._ws.recv(), timeout=timeout) + if isinstance(raw, (bytes, bytearray)): + raise ValueError( + f"Expected JSON, received binary ({len(raw)} bytes)" + ) + return json.loads(raw) + + async def _connect(self) -> None: + if self._ws is not None: + try: + await self._ws.close() + except Exception: # pylint: disable=broad-exception-caught + pass + self._ws = None + self._ready = False + + url = self._ws_url() + self.ten_env.log_info( + f"BlazeTTS connecting {url.split('?')[0]}", + category=LOG_CATEGORY_VENDOR, + ) + # Docs example: ping_interval=None + self._ws = await websockets.connect( + url, + max_size=16 * 1024 * 1024, + ping_interval=None, + open_timeout=20, + ) + + # 1) successful-connection + hello = await self._recv_json(timeout=15) + if hello.get("type") != "successful-connection": + raise RuntimeError(f"Connection failed: {hello}") + + # 2) auth — strategy MUST be "request" (not "request_base") + auth = { + "token": self.config.params.get("api_key", ""), + "strategy": "request", + } + await self._ws.send(json.dumps(auth)) + + # 3) successful-authentication (required before queries) + auth_ok = await self._recv_json(timeout=20) + if auth_ok.get("type") != "successful-authentication": + raise RuntimeError(f"Authentication failed: {auth_ok}") + + self._ready = True + self.ten_env.log_info( + "BlazeTTS realtime authenticated (strategy=request)", + category=LOG_CATEGORY_VENDOR, + ) + + async def _ensure_ws(self) -> None: + if self._ws is not None and self._ready: + return + await self._connect() + + def _build_query(self, text: str) -> dict: + # Match official docs field types exactly + return { + "query": text, + "language": self.config.params.get("language", "vi"), + "audio_format": self.config.params.get("audio_format", "pcm"), + "audio_quality": int(self.config.params.get("audio_quality", 64)), + # docs use string speed + "audio_speed": str(self.config.params.get("audio_speed", "1")), + "speaker_id": self.config.params.get("speaker_id"), + "normalization": self.config.params.get("normalization", "basic"), + "model": self.config.params.get("model", "2.0-realtime"), + "sample_rate": int(self.config.params.get("sample_rate", 24000)), + } + + async def get( + self, text: str, request_id: str + ) -> AsyncIterator[Tuple[bytes | None, TTS2HttpResponseEventType]]: + self._is_cancelled = False + + if not text or not text.strip(): + yield None, TTS2HttpResponseEventType.END + return + + async with self._lock: + try: + await self._ensure_ws() + assert self._ws is not None + + payload = self._build_query(text) + self.ten_env.log_debug( + f"BlazeTTS query request_id={request_id} " + f"model={payload['model']} chars={len(text)}", + category=LOG_CATEGORY_VENDOR, + ) + await self._ws.send(json.dumps(payload)) + + # processing-request + msg = await self._recv_json(timeout=30) + mtype = msg.get("type") or msg.get("status") + if mtype in ( + "failed-request", + "bad-request", + "internal-error", + "failed-authentication", + ): + detail = ( + msg.get("details") or msg.get("message") or str(msg) + ) + self.ten_env.log_error( + f"vendor_error: {mtype} {detail} " + f"request_id={request_id}", + category=LOG_CATEGORY_VENDOR, + ) + yield str(detail).encode( + "utf-8" + ), TTS2HttpResponseEventType.ERROR + self._ready = False + return + if mtype != "processing-request": + # Some paths may skip straight to started-byte-stream + if (msg.get("status") or msg.get("type")) != ( + "started-byte-stream" + ): + self.ten_env.log_warn( + f"BlazeTTS unexpected after query: {msg}", + category=LOG_CATEGORY_VENDOR, + ) + + # started-byte-stream (if not already) + if (msg.get("status") or msg.get("type")) != ( + "started-byte-stream" + ): + msg = await self._recv_json(timeout=60) + st = msg.get("status") or msg.get("type") + if st in ( + "failed-request", + "bad-request", + "internal-error", + ): + detail = ( + msg.get("details") or msg.get("message") or str(msg) + ) + self.ten_env.log_error( + f"vendor_error: {st} {detail} " + f"request_id={request_id}", + category=LOG_CATEGORY_VENDOR, + ) + yield str(detail).encode( + "utf-8" + ), TTS2HttpResponseEventType.ERROR + self._ready = False + return + if st != "started-byte-stream": + self.ten_env.log_warn( + f"BlazeTTS expected started-byte-stream, got {msg}", + category=LOG_CATEGORY_VENDOR, + ) + + # binary chunks until finished-byte-stream + while True: + if self._is_cancelled: + yield None, TTS2HttpResponseEventType.FLUSH + return + + raw = await asyncio.wait_for(self._ws.recv(), timeout=90) + + if isinstance(raw, (bytes, bytearray)): + if self._is_cancelled: + yield None, TTS2HttpResponseEventType.FLUSH + return + if len(raw) > 0: + yield ( + bytes(raw), + TTS2HttpResponseEventType.RESPONSE, + ) + continue + + data = json.loads(raw) + event = data.get("type") or data.get("status") + if event == "finished-byte-stream": + yield None, TTS2HttpResponseEventType.END + return + if event in ( + "failed-request", + "bad-request", + "internal-error", + ): + detail = ( + data.get("details") + or data.get("message") + or str(data) + ) + self.ten_env.log_error( + f"vendor_error: {event} {detail} " + f"request_id={request_id}", + category=LOG_CATEGORY_VENDOR, + ) + yield str(detail).encode( + "utf-8" + ), TTS2HttpResponseEventType.ERROR + self._ready = False + return + # ignore completed-request / speech-* if any + if event == "completed-request": + yield None, TTS2HttpResponseEventType.END + return + + except ConnectionClosed as exc: + self._ready = False + self._ws = None + msg = f"WS closed: {exc}" + self.ten_env.log_error( + f"vendor_error: {msg} request_id={request_id}", + category=LOG_CATEGORY_VENDOR, + ) + yield msg.encode("utf-8"), TTS2HttpResponseEventType.ERROR + except Exception as exc: # pylint: disable=broad-exception-caught + self._ready = False + msg = str(exc) + self.ten_env.log_error( + f"vendor_error: {msg} request_id={request_id}", + category=LOG_CATEGORY_VENDOR, + ) + if "401" in msg or "403" in msg or "Authentication" in msg: + yield msg.encode( + "utf-8" + ), TTS2HttpResponseEventType.INVALID_KEY_ERROR + else: + yield msg.encode("utf-8"), TTS2HttpResponseEventType.ERROR + + async def clean(self): + self.ten_env.log_debug("BlazeTTS: clean() called.") + self._ready = False + if self._ws is not None: + try: + await self._ws.close() + except Exception: # pylint: disable=broad-exception-caught + pass + self._ws = None + + def get_extra_metadata(self) -> dict[str, Any]: + return { + "speaker_id": self.config.params.get("speaker_id", ""), + "model": self.config.params.get("model", ""), + "language": self.config.params.get("language", ""), + "transport": "websocket-realtime", + } diff --git a/ai_agents/scripts/smoke_blaze.py b/ai_agents/scripts/smoke_blaze.py new file mode 100755 index 0000000000..7609a522b7 --- /dev/null +++ b/ai_agents/scripts/smoke_blaze.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +""" +Smoke-test Blaze STT and TTS extensions against a live Blaze API. + +Loads credentials from (first match wins): + 1) process environment + 2) ai_agents/.env + 3) repo-root .env + +Usage (from repo root or ai_agents/): + + python ai_agents/scripts/smoke_blaze.py + python ai_agents/scripts/smoke_blaze.py --audio /path/to/sample.wav + python ai_agents/scripts/smoke_blaze.py --text "Xin chào" --skip-stt + +Exit code 0 if selected checks pass. +""" + +from __future__ import annotations + +import argparse +import io +import math +import os +import struct +import sys +import time +import wave +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +AI_AGENTS = REPO_ROOT / "ai_agents" +EXT_ROOT = AI_AGENTS / "agents" / "ten_packages" / "extension" + + +def _load_dotenv(path: Path) -> None: + if not path.is_file(): + return + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip().strip('"').strip("'") + os.environ.setdefault(key, value) + + +def load_env() -> None: + _load_dotenv(AI_AGENTS / ".env") + _load_dotenv(REPO_ROOT / ".env") + # Map unified key if vendor-specific missing + key = os.environ.get("BLAZE_API_KEY", "") + if key: + os.environ.setdefault("BLAZE_STT_API_KEY", key) + os.environ.setdefault("BLAZE_TTS_API_KEY", key) + os.environ.setdefault("BLAZE_STT_API_URL", "https://api.blaze.vn") + os.environ.setdefault("BLAZE_TTS_API_URL", "https://api.blaze.vn") + + +def make_tone_wav(seconds: float = 0.5, sr: int = 16000, freq: float = 440.0) -> bytes: + n = int(sr * seconds) + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(sr) + frames = bytearray() + for i in range(n): + val = int(8000 * math.sin(2 * math.pi * freq * i / sr)) + frames += struct.pack(" str: + if not key: + return "" + if len(key) <= 8: + return "***" + return f"{key[:4]}...{key[-4:]} (len={len(key)})" + + +def smoke_tts(api_url: str, api_key: str, text: str, speaker_id: str, out_path: Path) -> bool: + sys.path.insert(0, str(EXT_ROOT / "blaze_tts_python")) + from blaze_tts import BlazeTTSExtension # type: ignore # pylint: disable=import-error + + print(f"[TTS] api_url={api_url}") + print(f"[TTS] api_key={redacted(api_key)}") + tts = BlazeTTSExtension( + config={ + "api_url": api_url, + "api_key": api_key, + "language": "vi", + "speaker_id": speaker_id, + } + ) + + speakers = tts.get_speakers() + lst = speakers.get("list_speakers") or [] + print(f"[TTS] speakers: {len(lst)}") + if not speaker_id and lst: + speaker_id = lst[0]["id"] + print(f"[TTS] using first speaker: {speaker_id}") + + result = tts.synthesize( + text=text, + speaker_id=speaker_id, + language="vi", + audio_format="wav", + media_type="audio/ogg; codecs=opus", + ) + job_id = result.get("id") or result.get("job_id") + print(f"[TTS] job_id={job_id} raw={result}") + if not job_id: + print("[TTS] FAIL: no job id") + return False + + # Poll download (Blaze often returns 425 while processing) + audio = None + for attempt in range(1, 61): + try: + audio = tts.download_audio(job_id) + print(f"[TTS] download attempt {attempt}: OK {len(audio)} bytes") + break + except Exception as exc: # pylint: disable=broad-exception-caught + status = getattr(getattr(exc, "response", None), "status_code", None) + print(f"[TTS] download attempt {attempt}: {status or type(exc).__name__}") + if status not in (None, 425): + print(f"[TTS] FAIL: {exc}") + return False + time.sleep(2) + + if not audio: + print("[TTS] FAIL: download timeout") + return False + + out_path.write_bytes(audio) + print(f"[TTS] saved {out_path} ({out_path.stat().st_size} bytes)") + + # Basic WAV sanity + try: + with wave.open(io.BytesIO(audio), "rb") as w: + print( + f"[TTS] wav channels={w.getnchannels()} " + f"width={w.getsampwidth()} rate={w.getframerate()} " + f"frames={w.getnframes()}" + ) + except wave.Error as exc: + print(f"[TTS] WARN: not a WAV file ({exc}); still counted as pass if bytes>0") + + print("[TTS] PASS") + return True + + +def smoke_stt(api_url: str, api_key: str, audio_bytes: bytes) -> bool: + sys.path.insert(0, str(EXT_ROOT / "blaze_stt_python")) + from blaze_stt import BlazeSTTExtension # type: ignore # pylint: disable=import-error + + print(f"[STT] api_url={api_url}") + print(f"[STT] api_key={redacted(api_key)}") + print(f"[STT] audio_bytes={len(audio_bytes)}") + + stt = BlazeSTTExtension( + config={ + "api_url": api_url, + "api_key": api_key, + "language": "vi", + } + ) + result = stt.process( + { + "audio_data": audio_bytes, + "audio_content_type": "audio/wav", + "language": "vi", + } + ) + text = result.get("transcription") or "" + print(f"[STT] status={result.get('status')} job_id={result.get('job_id')}") + print(f"[STT] transcription[:240]={text[:240]!r}") + if not text and result.get("status") not in ("completed", "processing"): + print("[STT] FAIL") + return False + # Empty text can be valid for tone-only audio; still require no exception + print("[STT] PASS") + return True + + +def main() -> int: + parser = argparse.ArgumentParser(description="Smoke test Blaze STT/TTS") + parser.add_argument( + "--audio", + type=Path, + default=None, + help="WAV file for STT (default: /home/trung/test/2.wav or tone)", + ) + parser.add_argument( + "--text", + default="Xin chào, đây là bài kiểm tra Blaze extension.", + help="Text for TTS", + ) + parser.add_argument( + "--speaker-id", + default=os.environ.get("BLAZE_TTS_SPEAKER_ID", "HN-Nam-2-BL"), + help="Blaze speaker_id for TTS", + ) + parser.add_argument( + "--out", + type=Path, + default=Path("/tmp/blaze_smoke_tts.wav"), + help="Where to write downloaded TTS audio", + ) + parser.add_argument("--skip-stt", action="store_true") + parser.add_argument("--skip-tts", action="store_true") + args = parser.parse_args() + + load_env() + stt_key = os.environ.get("BLAZE_STT_API_KEY") or os.environ.get("BLAZE_API_KEY", "") + tts_key = os.environ.get("BLAZE_TTS_API_KEY") or os.environ.get("BLAZE_API_KEY", "") + stt_url = os.environ.get("BLAZE_STT_API_URL", "https://api.blaze.vn") + tts_url = os.environ.get("BLAZE_TTS_API_URL", "https://api.blaze.vn") + + if not args.skip_stt and not stt_key: + print("ERROR: BLAZE_STT_API_KEY / BLAZE_API_KEY not set") + return 2 + if not args.skip_tts and not tts_key: + print("ERROR: BLAZE_TTS_API_KEY / BLAZE_API_KEY not set") + return 2 + + ok = True + if not args.skip_tts: + try: + ok = smoke_tts(tts_url, tts_key, args.text, args.speaker_id, args.out) and ok + except Exception as exc: # pylint: disable=broad-exception-caught + print(f"[TTS] FAIL: {exc}") + ok = False + + if not args.skip_stt: + if args.audio and args.audio.is_file(): + audio = args.audio.read_bytes() + print(f"[STT] using --audio {args.audio}") + else: + default = Path("/home/trung/test/2.wav") + if default.is_file(): + audio = default.read_bytes() + print(f"[STT] using {default}") + else: + audio = make_tone_wav() + print("[STT] using synthetic tone WAV (no sample file found)") + try: + ok = smoke_stt(stt_url, stt_key, audio) and ok + except Exception as exc: # pylint: disable=broad-exception-caught + print(f"[STT] FAIL: {exc}") + ok = False + + print("\n" + ("ALL CHECKS PASSED" if ok else "SOME CHECKS FAILED")) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main())