From 8db6375a6213b8b7db2260db10017e1e388ce446 Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Tue, 30 Sep 2025 17:09:32 +0300 Subject: [PATCH 01/19] feat(chat): add speech to text --- package-lock.json | 157 ++++++++++++++++- package.json | 1 + src/components/chat/chat-input.ts | 121 ++++++++++++- src/components/chat/chat.ts | 4 + src/components/chat/extras/stt-client.ts | 191 +++++++++++++++++++++ src/components/chat/themes/input.base.scss | 45 +++++ src/components/chat/types.ts | 15 ++ stories/chat.stories.ts | 5 + 8 files changed, 535 insertions(+), 4 deletions(-) create mode 100644 src/components/chat/extras/stt-client.ts diff --git a/package-lock.json b/package-lock.json index 98ef2aac27..b7b52d38fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@floating-ui/dom": "^1.7.4", "@lit-labs/virtualizer": "^2.1.1", "@lit/context": "^1.1.6", + "@microsoft/signalr": "^9.0.6", "lit": "^3.3.1" }, "devDependencies": { @@ -1364,6 +1365,19 @@ "react": ">=16" } }, + "node_modules/@microsoft/signalr": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@microsoft/signalr/-/signalr-9.0.6.tgz", + "integrity": "sha512-DrhgzFWI9JE4RPTsHYRxh4yr+OhnwKz8bnJe7eIi7mLLjqhJpEb62CiUy/YbFvLqLzcGzlzz1QWgVAW0zyipMQ==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "eventsource": "^2.0.2", + "fetch-cookie": "^2.0.3", + "node-fetch": "^2.6.7", + "ws": "^7.5.10" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -4014,6 +4028,18 @@ "node": ">=14" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -6472,6 +6498,15 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/eventemitter3": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", @@ -6479,6 +6514,15 @@ "dev": true, "license": "MIT" }, + "node_modules/eventsource": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-2.0.2.tgz", + "integrity": "sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==", + "license": "MIT", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -6650,6 +6694,16 @@ "pend": "~1.2.0" } }, + "node_modules/fetch-cookie": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/fetch-cookie/-/fetch-cookie-2.2.0.tgz", + "integrity": "sha512-h9AgfjURuCgA2+2ISl8GbavpUdR+WGAM2McW/ovn4tVccegp8ZqCKWSBR8uRdM8dDNlx5WdKRWxBYUwteLDCNQ==", + "license": "Unlicense", + "dependencies": { + "set-cookie-parser": "^2.4.8", + "tough-cookie": "^4.0.0" + } + }, "node_modules/file-entry-cache": { "version": "10.1.4", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-10.1.4.tgz", @@ -9414,6 +9468,48 @@ "license": "MIT", "optional": true }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/node-releases": { "version": "2.0.21", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.21.tgz", @@ -10363,6 +10459,18 @@ "dev": true, "license": "MIT" }, + "node_modules/psl": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz", + "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "funding": { + "url": "https://github.com/sponsors/lupomontero" + } + }, "node_modules/pump": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", @@ -10378,7 +10486,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -10476,6 +10583,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "license": "MIT" + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -10763,7 +10876,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", - "dev": true, "license": "MIT" }, "node_modules/resolve": { @@ -11782,6 +11894,12 @@ "dev": true, "license": "ISC" }, + "node_modules/set-cookie-parser": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.1.tgz", + "integrity": "sha512-IOc8uWeOZgnb3ptbCURJWNjWUPcO3ZnTTdzsurqERrP6nPyv+paC55vJM0LpOlT2ne+Ix+9+CRG1MNLlyZ4GjQ==", + "license": "MIT" + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -13216,6 +13334,30 @@ "node": ">=0.6" } }, + "node_modules/tough-cookie": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz", + "integrity": "sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==", + "license": "BSD-3-Clause", + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tough-cookie/node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "license": "MIT", + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/tr46": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", @@ -13701,6 +13843,16 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "license": "MIT", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -14222,7 +14374,6 @@ "version": "7.5.10", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.3.0" diff --git a/package.json b/package.json index dd42fd640e..1beba75c23 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "@floating-ui/dom": "^1.7.4", "@lit-labs/virtualizer": "^2.1.1", "@lit/context": "^1.1.6", + "@microsoft/signalr": "^9.0.6", "lit": "^3.3.1" }, "devDependencies": { diff --git a/src/components/chat/chat-input.ts b/src/components/chat/chat-input.ts index 27d457e0ff..d9c6f03593 100644 --- a/src/components/chat/chat-input.ts +++ b/src/components/chat/chat-input.ts @@ -1,6 +1,6 @@ import { consume } from '@lit/context'; import { html, LitElement, nothing } from 'lit'; -import { query, state } from 'lit/decorators.js'; +import { property, query, state } from 'lit/decorators.js'; import { cache } from 'lit/directives/cache.js'; import { ifDefined } from 'lit/directives/if-defined.js'; import { until } from 'lit/directives/until.js'; @@ -15,6 +15,7 @@ import { bindIf, hasFiles, isEmpty, trimmedHtml } from '../common/util.js'; import IgcIconComponent from '../icon/icon.js'; import IgcTextareaComponent from '../textarea/textarea.js'; import type { ChatState } from './chat-state.js'; +import { SttClient } from './extras/stt-client.js'; import { styles } from './themes/input.base.css.js'; import { all } from './themes/input.js'; import { styles as shared } from './themes/shared/input/input.common.css.js'; @@ -33,6 +34,7 @@ type DefaultInputRenderers = { inputActionsStart: ChatTemplateRenderer; inputAttachments: ChatTemplateRenderer; fileUploadButton: ChatTemplateRenderer; + speechToTextButton: ChatTemplateRenderer; sendButton: ChatTemplateRenderer; }; @@ -83,6 +85,7 @@ export default class IgcChatInputComponent extends LitElement { private readonly _defaults: Readonly = Object.freeze({ fileUploadButton: () => this._renderFileUploadButton(), + speechToTextButton: () => this._renderSpeechToTextButton(), input: () => this._renderTextArea(), inputActions: () => this._renderActionsArea(), inputActionsEnd: () => nothing, @@ -110,6 +113,85 @@ export default class IgcChatInputComponent extends LitElement { return this._state.acceptedFileTypes; } + private _sttClient?: SttClient; + + @property() + isRecording = false; + + @property() + isStopInProgress = false; + + onPulseSignal = () => { + const el = this.renderRoot.querySelector( + 'label[part="speech-to-text"]' + )?.firstElementChild; + if (!el) return; + + el.classList.add('pulsate'); + el.addEventListener('animationend', () => el.classList.remove('pulsate'), { + once: true, + }); + }; + + onStartCountdown = (ms: number | null) => { + if (ms) { + const circle = this.renderRoot.querySelector( + 'svg.countdown-ring .ring-progress' + ); + if (!circle) return; + + circle.style.transition = 'none'; + circle.style.strokeDashoffset = '100'; + void circle.getBoundingClientRect(); // reflow + + circle.style.transition = `stroke-dashoffset ${ms}ms linear`; + circle.style.strokeDashoffset = '0'; + } else { + const circle = this.renderRoot.querySelector( + 'svg.countdown-ring .ring-progress' + ); + if (circle) { + circle.style.transition = 'none'; + circle.style.strokeDashoffset = '100'; + } + } + }; + + onStopInProgress = () => { + this.isStopInProgress = true; + }; + + onTranscript = (text: string) => { + this._state.inputValue = text; + }; + + onFinishedTranscribing = (finished: string) => { + this.isStopInProgress = false; + this.isRecording = false; + if (finished === 'auto') { + this._sendMessage(); + } + }; + + async _toggleMic() { + if (!this.isRecording) { + this._sttClient = new SttClient( + this._state.options?.sttOptions?.serviceUri!, + 'this._state.host.sttToken', + this.onPulseSignal, + this.onStartCountdown, + this.onTranscript, + this.onStopInProgress, + this.onFinishedTranscribing + ); + await this._sttClient.start(this._state.options?.sttOptions?.lang); + this.isRecording = true; + } else { + this.isStopInProgress = true; + this._sttClient?.stop(); + } + } + constructor() { super(); addThemingController(this, all); @@ -315,6 +397,40 @@ export default class IgcChatInputComponent extends LitElement { )}`; } + private _renderSpeechToTextButton() { + const sttEnabled = this._state.options?.sttOptions?.enable; + + return html`${cache( + sttEnabled + ? html`
+ + ${this.isRecording && !this.isStopInProgress + ? html` + + + + + ` + : nothing} +
` + : nothing + )}`; + } + private _renderSendButton() { const enabled = this._state.hasInputValue || this._state.hasInputAttachments; @@ -338,6 +454,9 @@ export default class IgcChatInputComponent extends LitElement {
${this._getRenderer('fileUploadButton')(ctx)}
+
+ ${this._getRenderer('speechToTextButton')(ctx)} +
${this._getRenderer('inputActionsStart')(ctx)}
diff --git a/src/components/chat/chat.ts b/src/components/chat/chat.ts index c298da8729..9640192d10 100644 --- a/src/components/chat/chat.ts +++ b/src/components/chat/chat.ts @@ -165,6 +165,8 @@ const Slots = setSlots( * @csspart input-actions-end - Styles the group of actions at the end of the input. * @csspart file-upload-container - Styles the container for the file upload input. * @csspart file-upload - Styles the file upload input itself. + * @csspart speech-to-text-container - Styles the container for the speech to text buttons. + * @csspart speech-to-text - Styles the speech to text buttons. * @csspart send-button-container - Styles the container around the send button. * @csspart send-button - Styles the send button. * @@ -516,6 +518,8 @@ export default class IgcChatComponent extends EventEmitterMixin< input-actions-end, file-upload-container, file-upload, + speech-to-text-container, + speech-to-text, send-button-container, send-button" > diff --git a/src/components/chat/extras/stt-client.ts b/src/components/chat/extras/stt-client.ts new file mode 100644 index 0000000000..b00b682832 --- /dev/null +++ b/src/components/chat/extras/stt-client.ts @@ -0,0 +1,191 @@ +import * as signalR from '@microsoft/signalr'; + +const HUB_TRANSCRIBE_AUDIO_CHUNK = 'TranscribeAudioChunk'; +const HUB_RECEIVE_TRANSCRIPT = 'ReceiveTranscript'; +const HUB_COMPLETE_TRANSCRIBE = 'FinalizeTranscription'; +const SILENCE_TIMEOUT_MS = 3000; +const SILENCE_GRACE_PERIOD = 1000; + +export class SttClient { + private hubConnection?: signalR.HubConnection; + private mediaRecorder?: MediaRecorder; + private isRecording = false; + private isStopInProgress = false; + private isStopCompleted = false; + private stopHubTimeout: any; + private silenceTimeout: any; + private silenceGraceTimeout: any; + private isAutoFinished = false; + private isCountdownRunning = false; + + constructor( + private hubUrl: string, + private token: string, + private onPulseSignal: () => void, + private onStartCountdown: (ms: number | null) => void, + private onTranscript: (text: string) => void, + private onStopInProgress: () => void, + private onFinishedTranscribing: (finish: string) => void + ) {} + + async start(language = 'en-US') { + if (this.isRecording) { + return; + } + + this.onTranscript(''); + const transcript = { + confirmed: '', + tentative: '', + }; + + this.hubConnection = this.createHubConnection(); + + this.hubConnection.on( + HUB_RECEIVE_TRANSCRIPT, + (data: { + transcription: string; + isFinal: boolean; + isCompleted: boolean; + }) => { + this.handleTranscriptEvent(transcript, data); + } + ); + + await this.hubConnection.start(); + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + sampleRate: 48000, + noiseSuppression: true, + echoCancellation: true, + autoGainControl: true, + }, + }); + this.mediaRecorder = new MediaRecorder(stream, { + mimeType: 'audio/webm;codecs=opus', + }); + + this.mediaRecorder.ondataavailable = async (event) => { + this.sendForTranscription(event.data, language); + if (this.isStopInProgress) { + await this.hubConnection?.invoke(HUB_COMPLETE_TRANSCRIBE); + } + }; + + this.mediaRecorder.start(100); + this.isRecording = true; + this.isStopInProgress = false; + this.isStopCompleted = false; + this.resetSilenceTimer(); + this.restartGracePeriod(); + } + + stop() { + this.isStopInProgress = true; + this.onStopInProgress(); + this.mediaRecorder?.stop(); + this.clearSilenceTimer(); + if (this.stopHubTimeout) { + clearTimeout(this.stopHubTimeout); + } + this.stopHubTimeout = setTimeout(() => { + this.stopHubConnection(); + this.stopHubTimeout = null; + }, 1500); + } + + private async sendForTranscription(data: Blob, language: string) { + if ( + data.size > 0 && + this.hubConnection?.state === signalR.HubConnectionState.Connected + ) { + const buffer = await data.arrayBuffer(); + const base64Audio = btoa(String.fromCharCode(...new Uint8Array(buffer))); + await this.hubConnection.invoke( + HUB_TRANSCRIBE_AUDIO_CHUNK, + base64Audio, + language + ); + } + } + + private handleTranscriptEvent( + currentTranscript: any, + newTranscriptData: any + ) { + currentTranscript.tentative = newTranscriptData.transcription; + this.onTranscript( + currentTranscript.confirmed + currentTranscript.tentative + ); + this.resetSilenceTimer(); + if (newTranscriptData.isFinal) { + currentTranscript.confirmed += currentTranscript.tentative; + } + + this.onPulseSignal(); + + if (this.isCountdownRunning) { + this.onStartCountdown(null); + this.isCountdownRunning = false; + } + + this.restartGracePeriod(); + + if (newTranscriptData.isCompleted) { + this.stopHubConnection(); + } + } + + private createHubConnection() { + const hubConnection = new signalR.HubConnectionBuilder() + .withUrl(this.hubUrl, { accessTokenFactory: () => this.token }) + .configureLogging(signalR.LogLevel.Information) + .build(); + + return hubConnection; + } + + private stopHubConnection() { + if (this.isStopInProgress && !this.isStopCompleted) { + this.isRecording = false; + this.hubConnection?.off(HUB_RECEIVE_TRANSCRIPT); + this.hubConnection?.stop(); + this.isStopCompleted = true; + this.isStopInProgress = false; + if (this.stopHubTimeout) { + clearTimeout(this.stopHubTimeout); + } + + this.onFinishedTranscribing(this.isAutoFinished ? 'auto' : 'manual'); + this.isAutoFinished = false; + } + } + + private restartGracePeriod() { + if (this.silenceGraceTimeout) { + clearTimeout(this.silenceGraceTimeout); + } + + this.silenceGraceTimeout = setTimeout(() => { + this.isCountdownRunning = true; + this.onStartCountdown(SILENCE_TIMEOUT_MS - SILENCE_GRACE_PERIOD); + }, SILENCE_GRACE_PERIOD); + } + + private resetSilenceTimer() { + this.clearSilenceTimer(); + this.silenceTimeout = setTimeout(() => { + this.isAutoFinished = true; + this.stop(); + }, SILENCE_TIMEOUT_MS); + } + + private clearSilenceTimer() { + if (this.silenceTimeout) { + clearTimeout(this.silenceTimeout); + this.silenceTimeout = null; + } + } +} diff --git a/src/components/chat/themes/input.base.scss b/src/components/chat/themes/input.base.scss index 61630e6307..4318823b01 100644 --- a/src/components/chat/themes/input.base.scss +++ b/src/components/chat/themes/input.base.scss @@ -74,3 +74,48 @@ igc-textarea { margin-inline: rem(12px); } } + +@keyframes pulse { + 0% { transform: scale(1); } + 50% { transform: scale(1.3); } + 100% { transform: scale(1); } +} + +.pulsate { + animation: pulse 0.2s ease-in-out; +} + +.stop-btn-wrapper { + position: relative; + display: inline-block; +} + +label[part="speech-to-text"] igc-icon-button[disabled] { + opacity: 0.5; + cursor: not-allowed; +} + +.countdown-ring { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + transform: rotate(-90deg); /* start progress from top */ + pointer-events: none; +} + +.ring-bg { + fill: none; + stroke: transparent; + //stroke-width: 3; +} + +.ring-progress { + fill: none; + stroke: rgba(0,0,0,0.5); /* match your disabled color */ + stroke-width: 3; + stroke-dasharray: 100; + stroke-dashoffset: 100; + transition: stroke-dashoffset linear; +} diff --git a/src/components/chat/types.ts b/src/components/chat/types.ts index e4921d2550..9bc709c9af 100644 --- a/src/components/chat/types.ts +++ b/src/components/chat/types.ts @@ -155,8 +155,19 @@ export type IgcChatOptions = { * An object containing a collection of custom renderers for different parts of the chat UI. */ renderers?: ChatRenderers; + + /** + * Configuration options for enabling and customizing speech-to-text functionality. + * If provided, it enables a button in the chat input area that allows users to dictate messages using their voice. + */ + sttOptions?: SpeechToTextOptions; }; +export interface SpeechToTextOptions { + enable: boolean; + lang?: string; + serviceUri?: string; +} /** * Represents a user's reaction to a specific chat message. */ @@ -192,6 +203,10 @@ export interface ChatRenderers { * Custom renderer for the file upload button in the input area. */ fileUploadButton?: ChatTemplateRenderer; + /** + * Custom renderer for the speech to text button in the input area. + */ + speechToTextButton?: ChatTemplateRenderer; /** * Custom renderer for the main chat input field. */ diff --git a/stories/chat.stories.ts b/stories/chat.stories.ts index 8a6ecf4c95..aaed5d923d 100644 --- a/stories/chat.stories.ts +++ b/stories/chat.stories.ts @@ -144,6 +144,11 @@ const chat_options: IgcChatOptions = { suggestions: ['Hello', 'Hi', 'How are you?'], inputPlaceholder: 'Type your message here...', headerText: 'Chat', + sttOptions: { + enable: true, + lang: 'en-US', + serviceUri: 'https://localhost:5000/sttHub', + }, renderers: { messageContent: async ({ message }) => _markdownRenderer(message), }, From 10fe0f2385c0c79f18c99882011c67396481f5da Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Tue, 30 Sep 2025 17:46:44 +0300 Subject: [PATCH 02/19] feat(chat): add stt placeholder --- src/components/chat/chat-input.ts | 5 ++++- src/components/chat/extras/stt-client.ts | 2 +- src/components/chat/types.ts | 5 +++++ stories/chat.stories.ts | 3 ++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/components/chat/chat-input.ts b/src/components/chat/chat-input.ts index d9c6f03593..d7bd5850e7 100644 --- a/src/components/chat/chat-input.ts +++ b/src/components/chat/chat-input.ts @@ -186,6 +186,7 @@ export default class IgcChatInputComponent extends LitElement { ); await this._sttClient.start(this._state.options?.sttOptions?.lang); this.isRecording = true; + this.isStopInProgress = false; } else { this.isStopInProgress = true; this._sttClient?.stop(); @@ -355,7 +356,9 @@ export default class IgcChatInputComponent extends LitElement { html` - ๐ŸŽค + ๐Ÿ“ `; const _actionsEndTemplate = (ctx: ChatRenderContext) => html`
From 66bbc30217aa7317fb91ee414a69a64639716435 Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Mon, 13 Oct 2025 15:16:31 +0300 Subject: [PATCH 03/19] feat(chat): add webspeech speech-to-text implementation --- src/components/chat/chat-input.ts | 48 +++++++---- .../{stt-client.ts => stt-client-backend.ts} | 81 ++++++++----------- src/components/chat/extras/stt-client-base.ts | 79 ++++++++++++++++++ .../chat/extras/stt-client-webspeech.ts | 62 ++++++++++++++ src/components/chat/types.ts | 3 +- stories/chat.stories.ts | 3 +- 6 files changed, 209 insertions(+), 67 deletions(-) rename src/components/chat/extras/{stt-client.ts => stt-client-backend.ts} (71%) create mode 100644 src/components/chat/extras/stt-client-base.ts create mode 100644 src/components/chat/extras/stt-client-webspeech.ts diff --git a/src/components/chat/chat-input.ts b/src/components/chat/chat-input.ts index d7bd5850e7..055739b92f 100644 --- a/src/components/chat/chat-input.ts +++ b/src/components/chat/chat-input.ts @@ -15,7 +15,9 @@ import { bindIf, hasFiles, isEmpty, trimmedHtml } from '../common/util.js'; import IgcIconComponent from '../icon/icon.js'; import IgcTextareaComponent from '../textarea/textarea.js'; import type { ChatState } from './chat-state.js'; -import { SttClient } from './extras/stt-client.js'; +import { BackendSttClient } from './extras/stt-client-backend.js'; +import type { ISttClient } from './extras/stt-client-base.js'; +import { WebSpeechSttClient } from './extras/stt-client-webspeech.js'; import { styles } from './themes/input.base.css.js'; import { all } from './themes/input.js'; import { styles as shared } from './themes/shared/input/input.common.css.js'; @@ -113,7 +115,7 @@ export default class IgcChatInputComponent extends LitElement { return this._state.acceptedFileTypes; } - private _sttClient?: SttClient; + private _sttClient?: ISttClient; @property() isRecording = false; @@ -175,20 +177,34 @@ export default class IgcChatInputComponent extends LitElement { async _toggleMic() { if (!this.isRecording) { - this._sttClient = new SttClient( - this._state.options?.sttOptions?.serviceUri!, - 'this._state.host.sttToken', - this.onPulseSignal, - this.onStartCountdown, - this.onTranscript, - this.onStopInProgress, - this.onFinishedTranscribing - ); - await this._sttClient.start(this._state.options?.sttOptions?.lang); + if (this._state.options?.speechToText?.serviceProvider === 'webspeech') { + this._sttClient = new WebSpeechSttClient( + this.onPulseSignal, + this.onStartCountdown, + this.onTranscript, + this.onStopInProgress, + this.onFinishedTranscribing + ); + } else if ( + this._state.options?.speechToText?.serviceProvider === 'backend' && + this._state.options?.speechToText?.serviceUri + ) { + this._sttClient = new BackendSttClient( + this._state.options?.speechToText?.serviceUri!, + this.onPulseSignal, + this.onStartCountdown, + this.onTranscript, + this.onStopInProgress, + this.onFinishedTranscribing + ); + } else { + // console.error('No STT service configured'); + } + + await this._sttClient.start(this._state.options?.speechToText?.lang); this.isRecording = true; this.isStopInProgress = false; } else { - this.isStopInProgress = true; this._sttClient?.stop(); } } @@ -401,7 +417,7 @@ export default class IgcChatInputComponent extends LitElement { } private _renderSpeechToTextButton() { - const sttEnabled = this._state.options?.sttOptions?.enable; + const sttEnabled = this._state.options?.speechToText?.enable; return html`${cache( sttEnabled @@ -419,12 +435,12 @@ export default class IgcChatInputComponent extends LitElement { ${this.isRecording && !this.isStopInProgress ? html` - + ` diff --git a/src/components/chat/extras/stt-client.ts b/src/components/chat/extras/stt-client-backend.ts similarity index 71% rename from src/components/chat/extras/stt-client.ts rename to src/components/chat/extras/stt-client-backend.ts index 81adb0f45b..ca8274e908 100644 --- a/src/components/chat/extras/stt-client.ts +++ b/src/components/chat/extras/stt-client-backend.ts @@ -1,32 +1,36 @@ import * as signalR from '@microsoft/signalr'; +import { BaseSttClient } from './stt-client-base.js'; const HUB_TRANSCRIBE_AUDIO_CHUNK = 'TranscribeAudioChunk'; const HUB_RECEIVE_TRANSCRIPT = 'ReceiveTranscript'; const HUB_COMPLETE_TRANSCRIBE = 'FinalizeTranscription'; -const SILENCE_TIMEOUT_MS = 4000; -const SILENCE_GRACE_PERIOD = 1000; -export class SttClient { +export class BackendSttClient extends BaseSttClient { private hubConnection?: signalR.HubConnection; private mediaRecorder?: MediaRecorder; - private isRecording = false; private isStopInProgress = false; private isStopCompleted = false; private stopHubTimeout: any; - private silenceTimeout: any; - private silenceGraceTimeout: any; - private isAutoFinished = false; - private isCountdownRunning = false; constructor( - private hubUrl: string, - private token: string, - private onPulseSignal: () => void, - private onStartCountdown: (ms: number | null) => void, - private onTranscript: (text: string) => void, - private onStopInProgress: () => void, - private onFinishedTranscribing: (finish: string) => void - ) {} + hubUrl: string, + onPulseSignal: () => void, + onStartCountdown: (ms: number | null) => void, + onTranscript: (text: string) => void, + onStopInProgress: () => void, + onFinishedTranscribing: (finish: 'auto' | 'manual') => void + ) { + super( + onPulseSignal, + onStartCountdown, + onTranscript, + onStopInProgress, + onFinishedTranscribing + ); + this.hubUrl = hubUrl; + } + + private hubUrl: string; async start(language = 'en-US') { if (this.isRecording) { @@ -103,11 +107,16 @@ export class SttClient { ) { const buffer = await data.arrayBuffer(); const base64Audio = btoa(String.fromCharCode(...new Uint8Array(buffer))); - await this.hubConnection.invoke( - HUB_TRANSCRIBE_AUDIO_CHUNK, - base64Audio, - language - ); + try { + await this.hubConnection.invoke( + HUB_TRANSCRIBE_AUDIO_CHUNK, + base64Audio, + language + ); + } catch { + //report.error("STT invoke failed:", err); TOTO + this.stop(); + } } } @@ -140,8 +149,8 @@ export class SttClient { private createHubConnection() { const hubConnection = new signalR.HubConnectionBuilder() - .withUrl(this.hubUrl, { accessTokenFactory: () => this.token }) - .configureLogging(signalR.LogLevel.Information) + .withUrl(this.hubUrl) + .configureLogging(signalR.LogLevel.Warning) .build(); return hubConnection; @@ -162,30 +171,4 @@ export class SttClient { this.isAutoFinished = false; } } - - private restartGracePeriod() { - if (this.silenceGraceTimeout) { - clearTimeout(this.silenceGraceTimeout); - } - - this.silenceGraceTimeout = setTimeout(() => { - this.isCountdownRunning = true; - this.onStartCountdown(SILENCE_TIMEOUT_MS - SILENCE_GRACE_PERIOD); - }, SILENCE_GRACE_PERIOD); - } - - private resetSilenceTimer() { - this.clearSilenceTimer(); - this.silenceTimeout = setTimeout(() => { - this.isAutoFinished = true; - this.stop(); - }, SILENCE_TIMEOUT_MS); - } - - private clearSilenceTimer() { - if (this.silenceTimeout) { - clearTimeout(this.silenceTimeout); - this.silenceTimeout = null; - } - } } diff --git a/src/components/chat/extras/stt-client-base.ts b/src/components/chat/extras/stt-client-base.ts new file mode 100644 index 0000000000..9baa90e9f8 --- /dev/null +++ b/src/components/chat/extras/stt-client-base.ts @@ -0,0 +1,79 @@ +export interface ISttClient { + /** Start recording and transcribing */ + start(language?: string): Promise; + + /** Stop recording/transcribing */ + stop(): void; + + /** Whether recording is currently active */ + readonly isRecording: boolean; + + /** Signal emitted when audio activity is detected */ + readonly onPulseSignal: () => void; + + /** Called when the countdown should start or stop (pass null to stop) */ + readonly onStartCountdown: (ms: number | null) => void; + + /** Called when a partial or final transcript is produced */ + readonly onTranscript: (text: string) => void; + + /** Called when stop is in progress (manual or automatic) */ + readonly onStopInProgress: () => void; + + /** Called when transcription fully finishes */ + readonly onFinishedTranscribing: (finish: 'auto' | 'manual') => void; +} + +export abstract class BaseSttClient implements ISttClient { + protected static readonly SILENCE_TIMEOUT_MS = 4000; + protected static readonly SILENCE_GRACE_PERIOD = 1000; + + protected silenceTimeout: any = null; + protected silenceGraceTimeout: any = null; + protected isCountdownRunning = false; + protected isAutoFinished = false; + + isRecording = false; + + constructor( + public onPulseSignal: () => void, + public onStartCountdown: (ms: number | null) => void, + public onTranscript: (text: string) => void, + public onStopInProgress: () => void, + public onFinishedTranscribing: (finish: 'auto' | 'manual') => void + ) {} + + abstract start(language?: string): Promise; + abstract stop(): void; + + /** Clears the silence timeout */ + protected clearSilenceTimer() { + if (this.silenceTimeout) { + clearTimeout(this.silenceTimeout); + this.silenceTimeout = null; + } + } + + /** Resets silence timer and auto-stop trigger */ + protected resetSilenceTimer() { + this.clearSilenceTimer(); + this.silenceTimeout = setTimeout(() => { + this.isAutoFinished = true; + this.stop(); + }, BaseSttClient.SILENCE_TIMEOUT_MS); + } + + /** Restarts grace period before countdown starts */ + protected restartGracePeriod() { + if (this.silenceGraceTimeout) { + clearTimeout(this.silenceGraceTimeout); + } + + this.silenceGraceTimeout = setTimeout(() => { + this.isCountdownRunning = true; + this.onStartCountdown( + BaseSttClient.SILENCE_TIMEOUT_MS - BaseSttClient.SILENCE_GRACE_PERIOD + ); + }, BaseSttClient.SILENCE_GRACE_PERIOD); + } +} diff --git a/src/components/chat/extras/stt-client-webspeech.ts b/src/components/chat/extras/stt-client-webspeech.ts new file mode 100644 index 0000000000..d0eae02f31 --- /dev/null +++ b/src/components/chat/extras/stt-client-webspeech.ts @@ -0,0 +1,62 @@ +import { BaseSttClient } from './stt-client-base.js'; + +export class WebSpeechSttClient extends BaseSttClient { + private recognition?: SpeechRecognition; + + async start(language = 'en-US') { + if (this.isRecording) { + return; + } + + this.onTranscript(''); + + const SpeechRecognition = + window.SpeechRecognition || (window as any).webkitSpeechRecognition; + if (!SpeechRecognition) { + throw new Error('Web Speech API not supported in this browser.'); + } + this.recognition = new SpeechRecognition(); + this.recognition.continuous = true; + this.recognition.interimResults = true; + this.recognition.lang = language; + + this.recognition.onresult = (event: any) => { + this.handleTranscriptEvent(event); + }; + + this.recognition.onerror = () => { + //console.error("Speech recognition error", e); + }; + + this.recognition?.start(); + + this.isRecording = true; + this.resetSilenceTimer(); + this.restartGracePeriod(); + } + + stop() { + if (!this.isRecording) { + return; + } + this.recognition?.stop(); + this.isRecording = false; + this.onFinishedTranscribing(this.isAutoFinished ? 'auto' : 'manual'); + this.isAutoFinished = false; + } + + private handleTranscriptEvent(event: any) { + let transcript = ''; + for (let i = 0; i < event.results.length; i++) { + transcript += event.results[i][0].transcript; + } + this.onTranscript(transcript); + this.resetSilenceTimer(); + this.onPulseSignal(); + if (this.isCountdownRunning) { + this.onStartCountdown(null); + this.isCountdownRunning = false; + } + this.restartGracePeriod(); + } +} diff --git a/src/components/chat/types.ts b/src/components/chat/types.ts index 0835a21651..65344b10b2 100644 --- a/src/components/chat/types.ts +++ b/src/components/chat/types.ts @@ -165,12 +165,13 @@ export type IgcChatOptions = { * Configuration options for enabling and customizing speech-to-text functionality. * If provided, it enables a button in the chat input area that allows users to dictate messages using their voice. */ - sttOptions?: SpeechToTextOptions; + speechToText?: SpeechToTextOptions; }; export interface SpeechToTextOptions { enable: boolean; lang?: string; + serviceProvider: 'webspeech' | 'backend'; serviceUri?: string; } /** diff --git a/stories/chat.stories.ts b/stories/chat.stories.ts index fab7de7c3c..cea88fcb0e 100644 --- a/stories/chat.stories.ts +++ b/stories/chat.stories.ts @@ -145,9 +145,10 @@ const chat_options: IgcChatOptions = { inputPlaceholder: 'Type your message here...', speakPlaceholder: 'Speak...', headerText: 'Chat', - sttOptions: { + speechToText: { enable: true, lang: 'en-US', + serviceProvider: 'webspeech', // 'webspeech' | 'backend' serviceUri: 'https://localhost:5000/sttHub', }, renderers: { From 7fb833759ff0c30948c3396c2bb8c30979f68721 Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Wed, 15 Oct 2025 17:46:51 +0300 Subject: [PATCH 04/19] fix(chat): speech to text lint errors --- src/components/chat/chat-input.ts | 4 ++-- src/components/chat/extras/stt-client-backend.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/components/chat/chat-input.ts b/src/components/chat/chat-input.ts index 055739b92f..18c68dfa2c 100644 --- a/src/components/chat/chat-input.ts +++ b/src/components/chat/chat-input.ts @@ -117,10 +117,10 @@ export default class IgcChatInputComponent extends LitElement { private _sttClient?: ISttClient; - @property() + @property({ type: Boolean }) isRecording = false; - @property() + @property({ type: Boolean }) isStopInProgress = false; onPulseSignal = () => { diff --git a/src/components/chat/extras/stt-client-backend.ts b/src/components/chat/extras/stt-client-backend.ts index ca8274e908..efe8052200 100644 --- a/src/components/chat/extras/stt-client-backend.ts +++ b/src/components/chat/extras/stt-client-backend.ts @@ -114,7 +114,7 @@ export class BackendSttClient extends BaseSttClient { language ); } catch { - //report.error("STT invoke failed:", err); TOTO + //console.error("STT invoke failed:", err); //TODO this.stop(); } } From c2352c40e4e85b6effe3e5840cf37baaf77764ad Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Tue, 21 Oct 2025 11:39:10 +0300 Subject: [PATCH 05/19] fix(chat): speech to text lint errors --- src/components/chat/chat-input.ts | 8 +++++--- src/components/chat/themes/input.base.scss | 7 +++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/components/chat/chat-input.ts b/src/components/chat/chat-input.ts index 18c68dfa2c..aac4c6c84f 100644 --- a/src/components/chat/chat-input.ts +++ b/src/components/chat/chat-input.ts @@ -372,9 +372,11 @@ export default class IgcChatInputComponent extends LitElement { Date: Tue, 21 Oct 2025 11:48:23 +0300 Subject: [PATCH 06/19] fix(chat): speech to text lint errors --- src/components/chat/themes/input.base.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/chat/themes/input.base.scss b/src/components/chat/themes/input.base.scss index db22fb8a34..e938bf9890 100644 --- a/src/components/chat/themes/input.base.scss +++ b/src/components/chat/themes/input.base.scss @@ -112,7 +112,7 @@ label[part="speech-to-text"] igc-icon-button[disabled] { .ring-progress { fill: none; - stroke: rgba(0,0,0,0.5); + stroke: rgb(0 0 0 / 50%); stroke-width: 3; stroke-dasharray: 100; stroke-dashoffset: 100; From c36e2f124e19a40cb2e16734b548caacd431e22f Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Tue, 21 Oct 2025 13:25:57 +0300 Subject: [PATCH 07/19] fix(chat): speech to text interface declaration --- src/components/chat/chat-input.ts | 4 ++ .../chat/extras/stt-client-webspeech.ts | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/components/chat/chat-input.ts b/src/components/chat/chat-input.ts index aac4c6c84f..461f4f2250 100644 --- a/src/components/chat/chat-input.ts +++ b/src/components/chat/chat-input.ts @@ -201,6 +201,10 @@ export default class IgcChatInputComponent extends LitElement { // console.error('No STT service configured'); } + if (!this._sttClient) { + return; + } + await this._sttClient.start(this._state.options?.speechToText?.lang); this.isRecording = true; this.isStopInProgress = false; diff --git a/src/components/chat/extras/stt-client-webspeech.ts b/src/components/chat/extras/stt-client-webspeech.ts index d0eae02f31..13fda42839 100644 --- a/src/components/chat/extras/stt-client-webspeech.ts +++ b/src/components/chat/extras/stt-client-webspeech.ts @@ -60,3 +60,40 @@ export class WebSpeechSttClient extends BaseSttClient { this.restartGracePeriod(); } } + +//Fix. Add own interface declaration for SpeechRecognition +declare global { + interface Window { + SpeechRecognition: typeof SpeechRecognition; + webkitSpeechRecognition: typeof SpeechRecognition; + } + + // The browserโ€™s Web Speech API type (simplified) + var SpeechRecognition: { + new (): SpeechRecognition; + }; + + interface SpeechRecognition extends EventTarget { + continuous: boolean; + interimResults: boolean; + lang: string; + start(): void; + stop(): void; + abort(): void; + onaudioend?: (this: SpeechRecognition, ev: Event) => any; + onaudiostart?: (this: SpeechRecognition, ev: Event) => any; + onend?: (this: SpeechRecognition, ev: Event) => any; + onerror?: (this: SpeechRecognition, ev: Event) => any; + onnomatch?: (this: SpeechRecognition, ev: Event) => any; + onresult?: (this: SpeechRecognition, ev: SpeechRecognitionEvent) => any; + onsoundend?: (this: SpeechRecognition, ev: Event) => any; + onsoundstart?: (this: SpeechRecognition, ev: Event) => any; + onspeechend?: (this: SpeechRecognition, ev: Event) => any; + onspeechstart?: (this: SpeechRecognition, ev: Event) => any; + onstart?: (this: SpeechRecognition, ev: Event) => any; + } + + interface SpeechRecognitionEvent extends Event { + results: SpeechRecognitionResultList; + } +} From 54ffb7d994be7d0345f95d57c4d0adfdfe06dd5c Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Wed, 10 Jun 2026 11:40:51 +0300 Subject: [PATCH 08/19] fix(stt): lint errors --- src/components/chat/chat-input.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/chat/chat-input.ts b/src/components/chat/chat-input.ts index d54c80a6ed..7b4381d97e 100644 --- a/src/components/chat/chat-input.ts +++ b/src/components/chat/chat-input.ts @@ -215,7 +215,7 @@ export default class IgcChatInputComponent extends LitElement { this._state.options?.speechToText?.serviceUri ) { this._sttClient = new BackendSttClient( - this._state.options?.speechToText?.serviceUri!, + this._state.options?.speechToText?.serviceUri ?? undefined, this.onPulseSignal, this.onStartCountdown, this.onTranscript, From a0d40d554469ebefe152eb70bae066a8b6509882 Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Wed, 10 Jun 2026 11:43:34 +0300 Subject: [PATCH 09/19] fix(stt): lint warning --- stories/chat.stories.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stories/chat.stories.ts b/stories/chat.stories.ts index bf35efb001..49c5611a50 100644 --- a/stories/chat.stories.ts +++ b/stories/chat.stories.ts @@ -411,11 +411,11 @@ export const Basic: Story = { 'How do events work?', ], speechToText: { - enable: true, - lang: 'en-US', - serviceProvider: 'webspeech', // 'webspeech' | 'backend' - serviceUri: undefined, //e.g. 'https://localhost:5000/sttHub', - }, + enable: true, + lang: 'en-US', + serviceProvider: 'webspeech', // 'webspeech' | 'backend' + serviceUri: undefined, //e.g. 'https://localhost:5000/sttHub', + }, renderers: { messageContent: async ({ message }) => _markdownRenderer(message), }, From 208a938b9cbe8e7178fd6920b093ed2b066963f1 Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Wed, 10 Jun 2026 13:15:29 +0300 Subject: [PATCH 10/19] fix(stt): declaration error --- src/components/chat/extras/stt-client-webspeech.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/chat/extras/stt-client-webspeech.ts b/src/components/chat/extras/stt-client-webspeech.ts index 13fda42839..67b2060107 100644 --- a/src/components/chat/extras/stt-client-webspeech.ts +++ b/src/components/chat/extras/stt-client-webspeech.ts @@ -94,6 +94,6 @@ declare global { } interface SpeechRecognitionEvent extends Event { - results: SpeechRecognitionResultList; + readonly results: SpeechRecognitionResultList; } } From 9d3019dfc2a491d0d74625e3a28bdc280b285ca7 Mon Sep 17 00:00:00 2001 From: ivanvpetrov <110455887+ivanvpetrov@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:06:32 +0300 Subject: [PATCH 11/19] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/components/chat/chat-input.ts | 36 +++++++++++-------- .../chat/extras/stt-client-backend.ts | 2 ++ src/components/chat/extras/stt-client-base.ts | 4 +-- .../chat/extras/stt-client-webspeech.ts | 10 +++--- src/components/chat/themes/input.base.scss | 2 +- 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/components/chat/chat-input.ts b/src/components/chat/chat-input.ts index 7b4381d97e..c865736212 100644 --- a/src/components/chat/chat-input.ts +++ b/src/components/chat/chat-input.ts @@ -142,16 +142,15 @@ export default class IgcChatInputComponent extends LitElement { private _sttClient?: ISttClient; - @property({ type: Boolean }) - isRecording = false; - - @property({ type: Boolean }) - isStopInProgress = false; + @state() + private isRecording = false; + @state() + private isStopInProgress = false; onPulseSignal = () => { - const el = this.renderRoot.querySelector( - 'label[part="speech-to-text"]' - )?.firstElementChild; + const el = this.renderRoot.querySelector( + 'igc-icon-button[part="speech-to-text"]' + ); if (!el) return; el.classList.add('pulsate'); @@ -192,7 +191,7 @@ export default class IgcChatInputComponent extends LitElement { this._state.inputValue = text; }; - onFinishedTranscribing = (finished: string) => { + onFinishedTranscribing = (finished: 'auto' | 'manual') => { this.isStopInProgress = false; this.isRecording = false; if (finished === 'auto') { @@ -200,7 +199,7 @@ export default class IgcChatInputComponent extends LitElement { } }; - async _toggleMic() { + private async _toggleMic(): Promise { if (!this.isRecording) { if (this._state.options?.speechToText?.serviceProvider === 'webspeech') { this._sttClient = new WebSpeechSttClient( @@ -230,9 +229,15 @@ export default class IgcChatInputComponent extends LitElement { return; } - await this._sttClient.start(this._state.options?.speechToText?.lang); - this.isRecording = true; - this.isStopInProgress = false; + try { + await this._sttClient.start(this._state.options?.speechToText?.lang); + this.isRecording = true; + this.isStopInProgress = false; + } catch { + this._sttClient = undefined; + this.isRecording = false; + this.isStopInProgress = false; + } } else { this._sttClient?.stop(); } @@ -443,7 +448,8 @@ export default class IgcChatInputComponent extends LitElement { aria-label="Chat text input" placeholder=${ifDefined( this.isRecording - ? this._state.options?.speakPlaceholder + ? this._state.options?.speakPlaceholder ?? + this._state.options?.inputPlaceholder : this._state.options?.inputPlaceholder )} resize="auto" @@ -505,7 +511,7 @@ export default class IgcChatInputComponent extends LitElement { ${this.isRecording && !this.isStopInProgress ? html` - +
- + + ${this.isRecording ? '๐Ÿ›‘' : '๐ŸŽค'} + ${this.isRecording && !this.isStopInProgress ? html` Date: Thu, 11 Jun 2026 09:14:33 +0000 Subject: [PATCH 17/19] Add chat STT placeholder and button state tests --- src/components/chat/chat.spec.ts | 82 ++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/src/components/chat/chat.spec.ts b/src/components/chat/chat.spec.ts index a8416bba8c..a960ebc426 100644 --- a/src/components/chat/chat.spec.ts +++ b/src/components/chat/chat.spec.ts @@ -225,6 +225,88 @@ describe('Chat', () => { ); }); + it('should apply `speakPlaceholder` while recording', async () => { + chat.options = { + inputPlaceholder: 'Type message here...', + speakPlaceholder: 'Listening...', + speechToText: { enable: true, serviceProvider: 'webspeech' }, + }; + await elementUpdated(chat); + + const { input } = getChatDOM(chat); + expect(input.textarea.placeholder).to.equal( + chat.options.inputPlaceholder + ); + + Reflect.set(input.self, 'isRecording', true); + input.self.requestUpdate(); + await elementUpdated(input.self); + + expect(input.textarea.placeholder).to.equal( + chat.options.speakPlaceholder + ); + }); + + it('should fallback to `inputPlaceholder` while recording if `speakPlaceholder` is not set', async () => { + chat.options = { + inputPlaceholder: 'Type message here...', + speechToText: { enable: true, serviceProvider: 'webspeech' }, + }; + await elementUpdated(chat); + + const { input } = getChatDOM(chat); + Reflect.set(input.self, 'isRecording', true); + input.self.requestUpdate(); + await elementUpdated(input.self); + + expect(input.textarea.placeholder).to.equal( + chat.options.inputPlaceholder + ); + }); + + it('should render the speech-to-text button only when enabled', async () => { + chat.options = { + speechToText: { enable: true, serviceProvider: 'webspeech' }, + }; + await elementUpdated(chat); + + let speechToTextButton = getChatDOM( + chat + ).input.self.renderRoot.querySelector( + 'igc-icon-button[part="speech-to-text"]' + ); + expect(speechToTextButton).not.to.be.null; + + chat.options = { + speechToText: { enable: false, serviceProvider: 'webspeech' }, + }; + await elementUpdated(chat); + + speechToTextButton = getChatDOM(chat).input.self.renderRoot.querySelector( + 'igc-icon-button[part="speech-to-text"]' + ); + expect(speechToTextButton).to.be.null; + }); + + it('should pass an accessibility audit with speech-to-text enabled and disabled', async () => { + chat.options = { + inputPlaceholder: 'Type message here...', + speakPlaceholder: 'Listening...', + speechToText: { enable: true, serviceProvider: 'webspeech' }, + }; + await elementUpdated(chat); + await expect(chat).to.be.accessible(); + await expect(chat).shadowDom.to.be.accessible(); + + chat.options = { + inputPlaceholder: 'Type message here...', + speechToText: { enable: false, serviceProvider: 'webspeech' }, + }; + await elementUpdated(chat); + await expect(chat).to.be.accessible(); + await expect(chat).shadowDom.to.be.accessible(); + }); + it('should enable/disable the send button properly', async () => { const { textarea, sendButton, fileInput } = getChatDOM(chat).input; From 300c6c3d9dfa1f01f94651581629a032defe9ec3 Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Mon, 15 Jun 2026 11:20:48 +0300 Subject: [PATCH 18/19] fix(stt): signalR no longer peer dep --- .dependency-cruiser.cjs | 3 ++- package-lock.json | 26 +++++++++++++++++++++++++- package.json | 6 ------ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.dependency-cruiser.cjs b/.dependency-cruiser.cjs index c90c58c967..c68bed9291 100644 --- a/.dependency-cruiser.cjs +++ b/.dependency-cruiser.cjs @@ -182,7 +182,8 @@ module.exports = { severity: 'warn', from: { // src/extras uses peer-deps by design (e.g. shiki, marked for chat-markdown-renderer) - pathNot: '^src/extras/', + // src/components/chat/extras uses @microsoft/signalr as an optional peer dep for backend STT + pathNot: ['^src/extras/', '^src/components/chat/extras/'], }, to: { dependencyTypes: ['npm-peer'], diff --git a/package-lock.json b/package-lock.json index fda2e4ad20..95b5b5e8b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,6 @@ "@floating-ui/dom": "^1.7.6", "@lit-labs/virtualizer": "^2.1.1", "@lit/context": "^1.1.6", - "@microsoft/signalr": "^10.0.0", "igniteui-i18n-core": "^1.0.5", "lit": "^3.3.3" }, @@ -20,6 +19,7 @@ "@biomejs/biome": "~2.4.16", "@custom-elements-manifest/analyzer": "^0.11.0", "@igniteui/material-icons-extended": "^3.1.0", + "@microsoft/signalr": "^10.0.0", "@open-wc/testing": "^4.0.0", "@storybook/addon-a11y": "^10.4.2", "@storybook/addon-docs": "^10.4.2", @@ -61,10 +61,16 @@ "vite": "^8.0.16" }, "peerDependencies": { + "@microsoft/signalr": "^10.0.0", "dompurify": "^3.4.8", "marked": "^18.0.5", "marked-shiki": "^1.2.1", "shiki": "^4.2.0" + }, + "peerDependenciesMeta": { + "@microsoft/signalr": { + "optional": true + } } }, "node_modules/@adobe/css-tools": { @@ -1251,6 +1257,7 @@ "version": "10.0.0", "resolved": "https://registry.npmjs.org/@microsoft/signalr/-/signalr-10.0.0.tgz", "integrity": "sha512-0BRqz/uCx3JdrOqiqgFhih/+hfTERaUfCZXFB52uMaZJrKaPRzHzMuqVsJC/V3pt7NozcNXGspjKiQEK+X7P2w==", + "dev": true, "license": "MIT", "dependencies": { "abort-controller": "^3.0.0", @@ -1264,6 +1271,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-2.0.2.tgz", "integrity": "sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==", + "dev": true, "license": "MIT", "engines": { "node": ">=12.0.0" @@ -4586,6 +4594,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, "license": "MIT", "dependencies": { "event-target-shim": "^5.0.0" @@ -6798,6 +6807,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -7126,6 +7136,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/fetch-cookie/-/fetch-cookie-2.2.0.tgz", "integrity": "sha512-h9AgfjURuCgA2+2ISl8GbavpUdR+WGAM2McW/ovn4tVccegp8ZqCKWSBR8uRdM8dDNlx5WdKRWxBYUwteLDCNQ==", + "dev": true, "license": "Unlicense", "dependencies": { "set-cookie-parser": "^2.4.8", @@ -10462,6 +10473,7 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, "license": "MIT", "dependencies": { "whatwg-url": "^5.0.0" @@ -10482,18 +10494,21 @@ "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, "license": "MIT" }, "node_modules/node-fetch/node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, "license": "BSD-2-Clause" }, "node_modules/node-fetch/node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, "license": "MIT", "dependencies": { "tr46": "~0.0.3", @@ -11284,6 +11299,7 @@ "version": "1.15.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz", "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==", + "dev": true, "license": "MIT", "dependencies": { "punycode": "^2.3.1" @@ -11307,6 +11323,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -11403,6 +11420,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true, "license": "MIT" }, "node_modules/queue-microtask": { @@ -11637,6 +11655,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true, "license": "MIT" }, "node_modules/resolve": { @@ -12535,6 +12554,7 @@ "version": "2.7.2", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "dev": true, "license": "MIT" }, "node_modules/setprototypeof": { @@ -13648,6 +13668,7 @@ "version": "4.1.4", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz", "integrity": "sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "psl": "^1.1.33", @@ -13663,6 +13684,7 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 4.0.0" @@ -14090,6 +14112,7 @@ "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, "license": "MIT", "dependencies": { "querystringify": "^2.1.1", @@ -14496,6 +14519,7 @@ "version": "7.5.11", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.11.tgz", "integrity": "sha512-zS54Oen9bITtp7kp2XM3AydrCIq1D+HwJOuH+c+e4LfpL/lotP5osijd+UoMnxwAam1GN8R4KtLAyIrIcBNpiA==", + "dev": true, "license": "MIT", "engines": { "node": ">=8.3.0" diff --git a/package.json b/package.json index ddc00c3aad..a1c178dc6f 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,6 @@ "devDependencies": { "@biomejs/biome": "~2.4.16", "@custom-elements-manifest/analyzer": "^0.11.0", - "@microsoft/signalr": "^10.0.0", "@igniteui/material-icons-extended": "^3.1.0", "@open-wc/testing": "^4.0.0", "@storybook/addon-a11y": "^10.4.2", @@ -111,11 +110,6 @@ "marked-shiki": "^1.2.1", "shiki": "^4.2.0" }, - "peerDependenciesMeta": { - "@microsoft/signalr": { - "optional": true - } - }, "browserslist": [ "defaults" ], From 30508a454edf3b33877a089aaaa6a809d8563385 Mon Sep 17 00:00:00 2001 From: "INFRAGISTICS\\IPetrov" Date: Tue, 16 Jun 2026 12:11:59 +0300 Subject: [PATCH 19/19] test(stt): covarage improvements --- src/components/chat/chat.spec.ts | 124 +++++++- .../chat/extras/stt-clients.spec.ts | 281 ++++++++++++++++++ 2 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 src/components/chat/extras/stt-clients.spec.ts diff --git a/src/components/chat/chat.spec.ts b/src/components/chat/chat.spec.ts index 13b24c326a..1764e405a7 100644 --- a/src/components/chat/chat.spec.ts +++ b/src/components/chat/chat.spec.ts @@ -1,6 +1,6 @@ import { elementUpdated, expect, fixture } from '@open-wc/testing'; import { html, nothing } from 'lit'; -import { spy, stub, useFakeTimers } from 'sinon'; +import { fake, spy, stub, useFakeTimers } from 'sinon'; import { configureTheme } from '../../theming/config.js'; import type IgcIconButtonComponent from '../button/icon-button.js'; import IgcChipComponent from '../chip/chip.js'; @@ -429,6 +429,128 @@ describe('Chat', () => { }); }); + describe('Speech-to-text callbacks', () => { + beforeEach(async () => { + chat.options = { + speechToText: { enable: true, serviceProvider: 'webspeech' }, + }; + await elementUpdated(chat); + }); + + it('onPulseSignal() adds the pulsate class to the mic button', async () => { + const { input } = getChatDOM(chat); + const micButton = input.self.renderRoot.querySelector( + 'igc-icon-button[part="speech-to-text"]' + )!; + + input.self.onPulseSignal(); + + expect(micButton.classList.contains('pulsate')).to.be.true; + }); + + it('onStopInProgress() sets isStopInProgress to true', async () => { + const { input } = getChatDOM(chat); + + input.self.onStopInProgress(); + await elementUpdated(input.self); + + const micButton = input.self.renderRoot.querySelector( + 'igc-icon-button[part="speech-to-text"]' + )!; + expect(micButton.hasAttribute('disabled')).to.be.true; + }); + + it('onTranscript() updates the input value', async () => { + const { input } = getChatDOM(chat); + + input.self.onTranscript('hello world'); + await elementUpdated(chat); + + expect(input.textarea.value).to.equal('hello world'); + }); + + it('onFinishedTranscribing("manual") clears the recording state', async () => { + const { input } = getChatDOM(chat); + Reflect.set(input.self, 'isRecording', true); + Reflect.set(input.self, 'isStopInProgress', true); + input.self.requestUpdate(); + await elementUpdated(input.self); + + input.self.onFinishedTranscribing('manual'); + await elementUpdated(input.self); + + const micButton = input.self.renderRoot.querySelector( + 'igc-icon-button[part="speech-to-text"]' + )!; + expect(micButton.hasAttribute('disabled')).to.be.false; + }); + + it('onFinishedTranscribing("auto") clears recording state and sends the message', async () => { + const { input } = getChatDOM(chat); + Reflect.set(input.self, 'isRecording', true); + input.self.requestUpdate(); + await elementUpdated(input.self); + + // Set a pending input value so _sendMessage actually dispatches a message. + input.self.onTranscript('voice message'); + + const sendSpy = spy(input.self as any, '_sendMessage'); + input.self.onFinishedTranscribing('auto'); + + expect(sendSpy.calledOnce).to.be.true; + sendSpy.restore(); + }); + + it('onStartCountdown(ms) starts the SVG ring animation', async () => { + const { input } = getChatDOM(chat); + Reflect.set(input.self, 'isRecording', true); + input.self.requestUpdate(); + await elementUpdated(input.self); + + const circle = input.self.renderRoot.querySelector( + 'svg.countdown-ring .ring-progress' + )!; + expect(circle).not.to.be.null; + + input.self.onStartCountdown(3000); + + expect(circle.style.strokeDashoffset).to.equal('0'); + }); + + it('onStartCountdown(null) resets the SVG ring', async () => { + const { input } = getChatDOM(chat); + Reflect.set(input.self, 'isRecording', true); + input.self.requestUpdate(); + await elementUpdated(input.self); + + const circle = input.self.renderRoot.querySelector( + 'svg.countdown-ring .ring-progress' + )!; + + // First start the animation, then reset it. + input.self.onStartCountdown(3000); + input.self.onStartCountdown(null); + + expect(circle.style.strokeDashoffset).to.equal('100'); + }); + + it('clicking the mic button while recording stops the current client', async () => { + const { input } = getChatDOM(chat); + const stopFake = fake(); + Reflect.set(input.self, '_sttClient', { stop: stopFake }); + Reflect.set(input.self, 'isRecording', true); + input.self.requestUpdate(); + await elementUpdated(input.self); + + const micButton = input.self.renderRoot.querySelector( + 'igc-icon-button[part="speech-to-text"]' + )!; + simulateClick(micButton); + + expect(stopFake.calledOnce).to.be.true; + }); + }); + describe('Slots', () => { const getSlottedElements = (slotName: string) => { const prefixSlot = chat.shadowRoot?.querySelector( diff --git a/src/components/chat/extras/stt-clients.spec.ts b/src/components/chat/extras/stt-clients.spec.ts new file mode 100644 index 0000000000..21c9a390d6 --- /dev/null +++ b/src/components/chat/extras/stt-clients.spec.ts @@ -0,0 +1,281 @@ +import { expect } from '@open-wc/testing'; +import { fake, useFakeTimers } from 'sinon'; +import { BaseSttClient } from './stt-client-base.js'; +import { WebSpeechSttClient } from './stt-client-webspeech.js'; + +const SILENCE_TIMEOUT = 4000; +const GRACE_PERIOD = 1000; + +// Minimal concrete implementation of BaseSttClient for testing. +class ConcreteClient extends BaseSttClient { + async start(): Promise { + if (this.isRecording) return; + this.isRecording = true; + this.resetSilenceTimer(); + this.restartGracePeriod(); + } + + stop(): void { + this.clearSilenceTimer(); + this.clearSilenceGraceTimer(); + if (this.isCountdownRunning) { + this.onStartCountdown(null); + this.isCountdownRunning = false; + } + this.isRecording = false; + this.onFinishedTranscribing(this.isAutoFinished ? 'auto' : 'manual'); + this.isAutoFinished = false; + } +} + +// Minimal mock of the browser SpeechRecognition API. +class MockRecognition { + continuous = false; + interimResults = false; + lang = ''; + onresult: ((event: any) => void) | undefined; + onerror: (() => void) | undefined; + start = fake(); + stop = fake(); +} + +function makeWebSpeechClient( + opts: { + onPulse?: () => void; + onCountdown?: (ms: number | null) => void; + onTranscript?: (text: string) => void; + onStopInProgress?: () => void; + onFinished?: (f: 'auto' | 'manual') => void; + } = {} +) { + const noop = () => {}; + return new WebSpeechSttClient( + opts.onPulse ?? noop, + opts.onCountdown ?? noop, + opts.onTranscript ?? noop, + opts.onStopInProgress ?? noop, + opts.onFinished ?? noop + ); +} + +describe('BaseSttClient', () => { + const noop = () => {}; + let clock: ReturnType; + + beforeEach(() => { + clock = useFakeTimers(); + }); + + afterEach(() => { + clock.restore(); + }); + + it('initializes with isRecording set to false', () => { + const client = new ConcreteClient(noop, noop, noop, noop, noop); + expect(client.isRecording).to.be.false; + }); + + it('auto-stops and fires onFinishedTranscribing("auto") after silence timeout', async () => { + const onFinished = fake(); + const client = new ConcreteClient(noop, noop, noop, noop, onFinished); + await client.start(); + + clock.tick(SILENCE_TIMEOUT + 100); + + expect(onFinished.calledOnce).to.be.true; + expect(onFinished.firstCall.args[0]).to.equal('auto'); + }); + + it('stop() prevents auto-stop and fires onFinishedTranscribing("manual")', async () => { + const onFinished = fake(); + const client = new ConcreteClient(noop, noop, noop, noop, onFinished); + await client.start(); + + client.stop(); + clock.tick(SILENCE_TIMEOUT + 100); + + // Called exactly once by stop(), not again by the timer. + expect(onFinished.callCount).to.equal(1); + expect(onFinished.firstCall.args[0]).to.equal('manual'); + }); + + it('fires onStartCountdown after the grace period', async () => { + const onCountdown = fake(); + const client = new ConcreteClient(noop, onCountdown, noop, noop, noop); + await client.start(); + + clock.tick(GRACE_PERIOD + 100); + + expect(onCountdown.calledWith(SILENCE_TIMEOUT - GRACE_PERIOD)).to.be.true; + }); + + it('stop() resets the countdown when it is running', async () => { + const onCountdown = fake(); + const client = new ConcreteClient(noop, onCountdown, noop, noop, noop); + await client.start(); + + clock.tick(GRACE_PERIOD + 100); + client.stop(); + + expect(onCountdown.calledWith(null)).to.be.true; + }); + + it('start() is a no-op when already recording', async () => { + const client = new ConcreteClient(noop, noop, noop, noop, noop); + await client.start(); + const firstIsRecording = client.isRecording; + await client.start(); + + expect(firstIsRecording).to.be.true; + expect(client.isRecording).to.be.true; + }); +}); + +describe('WebSpeechSttClient', () => { + let clock: ReturnType; + let recognition!: MockRecognition; + + beforeEach(() => { + clock = useFakeTimers(); + recognition = new MockRecognition(); + (window as any).SpeechRecognition = class extends MockRecognition { + constructor() { + super(); + recognition = this; + } + }; + delete (window as any).webkitSpeechRecognition; + }); + + afterEach(() => { + clock.restore(); + delete (window as any).SpeechRecognition; + }); + + it('start() creates a recognition instance with correct settings', async () => { + const client = makeWebSpeechClient(); + await client.start('de-DE'); + + expect(client.isRecording).to.be.true; + expect(recognition.continuous).to.be.true; + expect(recognition.interimResults).to.be.true; + expect(recognition.lang).to.equal('de-DE'); + expect(recognition.start.calledOnce).to.be.true; + }); + + it('start() is a no-op when already recording', async () => { + const client = makeWebSpeechClient(); + await client.start(); + await client.start(); + + expect(recognition.start.calledOnce).to.be.true; + }); + + it('start() throws when SpeechRecognition is not available', async () => { + delete (window as any).SpeechRecognition; + delete (window as any).webkitSpeechRecognition; + const client = makeWebSpeechClient(); + + try { + await client.start(); + expect.fail('Expected an error to be thrown'); + } catch (e) { + expect((e as Error).message).to.include('not supported'); + } + }); + + it('stop() clears recording state and fires onFinishedTranscribing("manual")', async () => { + const onFinished = fake(); + const client = makeWebSpeechClient({ onFinished }); + await client.start(); + + client.stop(); + + expect(client.isRecording).to.be.false; + expect(recognition.stop.calledOnce).to.be.true; + expect(onFinished.calledWith('manual')).to.be.true; + }); + + it('stop() is a no-op when not recording', () => { + const onFinished = fake(); + const client = makeWebSpeechClient({ onFinished }); + + client.stop(); + + expect(onFinished.called).to.be.false; + expect(recognition.stop.called).to.be.false; + }); + + it('onresult handler updates transcript and fires onPulseSignal', async () => { + const onTranscript = fake(); + const onPulse = fake(); + const client = makeWebSpeechClient({ onTranscript, onPulse }); + await client.start(); + + recognition.onresult!({ results: [[{ transcript: 'hello world' }]] }); + + expect(onTranscript.calledWith('hello world')).to.be.true; + expect(onPulse.calledOnce).to.be.true; + }); + + it('onresult handler concatenates transcripts from multiple results', async () => { + const onTranscript = fake(); + const client = makeWebSpeechClient({ onTranscript }); + await client.start(); + + recognition.onresult!({ + results: [[{ transcript: 'hello ' }], [{ transcript: 'world' }]], + }); + + expect(onTranscript.lastCall.args[0]).to.equal('hello world'); + }); + + it('onerror handler stops the client', async () => { + const onFinished = fake(); + const client = makeWebSpeechClient({ onFinished }); + await client.start(); + + recognition.onerror!(); + + expect(client.isRecording).to.be.false; + expect(onFinished.called).to.be.true; + }); + + it('auto-stops after silence timeout with onFinishedTranscribing("auto")', async () => { + const onFinished = fake(); + const client = makeWebSpeechClient({ onFinished }); + await client.start(); + + clock.tick(SILENCE_TIMEOUT + 100); + + expect(onFinished.calledWith('auto')).to.be.true; + }); + + it('stop() resets countdown when it is running', async () => { + const onCountdown = fake(); + const client = makeWebSpeechClient({ onCountdown }); + await client.start(); + + clock.tick(GRACE_PERIOD + 100); + expect(onCountdown.calledWith(SILENCE_TIMEOUT - GRACE_PERIOD)).to.be.true; + + client.stop(); + expect(onCountdown.calledWith(null)).to.be.true; + }); + + it('falls back to webkitSpeechRecognition when SpeechRecognition is absent', async () => { + delete (window as any).SpeechRecognition; + (window as any).webkitSpeechRecognition = class extends MockRecognition { + constructor() { + super(); + recognition = this; + } + }; + + const client = makeWebSpeechClient(); + await client.start(); + + expect(recognition.start.calledOnce).to.be.true; + expect(client.isRecording).to.be.true; + }); +});