From 766ca871ec9eee423918a712191c2f2fc6027d1e Mon Sep 17 00:00:00 2001 From: WEIFENG2333 Date: Thu, 5 Mar 2026 14:44:55 +0800 Subject: [PATCH 1/2] docs: add qwen asr availability report --- docs/qwen-asr-fit-and-deployment-report.md | 209 +++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 docs/qwen-asr-fit-and-deployment-report.md diff --git a/docs/qwen-asr-fit-and-deployment-report.md b/docs/qwen-asr-fit-and-deployment-report.md new file mode 100644 index 00000000..545441e8 --- /dev/null +++ b/docs/qwen-asr-fit-and-deployment-report.md @@ -0,0 +1,209 @@ +# VideoCaptioner × Qwen ASR 契合度调研与部署报告 + +- 日期：2026-03-02 +- 项目：VideoCaptioner（master） +- 结论先行：**高契合（推荐优先走“OpenAI 兼容接入”）** + +## 1. 调研范围与依据 + +本报告基于以下两类事实来源： + +1) 项目代码与配置（本地仓库） +- ASR 统一入口与模型分发：`app/core/asr/transcribe.py` +- OpenAI 兼容 ASR 适配器：`app/core/asr/whisper_api.py` +- 转录模型枚举与语言能力：`app/core/entities.py` +- 转录配置项：`app/common/config.py` +- Whisper API 设置界面：`app/components/WhisperAPISettingWidget.py` +- Whisper 连通性测试：`app/core/llm/check_whisper.py` + +2) Qwen ASR 公开资料（2026-03-02 检索） +- Qwen3-ASR 模型页（HF） +- Qwen3-ASR Toolkit +- 阿里云 Model Studio（Fun-ASR 实时 WebSocket 文档） + +## 2. 项目现状（与接入相关） + +### 2.1 现有 ASR 架构特征 + +- `transcribe()` 通过 `TranscribeModelEnum` 分发到不同 ASR 实现。 +- `WhisperAPI` 已支持 OpenAI SDK 的 `audio.transcriptions.create()` 调用。 +- 结果统一转为 `ASRData/ASRDataSeg`，后续断句、优化、翻译、渲染复用同一数据结构。 +- 长音频统一通过 `ChunkedASR` 切块并合并，天然适配 API 限制场景。 + +### 2.2 对“可替换 ASR”的关键接口要求 + +- 必须可返回 `verbose_json` 风格结果，至少包含： + - `segments[*].text/start/end` + - 可选 `words[*].word/start/end`（用于词级时间戳） +- 需要支持语言参数（空值时可自动检测） +- 建议支持提示词（术语上下文） + +## 3. Qwen ASR 能力画像（对接视角） + +### 3.1 能力亮点（与本项目强相关） + +- 支持多语种与方言识别。 +- 支持流式/离线统一推理。 +- 支持长音频处理能力（官方 toolkit 也专门面向长音频并行切分）。 +- 可通过 forced aligner 获取更细粒度时间戳（需额外部署）。 + +### 3.2 可部署形态（对接可选） + +- 形态 A：本地/私有化 `qwen-asr`（transformers/vLLM） +- 形态 B：vLLM 服务化，并走 OpenAI 风格接口 +- 形态 C：DashScope（含实时 WebSocket 方案） + +## 4. 契合度评估 + +### 4.1 评分（10 分制） + +- 接口兼容性：**9/10**（已有 OpenAI 兼容通道） +- 工程改造成本：**8/10**（最小可做到零代码） +- 字幕链路适配性：**8/10**（核心取决于时间戳字段一致性） +- 运维复杂度：**7/10**（本地 GPU 与模型管理有门槛） +- 综合：**8.0/10（高契合）** + +### 4.2 关键判断 + +- **短期最优路径**：直接复用现有 `Whisper [API]` 通道，把 Qwen ASR 服务端当作 OpenAI 兼容 ASR 端点。 +- **中期最优路径**：新增 `QWEN_ASR` 模型枚举与专属设置页，沉淀为一等公民能力。 +- **实时字幕路径**：若要“边播边出字”，再引入 DashScope WebSocket 或 vLLM streaming 独立链路。 + +## 5. 完整部署思路（推荐方案） + +> 目标：先“低风险可用”，再“稳定提效”，最后“实时化”。 + +### Phase 0：基线与验收口径（半天） + +- 选 3 类测试集： + - 普通中文讲解（10-20 分钟） + - 中英夹杂技术内容（10-20 分钟） + - 噪声/背景音乐明显样本（5-10 分钟） +- 固定验收指标： + - WER/CER（可抽样） + - 时间戳偏差（句级 P50/P95） + - 端到端耗时（音频时长 vs 处理时长） + - 失败率与重试率 + +### Phase 1：零代码接入（1 天，推荐先做） + +#### 1) 部署 Qwen ASR 服务（vLLM/OpenAI 兼容） + +- 以官方 `qwen-asr` / `vLLM` 路径拉起服务。 +- 对外暴露 `http://:/v1`。 +- 使用模型名如 `Qwen/Qwen3-ASR-1.7B`（或 0.6B）。 + +#### 2) 在 VideoCaptioner 中配置 + +- 转录模型选择：`Whisper [API] ✨` +- `Whisper API Base`：Qwen 服务地址（`.../v1`） +- `Whisper API Key`：按服务要求填写（本地可用占位） +- `Whisper 模型`：填写 Qwen 模型名 +- 语言：先固定 `中文/英语` 做 A/B，再测试自动识别 + +#### 3) POC 验证重点 + +- 连通性：设置页“测试连接”通过 +- 字段兼容性：确认 `verbose_json` 下 `segments/words` 可被当前解析器直接消费 +- 长音频：确认 `ChunkedASR` 与服务端并行策略不会重复切分导致效率下降 + +### Phase 2：稳定性与性能优化（2-4 天） + +- 模型分层策略： + - 速度优先：0.6B + - 质量优先：1.7B +- 资源策略： + - GPU 显存水位（`gpu_memory_utilization`） + - 并发与批次平衡（避免 OOM） +- 业务策略： + - 噪声场景保持 VAD + - 术语场景补充 prompt/context +- 可观测性： + - 记录请求耗时、失败原因、切块数量、平均 chunk 时长 + +### Phase 3：产品化接入（3-5 天，可选） + +新增 `QWEN_ASR` 原生模型选项，而不是借道 `Whisper API`： + +- 新增枚举与配置分组： + - `TranscribeModelEnum.QWEN_ASR` + - `QwenASR` 专属配置（服务类型、本地/云端、模型、是否开启 aligner） +- 新增适配器： + - `app/core/asr/qwen_asr.py` + - 若接口仍为 OpenAI 兼容，可复用 `WhisperAPI` 基类逻辑 +- 新增 UI： + - 设置页模型切换时展示 Qwen 专属配置卡片 +- 回归测试： + - `transcribe.py` 分发逻辑 + - 词级时间戳路径 + - 失败重试与错误提示 + +### Phase 4：实时字幕（按需，1-2 周） + +- 路线 A：DashScope WebSocket（官方实时） +- 路线 B：Qwen vLLM streaming +- 与现有离线链路并行共存： + - “离线高质量”与“实时低延迟”双模式 + +## 6. 部署拓扑建议 + +### 6.1 单机版（MVP） + +- VideoCaptioner（桌面端） +- Qwen ASR 服务（同机） +- 可选：独立 GPU + +适合：个人或小团队，快速验证。 + +### 6.2 服务化版（团队） + +- 前端桌面客户端（多端） +- 内网 Qwen ASR 网关（鉴权、限流、监控） +- 后端推理节点（vLLM + 模型仓） +- 对象存储（可选，缓存音频与结果） + +适合：多人并发、成本可控、统一运维。 + +## 7. 风险与对策 + +1) **返回格式不完全兼容** +- 风险：`words/segments` 字段名或层级差异。 +- 对策：在 `WhisperAPI._make_segments()` 增加“Qwen 返回格式兜底解析”。 + +2) **时间戳精度不达标** +- 风险：字幕切分后观感不佳。 +- 对策：启用 forced aligner；或保留现有分句优化与时间轴后处理。 + +3) **GPU 资源波动/OOM** +- 风险：高峰期失败率上升。 +- 对策：降低 batch、限制并发、按模型分级路由。 + +4) **长音频双重切分导致吞吐下降** +- 风险：客户端切块 + 服务端切块重复。 +- 对策：只保留一层主切分策略（优先客户端 `ChunkedASR`）。 + +## 8. 验收标准（建议） + +- 功能： + - 可完成 60 分钟以内音频的稳定转录 + - 可输出可用 SRT，且时间轴连续无倒退 +- 质量： + - 中文样本 CER 相对现网不劣于 +5% + - 中英混合术语误识率下降 +- 性能： + - 20 分钟音频端到端处理时间 < 8 分钟（参考 GPU 机型可调整） +- 稳定： + - 20 次批量任务成功率 ≥ 95% + +## 9. 落地执行清单（可直接排期） + +- D1：POC 接入（零代码）+ 3 组样本回归 +- D2：兼容性补丁（必要时）+ 指标采集 +- D3-D4：性能调优 + 异常处理 +- D5：是否进入 `QWEN_ASR` 原生化改造评审 + +## 10. 最终建议 + +- **建议立即执行 Phase 1（零代码接入）**，用最小成本验证质量和速度。 +- 若 POC 通过，再进入 Phase 3 做“Qwen ASR 原生接入”，把它从“Whisper API 借道”升级为“产品内一等模型”。 +- 实时字幕不是本轮必需项，建议在离线链路稳定后再立项。 From 4f827fcc093543a193410c30eea98f292b117c77 Mon Sep 17 00:00:00 2001 From: WEIFENG2333 Date: Thu, 5 Mar 2026 15:22:38 +0800 Subject: [PATCH 2/2] feat(asr): keep qwen asr focused changes --- app/common/config.py | 55 ++ app/components/QwenASRSettingWidget.py | 336 +++++++++++ app/components/transcription_setting_card.py | 5 + app/core/asr/__init__.py | 2 + app/core/asr/asr_data.py | 20 +- app/core/asr/qwen_asr.py | 602 +++++++++++++++++++ app/core/asr/transcribe.py | 31 + app/core/entities.py | 37 ++ app/core/task_factory.py | 29 +- app/thread/transcript_thread.py | 21 +- app/view/setting_interface.py | 27 + docs/config/asr.md | 21 +- pyproject.toml | 3 + tests/test_asr/test_qwen_asr.py | 191 ++++++ 14 files changed, 1365 insertions(+), 15 deletions(-) create mode 100644 app/components/QwenASRSettingWidget.py create mode 100644 app/core/asr/qwen_asr.py create mode 100644 tests/test_asr/test_qwen_asr.py diff --git a/app/common/config.py b/app/common/config.py index 9ce62067..14bb8299 100644 --- a/app/common/config.py +++ b/app/common/config.py @@ -221,6 +221,61 @@ class Config(QConfig): whisper_api_model = OptionsConfigItem("WhisperAPI", "WhisperApiModel", "") whisper_api_prompt = ConfigItem("WhisperAPI", "WhisperApiPrompt", "") + # ------------------- Qwen ASR 配置 ------------------- + qwen_asr_backend = OptionsConfigItem( + "QwenASR", + "Backend", + "transformers", + OptionsValidator(["transformers", "vllm"]), + ) + qwen_asr_model = ConfigItem("QwenASR", "Model", "Qwen/Qwen3-ASR-0.6B") + qwen_asr_aligner_model = ConfigItem( + "QwenASR", "AlignerModel", "Qwen/Qwen3-ForcedAligner-0.6B" + ) + qwen_asr_api_base = ConfigItem("QwenASR", "ApiBase", "") + qwen_asr_api_key = ConfigItem("QwenASR", "ApiKey", "") + qwen_asr_prompt = ConfigItem("QwenASR", "Prompt", "") + qwen_asr_word_timestamp = ConfigItem( + "QwenASR", "WordTimestamp", True, BoolValidator() + ) + qwen_asr_max_new_tokens = RangeConfigItem( + "QwenASR", "MaxNewTokens", 1024, RangeValidator(16, 8192) + ) + qwen_asr_timestamp_mode = OptionsConfigItem( + "QwenASR", + "TimestampMode", + "forced_aligner_word", + OptionsValidator(["forced_aligner_word", "segment_only"]), + ) + qwen_asr_compute_dtype = OptionsConfigItem( + "QwenASR", + "ComputeDType", + "bfloat16", + OptionsValidator(["bfloat16", "float16", "float32"]), + ) + qwen_asr_language_mode = OptionsConfigItem( + "QwenASR", + "LanguageMode", + "auto", + OptionsValidator(["auto", "force"]), + ) + qwen_asr_force_language = OptionsConfigItem( + "QwenASR", + "ForceLanguage", + TranscribeLanguageEnum.AUTO, + OptionsValidator(TranscribeLanguageEnum), + EnumSerializer(TranscribeLanguageEnum), + ) + qwen_asr_timestamp_rounding = OptionsConfigItem( + "QwenASR", + "TimestampRounding", + "round", + OptionsValidator(["round", "floor"]), + ) + qwen_asr_vocal_separation = ConfigItem( + "QwenASR", "VocalSeparation", False, BoolValidator() + ) + # ------------------- 字幕配置 ------------------- need_optimize = ConfigItem("Subtitle", "NeedOptimize", False, BoolValidator()) need_translate = ConfigItem("Subtitle", "NeedTranslate", False, BoolValidator()) diff --git a/app/components/QwenASRSettingWidget.py b/app/components/QwenASRSettingWidget.py new file mode 100644 index 00000000..34cb0196 --- /dev/null +++ b/app/components/QwenASRSettingWidget.py @@ -0,0 +1,336 @@ +from pathlib import Path + +from PyQt5.QtCore import Qt, QThread, pyqtSignal +from PyQt5.QtWidgets import QVBoxLayout, QWidget +from qfluentwidgets import ( + ComboBoxSettingCard, + InfoBar, + InfoBarPosition, + PushSettingCard, + SettingCardGroup, + SingleDirectionScrollArea, + SwitchSettingCard, +) +from qfluentwidgets import FluentIcon as FIF + +from ..common.config import cfg +from ..config import ASSETS_PATH +from ..core.constant import INFOBAR_DURATION_ERROR, INFOBAR_DURATION_SUCCESS +from ..core.entities import TranscribeLanguageEnum +from .EditComboBoxSettingCard import EditComboBoxSettingCard +from .LineEditSettingCard import LineEditSettingCard +from .SpinBoxSettingCard import SpinBoxSettingCard + +DEFAULT_QWEN_MODELS = [ + "Qwen/Qwen3-ASR-0.6B", + "Qwen/Qwen3-ASR-1.7B", +] + +DEFAULT_ALIGNER_MODELS = [ + "Qwen/Qwen3-ForcedAligner-0.6B", +] + + +class QwenASRSettingWidget(QWidget): + def __init__(self, parent=None): + super().__init__(parent) + self.setup_ui() + self._connect_signals() + self._update_backend_dependent_ui(cfg.qwen_asr_backend.value) + + def setup_ui(self): + self.main_layout = QVBoxLayout(self) + + self.scrollArea = SingleDirectionScrollArea(orient=Qt.Vertical, parent=self) # type: ignore + self.scrollArea.setStyleSheet( + "QScrollArea{background: transparent; border: none}" + ) + + self.container = QWidget(self) + self.container.setStyleSheet("QWidget{background: transparent}") + self.containerLayout = QVBoxLayout(self.container) + + self.setting_group = SettingCardGroup(self.tr("Qwen ASR 设置"), self) + + self.backend_card = ComboBoxSettingCard( + cfg.qwen_asr_backend, + FIF.CLOUD, + self.tr("推理后端"), + self.tr("transformers(本地) / vllm(服务)"), + ["transformers", "vllm"], + self.setting_group, + ) + + self.model_card = EditComboBoxSettingCard( + cfg.qwen_asr_model, + FIF.ROBOT, + self.tr("ASR 模型"), + self.tr("可直接输入 HuggingFace 模型名"), + DEFAULT_QWEN_MODELS, + self.setting_group, + ) + + self.aligner_model_card = EditComboBoxSettingCard( + cfg.qwen_asr_aligner_model, + FIF.TAG, + self.tr("对齐器模型"), + self.tr("词级时间戳使用的 ForcedAligner 模型"), + DEFAULT_ALIGNER_MODELS, + self.setting_group, + ) + + self.word_timestamp_card = SwitchSettingCard( + FIF.UNIT, + self.tr("词级时间戳"), + self.tr("开启后输出词/字级时间戳（需 ForcedAligner）"), + cfg.qwen_asr_word_timestamp, + self.setting_group, + ) + self.vocal_separation_card = SwitchSettingCard( + FIF.MUSIC, + self.tr("人声分离"), + self.tr("转录前尝试分离人声（需安装 demucs）"), + cfg.qwen_asr_vocal_separation, + self.setting_group, + ) + self.timestamp_mode_card = ComboBoxSettingCard( + cfg.qwen_asr_timestamp_mode, + FIF.HISTORY, + self.tr("时间戳模式"), + self.tr("forced_aligner_word(词级) / segment_only(分段)"), + ["forced_aligner_word", "segment_only"], + self.setting_group, + ) + self.compute_dtype_card = ComboBoxSettingCard( + cfg.qwen_asr_compute_dtype, + FIF.FILTER, + self.tr("计算精度"), + self.tr("bfloat16(推荐) / float16(更快) / float32(更稳)"), + ["bfloat16", "float16", "float32"], + self.setting_group, + ) + self.language_mode_card = ComboBoxSettingCard( + cfg.qwen_asr_language_mode, + FIF.LANGUAGE, + self.tr("语言模式"), + self.tr("auto(自动检测) / force(使用下方 Qwen 强制语言)"), + ["auto", "force"], + self.setting_group, + ) + self.force_language_card = ComboBoxSettingCard( + cfg.qwen_asr_force_language, + FIF.LANGUAGE, + self.tr("Qwen 强制语言"), + self.tr("仅在语言模式为 force 时生效"), + [lang.value for lang in TranscribeLanguageEnum], + self.setting_group, + ) + self.timestamp_rounding_card = ComboBoxSettingCard( + cfg.qwen_asr_timestamp_rounding, + FIF.FILTER, + self.tr("时间戳取整"), + self.tr("round(四舍五入) / floor(向下取整)"), + ["round", "floor"], + self.setting_group, + ) + + self.api_base_card = LineEditSettingCard( + cfg.qwen_asr_api_base, + FIF.LINK, + self.tr("API Base URL"), + self.tr("vllm/OpenAI 兼容地址，例: http://127.0.0.1:8000/v1"), + "http://127.0.0.1:8000/v1", + self.setting_group, + ) + + self.api_key_card = LineEditSettingCard( + cfg.qwen_asr_api_key, + FIF.FINGERPRINT, + self.tr("API Key"), + self.tr("vllm/OpenAI 兼容 API Key"), + "EMPTY", + self.setting_group, + ) + + self.prompt_card = LineEditSettingCard( + cfg.qwen_asr_prompt, + FIF.CHAT, + self.tr("提示词"), + self.tr("可选：术语上下文，提升专有名词识别"), + "", + self.setting_group, + ) + + self.max_tokens_card = SpinBoxSettingCard( + cfg.qwen_asr_max_new_tokens, + FIF.FILTER, + self.tr("Max New Tokens"), + self.tr("生成上限，过小可能截断，过大可能变慢"), + minimum=16, + maximum=8192, + parent=self.setting_group, + ) + self.check_connection_card = PushSettingCard( + self.tr("测试连接"), + FIF.CONNECT, + self.tr("测试 Qwen ASR 连接"), + self.tr("点击测试 vllm/OpenAI 兼容接口"), + self.setting_group, + ) + + self.setting_group.addSettingCard(self.backend_card) + self.setting_group.addSettingCard(self.model_card) + self.setting_group.addSettingCard(self.aligner_model_card) + self.setting_group.addSettingCard(self.word_timestamp_card) + self.setting_group.addSettingCard(self.vocal_separation_card) + self.setting_group.addSettingCard(self.timestamp_mode_card) + self.setting_group.addSettingCard(self.compute_dtype_card) + self.setting_group.addSettingCard(self.language_mode_card) + self.setting_group.addSettingCard(self.force_language_card) + self.setting_group.addSettingCard(self.timestamp_rounding_card) + self.setting_group.addSettingCard(self.api_base_card) + self.setting_group.addSettingCard(self.api_key_card) + self.setting_group.addSettingCard(self.prompt_card) + self.setting_group.addSettingCard(self.max_tokens_card) + self.setting_group.addSettingCard(self.check_connection_card) + + self.containerLayout.addWidget(self.setting_group) + self.containerLayout.addStretch(1) + + self.backend_card.comboBox.setMinimumWidth(200) + self.model_card.comboBox.setMinimumWidth(200) + self.aligner_model_card.comboBox.setMinimumWidth(200) + self.timestamp_mode_card.comboBox.setMinimumWidth(200) + self.compute_dtype_card.comboBox.setMinimumWidth(200) + self.language_mode_card.comboBox.setMinimumWidth(200) + self.force_language_card.comboBox.setMinimumWidth(200) + self.timestamp_rounding_card.comboBox.setMinimumWidth(200) + self.api_base_card.lineEdit.setMinimumWidth(200) + self.api_key_card.lineEdit.setMinimumWidth(200) + self.prompt_card.lineEdit.setMinimumWidth(200) + + self.scrollArea.setWidget(self.container) + self.scrollArea.setWidgetResizable(True) + self.main_layout.addWidget(self.scrollArea) + + def _connect_signals(self): + self.backend_card.comboBox.currentTextChanged.connect( + self._update_backend_dependent_ui + ) + self.language_mode_card.comboBox.currentTextChanged.connect( + self._update_language_mode_ui + ) + self.check_connection_card.clicked.connect(self._on_check_connection) + self._update_language_mode_ui(self.language_mode_card.comboBox.currentText()) + + def _update_backend_dependent_ui(self, backend: str): + is_vllm = backend == "vllm" + self.api_base_card.setVisible(is_vllm) + self.api_key_card.setVisible(is_vllm) + self.prompt_card.setVisible(is_vllm) + self.check_connection_card.setVisible(is_vllm) + + def _update_language_mode_ui(self, language_mode: str): + self.force_language_card.setEnabled(language_mode == "force") + + def _on_check_connection(self): + if self.backend_card.comboBox.currentText() != "vllm": + InfoBar.warning( + self.tr("当前后端不支持"), + self.tr("仅 vllm 后端需要 API 连通性测试"), + duration=INFOBAR_DURATION_ERROR, + position=InfoBarPosition.BOTTOM, + parent=self.window(), + ) + return + + base_url = self.api_base_card.lineEdit.text().strip() + api_key = self.api_key_card.lineEdit.text().strip() + model = self.model_card.comboBox.currentText().strip() + if not base_url or not api_key or not model: + InfoBar.warning( + self.tr("配置不完整"), + self.tr("请填写 API Base URL、API Key、模型名"), + duration=INFOBAR_DURATION_ERROR, + position=InfoBarPosition.BOTTOM, + parent=self.window(), + ) + return + + self.check_connection_card.button.setEnabled(False) + self.check_connection_card.button.setText(self.tr("测试中...")) + self.connection_thread = QwenConnectionThread(base_url, api_key, model) + self.connection_thread.finished.connect(self._on_check_connection_finished) + self.connection_thread.error.connect(self._on_check_connection_error) + self.connection_thread.start() + + def _on_check_connection_finished(self, success: bool, message: str): + self.check_connection_card.button.setEnabled(True) + self.check_connection_card.button.setText(self.tr("测试连接")) + if success: + InfoBar.success( + self.tr("连接成功"), + message, + duration=INFOBAR_DURATION_SUCCESS, + position=InfoBarPosition.BOTTOM, + parent=self.window(), + ) + else: + InfoBar.error( + self.tr("连接失败"), + message, + duration=INFOBAR_DURATION_ERROR, + position=InfoBarPosition.BOTTOM, + parent=self.window(), + ) + + def _on_check_connection_error(self, message: str): + self.check_connection_card.button.setEnabled(True) + self.check_connection_card.button.setText(self.tr("测试连接")) + InfoBar.error( + self.tr("连接错误"), + message, + duration=INFOBAR_DURATION_ERROR, + position=InfoBarPosition.BOTTOM, + parent=self.window(), + ) + + +class QwenConnectionThread(QThread): + finished = pyqtSignal(bool, str) + error = pyqtSignal(str) + + def __init__(self, base_url: str, api_key: str, model: str): + super().__init__() + self.base_url = base_url + self.api_key = api_key + self.model = model + + def run(self): + try: + from openai import OpenAI + + from app.core.llm.client import normalize_base_url + + test_audio = Path(ASSETS_PATH) / "en.mp3" + if not test_audio.exists(): + self.finished.emit(False, f"测试音频不存在: {test_audio}") + return + + client = OpenAI( + base_url=normalize_base_url(self.base_url), + api_key=self.api_key, + timeout=20, + ) + with open(test_audio, "rb") as f: + resp = client.audio.transcriptions.create( + model=self.model, + file=f, + response_format="verbose_json", + timestamp_granularities=["segment"], + ) + + text = getattr(resp, "text", "") or "ok" + self.finished.emit(True, f"API 可用，返回: {text[:50]}") + except Exception as e: + self.error.emit(str(e)) diff --git a/app/components/transcription_setting_card.py b/app/components/transcription_setting_card.py index b2ca8abf..bd5d4720 100644 --- a/app/components/transcription_setting_card.py +++ b/app/components/transcription_setting_card.py @@ -11,6 +11,7 @@ ) from ..core.utils.platform_utils import is_macos from .FasterWhisperSettingWidget import FasterWhisperSettingWidget +from .QwenASRSettingWidget import QwenASRSettingWidget from .WhisperAPISettingWidget import WhisperAPISettingWidget from .WhisperCppSettingWidget import WhisperCppSettingWidget @@ -31,6 +32,7 @@ def setup_ui(self): self.empty_widget = QWidget(self) # 添加空白页面作为默认显示 self.whisper_cpp_widget = WhisperCppSettingWidget(self) self.whisper_api_widget = WhisperAPISettingWidget(self) + self.qwen_asr_widget = QwenASRSettingWidget(self) # FasterWhisper 在 macOS 上不可用 self.faster_whisper_widget: Optional[FasterWhisperSettingWidget] = None @@ -40,6 +42,7 @@ def setup_ui(self): self.stacked_widget.addWidget(self.empty_widget) # 添加空白页面 self.stacked_widget.addWidget(self.whisper_cpp_widget) self.stacked_widget.addWidget(self.whisper_api_widget) + self.stacked_widget.addWidget(self.qwen_asr_widget) if self.faster_whisper_widget is not None: self.stacked_widget.addWidget(self.faster_whisper_widget) @@ -51,6 +54,8 @@ def on_model_changed(self, value): self.stacked_widget.setCurrentWidget(self.whisper_cpp_widget) elif value == TranscribeModelEnum.WHISPER_API.value: self.stacked_widget.setCurrentWidget(self.whisper_api_widget) + elif value == TranscribeModelEnum.QWEN_ASR.value: + self.stacked_widget.setCurrentWidget(self.qwen_asr_widget) elif value == TranscribeModelEnum.FASTER_WHISPER.value: self.stacked_widget.setCurrentWidget(self.faster_whisper_widget) else: diff --git a/app/core/asr/__init__.py b/app/core/asr/__init__.py index 1bc9703e..ab01095d 100644 --- a/app/core/asr/__init__.py +++ b/app/core/asr/__init__.py @@ -2,6 +2,7 @@ from .chunked_asr import ChunkedASR from .faster_whisper import FasterWhisperASR from .jianying import JianYingASR +from .qwen_asr import QwenASR from .status import ASRStatus from .transcribe import transcribe from .whisper_api import WhisperAPI @@ -12,6 +13,7 @@ "ChunkedASR", "FasterWhisperASR", "JianYingASR", + "QwenASR", "WhisperAPI", "WhisperCppASR", "transcribe", diff --git a/app/core/asr/asr_data.py b/app/core/asr/asr_data.py index 4ae14bad..61d2ab20 100644 --- a/app/core/asr/asr_data.py +++ b/app/core/asr/asr_data.py @@ -40,13 +40,19 @@ def handle_long_path(path: str) -> str: Returns: Path with \\?\ prefix if needed (Windows only) """ - if ( - platform.system() == "Windows" - and len(path) > 260 - and not path.startswith(r"\\?\ ") - ): - return rf"\\?\{os.path.abspath(path)}" - return path + if platform.system() != "Windows": + return path + + abs_path = os.path.abspath(path) + + # 已是 Windows 长路径前缀，直接返回，避免重复前缀导致路径非法 + if abs_path.startswith("\\\\?\\"): + return abs_path + + if len(abs_path) > 260: + return rf"\\?\{abs_path}" + + return abs_path class ASRDataSeg: diff --git a/app/core/asr/qwen_asr.py b/app/core/asr/qwen_asr.py new file mode 100644 index 00000000..8b572b78 --- /dev/null +++ b/app/core/asr/qwen_asr.py @@ -0,0 +1,602 @@ +from __future__ import annotations + +import gc +import os +import tempfile +from typing import Any, Callable, Optional, Union + +from openai import APIConnectionError, OpenAI + +from app.core.llm.client import normalize_base_url +from app.core.utils.logger import setup_logger + +from .asr_data import ASRDataSeg +from .base import BaseASR + +logger = setup_logger("qwen_asr") + +_QWEN_LANGUAGE_MAP = { + "zh": "Chinese", + "en": "English", + "ja": "Japanese", + "ko": "Korean", + "yue": "Cantonese", +} + + +class QwenASR(BaseASR): + """Qwen-ASR implementation based on qwen-asr python package.""" + + def __init__( + self, + audio_input: Union[str, bytes], + model_name: str, + aligner_model_name: str = "Qwen/Qwen3-ForcedAligner-0.6B", + backend: str = "transformers", + language: str = "", + api_base: str = "", + api_key: str = "", + prompt: str = "", + need_word_time_stamp: bool = True, + max_new_tokens: int = 1024, + timestamp_mode: str = "forced_aligner_word", + compute_dtype: str = "bfloat16", + language_mode: str = "auto", + force_language: str = "", + timestamp_rounding: str = "round", + use_cache: bool = False, + ): + super().__init__(audio_input, use_cache, need_word_time_stamp) + self.model_name = model_name.strip() + self.aligner_model_name = aligner_model_name.strip() + self.backend = backend.strip() or "transformers" + self.language = language.strip() + self.api_base = api_base.strip() + self.api_key = api_key.strip() + self.prompt = prompt.strip() + self.need_word_time_stamp = need_word_time_stamp + self.max_new_tokens = max_new_tokens + self.timestamp_mode = timestamp_mode.strip() or "forced_aligner_word" + self.compute_dtype = compute_dtype.strip() or "bfloat16" + self.language_mode = language_mode.strip() or "auto" + self.force_language = force_language.strip() + self.timestamp_rounding = timestamp_rounding.strip() or "round" + + if not self.model_name: + raise ValueError("Qwen ASR model name must be set") + + def _get_key(self) -> str: + return ( + f"{self.crc32_hex}-{self.model_name}-{self.backend}-" + f"{self.language}-{self.need_word_time_stamp}-{self.max_new_tokens}-" + f"{self.api_base}-{self.prompt}-{self.timestamp_mode}-" + f"{self.compute_dtype}-{self.language_mode}-{self.force_language}-" + f"{self.timestamp_rounding}" + ) + + def _run( + self, callback: Optional[Callable[[int, str], None]] = None, **kwargs: Any + ) -> dict: + if self.backend == "vllm": + return self._run_vllm(callback) + return self._run_transformers(callback) + + def _run_vllm( + self, callback: Optional[Callable[[int, str], None]] = None + ) -> dict: + base_url = normalize_base_url(self.api_base) + if not base_url or not self.api_key: + raise RuntimeError( + "Qwen ASR(vllm) 需要配置 API Base URL 和 API Key（OpenAI 兼容）" + ) + + if callback: + callback(30, "Qwen-ASR(vllm) 推理中") + + client = OpenAI(base_url=base_url, api_key=self.api_key) + api_kwargs: dict[str, Any] = { + "model": self.model_name, + "response_format": "verbose_json", + "file": ("audio.mp3", self.file_binary or b"", "audio/mp3"), + "timestamp_granularities": ["word", "segment"], + } + if self.language: + api_kwargs["language"] = self.language + if self.prompt: + api_kwargs["prompt"] = self.prompt + + try: + completion = client.audio.transcriptions.create(**api_kwargs) + except APIConnectionError as e: + raise RuntimeError( + f"Qwen ASR(vllm) 连接失败：{base_url}。请先启动服务并确认端口可访问。" + ) from e + if isinstance(completion, str): + raise RuntimeError( + "Qwen ASR(vllm) 返回异常，请检查 OpenAI 兼容 API 地址与模型配置" + ) + + if callback: + callback(95, "Qwen-ASR(vllm) 结果整理中") + + return completion.to_dict() + + def _run_transformers( + self, callback: Optional[Callable[[int, str], None]] = None + ) -> dict: + language = self._resolve_qwen_language() + use_forced_aligner = self._need_forced_aligner() + + if callback: + callback(30, "Qwen-ASR 推理中") + + result: Any + resources_to_release: list[Any] = [] + try: + # qwen-asr>=0.0.6 API + from qwen_asr import Qwen3ASRModel + except Exception as import_error: + try: + # qwen-asr early API fallback + from qwen_asr import Qwen3ASR + from qwen_asr.models.qwen3_asr import Qwen3ASRConfig + from qwen_asr.models.qwen3_asr_toolkit import Qwen3ASRToolkit + except Exception: + raise RuntimeError( + "Qwen ASR(transformers) 依赖加载失败，请检查 qwen-asr/torch 环境与版本兼容" + ) from import_error + + config = Qwen3ASRConfig( + model_name_or_path=self.model_name, + llm_backend=self.backend, + max_new_tokens=self.max_new_tokens, + ) + model = Qwen3ASR(config=config) + resources_to_release.append(model) + toolkit = None + if use_forced_aligner: + toolkit = Qwen3ASRToolkit.forced_aligner(self.aligner_model_name) + resources_to_release.append(toolkit) + try: + result = model.transcribe( + self.file_binary, + language=language, + toolkit=toolkit, + ) + except Exception as old_api_error: + raise RuntimeError( + f"Qwen ASR(transformers) 推理失败：{old_api_error}" + ) from old_api_error + finally: + self._release_resources(resources_to_release) + else: + audio_input_for_qwen, temp_audio_path = self._prepare_audio_input_for_qwen() + try: + model_load_kwargs: dict[str, Any] = {} + try: + import torch + + if torch.cuda.is_available(): + model_load_kwargs["device_map"] = "cuda:0" + dtype_map = { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float32": torch.float32, + } + model_load_kwargs["torch_dtype"] = dtype_map.get( + self.compute_dtype, torch.bfloat16 + ) + logger.info("Qwen ASR using CUDA device: cuda:0") + except Exception: + pass + + model = Qwen3ASRModel.from_pretrained( + self.model_name, + forced_aligner=(self.aligner_model_name if use_forced_aligner else None), + max_new_tokens=self.max_new_tokens, + **model_load_kwargs, + ) + resources_to_release.append(model) + results = model.transcribe( + audio_input_for_qwen, + language=language, + return_time_stamps=use_forced_aligner, + ) + result = ( + results[0] if isinstance(results, list) and results else results + ) + except Exception as inference_error: + err_text = str(inference_error) + if ( + use_forced_aligner + and ( + "Qwen3-ForcedAligner" in err_text + or "Repository Not Found" in err_text + or "401" in err_text + ) + ): + fallback_aligner = "Qwen/Qwen3-ForcedAligner-0.6B" + if self.aligner_model_name != fallback_aligner: + logger.warning( + "Qwen aligner `%s` unavailable, fallback to `%s`", + self.aligner_model_name, + fallback_aligner, + ) + try: + model = Qwen3ASRModel.from_pretrained( + self.model_name, + forced_aligner=fallback_aligner, + max_new_tokens=self.max_new_tokens, + ) + resources_to_release.append(model) + results = model.transcribe( + audio_input_for_qwen, + language=language, + return_time_stamps=True, + ) + result = ( + results[0] + if isinstance(results, list) and results + else results + ) + return self._normalize_result(result) + except Exception: + pass + raise RuntimeError( + "Qwen ASR(transformers) 对齐器模型不可用：请把 Aligner 改为 " + "`Qwen/Qwen3-ForcedAligner-0.6B`，或关闭“词级时间戳”。" + ) from inference_error + raise RuntimeError( + f"Qwen ASR(transformers) 推理失败：{inference_error}" + ) from inference_error + finally: + if temp_audio_path: + try: + os.remove(temp_audio_path) + except OSError: + pass + self._release_resources(resources_to_release) + + if callback: + callback(95, "Qwen-ASR 结果整理中") + + normalized = self._normalize_result(result) + if normalized: + return normalized + + raise TypeError("Qwen ASR returned unsupported result type") + + def _normalize_result(self, result: Any) -> dict: + if isinstance(result, dict): + return result + + if hasattr(result, "model_dump"): + dumped = result.model_dump() + if isinstance(dumped, dict): + return dumped + + if hasattr(result, "__dict__"): + normalized: dict[str, Any] = dict(result.__dict__) + time_stamps = normalized.get("time_stamps") + if time_stamps is not None: + items = getattr(time_stamps, "items", time_stamps) + converted = [] + for item in items or []: + if isinstance(item, dict): + text = item.get("text", "") + start_value = item.get("start_time", item.get("start", 0)) + end_value = item.get("end_time", item.get("end", start_value)) + else: + text = getattr(item, "text", "") + start_value = getattr( + item, "start_time", getattr(item, "start", 0) + ) + end_value = getattr( + item, + "end_time", + getattr(item, "end", start_value), + ) + + try: + start_time = float(start_value) + end_time = float(end_value) + except (TypeError, ValueError): + continue + + converted.append( + { + "text": text, + "word": text, + "start": start_time, + "end": end_time, + "start_time": start_time, + "end_time": end_time, + } + ) + normalized["time_stamps"] = converted + normalized["words"] = converted + return normalized + + return {} + + def _prepare_audio_input_for_qwen(self) -> tuple[Union[str, bytes], Optional[str]]: + if isinstance(self.audio_input, str) and os.path.exists(self.audio_input): + return self.audio_input, None + + if not self.file_binary: + raise RuntimeError("Qwen ASR(transformers) 音频输入为空") + + with tempfile.NamedTemporaryFile( + suffix=".wav", delete=False + ) as temp_audio_file: + temp_audio_file.write(self.file_binary) + return temp_audio_file.name, temp_audio_file.name + + def _release_resources(self, resources: list[Any]) -> None: + for resource in list(resources): + self._release_single_resource(resource) + resources.clear() + gc.collect() + try: + import torch + + if torch.cuda.is_available(): + torch.cuda.synchronize() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except Exception: + pass + + def _release_single_resource(self, resource: Any) -> None: + try: + # qwen-asr wrapper objects keep heavy references in these fields. + for field in ("model", "forced_aligner", "processor", "sampling_params"): + if hasattr(resource, field): + inner = getattr(resource, field) + self._release_torch_object(inner) + try: + setattr(resource, field, None) + except Exception: + pass + self._release_torch_object(resource) + except Exception: + pass + + def _release_torch_object(self, obj: Any) -> None: + if obj is None: + return + try: + # Move modules to CPU before dropping refs to encourage CUDA allocator release. + to_fn = getattr(obj, "to", None) + if callable(to_fn): + try: + to_fn("cpu") + except Exception: + pass + except Exception: + pass + + def _make_segments(self, resp_data: dict) -> list[ASRDataSeg]: + words = resp_data.get("words") or [] + if self._need_forced_aligner() and words: + segments: list[ASRDataSeg] = [] + for word in words: + if not isinstance(word, dict): + continue + token_text = (word.get("word") or word.get("text") or "").strip() + if not token_text: + continue + start = word.get("start", word.get("start_time", 0)) + end = word.get("end", word.get("end_time", start)) + try: + start_ms = self._to_ms(start) + end_ms = self._to_ms(end) + except (TypeError, ValueError): + continue + segments.append( + ASRDataSeg( + text=token_text, + start_time=start_ms, + end_time=end_ms, + ) + ) + if segments: + if self._needs_synthetic_timeline(segments): + logger.warning( + "Qwen ASR words timestamps are degenerate (all zero), fallback to synthetic timeline" + ) + return self._synthesize_timeline(segments) + return self._repair_partial_zero_timestamps(segments) + + api_segments = resp_data.get("segments") or [] + if api_segments: + segments: list[ASRDataSeg] = [] + for seg in api_segments: + if not isinstance(seg, dict): + continue + seg_text = (seg.get("text") or "").strip() + start = seg.get("start", 0) + end = seg.get("end", start) + try: + start_ms = self._to_ms(start) + end_ms = self._to_ms(end) + except (TypeError, ValueError): + continue + segments.append( + ASRDataSeg( + text=seg_text, + start_time=start_ms, + end_time=end_ms, + ) + ) + if segments: + if self._needs_synthetic_timeline(segments): + logger.warning( + "Qwen ASR segment timestamps are degenerate (all zero), fallback to synthetic timeline" + ) + return self._synthesize_timeline(segments) + return self._repair_partial_zero_timestamps(segments) + + time_stamps = resp_data.get("time_stamps") or resp_data.get("timestamps") or [] + text = (resp_data.get("text") or "").strip() + + if self._need_forced_aligner() and time_stamps: + segments: list[ASRDataSeg] = [] + for item in time_stamps: + if not isinstance(item, dict): + continue + + token_text = ( + item.get("text") + or item.get("word") + or item.get("token") + or "" + ).strip() + if not token_text: + continue + + start = item.get("start_time", item.get("start", 0)) + end = item.get("end_time", item.get("end", start)) + try: + start_ms = self._to_ms(start) + end_ms = self._to_ms(end) + except (TypeError, ValueError): + continue + + segments.append( + ASRDataSeg( + text=token_text, + start_time=start_ms, + end_time=end_ms, + ) + ) + + if segments: + if self._needs_synthetic_timeline(segments): + logger.warning( + "Qwen ASR time_stamps are degenerate (all zero), fallback to synthetic timeline" + ) + return self._synthesize_timeline(segments) + return self._repair_partial_zero_timestamps(segments) + + if text: + max_end_ms = 0 + for item in time_stamps: + if not isinstance(item, dict): + continue + end = item.get("end_time", item.get("end", 0)) + try: + max_end_ms = max(max_end_ms, self._to_ms(end)) + except (TypeError, ValueError): + continue + + if max_end_ms <= 0: + max_end_ms = int(max(self.audio_duration, 0.1) * 1000) + + return [ASRDataSeg(text=text, start_time=0, end_time=max_end_ms)] + + logger.warning("Qwen ASR returned empty result") + return [ + ASRDataSeg( + text="", + start_time=0, + end_time=int(max(self.audio_duration, 0.1) * 1000), + ) + ] + + def _resolve_qwen_language(self) -> Optional[str]: + if self.language_mode != "force": + return None + preferred_language = self.force_language or self.language + if not preferred_language: + return None + mapped = _QWEN_LANGUAGE_MAP.get(preferred_language) + if mapped is None: + logger.warning( + "Qwen ASR force language mode is enabled but `%s` is unsupported, fallback to auto", + preferred_language, + ) + return mapped + + def _need_forced_aligner(self) -> bool: + return self.need_word_time_stamp and self.timestamp_mode == "forced_aligner_word" + + def _to_ms(self, value: Any) -> int: + ms = float(value) * 1000.0 + if self.timestamp_rounding == "floor": + return int(ms) + return int(round(ms)) + + def _needs_synthetic_timeline(self, segments: list[ASRDataSeg]) -> bool: + if not segments: + return False + + zero_pair_count = sum( + 1 for seg in segments if seg.start_time == 0 and seg.end_time == 0 + ) + zero_pair_ratio = zero_pair_count / len(segments) + max_end = max(seg.end_time for seg in segments) + + return zero_pair_ratio >= 0.9 or max_end <= 0 + + def _synthesize_timeline(self, segments: list[ASRDataSeg]) -> list[ASRDataSeg]: + if not segments: + return [] + + total_ms = int(max(self.audio_duration, 0.1) * 1000) + total_ms = max(total_ms, len(segments)) + + rebuilt: list[ASRDataSeg] = [] + for idx, seg in enumerate(segments): + start = int(idx * total_ms / len(segments)) + end = int((idx + 1) * total_ms / len(segments)) + if end <= start: + end = start + 1 + rebuilt.append( + ASRDataSeg( + text=seg.text, + start_time=start, + end_time=end, + ) + ) + + return rebuilt + + def _repair_partial_zero_timestamps( + self, segments: list[ASRDataSeg] + ) -> list[ASRDataSeg]: + if not segments: + return [] + + zero_pair_indexes = [ + idx + for idx, seg in enumerate(segments) + if seg.start_time == 0 and seg.end_time == 0 + ] + if not zero_pair_indexes: + return segments + + max_end = max(seg.end_time for seg in segments) + if max_end <= 0: + return segments + + rebuilt = [ + ASRDataSeg( + text=seg.text, + start_time=seg.start_time, + end_time=seg.end_time, + translated_text=seg.translated_text, + ) + for seg in segments + ] + synthetic = self._synthesize_timeline(segments) + + for idx in zero_pair_indexes: + rebuilt[idx].start_time = synthetic[idx].start_time + rebuilt[idx].end_time = synthetic[idx].end_time + + logger.warning( + "Qwen ASR repaired %d zero-pair timestamps with synthetic positions", + len(zero_pair_indexes), + ) + return rebuilt diff --git a/app/core/asr/transcribe.py b/app/core/asr/transcribe.py index 02e8944f..59f101cb 100644 --- a/app/core/asr/transcribe.py +++ b/app/core/asr/transcribe.py @@ -3,6 +3,7 @@ from app.core.asr.chunked_asr import ChunkedASR from app.core.asr.faster_whisper import FasterWhisperASR from app.core.asr.jianying import JianYingASR +from app.core.asr.qwen_asr import QwenASR from app.core.asr.whisper_api import WhisperAPI from app.core.asr.whisper_cpp import WhisperCppASR from app.core.entities import TranscribeConfig, TranscribeModelEnum @@ -68,6 +69,8 @@ def _create_asr_instance(audio_path: str, config: TranscribeConfig) -> ChunkedAS elif model_type == TranscribeModelEnum.FASTER_WHISPER: return _create_faster_whisper_asr(audio_path, config) + elif model_type == TranscribeModelEnum.QWEN_ASR: + return _create_qwen_asr(audio_path, config) else: raise ValueError(f"Invalid transcription model: {model_type}") @@ -158,6 +161,34 @@ def _create_faster_whisper_asr(audio_path: str, config: TranscribeConfig) -> Chu ) +def _create_qwen_asr(audio_path: str, config: TranscribeConfig) -> ChunkedASR: + """Create Qwen ASR instance with chunking support.""" + asr_kwargs = { + "use_cache": True, + "need_word_time_stamp": config.need_word_time_stamp, + "model_name": config.qwen_asr_model, + "aligner_model_name": config.qwen_asr_aligner_model, + "backend": config.qwen_asr_backend, + "language": config.transcribe_language, + "api_base": config.qwen_asr_api_base or "", + "api_key": config.qwen_asr_api_key or "", + "prompt": config.qwen_asr_prompt or "", + "max_new_tokens": config.qwen_asr_max_new_tokens, + "timestamp_mode": config.qwen_asr_timestamp_mode, + "compute_dtype": config.qwen_asr_compute_dtype, + "language_mode": config.qwen_asr_language_mode, + "force_language": config.qwen_asr_force_language, + "timestamp_rounding": config.qwen_asr_timestamp_rounding, + } + return ChunkedASR( + asr_class=QwenASR, + audio_path=audio_path, + asr_kwargs=asr_kwargs, + chunk_concurrency=1, + chunk_length=60 * 20, + ) + + if __name__ == "__main__": # 示例用法 from app.core.entities import WhisperModelEnum diff --git a/app/core/entities.py b/app/core/entities.py index 5eccab8e..82d8248d 100644 --- a/app/core/entities.py +++ b/app/core/entities.py @@ -121,6 +121,7 @@ class TranscribeModelEnum(Enum): WHISPER_API = "Whisper [API] ✨" FASTER_WHISPER = "FasterWhisper ✨" WHISPER_CPP = "WhisperCpp" + QWEN_ASR = "Qwen-ASR ✨" class TranslatorServiceEnum(Enum): @@ -505,6 +506,10 @@ def _get_all_languages_except_auto() -> list[TranscribeLanguageEnum]: supported_languages=_get_all_languages_except_auto(), supports_auto=True, ), + TranscribeModelEnum.QWEN_ASR: ASRLanguageCapability( + supported_languages=_get_all_languages_except_auto(), + supports_auto=True, + ), } @@ -562,6 +567,21 @@ class TranscribeConfig: whisper_api_base: Optional[str] = None whisper_api_model: Optional[str] = None whisper_api_prompt: Optional[str] = None + # Qwen ASR 配置 + qwen_asr_backend: str = "transformers" + qwen_asr_model: str = "Qwen/Qwen3-ASR-0.6B" + qwen_asr_aligner_model: str = "Qwen/Qwen3-ForcedAligner-0.6B" + qwen_asr_api_base: Optional[str] = None + qwen_asr_api_key: Optional[str] = None + qwen_asr_prompt: Optional[str] = None + qwen_asr_word_timestamp: bool = True + qwen_asr_max_new_tokens: int = 1024 + qwen_asr_timestamp_mode: str = "forced_aligner_word" + qwen_asr_compute_dtype: str = "bfloat16" + qwen_asr_language_mode: str = "auto" + qwen_asr_force_language: str = "" + qwen_asr_timestamp_rounding: str = "round" + qwen_asr_vocal_separation: bool = False # Faster Whisper 配置 faster_whisper_program: Optional[str] = None faster_whisper_model: Optional[FasterWhisperModelEnum] = None @@ -598,6 +618,23 @@ def print_config(self) -> str: lines.append(f"API Model: {self.whisper_api_model}") if self.whisper_api_prompt: lines.append(f"Prompt: {self.whisper_api_prompt[:30]}...") + elif self.transcribe_model == TranscribeModelEnum.QWEN_ASR: + lines.append(f"Backend: {self.qwen_asr_backend}") + lines.append(f"Model: {self.qwen_asr_model}") + lines.append(f"Aligner: {self.qwen_asr_aligner_model}") + if self.qwen_asr_backend == "vllm": + lines.append(f"API Base: {self.qwen_asr_api_base}") + lines.append(f"API Key: {self._mask_key(self.qwen_asr_api_key)}") + lines.append(f"Word Timestamp: {self.qwen_asr_word_timestamp}") + lines.append(f"Max New Tokens: {self.qwen_asr_max_new_tokens}") + lines.append(f"Timestamp Mode: {self.qwen_asr_timestamp_mode}") + lines.append(f"Compute DType: {self.qwen_asr_compute_dtype}") + lines.append(f"Language Mode: {self.qwen_asr_language_mode}") + lines.append( + f"Force Language: {self.qwen_asr_force_language or 'Auto'}" + ) + lines.append(f"Timestamp Rounding: {self.qwen_asr_timestamp_rounding}") + lines.append(f"Vocal Separation: {self.qwen_asr_vocal_separation}") elif self.transcribe_model == TranscribeModelEnum.FASTER_WHISPER: lines.append( diff --git a/app/core/task_factory.py b/app/core/task_factory.py index 76100a57..39cfd00f 100644 --- a/app/core/task_factory.py +++ b/app/core/task_factory.py @@ -13,6 +13,7 @@ SynthesisConfig, SynthesisTask, TranscribeConfig, + TranscribeModelEnum, TranscribeTask, TranscriptAndSubtitleTask, ) @@ -57,7 +58,12 @@ def create_transcribe_task( # 构建输出路径 if need_next_task: - need_word_time_stamp = cfg.need_split.value + if cfg.transcribe_model.value == TranscribeModelEnum.QWEN_ASR: + need_word_time_stamp = ( + cfg.need_split.value and cfg.qwen_asr_word_timestamp.value + ) + else: + need_word_time_stamp = cfg.need_split.value output_path = str( Path(cfg.work_dir.value) / file_name @@ -65,7 +71,11 @@ def create_transcribe_task( / f"【原始字幕】{file_name}-{cfg.transcribe_model.value.value}-{cfg.transcribe_language.value.value}.srt" ) else: - need_word_time_stamp = False + need_word_time_stamp = ( + cfg.qwen_asr_word_timestamp.value + if cfg.transcribe_model.value == TranscribeModelEnum.QWEN_ASR + else False + ) output_path = str(Path(file_path).parent / f"{file_name}.srt") config = TranscribeConfig( @@ -80,6 +90,21 @@ def create_transcribe_task( whisper_api_base=cfg.whisper_api_base.value, whisper_api_model=cfg.whisper_api_model.value, whisper_api_prompt=cfg.whisper_api_prompt.value, + # Qwen ASR 配置 + qwen_asr_backend=cfg.qwen_asr_backend.value, + qwen_asr_model=cfg.qwen_asr_model.value, + qwen_asr_aligner_model=cfg.qwen_asr_aligner_model.value, + qwen_asr_api_base=cfg.qwen_asr_api_base.value, + qwen_asr_api_key=cfg.qwen_asr_api_key.value, + qwen_asr_prompt=cfg.qwen_asr_prompt.value, + qwen_asr_word_timestamp=cfg.qwen_asr_word_timestamp.value, + qwen_asr_max_new_tokens=cfg.qwen_asr_max_new_tokens.value, + qwen_asr_timestamp_mode=cfg.qwen_asr_timestamp_mode.value, + qwen_asr_compute_dtype=cfg.qwen_asr_compute_dtype.value, + qwen_asr_language_mode=cfg.qwen_asr_language_mode.value, + qwen_asr_force_language=LANGUAGES[cfg.qwen_asr_force_language.value.value], + qwen_asr_timestamp_rounding=cfg.qwen_asr_timestamp_rounding.value, + qwen_asr_vocal_separation=cfg.qwen_asr_vocal_separation.value, # Faster Whisper 配置 faster_whisper_program=cfg.faster_whisper_program.value, faster_whisper_model=cfg.faster_whisper_model.value, diff --git a/app/thread/transcript_thread.py b/app/thread/transcript_thread.py index b3b089a4..444f809c 100644 --- a/app/thread/transcript_thread.py +++ b/app/thread/transcript_thread.py @@ -1,13 +1,14 @@ import datetime +import shutil import tempfile from pathlib import Path from PyQt5.QtCore import QThread, pyqtSignal from app.core.asr import transcribe -from app.core.entities import TranscribeOutputFormatEnum, TranscribeTask +from app.core.entities import TranscribeModelEnum, TranscribeOutputFormatEnum, TranscribeTask from app.core.utils.logger import setup_logger -from app.core.utils.video_utils import video2audio +from app.core.utils.video_utils import separate_vocals_with_demucs, video2audio logger = setup_logger("transcript_thread") @@ -90,6 +91,8 @@ def _perform_transcription(self): temp_audio_file = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) temp_audio_path = temp_audio_file.name temp_audio_file.close() # 立即关闭文件句柄，让 ffmpeg 可以写入 + demucs_temp_dir = "" + asr_audio_path = temp_audio_path try: # 转换音频文件 @@ -104,12 +107,22 @@ def _perform_transcription(self): logger.error("音频转换失败") raise RuntimeError(self.tr("音频转换失败")) + if ( + self.task.transcribe_config.transcribe_model == TranscribeModelEnum.QWEN_ASR + and self.task.transcribe_config.qwen_asr_vocal_separation + ): + self.progress.emit(12, self.tr("Qwen 人声分离中")) + logger.info("Qwen ASR 启用人声分离（demucs）") + asr_audio_path, demucs_temp_dir = separate_vocals_with_demucs( + temp_audio_path + ) + self.progress.emit(20, self.tr("语音转录中")) logger.info("开始语音转录") # 进行转录 asr_data = transcribe( - temp_audio_path, + asr_audio_path, self.task.transcribe_config, callback=self.progress_callback, ) @@ -143,6 +156,8 @@ def _perform_transcription(self): self.finished.emit(self.task) finally: Path(temp_audio_path).unlink(missing_ok=True) + if demucs_temp_dir: + shutil.rmtree(demucs_temp_dir, ignore_errors=True) def progress_callback(self, value, message): progress = min(20 + (value * 0.8), 100) diff --git a/app/view/setting_interface.py b/app/view/setting_interface.py index 0c3a2048..cca43dd8 100644 --- a/app/view/setting_interface.py +++ b/app/view/setting_interface.py @@ -116,6 +116,30 @@ def __initCards(self): texts=[lang.value for lang in cfg.target_language.validator.options], # type: ignore parent=self.translateGroup, ) + self.splitModelCard = LineEditSettingCard( + cfg.split_model, + FIF.ALIGNMENT, + self.tr("断句模型"), + self.tr("字幕断句阶段使用的模型，留空则使用 LLM 主模型"), + self.tr("例如: gpt-5-mini / deepseek-chat"), + self.translateGroup, + ) + self.optimizeModelCard = LineEditSettingCard( + cfg.optimize_model, + FIF.EDIT, + self.tr("优化模型"), + self.tr("字幕优化阶段使用的模型，留空则使用 LLM 主模型"), + self.tr("例如: gpt-5-mini / deepseek-v3"), + self.translateGroup, + ) + self.translateModelCard = LineEditSettingCard( + cfg.translate_model, + FIF.LANGUAGE, + self.tr("翻译模型"), + self.tr("LLM 翻译阶段使用的模型，留空则使用 LLM 主模型"), + self.tr("例如: gemini-2.0-flash / gpt-4o-mini"), + self.translateGroup, + ) # 字幕合成配置卡片 self.subtitleStyleCard = HyperlinkCard( @@ -239,6 +263,9 @@ def __initCards(self): self.translateGroup.addSettingCard(self.subtitleCorrectCard) self.translateGroup.addSettingCard(self.subtitleTranslateCard) self.translateGroup.addSettingCard(self.targetLanguageCard) + self.translateGroup.addSettingCard(self.splitModelCard) + self.translateGroup.addSettingCard(self.optimizeModelCard) + self.translateGroup.addSettingCard(self.translateModelCard) self.subtitleGroup.addSettingCard(self.subtitleStyleCard) self.subtitleGroup.addSettingCard(self.subtitleLayoutCard) diff --git a/docs/config/asr.md b/docs/config/asr.md index f955c72a..723b119d 100644 --- a/docs/config/asr.md +++ b/docs/config/asr.md @@ -9,15 +9,30 @@ | **FasterWhisper** | 准确度高，支持GPU | 推荐使用 | | **WhisperCpp** | 轻量级 | CPU环境 | | **Whisper API** | 云端服务 | 无需本地模型 | +| **Qwen-ASR** | 开源本地/服务化，支持词级时间戳 | 中文、多语、可调 max token | | **B接口/J接口** | 免费在线 | 快速测试 | -## 模型下载 +## Qwen-ASR 使用说明 -待补充... +1. 在「转录模型」选择 `Qwen-ASR ✨`。 +2. 在设置卡中选择后端： +- `transformers`：本地推理（默认） +- `vllm`：服务化推理 +3. 选择或手动填写 `ASR 模型`（默认 `Qwen/Qwen3-ASR-0.6B`）。 +4. 如需词级时间戳，开启 `词级时间戳` 并设置 `对齐器模型`（默认 `Qwen/Qwen3-ForcedAligner-0.6B`）。 +5. 通过 `Max New Tokens` 调整生成上限。 + +> 依赖安装：`pip install qwen-asr` ## 配置参数 -待补充... +| 参数 | 说明 | 默认值 | +|------|------|--------| +| 推理后端 | `transformers` / `vllm` | `transformers` | +| ASR 模型 | 主识别模型 | `Qwen/Qwen3-ASR-0.6B` | +| 对齐器模型 | 词级时间戳模型 | `Qwen/Qwen3-ForcedAligner-0.6B` | +| 词级时间戳 | 是否输出词/字级时间戳 | 开启 | +| Max New Tokens | 生成 token 上限 | `1024` | --- diff --git a/pyproject.toml b/pyproject.toml index 349a4608..bd28857f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,9 @@ dependencies = [ "GPUtil>=1.4.0", "pillow>=12.0.0", "fonttools>=4.61.1", + "qwen-asr", + "demucs>=4.0.1", + "torchcodec>=0.8.0", ] [project.urls] diff --git a/tests/test_asr/test_qwen_asr.py b/tests/test_asr/test_qwen_asr.py new file mode 100644 index 00000000..140d8612 --- /dev/null +++ b/tests/test_asr/test_qwen_asr.py @@ -0,0 +1,191 @@ +from pathlib import Path + +from app.core.asr.chunked_asr import ChunkedASR +from app.core.asr.qwen_asr import QwenASR +from app.core.asr.transcribe import _create_asr_instance +from app.core.entities import TranscribeConfig, TranscribeModelEnum + + +def test_qwen_asr_make_segments_word_level(): + asr = QwenASR( + audio_input=b"dummy-audio-bytes", + model_name="Qwen/Qwen3-ASR-0.6B", + need_word_time_stamp=True, + ) + + resp = { + "text": "你好世界", + "time_stamps": [ + {"text": "你好", "start": 0.0, "end": 0.4}, + {"text": "世界", "start": 0.4, "end": 0.9}, + ], + } + segments = asr._make_segments(resp) + + assert len(segments) == 2 + assert segments[0].text == "你好" + assert segments[0].start_time == 0 + assert segments[0].end_time == 400 + + +def test_qwen_asr_make_segments_sentence_level_fallback(): + asr = QwenASR( + audio_input=b"dummy-audio-bytes", + model_name="Qwen/Qwen3-ASR-0.6B", + need_word_time_stamp=False, + ) + + resp = { + "text": "这是一段句子级转录。", + "time_stamps": [{"start": 0.0, "end": 1.5}], + } + segments = asr._make_segments(resp) + + assert len(segments) == 1 + assert segments[0].text == "这是一段句子级转录。" + assert segments[0].start_time == 0 + assert segments[0].end_time == 1500 + + +def test_qwen_asr_make_segments_vllm_verbose_json_word_level(): + asr = QwenASR( + audio_input=b"dummy-audio-bytes", + model_name="Qwen/Qwen3-ASR-0.6B", + backend="vllm", + need_word_time_stamp=True, + ) + + resp = { + "text": "hello world", + "words": [ + {"word": "hello", "start": 0.0, "end": 0.5}, + {"word": "world", "start": 0.5, "end": 1.0}, + ], + } + segments = asr._make_segments(resp) + + assert len(segments) == 2 + assert segments[1].text == "world" + assert segments[1].start_time == 500 + assert segments[1].end_time == 1000 + + +def test_qwen_asr_normalize_result_legacy_start_end_fields(): + class LegacyTimestamp: + def __init__(self, text: str, start: float, end: float): + self.text = text + self.start = start + self.end = end + + class LegacyResult: + def __init__(self): + self.time_stamps = [ + LegacyTimestamp("今日", 0.0, 0.5), + LegacyTimestamp("のストーリー", 0.5, 1.2), + ] + + asr = QwenASR( + audio_input=b"dummy-audio-bytes", + model_name="Qwen/Qwen3-ASR-0.6B", + need_word_time_stamp=True, + ) + + normalized = asr._normalize_result(LegacyResult()) + segments = asr._make_segments(normalized) + + assert len(segments) == 2 + assert segments[0].start_time == 0 + assert segments[0].end_time == 500 + assert segments[1].start_time == 500 + assert segments[1].end_time == 1200 + + +def test_qwen_asr_make_segments_all_zero_timestamps_triggers_synthetic_timeline(): + asr = QwenASR( + audio_input=b"dummy-audio-bytes", + model_name="Qwen/Qwen3-ASR-0.6B", + need_word_time_stamp=True, + ) + + resp = { + "words": [ + {"word": "a", "start": 0.0, "end": 0.0}, + {"word": "b", "start": 0.0, "end": 0.0}, + ] + } + segments = asr._make_segments(resp) + + assert len(segments) == 2 + assert segments[0].start_time == 0 + assert segments[0].end_time > 0 + assert segments[1].start_time >= segments[0].end_time + + +def test_qwen_asr_make_segments_non_monotonic_but_non_zero_keeps_original_timestamps(): + asr = QwenASR( + audio_input=b"dummy-audio-bytes", + model_name="Qwen/Qwen3-ASR-0.6B", + need_word_time_stamp=True, + ) + + resp = { + "words": [ + {"word": "a", "start": 0.0, "end": 0.2}, + {"word": "b", "start": 0.15, "end": 0.4}, + ] + } + segments = asr._make_segments(resp) + + assert len(segments) == 2 + assert segments[0].start_time == 0 + assert segments[0].end_time == 200 + assert segments[1].start_time == 150 + assert segments[1].end_time == 400 + + +def test_qwen_asr_make_segments_partial_zero_timestamps_are_repaired(): + asr = QwenASR( + audio_input=b"dummy-audio-bytes", + model_name="Qwen/Qwen3-ASR-0.6B", + need_word_time_stamp=True, + ) + + resp = { + "words": [ + {"word": "今", "start": 0.0, "end": 0.0}, + {"word": "日", "start": 0.0, "end": 0.0}, + {"word": "は", "start": 8.0, "end": 8.5}, + {"word": "晴", "start": 9.0, "end": 9.5}, + ] + } + segments = asr._make_segments(resp) + + assert len(segments) == 4 + assert segments[0].end_time > 0 + assert segments[1].end_time > segments[1].start_time + assert segments[2].start_time == 8000 + assert segments[2].end_time == 8500 + assert segments[3].start_time == 9000 + assert segments[3].end_time == 9500 + + +def test_create_asr_instance_qwen_model(): + audio_path = Path(__file__).resolve().parent.parent / "fixtures" / "audio" / "en.mp3" + assert audio_path.exists() + + config = TranscribeConfig( + transcribe_model=TranscribeModelEnum.QWEN_ASR, + transcribe_language="zh", + need_word_time_stamp=True, + qwen_asr_model="Qwen/Qwen3-ASR-0.6B", + qwen_asr_aligner_model="Qwen/Qwen3-ForcedAligner-0.6B", + qwen_asr_backend="transformers", + qwen_asr_max_new_tokens=1024, + ) + + asr = _create_asr_instance(str(audio_path), config) + + assert isinstance(asr, ChunkedASR) + assert asr.asr_class is QwenASR + assert asr.asr_kwargs["model_name"] == "Qwen/Qwen3-ASR-0.6B" + assert asr.asr_kwargs["need_word_time_stamp"] is True