diff --git a/rl-tutorial/cosyvoice_llm/Dockerfile b/rl-tutorial/cosyvoice_llm/Dockerfile new file mode 100644 index 0000000..0585c20 --- /dev/null +++ b/rl-tutorial/cosyvoice_llm/Dockerfile @@ -0,0 +1,6 @@ +FROM verlai/verl:app-verl0.4-vllm0.8.5-mcore0.12.2-te2.2 +COPY requirements-cosyvoice.txt /myworkspace/requirements.txt +RUN pip install -r /myworkspace/requirements.txt +RUN pip install -U nvidia-pytriton +RUN git clone https://github.com/yuekaizhang/verl.git /myworkspace/verl -b thread && cd /myworkspace/verl && pip install --no-deps -e . +RUN git clone https://github.com/yuekaizhang/PytritonSenseVoice.git /myworkspace/PytritonSenseVoice && cd /myworkspace/PytritonSenseVoice && pip install -e . \ No newline at end of file diff --git a/rl-tutorial/cosyvoice_llm/README.md b/rl-tutorial/cosyvoice_llm/README.md index 41cce1c..1990358 100644 --- a/rl-tutorial/cosyvoice_llm/README.md +++ b/rl-tutorial/cosyvoice_llm/README.md @@ -132,9 +132,10 @@ The script synthesizes a tongue-twister using the merged checkpoint and prints t | Model | Seed-TTS `test_zh` CER | Cosyvoice3 `zero_shot_zh` |Comment | |-|------------------------------------------------------|------------------------|--------------------------------------------------------------------------------| | Official CosyVoice2 LLM | 1.45 % |4.08%| See the [paper](https://arxiv.org/abs/2412.10117) | -| SFT (initialized from Qwen2-0.5B-Instruct) | 1.81 % |4.83%| See [PR #1887](https://github.com/k2-fsa/icefall/pull/1887) | -| GRPO (this work, trained on AIShell-3) | **1.06 %** |4.03%| | - +| SFT (initialized from Qwen2-0.5B-Instruct) | 1.70 % |4.26%| See [PR #1887](https://github.com/k2-fsa/icefall/pull/1887) | +| GRPO (this work, trained on AIShell-3) | 1.06 % |3.01%| [Commit](https://github.com/nvidia-china-sae/mair-hub/commit/5659ee4d128d5902f2f1a2abb333bdd2e387268d) | +| GRPO (this work, trained on emilia_zh subset, using top_p=1, temperature=1.0) | 0.87% |2.63% (1800 steps)| | +| DAPO (this work, trained on emilia_zh subset, using top_p=1, temperature=1.0) | 0.83% |2.71% (700 steps)| See `run_dapo.sh`| ## Acknowledgement This work is inspired by the implementation in diff --git a/rl-tutorial/cosyvoice_llm/filter_data.py b/rl-tutorial/cosyvoice_llm/filter_data.py new file mode 100644 index 0000000..4499c38 --- /dev/null +++ b/rl-tutorial/cosyvoice_llm/filter_data.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from tqdm import tqdm +import re + +def load_jsonl(file_path: str): + """Load data from jsonl file.""" + data = [] + count = 0 + with open(file_path, 'r', encoding='utf-8') as f: + for line in tqdm(f): + item = json.loads(line.strip()) + if item["language"] == "zh": + # check if there is any english in the text + if item["duration"] < 30: + count += 1 + item["text"] = item["text"].lower() + if re.search(r'[a-z]', item["text"]): + print(item["text"]) + continue + else: + data.append(item) + if count > 80000: + break + print(f"Total data: {len(data)}") + return data + +if __name__ == "__main__": + jsonl_file = "data/emilia_zh.jsonl" + data = load_jsonl(jsonl_file) + with open(f"./data/{jsonl_file.split('/')[-1].split('.')[0]}-zh-filtered.jsonl", "w", encoding="utf-8") as f: + for item in data: + f.write(json.dumps(item, ensure_ascii=False) + "\n") \ No newline at end of file diff --git a/rl-tutorial/cosyvoice_llm/infer_dataset.py b/rl-tutorial/cosyvoice_llm/infer_dataset.py index 40c968d..6864c29 100644 --- a/rl-tutorial/cosyvoice_llm/infer_dataset.py +++ b/rl-tutorial/cosyvoice_llm/infer_dataset.py @@ -52,10 +52,6 @@ except RuntimeError: pass - -TEMPLATE = "{% for message in messages %}{%- if message['role'] == 'user' %}{{- '<|im_start|>' + message['role'] + '\n' + 'Convert the text to speech: ' + message['content'] + '<|im_end|>\n'}}{%- elif message['role'] == 'assistant' %}{{- '<|im_start|>' + message['role'] + '\n' + '<|SPEECH_GENERATION_START|>' + message['content']}}{%- endif %}{%- endfor %}" - - def audio_decode_cosyvoice2( audio_tokens, prompt_text, prompt_speech_16k, codec_decoder ): @@ -197,6 +193,12 @@ def data_collator(batch, tokenizer, s3_tokenizer): prompt_text_list.append(prompt_text) # Combine prompt and target text full_text = prompt_text + target_text + # remove the unnecessary punctuation for cosyvoice3 zero_shot_zh dataset + puncts = ['"', '(', ')', '“', '”', '‘', '（', '）', '\''] + for p in puncts: + if p in full_text: + full_text = full_text.replace(p, '') + print(f"removed {p} from {full_text}") # get prompt audio for CosyVoice2 (convert to 16kHz) ref_audio_org, ref_sr = ( @@ -234,8 +236,9 @@ def data_collator(batch, tokenizer, s3_tokenizer): {"role": "user", "content": full_text}, {"role": "assistant", "content": prompt_audio_cosy2_id_str} ] - if 'system' in tokenizer.chat_template: - tokenizer.chat_template = TEMPLATE + + assert 'system' not in tokenizer.chat_template, "system is not allowed in the chat template" + input_ids = tokenizer.apply_chat_template( chat, tokenize=True, @@ -301,7 +304,7 @@ def main(): prompt_speech_16k = load_wav(args.prompt_speech_path, 16000) else: prompt_speech_16k = None - s3_tokenizer = s3tokenizer.load_model("speech_tokenizer_v2_25hz").to(device) if 'zero' in args.split_name else None + s3_tokenizer = s3tokenizer.load_model(f"{args.token2wav_path}/speech_tokenizer_v2.onnx").to(device) if 'zero' in args.split_name else None dataset_name = "yuekai/CV3-Eval" if 'zero' in args.split_name else "yuekai/seed_tts_cosy2" dataset = load_dataset( dataset_name, diff --git a/rl-tutorial/cosyvoice_llm/prepare_data.py b/rl-tutorial/cosyvoice_llm/prepare_data.py index f6176c3..b7a51c6 100644 --- a/rl-tutorial/cosyvoice_llm/prepare_data.py +++ b/rl-tutorial/cosyvoice_llm/prepare_data.py @@ -23,7 +23,12 @@ from verl.utils.hdfs_io import copy, makedirs +from typing import List +import random +def code_to_solution_str(code_list: List[int]) -> str: + """Convert code list to solution string format.""" + return ''.join([f"<|s_{code}|>" for code in code_list]) if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -31,7 +36,7 @@ parser.add_argument("--test_file", required=True, help="Path to test JSON/JSONL file") parser.add_argument("--local_dir", default=None, required=True) parser.add_argument("--hdfs_dir", default=None) - parser.add_argument("--use_custom_template", action="store_true", help="Use custom template for training") + parser.add_argument("--use_speech_prefix", action="store_true", help="Use speech prefix") args = parser.parse_args() @@ -40,17 +45,21 @@ test_dataset = datasets.load_dataset("json", data_files=args.test_file)['train'] # add a row to each data item that represents a unique id - def make_map_fn(split, use_custom_template=False): + def make_map_fn(split): def process_fn(example, idx): text = example.pop("text") - if use_custom_template: - question = f"Convert the text to speech: {text}" - answer = "<|SPEECH_GENERATION_START|>" - print(f"use custom template for {split} {idx}") - else: - # use cosyvoice2 official huggingface compatible checkpoint template - question = text - answer = "" + + # use cosyvoice2 official huggingface compatible checkpoint template + question = text + answer = "" + # generate a random float between 0 and 1, then convert it to 0 to 5 + random_number = random.random() * 4 + 1 + speech_token_len = int(random_number * 25) + codes = example.pop("code") + prefix_speech_token = codes[:speech_token_len] + prefix_speech_str = code_to_solution_str(prefix_speech_token) + + answer = prefix_speech_str if args.use_speech_prefix else "" data = { "data_source": f"{args.train_file}_{args.test_file}", # Use file names as data source @@ -72,12 +81,15 @@ def process_fn(example, idx): "text": text, }, } + if args.use_speech_prefix: + data["extra_info"]["prefix_speech_str"] = prefix_speech_str + return data return process_fn - train_dataset = train_dataset.map(function=make_map_fn("train", use_custom_template=args.use_custom_template), with_indices=True) - test_dataset = test_dataset.map(function=make_map_fn("test", use_custom_template=args.use_custom_template), with_indices=True) + train_dataset = train_dataset.map(function=make_map_fn("train"), with_indices=True) + test_dataset = test_dataset.map(function=make_map_fn("test"), with_indices=True) local_dir = args.local_dir hdfs_dir = args.hdfs_dir diff --git a/rl-tutorial/cosyvoice_llm/requirements-cosyvoice.txt b/rl-tutorial/cosyvoice_llm/requirements-cosyvoice.txt index 6cbeffc..50f4edd 100644 --- a/rl-tutorial/cosyvoice_llm/requirements-cosyvoice.txt +++ b/rl-tutorial/cosyvoice_llm/requirements-cosyvoice.txt @@ -21,3 +21,11 @@ soundfile==0.12.1 tensorboard==2.14.0 wget==3.2 WeTextProcessing==1.0.3 +s3tokenizer +tensorrt +sherpa_onnx +jiwer +zhon +numpy==1.25.2 +pypinyin +openai-whisper \ No newline at end of file diff --git a/rl-tutorial/cosyvoice_llm/reward_tts.py b/rl-tutorial/cosyvoice_llm/reward_tts.py index f49dc6d..e4ae416 100644 --- a/rl-tutorial/cosyvoice_llm/reward_tts.py +++ b/rl-tutorial/cosyvoice_llm/reward_tts.py @@ -94,8 +94,11 @@ def compute_score( """ # Decode token IDs - ids = _parse_ids(solution_str) - + if "prefix_speech_str" in extra_info: + prefix_speech_str = extra_info["prefix_speech_str"] + ids = _parse_ids(prefix_speech_str + solution_str) + else: + ids = _parse_ids(solution_str) # Query remote server for reward try: reward = _remote_reward(ids, ground_truth) @@ -107,7 +110,7 @@ def compute_score( print( f"\033[92m[{data_source}] Remote reward: {reward:.4f}\033[0m" ) - + reward = {"score": reward} return reward # CLI quick test diff --git a/rl-tutorial/cosyvoice_llm/run.sh b/rl-tutorial/cosyvoice_llm/run.sh index 9422a18..ffecf8f 100644 --- a/rl-tutorial/cosyvoice_llm/run.sh +++ b/rl-tutorial/cosyvoice_llm/run.sh @@ -19,7 +19,9 @@ if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then # install verl git clone https://github.com/volcengine/verl.git cd verl - USE_MEGATRON=0 bash scripts/install_vllm_sglang_mcore.sh + USE_MEGATRON=0 USE_SGLANG=0 bash scripts/install_vllm_sglang_mcore.sh + # manually install flash attn above 2.7.4post1 + pip install -r requirements.txt pip install --no-deps -e . # install CosyVoice @@ -50,21 +52,25 @@ if [ $stage -le -1 ] && [ $stop_stage -ge -1 ]; then # If you would like to use the official CosyVoice2-0.5B LLM and do RL training, please see run_official.sh fi - +data_dir=data/parquet_emilia_zh_en_removed if [ $stage -le 0 ] && [ $stop_stage -ge 0 ]; then log "stage 0: prepare data into verl format" # yuekai/llasa_cosyvoice2_token_qwen_0.5b is the emilia zh trained model, please set use_custom_template=True to use the custom template # yuekai/cosyvoice2_llm is the official cosyvoice2 llm model, please set use_custom_template=False to use the official template + mkdir -p $data_dir + wget https://huggingface.co/datasets/SparkAudio/voxbox/resolve/main/metadata/emilia_zh.jsonl -O data/emilia_zh.jsonl + tail -n 100 data/emilia_zh.jsonl > data/emilia_test.jsonl + python3 filter_data.py python prepare_data.py \ - --train_file data/emilia_zh-cosy-tiny-train.jsonl \ - --test_file data/emilia_zh-cosy-tiny-test.jsonl \ - --local_dir data/parquet_tiny \ - --use_custom_template + --train_file data/emilia_zh-cosy-zh-filtered.jsonl \ + --test_file data/emilia_test.jsonl \ + --local_dir $data_dir fi +n_gpus=8 if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then log "stage 1: start token2wav asr server for reward function" - python3 token2wav_asr_server.py --number-of-devices 8 + python3 token2wav_asr_server.py --number-of-devices $n_gpus # log "Test the reward server" # python3 reward_tts.py \ @@ -76,18 +82,19 @@ if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then fi sft_model_path=/workspace/rl/llasa_cosyvoice2_token_qwen_0.5b/checkpoint-885000 - +exp_name=emilia_zh if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then log "stage 2: grpo train" + # wandb login export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" export MKL_SERVICE_FORCE_INTEL=TRUE - n_gpus_per_node=8 + n_gpus_per_node=$n_gpus micro_batch_size=4 train_batch_size=32 python3 -m verl.trainer.main_ppo \ algorithm.adv_estimator=grpo \ - data.train_files=data/parquet_aishell3_custom/train.parquet \ - data.val_files=data/parquet_aishell3_custom/test.parquet \ + data.train_files=$data_dir/train.parquet \ + data.val_files=$data_dir/test.parquet \ data.train_batch_size=$train_batch_size \ data.max_prompt_length=1024 \ data.max_response_length=1024 \ @@ -106,29 +113,34 @@ if [ $stage -le 2 ] && [ $stop_stage -ge 2 ]; then actor_rollout_ref.rollout.name=vllm \ actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \ actor_rollout_ref.rollout.do_sample=true \ - actor_rollout_ref.rollout.temperature=0.8 \ - actor_rollout_ref.rollout.top_p=0.9 \ - actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.top_p=1.0 \ + actor_rollout_ref.rollout.top_k=-1 \ + actor_rollout_ref.rollout.n=8 \ actor_rollout_ref.rollout.val_kwargs.do_sample=true \ - actor_rollout_ref.rollout.val_kwargs.temperature=0.8 \ - actor_rollout_ref.rollout.val_kwargs.top_p=0.9 \ + actor_rollout_ref.rollout.val_kwargs.temperature=1.0 \ + actor_rollout_ref.rollout.val_kwargs.top_p=1.0 \ + actor_rollout_ref.rollout.val_kwargs.top_k=-1 \ reward_model.reward_manager=prime \ custom_reward_function.path=reward_tts.py \ custom_reward_function.name=compute_score \ trainer.project_name='llasa_tts_grpo' \ - trainer.experiment_name='aishell3_reward_tts_prime_test' \ + trainer.experiment_name=${exp_name} \ trainer.logger=['console','wandb'] \ trainer.n_gpus_per_node=$n_gpus_per_node \ trainer.nnodes=1 \ - trainer.save_freq=100 \ - trainer.test_freq=400 \ + trainer.save_freq=50 \ + trainer.test_freq=50 \ trainer.resume_mode='auto' \ trainer.total_epochs=1 \ - trainer.val_before_train=False + trainer.val_before_train=False \ + algorithm.norm_adv_by_std_in_grpo=False fi -step=2100 -llm_path=./checkpoints/llasa_tts_grpo/aishell3_reward_tts_prime/global_step_${step} +steps=(1300 600 700 800) + +for step in ${steps[@]}; do +llm_path=./checkpoints/llasa_tts_grpo/${exp_name}/global_step_${step} if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then log "stage 3: merge the model" python -m verl.model_merger merge \ @@ -136,24 +148,26 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then --local_dir $llm_path/actor \ --target_dir $llm_path/merged_hf_model || exit 1 -fi +fi + +token2wav_path=/workspace/CosyVoice2-0.5B +model_path=$llm_path/merged_hf_model if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then log "stage 4: Test the model" - dataset=zero_shot_zh - output_dir=./outputs_rl_aishell3_step${step}_${dataset}_jit_trt_fp16_reward_tts - token2wav_path=/workspace/CosyVoice2-0.5B - model_path=$llm_path/merged_hf_model - + datasets=(zero_shot_zh test_zh) + for dataset in ${datasets[@]}; do + output_dir=./outputs_rl_${exp_name}_step${step}_${dataset} CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ - torchrun --nproc_per_node=8 \ + torchrun --nproc_per_node=$n_gpus \ infer_dataset.py \ --output-dir $output_dir \ --llm-model-name-or-path $model_path \ --token2wav-path $token2wav_path \ --split-name ${dataset} || exit 1 - bash scripts/compute_wer.sh $output_dir ${dataset} + done fi +done if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then log "stage 5: Infer with single case" @@ -163,4 +177,4 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then --prompt-speech-path ./assets/prompt_audio.wav \ --model-path $llm_path/merged_hf_model \ --input-text "扁担长，板凳宽，扁担绑在板凳上。吃葡萄不吐葡萄皮，不吃葡萄倒吐葡萄皮。" -fi \ No newline at end of file +fi diff --git a/rl-tutorial/cosyvoice_llm/run_dapo.sh b/rl-tutorial/cosyvoice_llm/run_dapo.sh new file mode 100644 index 0000000..106cb24 --- /dev/null +++ b/rl-tutorial/cosyvoice_llm/run_dapo.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +set -xeuo pipefail + +stage=$1 +stop_stage=$2 + +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +project_name='llasa_tts_grpo' +exp_name='dapo_emilia_zh' + +adv_estimator=grpo + +use_kl_in_reward=False +kl_coef=0.0 +use_kl_loss=False +kl_loss_coef=0.0 + +clip_ratio_low=0.2 +clip_ratio_high=0.28 + +max_prompt_length=$((1024 * 2)) +max_response_length=$((1024 * 2)) +enable_overlong_buffer=False +overlong_buffer_len=512 +overlong_penalty_factor=1.0 + +loss_agg_mode="token-mean" + +enable_filter_groups=True +filter_groups_metric=score +max_num_gen_batches=10 +train_prompt_bsz=32 +gen_prompt_bsz=$((train_prompt_bsz * 3)) +train_prompt_mini_bsz=16 +n_resp_per_prompt=8 + +# Ray +RAY_ADDRESS="http://localhost:8265" + + +WORKING_DIR=/myworkspace +RUNTIME_ENV=${RUNTIME_ENV:-"${WORKING_DIR}/verl/verl/trainer/runtime_env.yaml"} +NNODES=1 +# Paths +RAY_DATA_HOME=${RAY_DATA_HOME:-"${WORKING_DIR}/verl"} +MODEL_PATH=/workspace/rl/llasa_cosyvoice2_token_qwen_0.5b/checkpoint-885000 +CKPTS_DIR=/workspace/slam/mair-hub/rl-tutorial/cosyvoice_llm/checkpoints/${project_name}/${exp_name} +TRAIN_FILE=/workspace/slam/mair-hub/rl-tutorial/cosyvoice_llm/data/parquet_emilia_zh_new/train.parquet +TEST_FILE=/workspace/slam/mair-hub/rl-tutorial/cosyvoice_llm/data/parquet_emilia_zh_new/test.parquet + + +# Algorithm +temperature=1.0 +top_p=1.0 +top_k=-1 # 0 for HF rollout, -1 for vLLM rollout + +# Mathematically equivalent +use_dynamic_bsz=True +infer_micro_batch_size=null +train_micro_batch_size=null +offload=False + +if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then + python3 -m recipe.dapo.main_dapo \ + data.train_files="${TRAIN_FILE}" \ + data.val_files="${TEST_FILE}" \ + data.prompt_key=prompt \ + data.truncation='left' \ + data.max_prompt_length=${max_prompt_length} \ + data.max_response_length=${max_response_length} \ + data.gen_batch_size=${gen_prompt_bsz} \ + data.train_batch_size=${train_prompt_bsz} \ + actor_rollout_ref.rollout.n=${n_resp_per_prompt} \ + actor_rollout_ref.actor.use_kl_loss=${use_kl_loss} \ + actor_rollout_ref.actor.kl_loss_coef=${kl_loss_coef} \ + actor_rollout_ref.actor.clip_ratio_low=${clip_ratio_low} \ + actor_rollout_ref.actor.clip_ratio_high=${clip_ratio_high} \ + actor_rollout_ref.actor.clip_ratio_c=10.0 \ + algorithm.adv_estimator=${adv_estimator} \ + algorithm.use_kl_in_reward=${use_kl_in_reward} \ + algorithm.kl_ctrl.kl_coef=${kl_coef} \ + algorithm.filter_groups.enable=${enable_filter_groups} \ + algorithm.filter_groups.metric=${filter_groups_metric} \ + algorithm.filter_groups.max_num_gen_batches=${max_num_gen_batches} \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \ + actor_rollout_ref.actor.ppo_max_token_len_per_gpu=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.model.path="${MODEL_PATH}" \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.actor.optim.lr_warmup_steps=10 \ + actor_rollout_ref.actor.optim.weight_decay=0.1 \ + actor_rollout_ref.actor.ppo_mini_batch_size=${train_prompt_mini_bsz} \ + actor_rollout_ref.actor.ppo_micro_batch_size=${train_micro_batch_size} \ + actor_rollout_ref.actor.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=${offload} \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.actor.grad_clip=1.0 \ + actor_rollout_ref.actor.loss_agg_mode=${loss_agg_mode} \ + actor_rollout_ref.actor.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.enable_chunked_prefill=True \ + actor_rollout_ref.rollout.max_num_batched_tokens=$((max_prompt_length + max_response_length)) \ + actor_rollout_ref.rollout.temperature=${temperature} \ + actor_rollout_ref.rollout.top_p=${top_p} \ + actor_rollout_ref.rollout.top_k="${top_k}" \ + actor_rollout_ref.rollout.val_kwargs.temperature=${temperature} \ + actor_rollout_ref.rollout.val_kwargs.top_p=${top_p} \ + actor_rollout_ref.rollout.val_kwargs.top_k=${top_k} \ + actor_rollout_ref.rollout.val_kwargs.do_sample=True \ + actor_rollout_ref.rollout.val_kwargs.n=1 \ + actor_rollout_ref.ref.log_prob_micro_batch_size=${infer_micro_batch_size} \ + actor_rollout_ref.ref.fsdp_config.param_offload=${offload} \ + actor_rollout_ref.ref.ulysses_sequence_parallel_size=1 \ + actor_rollout_ref.actor.fsdp_config.fsdp_size=-1 \ + reward_model.reward_manager=prime \ + custom_reward_function.path=reward_tts.py \ + custom_reward_function.name=compute_score \ + trainer.logger='["console","wandb"]' \ + trainer.project_name="${project_name}" \ + trainer.experiment_name="${exp_name}" \ + trainer.n_gpus_per_node=8 \ + trainer.nnodes="${NNODES}" \ + trainer.val_before_train=True \ + trainer.test_freq=50 \ + trainer.save_freq=50 \ + trainer.total_epochs=1 \ + trainer.default_local_dir="${CKPTS_DIR}" \ + trainer.resume_mode=disable \ + trainer.val_before_train=False +fi + +steps=(600 650 750) +export PYTHONPATH=/workspace/CosyVoice + +for step in ${steps[@]}; do +llm_path=./checkpoints/llasa_tts_grpo/${exp_name}/global_step_${step} +if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then + log "stage 3: merge the model" + python -m verl.model_merger merge \ + --backend fsdp \ + --local_dir $llm_path/actor \ + --target_dir $llm_path/merged_hf_model || exit 1 + +fi + +token2wav_path=/workspace/CosyVoice2-0.5B +model_path=$llm_path/merged_hf_model +if [ $stage -le 4 ] && [ $stop_stage -ge 4 ]; then + log "stage 4: Test the model" + datasets=(zero_shot_zh test_zh) + datasets=(zero_shot_zh) + datasets=(test_zh) + for dataset in ${datasets[@]}; do + output_dir=./outputs_rl_${exp_name}_step${step}_${dataset} + CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ + torchrun --nproc_per_node=8 \ + infer_dataset.py \ + --output-dir $output_dir \ + --llm-model-name-or-path $model_path \ + --token2wav-path $token2wav_path \ + --split-name ${dataset} || exit 1 + bash scripts/compute_wer.sh $output_dir ${dataset} + done +fi +done \ No newline at end of file diff --git a/rl-tutorial/cosyvoice_llm/scripts/compute-wer.py b/rl-tutorial/cosyvoice_llm/scripts/compute-wer.py new file mode 100755 index 0000000..bdf6290 --- /dev/null +++ b/rl-tutorial/cosyvoice_llm/scripts/compute-wer.py @@ -0,0 +1,556 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import re, sys, unicodedata +import codecs +import argparse +from collections import defaultdict +from typing import List, Tuple, Dict, TextIO +import logging +from pypinyin import lazy_pinyin, Style + + +# gt_pinyin = lazy_pinyin( +# gt_norm, +# style=Style.TONE3, +# tone_sandhi=True, +# neutral_tone_with_five=True, +# ) + + +remove_tag = True +spacelist = [' ', '\t', '\r', '\n'] +puncts = [ + '!', ',', '?', '、', '。', '！', '，', '；', '？', '：', '「', '」', '︰', '『', '』', + '《', '》' +] + + +def characterize(string): + res = [] + i = 0 + while i < len(string): + char = string[i] + if char in puncts: + i += 1 + continue + cat1 = unicodedata.category(char) + #https://unicodebook.readthedocs.io/unicode.html#unicode-categories + if cat1 == 'Zs' or cat1 == 'Cn' or char in spacelist: # space or not assigned + i += 1 + continue + if cat1 == 'Lo': # letter-other + res.append(char) + i += 1 + else: + # some input looks like: , we want to separate it to two words. + sep = ' ' + if char == '<': sep = '>' + j = i + 1 + while j < len(string): + c = string[j] + if ord(c) >= 128 or (c in spacelist) or (c == sep): + break + j += 1 + if j < len(string) and string[j] == '>': + j += 1 + res.append(string[i:j]) + i = j + return res + + +def stripoff_tags(x): + if not x: return '' + chars = [] + i = 0 + T = len(x) + while i < T: + if x[i] == '<': + while i < T and x[i] != '>': + i += 1 + i += 1 + else: + chars.append(x[i]) + i += 1 + return ''.join(chars) + + +def normalize(sentence, ignore_words, cs, split=None): + """ sentence, ignore_words are both in unicode + """ + new_sentence = [] + for token in sentence: + x = token + if not cs: + x = x.upper() + if x in ignore_words: + continue + if remove_tag: + x = stripoff_tags(x) + if not x: + continue + if split and x in split: + new_sentence += split[x] + else: + new_sentence.append(x) + return new_sentence + + +class Calculator: + + def __init__(self): + self.data = {} + self.space = [] + self.cost = {} + self.cost['cor'] = 0 + self.cost['sub'] = 1 + self.cost['del'] = 1 + self.cost['ins'] = 1 + + def calculate(self, lab, rec): + # Initialization + lab.insert(0, '') + rec.insert(0, '') + while len(self.space) < len(lab): + self.space.append([]) + for row in self.space: + for element in row: + element['dist'] = 0 + element['error'] = 'non' + while len(row) < len(rec): + row.append({'dist': 0, 'error': 'non'}) + for i in range(len(lab)): + self.space[i][0]['dist'] = i + self.space[i][0]['error'] = 'del' + for j in range(len(rec)): + self.space[0][j]['dist'] = j + self.space[0][j]['error'] = 'ins' + self.space[0][0]['error'] = 'non' + for token in lab: + if token not in self.data and len(token) > 0: + self.data[token] = { + 'all': 0, + 'cor': 0, + 'sub': 0, + 'ins': 0, + 'del': 0 + } + for token in rec: + if token not in self.data and len(token) > 0: + self.data[token] = { + 'all': 0, + 'cor': 0, + 'sub': 0, + 'ins': 0, + 'del': 0 + } + # Computing edit distance + for i, lab_token in enumerate(lab): + for j, rec_token in enumerate(rec): + if i == 0 or j == 0: + continue + min_dist = sys.maxsize + min_error = 'none' + dist = self.space[i - 1][j]['dist'] + self.cost['del'] + error = 'del' + if dist < min_dist: + min_dist = dist + min_error = error + dist = self.space[i][j - 1]['dist'] + self.cost['ins'] + error = 'ins' + if dist < min_dist: + min_dist = dist + min_error = error + if lab_token == rec_token: + dist = self.space[i - 1][j - 1]['dist'] + self.cost['cor'] + error = 'cor' + else: + dist = self.space[i - 1][j - 1]['dist'] + self.cost['sub'] + error = 'sub' + if dist < min_dist: + min_dist = dist + min_error = error + self.space[i][j]['dist'] = min_dist + self.space[i][j]['error'] = min_error + # Tracing back + result = { + 'lab': [], + 'rec': [], + 'all': 0, + 'cor': 0, + 'sub': 0, + 'ins': 0, + 'del': 0 + } + i = len(lab) - 1 + j = len(rec) - 1 + while True: + if self.space[i][j]['error'] == 'cor': # correct + if len(lab[i]) > 0: + self.data[lab[i]]['all'] = self.data[lab[i]]['all'] + 1 + self.data[lab[i]]['cor'] = self.data[lab[i]]['cor'] + 1 + result['all'] = result['all'] + 1 + result['cor'] = result['cor'] + 1 + result['lab'].insert(0, lab[i]) + result['rec'].insert(0, rec[j]) + i = i - 1 + j = j - 1 + elif self.space[i][j]['error'] == 'sub': # substitution + if len(lab[i]) > 0: + self.data[lab[i]]['all'] = self.data[lab[i]]['all'] + 1 + self.data[lab[i]]['sub'] = self.data[lab[i]]['sub'] + 1 + result['all'] = result['all'] + 1 + result['sub'] = result['sub'] + 1 + result['lab'].insert(0, lab[i]) + result['rec'].insert(0, rec[j]) + i = i - 1 + j = j - 1 + elif self.space[i][j]['error'] == 'del': # deletion + if len(lab[i]) > 0: + self.data[lab[i]]['all'] = self.data[lab[i]]['all'] + 1 + self.data[lab[i]]['del'] = self.data[lab[i]]['del'] + 1 + result['all'] = result['all'] + 1 + result['del'] = result['del'] + 1 + result['lab'].insert(0, lab[i]) + result['rec'].insert(0, "") + i = i - 1 + elif self.space[i][j]['error'] == 'ins': # insertion + if len(rec[j]) > 0: + self.data[rec[j]]['ins'] = self.data[rec[j]]['ins'] + 1 + result['ins'] = result['ins'] + 1 + result['lab'].insert(0, "") + result['rec'].insert(0, rec[j]) + j = j - 1 + elif self.space[i][j]['error'] == 'non': # starting point + break + else: # shouldn't reach here + print( + 'this should not happen , i = {i} , j = {j} , error = {error}' + .format(i=i, j=j, error=self.space[i][j]['error'])) + return result + + def overall(self): + result = {'all': 0, 'cor': 0, 'sub': 0, 'ins': 0, 'del': 0} + for token in self.data: + result['all'] = result['all'] + self.data[token]['all'] + result['cor'] = result['cor'] + self.data[token]['cor'] + result['sub'] = result['sub'] + self.data[token]['sub'] + result['ins'] = result['ins'] + self.data[token]['ins'] + result['del'] = result['del'] + self.data[token]['del'] + return result + + def cluster(self, data): + result = {'all': 0, 'cor': 0, 'sub': 0, 'ins': 0, 'del': 0} + for token in data: + if token in self.data: + result['all'] = result['all'] + self.data[token]['all'] + result['cor'] = result['cor'] + self.data[token]['cor'] + result['sub'] = result['sub'] + self.data[token]['sub'] + result['ins'] = result['ins'] + self.data[token]['ins'] + result['del'] = result['del'] + self.data[token]['del'] + return result + + def keys(self): + return list(self.data.keys()) + + +def width(string): + return sum(1 + (unicodedata.east_asian_width(c) in "AFW") for c in string) + + +def default_cluster(word): + unicode_names = [unicodedata.name(char) for char in word] + for i in reversed(range(len(unicode_names))): + if unicode_names[i].startswith('DIGIT'): # 1 + unicode_names[i] = 'Number' # 'DIGIT' + elif (unicode_names[i].startswith('CJK UNIFIED IDEOGRAPH') + or unicode_names[i].startswith('CJK COMPATIBILITY IDEOGRAPH')): + # 明 / 郎 + unicode_names[i] = 'Mandarin' # 'CJK IDEOGRAPH' + elif (unicode_names[i].startswith('LATIN CAPITAL LETTER') + or unicode_names[i].startswith('LATIN SMALL LETTER')): + # A / a + unicode_names[i] = 'English' # 'LATIN LETTER' + elif unicode_names[i].startswith('HIRAGANA LETTER'): # は こ め + unicode_names[i] = 'Japanese' # 'GANA LETTER' + elif (unicode_names[i].startswith('AMPERSAND') + or unicode_names[i].startswith('APOSTROPHE') + or unicode_names[i].startswith('COMMERCIAL AT') + or unicode_names[i].startswith('DEGREE CELSIUS') + or unicode_names[i].startswith('EQUALS SIGN') + or unicode_names[i].startswith('FULL STOP') + or unicode_names[i].startswith('HYPHEN-MINUS') + or unicode_names[i].startswith('LOW LINE') + or unicode_names[i].startswith('NUMBER SIGN') + or unicode_names[i].startswith('PLUS SIGN') + or unicode_names[i].startswith('SEMICOLON')): + # & / ' / @ / ℃ / = / . / - / _ / # / + / ; + del unicode_names[i] + else: + return 'Other' + if len(unicode_names) == 0: + return 'Other' + if len(unicode_names) == 1: + return unicode_names[0] + for i in range(len(unicode_names) - 1): + if unicode_names[i] != unicode_names[i + 1]: + return 'Other' + return unicode_names[0] + + +def usage(): + print( + "compute-wer.py : compute word error rate (WER) and align recognition results and references." + ) + print( + " usage : python compute-wer.py [--cs={0,1}] [--cluster=foo] [--ig=ignore_file] [--char={0,1}] [--v={0,1}] [--padding-symbol={space,underline}] test.ref test.hyp > test.wer" + ) + print() # keep legacy usage for reference + + +# ============================================================================= +# New argparse-based entry point handling a single combined recog file +# ============================================================================= +WHITELIST = [ + ('地', '的'), + ('的', '地'), +] + +if __name__ == '__main__': + # helper to parse boolean-like CLI arguments + def str2bool(v): + if isinstance(v, bool): + return v + return str(v).lower() not in ('false', '0', 'no', 'n') + + parser = argparse.ArgumentParser( + description='Compute Word Error Rate (WER) using a combined recog file that contains both reference (ref= ...) and hypothesis (hyp= ...) lines.') + + # positional argument: combined file path + parser.add_argument('recog_file', + help='Path to combined recog.txt file. Each line must look like ":\tref=..." or ":\thyp=..."') + + # options corresponding to the legacy switches + parser.add_argument('--maxw', type=int, default=sys.maxsize, + help='Maximum number of words per printed line (legacy --maxw=)') + parser.add_argument('--rt', type=str2bool, default=True, + help='Remove XML/HTML-style tags before scoring (legacy --rt=)') + parser.add_argument('--cs', type=str2bool, default=False, + help='Case-sensitive evaluation (legacy --cs=)') + parser.add_argument('--cluster', dest='cluster_file', default='', + help='Optional word-cluster definition file (legacy --cluster=)') + parser.add_argument('--splitfile', default='', + help='Optional word split definition file (legacy --splitfile=)') + parser.add_argument('--ig', default='', + help='Optional ignore-word list file (legacy --ig=)') + parser.add_argument('--char', type=str2bool, default=True, + help='Tokenize at Unicode character level instead of word level (legacy --char=)') + parser.add_argument('-v', '--verbose', type=int, default=1, + help='Verbosity level 0/1/2 (legacy --v=)') + parser.add_argument('--padding-symbol', choices=['space', 'underline'], default='space', + help='Padding symbol when printing alignment (legacy --padding-symbol=)') + + args = parser.parse_args() + + # --------------------------------------------------------------------- + # Map parsed arguments to variables expected by the original algorithm + # --------------------------------------------------------------------- + remove_tag = args.rt + case_sensitive = args.cs + cluster_file = args.cluster_file + tochar = args.char + verbose = args.verbose + padding_symbol = ' ' if args.padding_symbol == 'space' else '_' + max_words_per_line = args.maxw + split = None + ignore_words = set() + + # --------------------------------------------------------------------- + # Load auxiliary resources (ignore list, split rules) + # --------------------------------------------------------------------- + if args.ig: + with codecs.open(args.ig, 'r', 'utf-8') as fh: + for line in fh: + token = line.strip() + if token: + ignore_words.add(token if case_sensitive else token.upper()) + + if args.splitfile: + split = {} + with codecs.open(args.splitfile, 'r', 'utf-8') as fh: + for line in fh: + words = line.strip().split() + if len(words) >= 2: + key = words[0] + values = words[1:] + if not case_sensitive: + key = key.upper() + values = [w.upper() for w in values] + split[key] = values + + # --------------------------------------------------------------------- + # Parse combined recog file into reference/hypothesis dictionaries + # --------------------------------------------------------------------- + ref_set, rec_set, utter_order = {}, {}, [] + combined_line_re = re.compile(r'([^:]+):\s*(ref|hyp)\s*=\s*(.*)', re.IGNORECASE) + + with codecs.open(args.recog_file, 'r', 'utf-8') as fh: + for raw_line in fh: + line = raw_line.rstrip('\n') + if not line: + continue + m = combined_line_re.match(line) + if not m: + continue # skip malformed lines + utt_id, kind, content = m.group(1).strip(), m.group(2).lower(), m.group(3).strip() + + tokens = characterize(content) if tochar else content.split() + norm_tokens = normalize(tokens, ignore_words, case_sensitive, split) + + if utt_id not in utter_order: + utter_order.append(utt_id) + if kind == 'ref': + ref_set[utt_id] = norm_tokens + else: + rec_set[utt_id] = norm_tokens + + # --------------------------------------------------------------------- + # WER computation + # --------------------------------------------------------------------- + calculator = Calculator() + default_clusters = {} + default_words = {} + + # dictionaries for global error distribution + subs = defaultdict(int) # all substitution errors + pinyin_subs = defaultdict(int) # substitution errors with different pinyin + ins = defaultdict(int) # insertion errors + dels = defaultdict(int) # deletion errors + + for utt_id in utter_order: + if utt_id not in ref_set or utt_id not in rec_set: + continue # incomplete pair + lab, rec = ref_set[utt_id], rec_set[utt_id] + + if verbose: + print(f"\nutt: {utt_id}") + + # accumulate default clusters for per-cluster WER later + for word in rec + lab: + if word not in default_words: + cls_name = default_cluster(word) + default_clusters.setdefault(cls_name, {})[word] = 1 + default_words[word] = cls_name + + result = calculator.calculate(lab, rec) + + # ----------------------------------------------------------------- + # Build arrow-style alignment and accumulate error statistics + # ----------------------------------------------------------------- + arrow_tokens = [] + for ltok, rtok in zip(result['lab'], result['rec']): + if ltok == rtok: + arrow_tokens.append(ltok) + elif ltok != '' and rtok != '': + arrow_tokens.append(f'({ltok}->{rtok})') + subs[(ltok, rtok)] += 1 + + # determine if pinyin differs + try: + p_ref = ''.join(lazy_pinyin(ltok, style=Style.TONE3, tone_sandhi=True, neutral_tone_with_five=True)) + p_hyp = ''.join(lazy_pinyin(rtok, style=Style.TONE3, tone_sandhi=True, neutral_tone_with_five=True)) + except Exception: + p_ref, p_hyp = ltok, rtok # fallback + if p_ref != p_hyp: + if (ltok, rtok) not in WHITELIST: + pinyin_subs[(ltok, rtok)] += 1 + elif ltok != '' and rtok == '': + arrow_tokens.append(f'({ltok}->*)') + dels[ltok] += 1 + elif ltok == '' and rtok != '': + arrow_tokens.append(f'(*->{rtok})') + ins[rtok] += 1 + + # ---------------------- per-utterance printout -------------------- + if verbose: + wer = 0.0 if result['all'] == 0 else (result['ins'] + result['sub'] + result['del']) * 100.0 / result['all'] + print(f"WER: {wer:4.2f} % N={result['all']} C={result['cor']} S={result['sub']} D={result['del']} I={result['ins']}") + print(' '.join(filter(lambda t: t != '', arrow_tokens))) + + # --------------------------------------------------------------------- + # Overall WER statistics (unchanged logic) + # --------------------------------------------------------------------- + if verbose: + print('=' * 75 + '\n') + + overall = calculator.overall() + + subs_total = overall['sub'] + pinyin_sub_total = sum(pinyin_subs.values()) + + overall_wer_sub = 0.0 if overall['all'] == 0 else (overall['ins'] + subs_total + overall['del']) * 100.0 / overall['all'] + overall_wer_pinyin = 0.0 if overall['all'] == 0 else (overall['ins'] + pinyin_sub_total + overall['del']) * 100.0 / overall['all'] + + if not verbose: + print() + + # ---------------------- global error distribution -------------------- + if verbose: + print() + print('SUBSTITUTIONS: count ref -> hyp') + for (ref_word, hyp_word), cnt in sorted(subs.items(), key=lambda kv: kv[1], reverse=True): + print(f"{cnt} {ref_word} -> {hyp_word}") + + print() # blank line + print('PINYIN_SUBSTITUTIONS: count ref -> hyp') + for (ref_word, hyp_word), cnt in sorted(pinyin_subs.items(), key=lambda kv: kv[1], reverse=True): + print(f"{cnt} {ref_word} -> {hyp_word}") + + print() # blank line + print('DELETIONS: count ref') + for ref_word, cnt in sorted(dels.items(), key=lambda kv: kv[1], reverse=True): + print(f"{cnt} {ref_word}") + + print() # blank line + print('INSERTIONS: count hyp') + for hyp_word, cnt in sorted(ins.items(), key=lambda kv: kv[1], reverse=True): + print(f"{cnt} {hyp_word}") + print() + + # --------------------------------------------------------------------- + # Per-cluster WER (same as original implementation) + # --------------------------------------------------------------------- + print(f"Overall (substitution) -> {overall_wer_sub:4.2f} % N={overall['all']} C={overall['cor']} S={subs_total} D={overall['del']} I={overall['ins']}") + print(f"Overall (pinyin_substitution) -> {overall_wer_pinyin:4.2f} % N={overall['all']} C={overall['cor']} S={pinyin_sub_total} D={overall['del']} I={overall['ins']}") + print('=' * 75) + if verbose: + for cluster_id in default_clusters: + stats = calculator.cluster(list(default_clusters[cluster_id].keys())) + + # compute substitution counts for this cluster + sub_cnt_cluster = stats['sub'] + p_sub_cnt = sum(cnt for (ref_word, _), cnt in pinyin_subs.items() if default_words.get(ref_word) == cluster_id) + + c_wer_sub = 0.0 if stats['all'] == 0 else (stats['ins'] + sub_cnt_cluster + stats['del']) * 100.0 / stats['all'] + c_wer_pinyin = 0.0 if stats['all'] == 0 else (stats['ins'] + p_sub_cnt + stats['del']) * 100.0 / stats['all'] + + print(f"{cluster_id} (substitution) -> {c_wer_sub:4.2f} % N={stats['all']} C={stats['cor']} S={sub_cnt_cluster} D={stats['del']} I={stats['ins']}") + print(f"{cluster_id} (pinyin_substitution) -> {c_wer_pinyin:4.2f} % N={stats['all']} C={stats['cor']} S={p_sub_cnt} D={stats['del']} I={stats['ins']}") + + # legacy explicit cluster file support + if cluster_file: + cluster_id, cluster_terms = '', [] + with open(cluster_file, 'r', encoding='utf-8') as fh: + for token in fh.read().split(): + # end of cluster like + if token.startswith('') and token.lstrip('') == cluster_id: + stats = calculator.cluster(cluster_terms) + c_wer = 0.0 if stats['all'] == 0 else (stats['ins'] + stats['sub'] + stats['del']) * 100.0 / stats['all'] + print(f"{cluster_id} -> {c_wer:4.2f} % N={stats['all']} C={stats['cor']} S={stats['sub']} D={stats['del']} I={stats['ins']}") + cluster_id, cluster_terms = '', [] + # begin of cluster like + elif token.startswith('<') and token.endswith('>') and not cluster_id: + cluster_id = token.lstrip('<').rstrip('>') + # regular term within a cluster + else: + cluster_terms.append(token) diff --git a/rl-tutorial/cosyvoice_llm/scripts/compute_wer.sh b/rl-tutorial/cosyvoice_llm/scripts/compute_wer.sh index 55ae1a7..8f4400c 100644 --- a/rl-tutorial/cosyvoice_llm/scripts/compute_wer.sh +++ b/rl-tutorial/cosyvoice_llm/scripts/compute_wer.sh @@ -10,6 +10,7 @@ model_path=models/sherpa-onnx-paraformer-zh-2023-09-14 if [ ! -d $model_path ]; then pip install sherpa-onnx wget -nc https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 + mkdir -p models tar xvf sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 -C models fi @@ -24,9 +25,6 @@ python3 scripts/offline-decode-files.py \ --feature-dim=80 \ --split-name $split_name \ --name sherpa_onnx \ - $wav_files + $wav_files || exit 1 -# python3 scripts/paraformer-pytriton-client.py \ -# --log-dir $wav_dir \ -# --split-name $split_name \ -# $wav_files \ No newline at end of file +python3 scripts/compute-wer.py "$wav_dir/recogs-sherpa_onnx.txt" > "$wav_dir/wer-sherpa-onnx.txt" || exit 1 \ No newline at end of file diff --git a/rl-tutorial/cosyvoice_llm/token2wav_asr_server.py b/rl-tutorial/cosyvoice_llm/token2wav_asr_server.py index a0bfe35..4dc0c86 100644 --- a/rl-tutorial/cosyvoice_llm/token2wav_asr_server.py +++ b/rl-tutorial/cosyvoice_llm/token2wav_asr_server.py @@ -70,7 +70,7 @@ def __call__(self, WAV: np.ndarray, WAV_LENS: np.ndarray, LANGUAGE: np.ndarray, results = self._model.transcribe_single_batch( wavs, - language="auto", + language="zh", textnorm="woitn", ) texts = [result.text for result in results] @@ -141,6 +141,13 @@ def get_random_prompt_from_dataset(dataset): prompt_text = prompt_text.replace(" ", "") return prompt_text, prompt_speech_16k +def get_reward_value(c): + k_pe = 12 + exponents = 1.5 + pow_exp_val = np.exp(-k_pe * c ** exponents) + # return 1.0 - np.tanh(3.0 * c) + return pow_exp_val + class _Token2Wav_ASR: """Wraps a single OmniSenseVoiceSmall model instance for Triton.""" @@ -203,7 +210,7 @@ def __call__(self, TOKENS: np.ndarray, TOKEN_LENS: np.ndarray, GT_TEXT: np.ndarr results = self.asr_model.transcribe_single_batch( wavs, - language="auto", + language="zh", textnorm="woitn", ) texts = [result.text for result in results] @@ -214,27 +221,32 @@ def __call__(self, TOKENS: np.ndarray, TOKEN_LENS: np.ndarray, GT_TEXT: np.ndarr gt_norm = zh_tn_model.normalize(gt_text).lower() hyp_norm = zh_tn_model.normalize(hyp_text).lower() - gt_pinyin = lazy_pinyin( - gt_norm, - style=Style.TONE3, - tone_sandhi=True, - neutral_tone_with_five=True, - ) - hyp_pinyin = lazy_pinyin( - hyp_norm, - style=Style.TONE3, - tone_sandhi=True, - neutral_tone_with_five=True, - ) + # gt_pinyin = lazy_pinyin( + # gt_norm, + # style=Style.TONE3, + # tone_sandhi=True, + # neutral_tone_with_five=True, + # ) + # hyp_pinyin = lazy_pinyin( + # hyp_norm, + # style=Style.TONE3, + # tone_sandhi=True, + # neutral_tone_with_five=True, + # ) + # don't compute the tone error + gt_pinyin = lazy_pinyin(gt_norm) + hyp_pinyin = lazy_pinyin(hyp_norm) c = float(wer(" ".join(gt_pinyin), " ".join(hyp_pinyin))) - reward_val = 1.0 - np.tanh(3.0 * c) + reward_val = get_reward_value(c) reward_val = max(0.0, min(1.0, reward_val)) rewards.append(reward_val) + print(f"gt_text: {gt_norm}, hyp_text: {hyp_norm}, reward_val: {reward_val}") transcripts = np.char.encode(np.array(texts).reshape(-1, 1), "utf-8") rewards_arr = np.array(rewards, dtype=np.float32).reshape(-1, 1) + return {"REWARDS": rewards_arr, "TRANSCRIPTS": transcripts}