diff --git a/examples/transformers/README.md b/examples/transformers/README.md index d1c75be2..dc55d7b5 100644 --- a/examples/transformers/README.md +++ b/examples/transformers/README.md @@ -54,6 +54,42 @@ per file. `keywords` and `prompt` are hints, not enforced vocabulary. Missing EO or empty text produces a nonzero exit after printing the diagnostic result. EOS is a generation boundary, not proof that every word was recognized. +## CUDA: an isolated, tested recipe + +Use a separate environment for this GPU recipe; do not install the CPU +requirements into it. It was functionally tested on Linux x86-64, Python 3.12, +an NVIDIA H100 80 GB and driver **550.127.08**, with PyTorch/torchaudio +**2.11.0+cu128** and Transformers **5.17.0**. Other GPU/driver combinations need +their own validation. + +```bash +python3.12 -m venv .venv-native-gpu +. .venv-native-gpu/bin/activate +python -m pip install --index-url https://download.pytorch.org/whl/cu128 'torch==2.11.0+cu128' 'torchaudio==2.11.0+cu128' +python -m pip install -r examples/transformers/requirements-gpu.txt +python -m pip check +python examples/transformers/transcribe.py --device cuda --dtype float32 +python examples/transformers/transcribe.py chinese.wav english.wav --language zh en --device cuda --dtype bfloat16 +``` + +The last command uses your local files. Without device flags, the CLI still uses +CPU float32. CUDA requests fail if CUDA is unavailable; they never silently fall +back to CPU. BF16 also requires device support. Model weights and processor +tensors are moved to the selected device without converting integer token IDs +to a floating dtype. JSON output records the actual device, dtype and versions. + +On 2026-09-10, both float32 and BF16 passed English, Chinese, Chinese with +keywords, and padded Chinese/English batch inference using the pinned public +samples. Outputs were nonempty and reached EOS. The Chinese sample still +transcribed `开放时间` as `开饭时间`, including with the keyword hint; these checks +do not establish accuracy or keyword benefit. They are not throughput, minimum +VRAM or concurrency benchmarks. Float16 and other accelerators were not tested. + +Transformers emitted an attention-implementation warning in this environment. +Successful inference does not verify the attention kernel of every component; +this recipe makes no Flash Attention or all-SDPA performance claim. The online +demo and notebook linked above do not imply a verified hosted GPU environment. + ## Select a backend, not just a suffix | Need | Artifact and entry point | @@ -66,8 +102,8 @@ EOS is a generation boundary, not proof that every word was recognized. The native `-hf` export produces transcription text. It does not add word timestamps, speaker identities, a streaming protocol or an HTTP server. The 31-language MLT checkpoint is a separate model, not an alternate name for this -zh/en/ja export. GPU dtype, attention kernels and throughput require separate -hardware validation; the CPU sample is not a GPU benchmark. +zh/en/ja export. The CUDA recipe above covers the stated functional cases only; +attention kernels and serving throughput require separate hardware evaluation. ## Verification diff --git a/examples/transformers/requirements-gpu.txt b/examples/transformers/requirements-gpu.txt new file mode 100644 index 00000000..15c64e6d --- /dev/null +++ b/examples/transformers/requirements-gpu.txt @@ -0,0 +1,8 @@ +transformers==5.17.0 +torch==2.11.0+cu128 +torchaudio==2.11.0+cu128 +numpy==1.26.4 +librosa==0.11.0 +soundfile==0.13.1 +huggingface-hub==1.30.0 +tokenizers==0.23.2 diff --git a/examples/transformers/transcribe.py b/examples/transformers/transcribe.py index d61389ec..a431bf2e 100644 --- a/examples/transformers/transcribe.py +++ b/examples/transformers/transcribe.py @@ -1,4 +1,4 @@ -"""Native Transformers CPU example for short, authorized recordings.""" +"""Native Transformers example for short, authorized recordings; CPU by default.""" import argparse import json from pathlib import Path @@ -38,6 +38,17 @@ def languages_for_batch(languages, count): return languages * count if len(languages) == 1 else languages +def validate_runtime(device, dtype, *, cuda_available=False, bf16_supported=False): + if device not in ("cpu", "cuda") or dtype not in ("float32", "bfloat16"): + raise ValueError("Choose cpu/cuda and float32/bfloat16 explicitly") + if device == "cpu" and dtype != "float32": + raise ValueError("This example supports float32 on CPU; use CUDA for bfloat16") + if device == "cuda" and not cuda_available: + raise ValueError("CUDA was requested but is unavailable; no CPU fallback") + if device == "cuda" and dtype == "bfloat16" and not bf16_supported: + raise ValueError("The selected CUDA device does not support bfloat16") + + def load_audio(path): import soundfile as sf @@ -55,10 +66,17 @@ def main(): parser.add_argument("--keywords", nargs="*", default=None) parser.add_argument("--prompt", default=None) parser.add_argument("--max-new-tokens", type=int, default=256) + parser.add_argument("--device", choices=["cpu", "cuda"], default="cpu") + parser.add_argument("--dtype", choices=["float32", "bfloat16"], default="float32") args = parser.parse_args() if not 1 <= args.max_new_tokens <= 1024: parser.error("--max-new-tokens must be between 1 and 1024") languages = languages_for_batch(args.language, len(args.audio) or 1) + import torch + + cuda_available = args.device == "cuda" and torch.cuda.is_available() + validate_runtime(args.device, args.dtype, cuda_available=cuda_available, + bf16_supported=cuda_available and torch.cuda.is_bf16_supported()) if not args.audio: from huggingface_hub import hf_hub_download @@ -67,7 +85,6 @@ def main(): if sum(len(item) for item in audio) > MAX_SECONDS * SAMPLE_RATE: raise ValueError("Keep total batch audio within 60 seconds for this example") - import torch import transformers from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor @@ -75,13 +92,13 @@ def main(): processor = AutoProcessor.from_pretrained(MODEL_ID, revision=REVISION, trust_remote_code=False, token=False) model = AutoModelForSpeechSeq2Seq.from_pretrained( MODEL_ID, revision=REVISION, trust_remote_code=False, token=False, - dtype=torch.float32, - ).to("cpu").eval() + dtype=getattr(torch, args.dtype), + ).to(args.device).eval() inputs = processor.apply_transcription_request( audio=audio, language=languages, keywords=args.keywords, prompt=args.prompt, processor_kwargs={"return_tensors": "pt", "audio_kwargs": {"sampling_rate": SAMPLE_RATE}, "text_kwargs": {"padding": True}}, - ) + ).to(args.device) with torch.inference_mode(): generated = model.generate(**inputs, max_new_tokens=args.max_new_tokens, do_sample=False) new_tokens = generated[:, inputs.input_ids.shape[1]:] @@ -93,7 +110,8 @@ def main(): results.append({"file": str(path), "language": language, "text": text, "reached_eos": any(token in eos_ids for token in tokens)}) print(json.dumps({"model": MODEL_ID, "revision": REVISION, "transformers": transformers.__version__, - "device": "cpu", "results": results}, ensure_ascii=False, indent=2)) + "torch": torch.__version__, "device": str(next(model.parameters()).device), + "dtype": str(model.dtype), "results": results}, ensure_ascii=False, indent=2)) if not all(row["reached_eos"] and row["text"].strip() for row in results): raise SystemExit("Incomplete generation: inspect empty text or missing EOS; do not treat this as a complete transcript") diff --git a/tests/test_transformers_quickstart.py b/tests/test_transformers_quickstart.py index 74c130e5..f04f8fb8 100644 --- a/tests/test_transformers_quickstart.py +++ b/tests/test_transformers_quickstart.py @@ -16,6 +16,34 @@ class NativeExampleTests(unittest.TestCase): + def test_runtime_choices_fail_closed_without_silent_cpu_fallback(self): + validate = getattr(native, 'validate_runtime', None) + self.assertTrue(callable(validate), 'Explicit runtime validation is missing') + validate('cpu', 'float32') + validate('cuda', 'float32', cuda_available=True) + validate('cuda', 'bfloat16', cuda_available=True, bf16_supported=True) + for device, dtype, available, bf16 in [ + ('cuda', 'float32', False, False), + ('cuda', 'bfloat16', True, False), + ('cpu', 'bfloat16', False, False), + ('auto', 'float32', False, False), + ('cuda', 'float16', True, True), + ]: + with self.subTest(device=device, dtype=dtype), self.assertRaises(ValueError): + validate(device, dtype, cuda_available=available, bf16_supported=bf16) + + def test_gpu_recipe_keeps_cpu_requirements_separate(self): + gpu = ROOT / 'examples/transformers/requirements-gpu.txt' + self.assertTrue(gpu.is_file()) + requirements = gpu.read_text() + self.assertIn('torch==2.11.0+cu128', requirements) + self.assertIn('torchaudio==2.11.0+cu128', requirements) + self.assertIn('transformers==5.17.0', requirements) + self.assertIn('torch==2.10.0', (ROOT / 'examples/transformers/requirements.txt').read_text()) + guide = (ROOT / 'examples/transformers/README.md').read_text() + for marker in ('--device cuda --dtype bfloat16', 'requirements-gpu.txt', 'H100', '550.127.08'): + self.assertIn(marker, guide) + def test_fixed_native_artifact(self): self.assertEqual(native.MODEL_ID, "FunAudioLLM/Fun-ASR-Nano-2512-hf") self.assertEqual(native.REVISION, "d93b302ee7fd505e1b3576120fc142fc6f7820e1")