From adffb37ae2f3f9022220c765c1f0703ce92bb3e1 Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Sat, 25 Jul 2026 18:34:42 -0400 Subject: [PATCH] feat(tts): Inflect v2 (Micro/Nano) CoreML backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultra-tiny VITS-family English TTS (9.4M / 4.0M params, 24 kHz). Ships FluidInference/inflect-v2-coreml: a fixed-shape encoder + duration predictor and 8 frame-bucketed HiFiGAN synthesizers per variant. Everything stochastic or dynamically shaped runs host-side in Swift (duration expansion, prior sampling), so both CoreML graphs are deterministic. - InflectManager (actor): text + ipa entry points, phoneme chunking, inter-chunk pauses, edge fade. - InflectSymbols: keithito/espeak symbol table + token encoder. Iterates Unicode scalars (a syllabic n̩ is two symbols, not one grapheme) and reproduces intersperse(cleaned_text_to_sequence(...)); golden-vector tested against the upstream Python. - InflectSynthesizer: encoder -> host expand+sample z_p -> smallest fitting bucket -> trim. InflectNoise: seedable Box-Muller Gaussian. - English frontend shared with StyleTTS2 (Misaki lexicon + BART G2P, espeak-approximated); synthesize(ipa:) bypasses it for faithful espeak input. espeak-ng itself (GPL) is not vendored. Validated end-to-end on M5 Pro: micro IPA and nano text both round-trip through Parakeet to the input text. Unit tests cover the symbol encoder (golden vectors), noise determinism, and repo/model wiring. --- Sources/FluidAudio/ModelNames.swift | 38 ++++ .../Inflect/Assets/InflectModelStore.swift | 88 ++++++++ .../Assets/InflectResourceDownloader.swift | 58 ++++++ .../TTS/Inflect/InflectConstants.swift | 46 +++++ .../FluidAudio/TTS/Inflect/InflectError.swift | 37 ++++ .../TTS/Inflect/InflectManager.swift | 193 ++++++++++++++++++ .../TTS/Inflect/InflectSymbols.swift | 79 +++++++ .../TTS/Inflect/InflectVariant.swift | 13 ++ .../TTS/Inflect/Pipeline/InflectNoise.swift | 53 +++++ .../Inflect/Pipeline/InflectSynthesizer.swift | 185 +++++++++++++++++ Sources/FluidAudio/TTS/TtsBackend.swift | 7 + .../FluidAudioCLI/Commands/TTSCommand.swift | 99 +++++++++ .../TTS/Inflect/InflectNoiseTests.swift | 51 +++++ .../TTS/Inflect/InflectSymbolsTests.swift | 62 ++++++ .../TTS/Inflect/InflectWiringTests.swift | 42 ++++ 15 files changed, 1051 insertions(+) create mode 100644 Sources/FluidAudio/TTS/Inflect/Assets/InflectModelStore.swift create mode 100644 Sources/FluidAudio/TTS/Inflect/Assets/InflectResourceDownloader.swift create mode 100644 Sources/FluidAudio/TTS/Inflect/InflectConstants.swift create mode 100644 Sources/FluidAudio/TTS/Inflect/InflectError.swift create mode 100644 Sources/FluidAudio/TTS/Inflect/InflectManager.swift create mode 100644 Sources/FluidAudio/TTS/Inflect/InflectSymbols.swift create mode 100644 Sources/FluidAudio/TTS/Inflect/InflectVariant.swift create mode 100644 Sources/FluidAudio/TTS/Inflect/Pipeline/InflectNoise.swift create mode 100644 Sources/FluidAudio/TTS/Inflect/Pipeline/InflectSynthesizer.swift create mode 100644 Tests/FluidAudioTests/TTS/Inflect/InflectNoiseTests.swift create mode 100644 Tests/FluidAudioTests/TTS/Inflect/InflectSymbolsTests.swift create mode 100644 Tests/FluidAudioTests/TTS/Inflect/InflectWiringTests.swift diff --git a/Sources/FluidAudio/ModelNames.swift b/Sources/FluidAudio/ModelNames.swift index cbab37ce..e6211a0a 100644 --- a/Sources/FluidAudio/ModelNames.swift +++ b/Sources/FluidAudio/ModelNames.swift @@ -72,6 +72,12 @@ public enum Repo: String, CaseIterable, Sendable { /// `tokens.txt` / `config.json`. Conversion lives in mobius /// (`models/tts/zipvoice`). case luxtts = "FluidInference/luxtts-coreml" + /// Inflect v2 (Micro / Nano) — ultra-tiny VITS-family English TTS. One repo + /// with two variant subdirectories (`micro/`, `nano/`), each holding + /// `encoder.mlmodelc` + 8 `synthesizer_f.mlmodelc` frame buckets. + /// Conversion lives in mobius (`models/tts/inflect-v2`). + case inflectMicro = "FluidInference/inflect-v2-coreml/micro" + case inflectNano = "FluidInference/inflect-v2-coreml/nano" /// Repository slug (without owner) public var name: String { @@ -142,6 +148,10 @@ public enum Repo: String, CaseIterable, Sendable { return "supertonic-3-coreml" case .luxtts: return "luxtts-coreml" + case .inflectMicro: + return "inflect-v2-coreml/micro" + case .inflectNano: + return "inflect-v2-coreml/nano" } } @@ -170,6 +180,8 @@ public enum Repo: String, CaseIterable, Sendable { return "FluidInference/cohere-transcribe-03-2026-coreml" case .styletts2: return "FluidInference/StyleTTS-2-coreml" + case .inflectMicro, .inflectNano: + return "FluidInference/inflect-v2-coreml" default: return "FluidInference/\(name)" } @@ -208,6 +220,10 @@ public enum Repo: String, CaseIterable, Sendable { return "q8" case .styletts2: return "iteration_3/compiled" + case .inflectMicro: + return "micro" + case .inflectNano: + return "nano" default: return nil } @@ -262,6 +278,10 @@ public enum Repo: String, CaseIterable, Sendable { return "styletts2" case .supertonic3: return "supertonic-3" + case .inflectMicro: + return "inflect-v2-coreml/micro" + case .inflectNano: + return "inflect-v2-coreml/nano" default: return name.replacingOccurrences(of: "-coreml", with: "") } @@ -1154,6 +1174,22 @@ public enum ModelNames { } } + /// Inflect v2 (Micro / Nano) — VITS-family English TTS. Each variant + /// subdirectory holds a fixed-shape `encoder` and 8 `synthesizer_f` + /// frame buckets. File names match `FluidInference/inflect-v2-coreml//`. + public enum Inflect { + public static let encoderFile = "encoder.mlmodelc" + + public static func synthesizerFile(frames: Int) -> String { + "synthesizer_f\(frames).mlmodelc" + } + + /// Encoder + all 8 synthesizer buckets (downloaded up front; buckets + /// load lazily). Buckets mirror `InflectConstants.frameBuckets`. + public static let requiredModels: Set = Set( + [encoderFile] + InflectConstants.frameBuckets.map { synthesizerFile(frames: $0) }) + } + /// LuxTTS (ZipVoice-Distill) model names. The HF repo publishes the same /// text encoder + flow-matching decoder in two graph layouts: /// - `gpu/` — original graph; fastest on Mac GPU (do NOT run on ANE: @@ -1452,6 +1488,8 @@ public enum ModelNames { case .luxtts: // Variants: "gpu" (macOS) / "ane" (iOS); nil → platform default. return ModelNames.LuxTts.requiredFiles(variant: variant) + case .inflectMicro, .inflectNano: + return ModelNames.Inflect.requiredModels } } } diff --git a/Sources/FluidAudio/TTS/Inflect/Assets/InflectModelStore.swift b/Sources/FluidAudio/TTS/Inflect/Assets/InflectModelStore.swift new file mode 100644 index 00000000..2d42944e --- /dev/null +++ b/Sources/FluidAudio/TTS/Inflect/Assets/InflectModelStore.swift @@ -0,0 +1,88 @@ +@preconcurrency import CoreML +import Foundation + +/// Actor store for one Inflect variant's CoreML bundles: the fixed-shape +/// `encoder` (loaded eagerly) and the eight `synthesizer_f` frame buckets +/// (loaded lazily and cached the first time a bucket is needed). +/// +/// All buckets are *downloaded* up front by `InflectResourceDownloader` +/// (they total ~8–19 MB), so bucket loads never hit the network. +public actor InflectModelStore { + + private let logger = AppLogger(category: "InflectModelStore") + + private let variant: InflectVariant + private let directory: URL? + private let computeUnits: MLComputeUnits + + private var encoderModel: MLModel? + private var bucketModels: [Int: MLModel] = [:] + private var repoDirectory: URL? + + public init( + variant: InflectVariant, + directory: URL? = nil, + computeUnits: MLComputeUnits = .cpuAndGPU + ) { + self.variant = variant + self.directory = directory + self.computeUnits = computeUnits + } + + /// Download (if missing) and load the encoder. Buckets stay lazy. + public func loadIfNeeded() async throws { + if encoderModel != nil { return } + + let repoDir = try await InflectResourceDownloader.ensureModels( + variant: variant, directory: directory) + self.repoDirectory = repoDir + + encoderModel = try loadModel( + repoDir: repoDir, fileName: ModelNames.Inflect.encoderFile) + logger.info("Inflect \(variant.rawValue) encoder loaded from \(repoDir.path)") + } + + public func encoder() throws -> MLModel { + guard let encoderModel else { throw InflectError.notInitialized } + return encoderModel + } + + /// Return the synthesizer for the smallest bucket ≥ `frames`, loading and + /// caching it on first use. + public func synthesizer(forFrames frames: Int) async throws -> (model: MLModel, bucket: Int) { + guard let bucket = InflectConstants.frameBuckets.first(where: { frames <= $0 }) else { + throw InflectError.durationOverflow( + frames: frames, maxFrames: InflectConstants.maxFrames) + } + if let cached = bucketModels[bucket] { + return (cached, bucket) + } + guard let repoDir = repoDirectory else { throw InflectError.notInitialized } + let model = try loadModel( + repoDir: repoDir, fileName: ModelNames.Inflect.synthesizerFile(frames: bucket)) + bucketModels[bucket] = model + logger.info("Inflect \(variant.rawValue) synthesizer bucket f\(bucket) loaded") + return (model, bucket) + } + + public func unload() { + encoderModel = nil + bucketModels.removeAll(keepingCapacity: false) + } + + // MARK: - Helpers + + private func loadModel(repoDir: URL, fileName: String) throws -> MLModel { + let url = repoDir.appendingPathComponent(fileName) + guard FileManager.default.fileExists(atPath: url.path) else { + throw InflectError.modelFileNotFound(fileName) + } + let config = MLModelConfiguration() + config.computeUnits = computeUnits + do { + return try MLModel(contentsOf: url, configuration: config) + } catch { + throw InflectError.corruptedModel(fileName, underlying: "\(error)") + } + } +} diff --git a/Sources/FluidAudio/TTS/Inflect/Assets/InflectResourceDownloader.swift b/Sources/FluidAudio/TTS/Inflect/Assets/InflectResourceDownloader.swift new file mode 100644 index 00000000..0aaec3f0 --- /dev/null +++ b/Sources/FluidAudio/TTS/Inflect/Assets/InflectResourceDownloader.swift @@ -0,0 +1,58 @@ +import Foundation + +/// Downloads Inflect v2 CoreML bundles from +/// `FluidInference/inflect-v2-coreml`. Each variant lives under its own +/// subdirectory (`micro/`, `nano/`) holding `encoder.mlmodelc` plus the eight +/// `synthesizer_f.mlmodelc` frame buckets. +public enum InflectResourceDownloader { + + private static let logger = AppLogger(category: "InflectResourceDownloader") + + /// Ensure the variant's encoder + all synthesizer buckets are cached + /// locally. Returns the directory that holds the `.mlmodelc` bundles. + @discardableResult + public static func ensureModels( + variant: InflectVariant, + directory: URL? = nil, + progressHandler: ProgressHandler? = nil + ) async throws -> URL { + let repo = variant.repo + let modelsRoot = try directory ?? defaultCacheRoot() + let repoDir = modelsRoot.appendingPathComponent(repo.folderName) + + let allPresent = ModelNames.Inflect.requiredModels.allSatisfy { entry in + FileManager.default.fileExists(atPath: repoDir.appendingPathComponent(entry).path) + } + + if allPresent { + logger.info("Inflect \(variant.rawValue) models found in cache at \(repoDir.path)") + return repoDir + } + + logger.info("Downloading Inflect \(variant.rawValue) CoreML models from HuggingFace…") + do { + try await ModelHub.download(repo, to: modelsRoot, progressHandler: progressHandler) + } catch { + throw InflectError.downloadFailed("\(error)") + } + return repoDir + } + + private static func defaultCacheRoot() throws -> URL { + let root = try TtsCacheDirectory.ensure().appendingPathComponent("Models") + if !FileManager.default.fileExists(atPath: root.path) { + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + } + return root + } +} + +extension InflectVariant { + /// HuggingFace repo case for this variant's subdirectory. + var repo: Repo { + switch self { + case .micro: return .inflectMicro + case .nano: return .inflectNano + } + } +} diff --git a/Sources/FluidAudio/TTS/Inflect/InflectConstants.swift b/Sources/FluidAudio/TTS/Inflect/InflectConstants.swift new file mode 100644 index 00000000..0b096227 --- /dev/null +++ b/Sources/FluidAudio/TTS/Inflect/InflectConstants.swift @@ -0,0 +1,46 @@ +import Foundation + +/// Fixed pipeline parameters for the Inflect v2 CoreML backend. Values mirror +/// the upstream `config.json` and the mobius conversion (`inflect-v2/coreml`). +public enum InflectConstants { + + /// Output sample rate (24 kHz mono, matches `data.sampling_rate`). + public static let sampleRate = 24_000 + + /// HiFiGAN total upsampling factor (8·8·2·2) — samples per mel frame. + public static let hopLength = 256 + + /// Fixed token axis of the encoder bundle (`tokens`/`x_mask` shape `[1, 512]`). + public static let encoderTokens = 512 + + /// Latent channel count fed to the synthesizer (`inter_channels`). + /// Micro = 192, Nano = 128. + public static func interChannels(for variant: InflectVariant) -> Int { + switch variant { + case .micro: return 192 + case .nano: return 128 + } + } + + /// Synthesizer frame buckets (each `synthesizer_f.mlmodelc`). The + /// smallest bucket ≥ the predicted frame length is used; audio is trimmed + /// to the exact length afterwards. + public static let frameBuckets = [256, 384, 512, 640, 768, 896, 1024, 2048] + + /// Largest bucket — a chunk whose predicted duration exceeds this throws. + public static var maxFrames: Int { frameBuckets.last! } + + /// Max phoneme characters per synthesis chunk. Interspersed with blanks + /// this stays under the 512-token encoder axis (`2·255 + 1 = 511`); the + /// text path splits longer input at punctuation/whitespace. + public static let maxPhonemeChunkChars = 240 + + /// Prior sampling temperature (`noise_scale`, VITS `z_p` stddev multiplier). + public static let defaultNoiseScale: Float = 0.667 + + /// Speech-rate multiplier. Durations scale by `1 / speed`. + public static let defaultSpeed: Float = 1.0 + + /// Edge fade applied to each chunk (ms), matching upstream `edge_fade`. + public static let edgeFadeMs: Double = 5.0 +} diff --git a/Sources/FluidAudio/TTS/Inflect/InflectError.swift b/Sources/FluidAudio/TTS/Inflect/InflectError.swift new file mode 100644 index 00000000..261c2060 --- /dev/null +++ b/Sources/FluidAudio/TTS/Inflect/InflectError.swift @@ -0,0 +1,37 @@ +import Foundation + +/// Errors thrown by the Inflect v2 TTS backend. +public enum InflectError: Error, LocalizedError { + case notInitialized + case downloadFailed(String) + case modelFileNotFound(String) + case corruptedModel(String, underlying: String) + case phonemizationFailed(String) + case inputProcessingFailed(String) + /// A chunk's predicted duration exceeded the largest synthesizer bucket. + case durationOverflow(frames: Int, maxFrames: Int) + case predictionFailed(String) + + public var errorDescription: String? { + switch self { + case .notInitialized: + return "Inflect backend is not initialized; call initialize() first." + case .downloadFailed(let detail): + return "Inflect model download failed: \(detail)" + case .modelFileNotFound(let name): + return "Inflect model file not found: \(name)" + case .corruptedModel(let name, let underlying): + return "Inflect model \(name) failed to load: \(underlying)" + case .phonemizationFailed(let detail): + return "Inflect phonemization failed: \(detail)" + case .inputProcessingFailed(let detail): + return "Inflect input processing failed: \(detail)" + case .durationOverflow(let frames, let maxFrames): + return + "Predicted duration \(frames) frames exceeds the largest bucket " + + "(\(maxFrames)); shorten the input chunk." + case .predictionFailed(let detail): + return "Inflect CoreML prediction failed: \(detail)" + } + } +} diff --git a/Sources/FluidAudio/TTS/Inflect/InflectManager.swift b/Sources/FluidAudio/TTS/Inflect/InflectManager.swift new file mode 100644 index 00000000..97be110c --- /dev/null +++ b/Sources/FluidAudio/TTS/Inflect/InflectManager.swift @@ -0,0 +1,193 @@ +@preconcurrency import CoreML +import Foundation + +/// Public API for the Inflect v2 (Micro / Nano) CoreML TTS backend. +/// +/// Ultra-tiny VITS-family English TTS (9.4M / 4.0M params, 24 kHz). The +/// pipeline is a fixed-shape encoder + duration predictor, host-side duration +/// expansion and prior sampling, then a bucketed HiFiGAN synthesizer. See +/// `FluidInference/inflect-v2-coreml`. +/// +/// > Phonemizer parity gap — same as StyleTTS2. +/// > Inflect trained on `phonemizer`/espeak-ng en-us IPA. FluidAudio can't +/// > ship the GPL espeak C library, so the text path reuses the shared +/// > Misaki-lexicon + BART G2P frontend (`StyleTTS2Phonemizer`), which +/// > approximates espeak IPA. Output is intelligible but stress markers can +/// > differ. Callers with a real espeak phonemizer should feed IPA directly +/// > via ``synthesize(ipa:)``. +public actor InflectManager { + + private let logger = AppLogger(category: "InflectManager") + + private let variant: InflectVariant + private let directory: URL? + private let computeUnits: MLComputeUnits + private let noiseScale: Float + private let speed: Float + + private var store: InflectModelStore? + private var synthesizer: InflectSynthesizer? + private var phonemizer: StyleTTS2Phonemizer? + + public nonisolated var sampleRate: Int { InflectConstants.sampleRate } + + public init( + variant: InflectVariant = .micro, + directory: URL? = nil, + computeUnits: MLComputeUnits = .cpuAndGPU, + noiseScale: Float = InflectConstants.defaultNoiseScale, + speed: Float = InflectConstants.defaultSpeed + ) { + self.variant = variant + self.directory = directory + self.computeUnits = computeUnits + self.noiseScale = noiseScale + self.speed = speed + } + + public var isAvailable: Bool { synthesizer != nil } + + /// Convenience factory: download assets and return a ready manager. + public static func downloadAndCreate( + variant: InflectVariant = .micro, + directory: URL? = nil, + computeUnits: MLComputeUnits = .cpuAndGPU + ) async throws -> InflectManager { + let manager = InflectManager( + variant: variant, directory: directory, computeUnits: computeUnits) + try await manager.initialize() + return manager + } + + /// Download + load the variant's CoreML bundles and the shared English + /// frontend (Misaki lexicon cache + BART G2P), then build the phonemizer. + public func initialize() async throws { + if synthesizer != nil { return } + + let store = InflectModelStore( + variant: variant, directory: directory, computeUnits: computeUnits) + try await store.loadIfNeeded() + self.store = store + self.synthesizer = InflectSynthesizer(store: store, variant: variant) + + // Reuse the shared English frontend (same espeak-approximation used by + // StyleTTS2). The lexicon cache + BART G2P assets live in the kokoro + // cache dir and are shared across backends. + do { + try await StyleTTS2ResourceDownloader.ensureG2PAssets(directory: directory) + let kokoroDir = try await StyleTTS2ResourceDownloader.ensureLexiconCache() + try await G2PModel.shared.ensureModelsAvailable() + + let allowedTokens = Set(InflectSymbols.symbols.map { String($0) }) + let lexiconCache = LexiconAssetCache() + try await lexiconCache.ensureLoaded( + kokoroDirectory: kokoroDir, allowedTokens: allowedTokens) + let lexicons = await lexiconCache.lexicons() + self.phonemizer = StyleTTS2Phonemizer( + wordToPhonemes: lexicons.word, + caseSensitiveWordToPhonemes: lexicons.caseSensitive) + logger.info( + "Inflect \(variant.rawValue) ready — lexicon \(lexicons.word.count) entries, " + + "compute \(computeUnits.inflectDescription)") + } catch { + logger.warning( + "English frontend load failed (\(error)); text synthesis unavailable, " + + "use synthesize(ipa:). Detail: \(error.localizedDescription)") + self.phonemizer = StyleTTS2Phonemizer() + } + } + + // MARK: - Text path + + /// Phonemize `text` (Misaki + BART G2P), split into encoder-sized chunks, + /// and synthesize. Returns 24 kHz mono Float32 PCM. + public func synthesize( + text: String, + noiseSeed: UInt64 = 0 + ) async throws -> [Float] { + guard let phonemizer else { throw InflectError.notInitialized } + let ipa: String + do { + ipa = try await phonemizer.phonemize(text) + } catch { + throw InflectError.phonemizationFailed("\(error)") + } + return try await synthesize(ipa: ipa, noiseSeed: noiseSeed) + } + + // MARK: - IPA path (espeak-parity escape hatch) + + /// Synthesize directly from an IPA phoneme string (keithito/espeak symbol + /// set). Bypasses the lexicon + G2P — feed the espeak IPA the model was + /// trained on for best quality. Chunks longer than the encoder axis are + /// split at punctuation/whitespace and concatenated with short pauses. + public func synthesize( + ipa: String, + noiseSeed: UInt64 = 0 + ) async throws -> [Float] { + guard let synthesizer else { throw InflectError.notInitialized } + let chunks = PhonemeChunker.chunk(ipa, maxLength: InflectConstants.maxPhonemeChunkChars) + guard !chunks.isEmpty else { + throw InflectError.inputProcessingFailed("no speakable phonemes in input") + } + + var samples: [Float] = [] + for (index, chunk) in chunks.enumerated() { + let tokens = InflectSymbols.encode(chunk) + guard tokens.count > 1 else { continue } + if index > 0 { + samples.append( + contentsOf: [Float]( + repeating: 0, count: pauseSamples(afterChunk: chunks[index - 1]))) + } + let chunkSamples = try await synthesizer.synthesize( + tokens: tokens, + noiseScale: noiseScale, + speed: speed, + noiseSeed: noiseSeed &+ UInt64(index)) + samples.append(contentsOf: chunkSamples) + } + guard !samples.isEmpty else { + throw InflectError.inputProcessingFailed("synthesis produced no audio") + } + return samples + } + + public func cleanup() async { + await store?.unload() + store = nil + synthesizer = nil + phonemizer = nil + } + + // MARK: - Helpers + + /// Inter-chunk silence, keyed on the preceding chunk's final punctuation + /// (mirrors upstream `boundary_pause_seconds`). + private func pauseSamples(afterChunk chunk: String) -> Int { + let ending = chunk.trimmingCharacters(in: .whitespaces).last + let seconds: Double + switch ending { + case "?": seconds = 0.28 + case "!": seconds = 0.24 + case ".": seconds = 0.22 + case ";": seconds = 0.16 + case ":": seconds = 0.13 + case ",": seconds = 0.09 + default: seconds = 0.08 + } + return Int(Double(InflectConstants.sampleRate) * seconds) + } +} + +extension MLComputeUnits { + fileprivate var inflectDescription: String { + switch self { + case .cpuOnly: return "cpuOnly" + case .cpuAndGPU: return "cpuAndGPU" + case .all: return "all" + case .cpuAndNeuralEngine: return "cpuAndNeuralEngine" + @unknown default: return "unknown" + } + } +} diff --git a/Sources/FluidAudio/TTS/Inflect/InflectSymbols.swift b/Sources/FluidAudio/TTS/Inflect/InflectSymbols.swift new file mode 100644 index 00000000..d90078a3 --- /dev/null +++ b/Sources/FluidAudio/TTS/Inflect/InflectSymbols.swift @@ -0,0 +1,79 @@ +import Foundation + +/// Inflect v2 symbol table + token encoder. +/// +/// Inflect uses the keithito/Tacotron symbol set (`runtime/text/symbols.py`): +/// a pad, punctuation, ASCII letters, then the full IPA inventory. Token IDs +/// are indices into that list. The four source literals are reproduced +/// verbatim; concatenating them and enumerating Unicode *scalars* reproduces +/// Python's `[_pad] + list(_punctuation) + list(_letters) + list(_letters_ipa)` +/// exactly (confirmed byte-identical against the shipped `symbols.py`). +/// +/// Two parity details that a naive port gets wrong: +/// 1. Iterate `unicodeScalars`, not `Character`s. A syllabic consonant like +/// `n̩` (n + U+0329) is one grapheme but two symbols in Python — the +/// combining mark has its own id. +/// 2. The apostrophe `'` appears twice in `_letters_ipa`; Python's +/// `{s: i for i, s in enumerate(symbols)}` keeps the *last* index, so the +/// scalar→id map overwrites rather than skipping duplicates. +enum InflectSymbols { + + /// keithito literals, verbatim from `runtime/text/symbols.py`. + private static let pad = "_" + private static let punctuation = ";:,.!?¡¿—…\"«»“” " + private static let letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + private static let lettersIPA = + "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ" + + /// Ordered symbol list; `count == 178`. Index == token id. + static let symbols: [Unicode.Scalar] = { + var out: [Unicode.Scalar] = [] + for literal in [pad, punctuation, letters, lettersIPA] { + out.append(contentsOf: literal.unicodeScalars) + } + return out + }() + + /// Scalar → token id. Later occurrences win (matches Python dict build). + private static let scalarToID: [Unicode.Scalar: Int32] = { + var map: [Unicode.Scalar: Int32] = [:] + for (index, scalar) in symbols.enumerated() { + map[scalar] = Int32(index) + } + return map + }() + + /// Pad / blank token id (`_`), interspersed between phonemes. + static let blankID: Int32 = 0 + + /// Map a phoneme string to token ids, dropping scalars outside the symbol + /// set (mirrors `cleaned_text_to_sequence`'s `if s in _symbol_to_id`). + static func sequence(for phonemes: String) -> [Int32] { + var ids: [Int32] = [] + ids.reserveCapacity(phonemes.unicodeScalars.count) + for scalar in phonemes.unicodeScalars { + if let id = scalarToID[scalar] { + ids.append(id) + } + } + return ids + } + + /// Intersperse `blankID` around every token: `commons.intersperse(seq, 0)` + /// produces `[0, t0, 0, t1, 0, …, tn, 0]` (length `2·count + 1`). + static func intersperse(_ sequence: [Int32]) -> [Int32] { + guard !sequence.isEmpty else { return [blankID] } + var out: [Int32] = [blankID] + out.reserveCapacity(sequence.count * 2 + 1) + for token in sequence { + out.append(token) + out.append(blankID) + } + return out + } + + /// Full text→token pipeline: `intersperse(cleaned_text_to_sequence(ipa))`. + static func encode(_ phonemes: String) -> [Int32] { + intersperse(sequence(for: phonemes)) + } +} diff --git a/Sources/FluidAudio/TTS/Inflect/InflectVariant.swift b/Sources/FluidAudio/TTS/Inflect/InflectVariant.swift new file mode 100644 index 00000000..1b9a67bd --- /dev/null +++ b/Sources/FluidAudio/TTS/Inflect/InflectVariant.swift @@ -0,0 +1,13 @@ +import Foundation + +/// Inflect v2 model size. Both share the same pipeline, symbol table, and +/// English frontend — only the CoreML weights and repo subdirectory differ. +public enum InflectVariant: String, Sendable, CaseIterable { + /// 9.36M params (~19 MB fp16). Higher quality. + case micro + /// 3.97M params (~8 MB fp16). Smallest footprint, fastest. + case nano + + /// Subdirectory under `FluidInference/inflect-v2-coreml/`. + public var subdirectory: String { rawValue } +} diff --git a/Sources/FluidAudio/TTS/Inflect/Pipeline/InflectNoise.swift b/Sources/FluidAudio/TTS/Inflect/Pipeline/InflectNoise.swift new file mode 100644 index 00000000..211bee01 --- /dev/null +++ b/Sources/FluidAudio/TTS/Inflect/Pipeline/InflectNoise.swift @@ -0,0 +1,53 @@ +import Foundation + +/// Deterministic standard-normal generator for the VITS prior sample +/// (`z_p = m + noise·exp(logs)·noise_scale`). Seedable so a given +/// (text, seed) renders identical audio and unit tests are reproducible. +/// +/// SplitMix64 uniform stream → Box–Muller Gaussian pairs. +struct InflectNoise { + + private var state: UInt64 + private var spare: Float? + + init(seed: UInt64) { + // Avoid the all-zero SplitMix64 fixed point. + self.state = seed == 0 ? 0x9E37_79B9_7F4A_7C15 : seed + } + + private mutating func nextUInt64() -> UInt64 { + state &+= 0x9E37_79B9_7F4A_7C15 + var z = state + z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9 + z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB + return z ^ (z >> 31) + } + + /// Uniform in (0, 1]. + private mutating func nextUnit() -> Float { + // Top 24 bits → [0, 1); shift to (0, 1] so log() is finite. + let bits = nextUInt64() >> 40 + return (Float(bits) + 1.0) / Float(1 << 24) + } + + /// Next N(0, 1) sample. + mutating func nextGaussian() -> Float { + if let s = spare { + spare = nil + return s + } + let u1 = nextUnit() + let u2 = nextUnit() + let radius = (-2.0 * Foundation.log(u1)).squareRoot() + let angle = 2.0 * Float.pi * u2 + spare = radius * Foundation.sin(angle) + return radius * Foundation.cos(angle) + } + + /// Fill `buffer` with N(0, 1) samples. + mutating func fill(_ buffer: inout [Float]) { + for i in buffer.indices { + buffer[i] = nextGaussian() + } + } +} diff --git a/Sources/FluidAudio/TTS/Inflect/Pipeline/InflectSynthesizer.swift b/Sources/FluidAudio/TTS/Inflect/Pipeline/InflectSynthesizer.swift new file mode 100644 index 00000000..8c945737 --- /dev/null +++ b/Sources/FluidAudio/TTS/Inflect/Pipeline/InflectSynthesizer.swift @@ -0,0 +1,185 @@ +@preconcurrency import CoreML +import Foundation + +/// Drives the two-stage Inflect CoreML pipeline for a single chunk: +/// encoder → host duration expansion + prior sampling → bucketed synthesizer. +/// Everything stochastic or dynamically shaped runs here in Swift; both CoreML +/// graphs are deterministic and fixed-shape. +struct InflectSynthesizer { + + private let store: InflectModelStore + private let variant: InflectVariant + + init(store: InflectModelStore, variant: InflectVariant) { + self.store = store + self.variant = variant + } + + /// Synthesize one chunk of already-encoded, blank-interspersed token ids. + /// `tokens.count` must be ≤ `InflectConstants.encoderTokens`. + func synthesize( + tokens: [Int32], + noiseScale: Float, + speed: Float, + noiseSeed: UInt64 + ) async throws -> [Float] { + let tText = InflectConstants.encoderTokens + let channels = InflectConstants.interChannels(for: variant) + let n = tokens.count + guard n > 0, n <= tText else { + throw InflectError.inputProcessingFailed( + "token count \(n) out of range (1...\(tText))") + } + + // --- Encoder --- + var tokenBuf = [Int32](repeating: InflectSymbols.blankID, count: tText) + var maskBuf = [Float](repeating: 0, count: tText) + for i in 0..= 1 else { + throw InflectError.inputProcessingFailed("predicted zero total duration") + } + guard yLen <= InflectConstants.maxFrames else { + throw InflectError.durationOverflow(frames: yLen, maxFrames: InflectConstants.maxFrames) + } + + let (synthModel, bucket) = try await store.synthesizer(forFrames: yLen) + + // --- Host: expand prior to frame rate + sample z_p (padded to bucket) --- + var noise = InflectNoise(seed: noiseSeed) + var zp = [Float](repeating: 0, count: channels * bucket) + var maskY = [Float](repeating: 0, count: bucket) + for f in 0.. 0 { + let col = i + for _ in 0.. 0 else { return } + for i in 0.. MLFeatureProvider { + do { + let provider = try MLDictionaryFeatureProvider( + dictionary: inputs.mapValues { MLFeatureValue(multiArray: $0) }) + return try model.prediction(from: provider) + } catch { + throw InflectError.predictionFailed("\(error)") + } + } + + private func multiArray(_ values: [Float], shape: [Int]) throws -> MLMultiArray { + let arr = try MLMultiArray(shape: shape.map(NSNumber.init), dataType: .float32) + let dst = arr.dataPointer.bindMemory(to: Float.self, capacity: values.count) + values.withUnsafeBufferPointer { dst.update(from: $0.baseAddress!, count: values.count) } + return arr + } + + private func multiArray(_ values: [Int32], shape: [Int]) throws -> MLMultiArray { + let arr = try MLMultiArray(shape: shape.map(NSNumber.init), dataType: .int32) + let dst = arr.dataPointer.bindMemory(to: Int32.self, capacity: values.count) + values.withUnsafeBufferPointer { dst.update(from: $0.baseAddress!, count: values.count) } + return arr + } + + /// Read a named output as `[Float]`, handling fp16 or fp32 backing. + private func floats(_ provider: MLFeatureProvider, _ name: String) throws -> [Float] { + guard let arr = provider.featureValue(for: name)?.multiArrayValue else { + throw InflectError.predictionFailed("missing output '\(name)'") + } + let count = arr.count + var out = [Float](repeating: 0, count: count) + switch arr.dataType { + case .float32: + let src = arr.dataPointer.bindMemory(to: Float.self, capacity: count) + out.withUnsafeMutableBufferPointer { $0.baseAddress!.update(from: src, count: count) } + case .float16: + let src = arr.dataPointer.bindMemory(to: UInt16.self, capacity: count) + for i in 0..> 15) & 0x1) + let exponent = Int((bits >> 10) & 0x1F) + let mantissa = Float(bits & 0x3FF) + let value: Float + if exponent == 0 { + value = mantissa / 1024.0 * Float(pow(2.0, -14.0)) + } else if exponent == 0x1F { + value = mantissa == 0 ? Float.infinity : Float.nan + } else { + value = (1.0 + mantissa / 1024.0) * Float(pow(2.0, Double(exponent - 15))) + } + self = (sign == 1 ? -1 : 1) * value + } +} diff --git a/Sources/FluidAudio/TTS/TtsBackend.swift b/Sources/FluidAudio/TTS/TtsBackend.swift index 978fd768..d961ab5e 100644 --- a/Sources/FluidAudio/TTS/TtsBackend.swift +++ b/Sources/FluidAudio/TTS/TtsBackend.swift @@ -36,4 +36,11 @@ public enum TtsBackend: Sendable { /// (`LuxTtsG2p`); pre-phonemized espeak IPA is accepted via /// `LuxTtsManager.synthesize(phonemes:...)`. case luxtts + /// Inflect v2 (Micro / Nano) — ultra-tiny VITS-family English TTS + /// (9.4M / 4.0M params, 24 kHz mono). Fixed-shape encoder + duration + /// predictor, host-side duration expansion + prior sampling, then a + /// bucketed HiFiGAN synthesizer. English frontend shared with StyleTTS2 + /// (espeak-approximated Misaki + BART G2P); feed IPA directly via + /// `InflectManager.synthesize(ipa:)` to bypass it. + case inflect } diff --git a/Sources/FluidAudioCLI/Commands/TTSCommand.swift b/Sources/FluidAudioCLI/Commands/TTSCommand.swift index d4819333..7317d84a 100644 --- a/Sources/FluidAudioCLI/Commands/TTSCommand.swift +++ b/Sources/FluidAudioCLI/Commands/TTSCommand.swift @@ -62,6 +62,10 @@ public struct TTS { // KokoroAne language variant — only consulted when backend == .kokoroAne. // Parsed from the `--variant` flag (en/english/zh/mandarin). var kokoroAneVariant: KokoroAneVariant = .english + // Inflect model size — only consulted when backend == .inflect. + // Parsed from `--variant` (micro/nano) or the backend token + // (inflect-micro / inflect-nano). + var inflectVariant: InflectVariant = .micro var lexiconPath: String? = nil var text: String? = nil // KokoroAne: treat the positional/`--text` value as a pre-computed @@ -136,6 +140,10 @@ public struct TTS { kokoroAneVariant = .mandarin case "ja", "japanese", "jp": kokoroAneVariant = .japanese + case "micro", "inflect-micro": + inflectVariant = .micro + case "nano", "inflect-nano": + inflectVariant = .nano default: logger.warning("Unknown variant preference '\(arguments[i + 1])'; ignoring") } @@ -160,6 +168,14 @@ public struct TTS { backend = .supertonic3 case "luxtts", "lux-tts", "lux", "zipvoice": backend = .luxtts + case "inflect", "inflect-v2": + backend = .inflect + case "inflect-micro": + backend = .inflect + inflectVariant = .micro + case "inflect-nano": + backend = .inflect + inflectVariant = .nano default: logger.warning("Unknown backend '\(arguments[i + 1])'; using kokoro-ane") } @@ -350,6 +366,89 @@ public struct TTS { treatAsPhonemes: treatAsPhonemes, speed: luxttsSpeed, seed: luxttsSeed, metricsPath: metricsPath) + case .inflect: + await runInflect( + text: text, output: output, + variant: inflectVariant, treatAsPhonemes: treatAsPhonemes, + seed: pocketSeed ?? 0, + metricsPath: metricsPath, cpuOnly: cpuOnly) + } + } + + /// Run Inflect v2 (Micro / Nano) TTS. With `--phonemes` the positional + /// text is treated as an espeak-style IPA string and fed straight to the + /// synthesizer (bypassing the Misaki + BART G2P frontend). + private static func runInflect( + text: String, output: String, + variant: InflectVariant, treatAsPhonemes: Bool, + seed: UInt64, + metricsPath: String?, cpuOnly: Bool + ) async { + do { + let tStart = Date() + let computeUnits: MLComputeUnits = cpuOnly ? .cpuOnly : .cpuAndGPU + let manager = InflectManager(variant: variant, computeUnits: computeUnits) + + let tLoad0 = Date() + try await manager.initialize() + let tLoad1 = Date() + + logger.info("Inflect \(variant.rawValue) \(treatAsPhonemes ? "IPA" : "text") synthesis") + let tSynth0 = Date() + let samples = + treatAsPhonemes + ? try await manager.synthesize(ipa: text, noiseSeed: seed) + : try await manager.synthesize(text: text, noiseSeed: seed) + let tSynth1 = Date() + + let outURL = resolveInputURL(output) + try FileManager.default.createDirectory( + at: outURL.deletingLastPathComponent(), withIntermediateDirectories: true) + let wav = try AudioWAV.data( + from: samples, sampleRate: Double(InflectConstants.sampleRate)) + try wav.write(to: outURL) + + let loadS = tLoad1.timeIntervalSince(tLoad0) + let synthS = tSynth1.timeIntervalSince(tSynth0) + let totalS = tSynth1.timeIntervalSince(tStart) + let audioSecs = Double(samples.count) / Double(InflectConstants.sampleRate) + let rtfx = synthS > 0 ? audioSecs / synthS : 0 + + logger.info("Inflect synthesis complete") + logger.info(" Load: \(String(format: "%.3f", loadS))s") + logger.info(" Synthesis: \(String(format: "%.3f", synthS))s") + logger.info(" Audio: \(String(format: "%.3f", audioSecs))s") + logger.info(" RTFx: \(String(format: "%.2f", rtfx))x") + logger.info(" Total: \(String(format: "%.3f", totalS))s") + logger.info(" Output: \(outURL.path)") + + if let metricsPath { + let metricsDict: [String: Any] = [ + "backend": "inflect-\(variant.rawValue)", + "text": text, + "phonemes_mode": treatAsPhonemes, + "seed": seed, + "output": outURL.path, + "model_load_time_s": loadS, + "inference_time_s": synthS, + "audio_duration_s": audioSecs, + "realtime_speed": rtfx, + "total_time_s": totalS, + ] + let artifactsRoot = try ensureArtifactsRoot() + let mURL = resolveOutputURL( + metricsPath, artifactsRoot: artifactsRoot, expectsDirectory: false) + try FileManager.default.createDirectory( + at: mURL.deletingLastPathComponent(), withIntermediateDirectories: true) + let json = try JSONSerialization.data( + withJSONObject: metricsDict, options: [.prettyPrinted]) + try json.write(to: mURL) + logger.info("Metrics saved: \(mURL.path)") + } + } catch { + logger.error("Inflect Error: \(error)") + print("Inflect failed: \(error)") + exit(1) } } diff --git a/Tests/FluidAudioTests/TTS/Inflect/InflectNoiseTests.swift b/Tests/FluidAudioTests/TTS/Inflect/InflectNoiseTests.swift new file mode 100644 index 00000000..e7bb9644 --- /dev/null +++ b/Tests/FluidAudioTests/TTS/Inflect/InflectNoiseTests.swift @@ -0,0 +1,51 @@ +import XCTest + +@testable import FluidAudio + +final class InflectNoiseTests: XCTestCase { + + func testSameSeedIsDeterministic() { + var a = InflectNoise(seed: 42) + var b = InflectNoise(seed: 42) + for _ in 0..<1000 { + XCTAssertEqual(a.nextGaussian(), b.nextGaussian()) + } + } + + func testDifferentSeedsDiverge() { + var a = InflectNoise(seed: 1) + var b = InflectNoise(seed: 2) + var anyDifferent = false + for _ in 0..<100 where a.nextGaussian() != b.nextGaussian() { + anyDifferent = true + } + XCTAssertTrue(anyDifferent) + } + + func testApproximatelyStandardNormal() { + var rng = InflectNoise(seed: 7) + let n = 50_000 + var sum: Double = 0 + var sumSq: Double = 0 + for _ in 0..