Dev.ap/phone based models

styletts2/cli/synthesize.py

@@ -13,7 +13,8 @@
 from everyvoice import logger
 from everyvoice.base_cli import command, default_typer_args
 from everyvoice.base_cli.interfaces import typer_file_option
-from everyvoice.model.feature_prediction.FastSpeech2_lightning.fs2.type_definitions import (
+from everyvoice.config.type_definitions import (
+    DatasetTextRepresentation,
     SynthesizeOutputFormats,
 )
 
@@ -73,6 +74,7 @@ def synthesize_one(
     acoustic_blend: float = 0.3,
     prosody_blend: float = 0.7,
     language: str | None = None,
+    text_representation: DatasetTextRepresentation | None = None,
 ):
     """Synthesize a single utterance and return a float32 numpy waveform.
 
@@ -82,21 +84,20 @@ def synthesize_one(
 
     ``language`` selects the language embedding for multilingual checkpoints
     (must be a key of ``module.lang2id``); ignored for monolingual checkpoints.
+    ``text_representation`` indicates whether ``text`` is raw characters (the
+    default) or already-phonemized IPA; see ``encode_text_for_inference``.
     """
     import torch
 
-    from ..text_utils import (
-        TextCleaner,
-    )
     from ..utils import (
         _load_reference_mel,
+        encode_text_for_inference,
     )
 
     with torch.no_grad():
-        text_cleaner = TextCleaner()
-        tokens = torch.LongTensor(text_cleaner(text)).unsqueeze(0).to(device)
-        if tokens.numel() == 0:
-            raise ValueError(f"Text produced no tokens: {text!r}")
+        tokens = encode_text_for_inference(
+            module, text, language, text_representation
+        ).to(device)
 
         input_lengths = torch.LongTensor([tokens.size(1)]).to(device)
         ref_mel = _load_reference_mel(reference_path, module.sr, mel_transform).to(
@@ -156,6 +157,14 @@ def synthesize(
         " Use --filelist instead if you want to synthesize a lot of sentences or"
         " have different speaker/language/reference per sentence.",
     ),
+    text_representation: DatasetTextRepresentation = typer.Option(
+        DatasetTextRepresentation.characters,
+        "--text-representation",
+        help="The representation of the text passed via --text: 'characters' or"
+        " 'phones' (already-phonemized IPA). The input type must be compatible"
+        " with your model. Only applies to --text; --filelist rows determine"
+        " this automatically from their 'characters'/'phones' column.",
+    ),
     filelist: Path | None = typer_file_option(
         None,
         "--filelist",
@@ -199,13 +208,16 @@ def synthesize(
         "-s",
         help="Speaker label written into output filenames.",
     ),
-    language: str = typer.Option(
-        "und",
+    language: str | None = typer.Option(
+        None,
         "--language",
         "-l",
-        help="Language tag written into output filenames. For multilingual "
-        "checkpoints (must match a language seen during training), this also "
-        "selects the language embedding; ignored for monolingual checkpoints.",
+        help="Language tag written into output filenames, and used to select "
+        "a g2p engine/normalization rules for text processing. For "
+        "multilingual checkpoints (must match a language seen during "
+        "training), this also selects the language embedding. Defaults to "
+        "the checkpoint's own (single) training language for monolingual "
+        "checkpoints.",
     ),
     diffusion_steps: int = typer.Option(
         5,
@@ -267,6 +279,7 @@ def synthesize(
         build_filelist_entries,
         build_text_entries,
         get_styletts2_synthesis_output_callbacks,
+        get_styletts2_text_split_params,
     )
 
     device = torch.device(
@@ -282,9 +295,15 @@ def synthesize(
     module, mel_transform = load_styletts2_model(model_path, device)
     module._mel_transform = mel_transform
 
+    language = language or next(iter(module.lang2id.keys()), None)
+
     state = torch.load(model_path, map_location="cpu", weights_only=True)
     global_step = int(state.get("global_step", 0))
 
+    split_text, split_params = get_styletts2_text_split_params(
+        module, language, text_representation
+    )
+
     try:
         if text:
             entries = build_text_entries(
@@ -296,6 +315,9 @@ def synthesize(
                 embedding_scale,
                 acoustic_blend,
                 prosody_blend,
+                text_representation=text_representation,
+                split_text=split_text,
+                split_params=split_params,
             )
         else:
             assert filelist is not None
@@ -309,6 +331,8 @@ def synthesize(
                 embedding_scale,
                 acoustic_blend,
                 prosody_blend,
+                split_text=split_text,
+                split_params=split_params,
             )
     except ValueError as e:
         logger.error(str(e))

styletts2/cli/train.py

@@ -72,16 +72,15 @@ def train(
 
     config = to_native_config(ev_config)
 
-    if ev_config.model.multilingual:
-        from everyvoice.text.lookups import lookuptables_from_config
+    from everyvoice.text.lookups import lookuptables_from_config
 
-        lang2id, _ = lookuptables_from_config(ev_config)
-        if not lang2id:
-            sys.exit(
-                "ERROR: model.multilingual is true but no 'language' values were "
-                "found in your training/validation filelists."
-            )
-        config["lang2id"] = lang2id
+    lang2id, _ = lookuptables_from_config(ev_config)
+    if ev_config.model.multilingual and not lang2id:
+        sys.exit(
+            "ERROR: model.multilingual is true but no 'language' values were "
+            "found in your training/validation filelists."
+        )
+    config["lang2id"] = lang2id
 
     tr = ev_config.training
     max_epochs = (

styletts2/cli/utils_heavy.py

@@ -13,65 +13,101 @@
 from everyvoice.base_cli.prediction_writing_callback import (
     BasePredictionWritingCallback,
 )
-from everyvoice.model.feature_prediction.FastSpeech2_lightning.fs2.type_definitions import (
+from everyvoice.config.type_definitions import (
+    DatasetTextRepresentation,
     SynthesizeOutputFormats,
 )
-from everyvoice.model.feature_prediction.FastSpeech2_lightning.fs2.utils import (
-    truncate_basename,
-)
-from everyvoice.utils import slugify
+from everyvoice.text.textsplit import chunk_text, resolve_split_params
+from everyvoice.utils import resolve_chunked_basename, slugify, truncate_basename
 from torch.utils.data import DataLoader, Dataset
 
 
+def get_styletts2_text_split_params(
+    module,
+    language: Optional[str],
+    text_representation: DatasetTextRepresentation | None = None,
+) -> "tuple[bool, tuple[int, int, str, str]]":
+    """Calculate text-chunking parameters for StyleTTS2.
+
+    StyleTTS2 has no per-model length statistics, so ``desired_length``/
+    ``max_length`` keep ``resolve_split_params``'s defaults; only the boundary
+    lookup depends on the checkpoint's ``TextConfig``. ``text_representation`` is
+    accepted for call-site symmetry with FastSpeech2 but unused here.
+    """
+    return resolve_split_params(module.config["ev_config"].text, language)
+
+
 def build_text_entries(
     texts: list[str],
     reference_path: str,
     speaker: str,
-    language: str,
+    language: Optional[str],
     diffusion_steps: int,
     embedding_scale: float,
     acoustic_blend: float,
     prosody_blend: float,
+    text_representation: DatasetTextRepresentation = DatasetTextRepresentation.characters,
+    split_text: bool = False,
+    split_params: tuple[int, int, str, str] = (100, 200, "", ""),
 ) -> list[dict]:
     """Build one synthesis entry per `--text` string, all sharing the same
-    reference/speaker/language."""
-    return [
-        {
-            "raw_text": t,
-            "basename": truncate_basename(slugify(t)),
-            "speaker": speaker,
-            "language": language,
-            "reference_path": reference_path,
-            "diffusion_steps": diffusion_steps,
-            "embedding_scale": embedding_scale,
-            "acoustic_blend": acoustic_blend,
-            "prosody_blend": prosody_blend,
-        }
-        for t in texts
-    ]
+    reference/speaker/language. When `split_text` is set, long strings are
+    split into multiple chunk entries (via `chunk_text`) sharing the same
+    basename, flagged with `is_last_input_chunk` for reassembly at write time."""
+    entries = []
+    for t in texts:
+        chunks = chunk_text(t, *split_params) if split_text else [t]
+        basename = truncate_basename(slugify(t))
+        for i, chunk in enumerate(chunks):
+            entries.append(
+                {
+                    "raw_text": chunk,
+                    "basename": basename,
+                    "speaker": speaker,
+                    "language": language,
+                    "reference_path": reference_path,
+                    "diffusion_steps": diffusion_steps,
+                    "embedding_scale": embedding_scale,
+                    "acoustic_blend": acoustic_blend,
+                    "prosody_blend": prosody_blend,
+                    "text_representation": text_representation,
+                    "is_last_input_chunk": i == len(chunks) - 1,
+                }
+            )
+    return entries
 
 
 def build_filelist_entries(
     rows: Iterable[dict],
     default_reference_path: Optional[str],
     default_speaker: str,
-    default_language: str,
+    default_language: Optional[str],
     diffusion_steps: int,
     embedding_scale: float,
     acoustic_blend: float,
     prosody_blend: float,
+    split_text: bool = False,
+    split_params: tuple[int, int, str, str] = (100, 200, "", ""),
 ) -> list[dict]:
-    """Build one synthesis entry per filelist row.
+    """Build one synthesis entry per filelist row (or per chunk of it, when
+    `split_text` is set).
 
     Each row may supply its own 'basename', 'speaker', 'language', and
     'reference'/'reference_path' columns; any that are absent fall back to
-    the corresponding default. Raises ValueError if a row has no
-    'characters'/'phones' column, or no reference path can be resolved.
+    the corresponding default. The row's text representation is inferred
+    from which of 'characters'/'phones' is present (matching precedence),
+    so it doesn't need to be passed in. Raises ValueError if a row has
+    neither column, or no reference path can be resolved.
     """
     entries = []
     for d in rows:
-        raw_text = d.get("characters") or d.get("phones")
-        if not raw_text:
+        if d.get("characters"):
+            raw_text = d["characters"]
+            text_representation = DatasetTextRepresentation.characters
+        elif d.get("phones"):
+            raw_text = d["phones"]
+            text_representation = DatasetTextRepresentation.ipa_phones
+        else:
             raise ValueError(
                 f"Filelist row is missing a 'characters' or 'phones' column: {d}"
             )
@@ -83,19 +119,26 @@ def build_filelist_entries(
                 "Missing --reference option, and this filelist row has no"
                 f" 'reference'/'reference_path' column: {d}"
             )
-        entries.append(
-            {
-                "raw_text": raw_text,
-                "basename": d.get("basename", truncate_basename(slugify(raw_text))),
-                "speaker": d.get("speaker", default_speaker),
-                "language": d.get("language", default_language),
-                "reference_path": reference_path,
-                "diffusion_steps": diffusion_steps,
-                "embedding_scale": embedding_scale,
-                "acoustic_blend": acoustic_blend,
-                "prosody_blend": prosody_blend,
-            }
-        )
+        basename = d.get("basename", truncate_basename(slugify(raw_text)))
+        speaker = d.get("speaker", default_speaker)
+        language = d.get("language", default_language)
+        chunks = chunk_text(raw_text, *split_params) if split_text else [raw_text]
+        for i, chunk in enumerate(chunks):
+            entries.append(
+                {
+                    "raw_text": chunk,
+                    "basename": basename,
+                    "speaker": speaker,
+                    "language": language,
+                    "reference_path": reference_path,
+                    "diffusion_steps": diffusion_steps,
+                    "embedding_scale": embedding_scale,
+                    "acoustic_blend": acoustic_blend,
+                    "prosody_blend": prosody_blend,
+                    "text_representation": text_representation,
+                    "is_last_input_chunk": i == len(chunks) - 1,
+                }
+            )
     return entries
 
 
@@ -142,6 +185,9 @@ def __init__(
             simple_filenames=simple_filenames,
         )
         self.last_file_written: Optional[str] = None
+        self.full_wav = torch.tensor(())  # Accumulates full wav before saving file
+        self.full_text: str = ""  # Accumulates full text before saving file
+        self.chunk_basenames: list[str] = []  # Accumulates per-chunk basenames
 
     def on_predict_batch_end(  # pyright: ignore [reportIncompatibleMethodOverride]
         self,
@@ -155,12 +201,22 @@ def on_predict_batch_end(  # pyright: ignore [reportIncompatibleMethodOverride]
         if outputs is None:
             return
         wav_tensor = torch.from_numpy(outputs["wav"]).unsqueeze(0)
-        filename = self.get_filename(
-            outputs["basename"], outputs["speaker"], outputs["language"]
-        )
+
+        # Concatenate the current chunk onto the accumulated wav/text for this
+        # utterance (a no-op when the input wasn't split into chunks, since
+        # is_last_input_chunk is True on every single-chunk entry).
+        self.full_wav = torch.cat((self.full_wav, wav_tensor), -1)
+        self.full_text += outputs["raw_text"]
+        self.chunk_basenames.append(outputs["basename"])
+
+        if not outputs.get("is_last_input_chunk", True):
+            return
+
+        basename = resolve_chunked_basename(self.chunk_basenames, self.full_text)
+        filename = self.get_filename(basename, outputs["speaker"], outputs["language"])
         torchaudio.save(
             filename,
-            wav_tensor,
+            self.full_wav,
             outputs["sample_rate"],
             format="wav",
             encoding="PCM_S",
@@ -169,6 +225,11 @@ def on_predict_batch_end(  # pyright: ignore [reportIncompatibleMethodOverride]
         self.last_file_written = filename
         logger.info(f"Saved WAV: {filename}")
 
+        # Reset the accumulator variables
+        self.full_wav = torch.tensor(())
+        self.full_text = ""
+        self.chunk_basenames = []
+
 
 def get_styletts2_synthesis_output_callbacks(
     output_type: Sequence[SynthesizeOutputFormats],

styletts2/dataset.py

@@ -56,8 +56,8 @@ def __init__(
         validation=False,
         OOD_data="data/OOD_texts.txt",
         OOD_min_length=50,
-        ood_data_paths: "dict[str, Any] | None" = None,
-        ood_val_list: "list[dict] | None" = None,
+        ood_data_paths: dict[str, Any] | None = None,
+        ood_val_list: list[dict] | None = None,
     ):
         pp = config["preprocess_params"]
         self.sr = pp.get("sr", 24000)
@@ -141,8 +141,8 @@ def __len__(self):
 
     def _load_ood_ev(
         self,
-        ood_data_paths: "dict[str, Any] | None",
-        ood_val_list: "list[dict] | None",
+        ood_data_paths: dict[str, Any] | None,
+        ood_val_list: list[dict] | None,
     ) -> None:
         """Populate self.ood_texts from per-language PSV paths or a val-fallback list."""
         from everyvoice.utils import generic_psv_filelist_reader