From d5c26bcdaa943efb063abd17d36059ff6626b09f Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 11:59:11 +0100 Subject: [PATCH 001/110] feat: OVOS-INTENT-2 resource loader and dialog renderer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the remaining two intent primitives. - resources.py — `LocaleResources`, the OVOS-INTENT-2 loader: the §3 common reader (UTF-8/BOM/CRLF, comments, blanks), recursive `locale//` search with case-insensitive tags, the user→skill→core override precedence (§2.1), and the five resource roles (§4). The user-data path is a parameter — no configuration import. Duplicate resources and empty files raise `MalformedResource`. - dialog.py — `render()`, the OVOS-INTENT-2 §4.2 dialog renderer: select a phrase, expand its variety, fill `{name}` slots from caller values. Expansion precedes fill so a slot value is never parsed as grammar. An unfilled slot raises `UnfilledSlot`. 20 more conformance tests (58 total). Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 36 +++++- ovos_intent_primitives/__init__.py | 26 +++- ovos_intent_primitives/dialog.py | 79 ++++++++++++ ovos_intent_primitives/resources.py | 190 ++++++++++++++++++++++++++++ test/test_dialog.py | 55 ++++++++ test/test_resources.py | 130 +++++++++++++++++++ 6 files changed, 509 insertions(+), 7 deletions(-) create mode 100644 ovos_intent_primitives/dialog.py create mode 100644 ovos_intent_primitives/resources.py create mode 100644 test/test_dialog.py create mode 100644 test/test_resources.py diff --git a/README.md b/README.md index 16ebf41..17dd451 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ implementation those components — and any third-party tool — can depend on. | Primitive | Spec | State | |-----------|------|-------| | Sentence template expander | OVOS-INTENT-1 v2 | available | -| Locale resource loader | OVOS-INTENT-2 | planned | -| Dialog renderer | OVOS-INTENT-2 §4.2 | planned | +| Locale resource loader | OVOS-INTENT-2 | available | +| Dialog renderer | OVOS-INTENT-2 §4.2 | available | ## The expander @@ -42,6 +42,38 @@ repeated slot name, or an undefined or cyclic vocabulary reference — raises Input is assumed already ASR-normalized (OVOS-INTENT-1 §2): lowercase, single-spaced, alphanumeric. This package expands; it does not normalize. +## The resource loader + +`LocaleResources` discovers and loads a skill's locale resource files +(OVOS-INTENT-2) — the five roles `.intent`, `.dialog`, `.entity`, `.voc`, +`.blacklist` — through the user → skill → core override precedence, searching +each `locale//` tree recursively. + +```python +from ovos_intent_primitives import LocaleResources + +res = LocaleResources("en-US", skill_locale="my-skill/locale") +res.load_intent("play") # sample set, named slots intact +res.load_dialog("weather") # phrase strings, not expanded (§4.2) +res.load_vocabulary("yes") # expanded phrase set +``` + +The user-data path of the override precedence is assistant-defined and passed +in (`user_locale=`); this package imports no configuration. + +## The dialog renderer + +`render()` selects one phrase from a loaded `.dialog`, expands its variety to a +single variant, and fills every `{name}` slot with a caller-supplied value +(OVOS-INTENT-2 §4.2). A phrase with an unfilled slot raises `UnfilledSlot`. + +```python +from ovos_intent_primitives import render + +render(res.load_dialog("weather"), slots={"temperature": 21}) +# 'It is 21 degrees.' +``` + ## Install ```bash diff --git a/ovos_intent_primitives/__init__.py b/ovos_intent_primitives/__init__.py index 3e4e56e..6b3e984 100644 --- a/ovos_intent_primitives/__init__.py +++ b/ovos_intent_primitives/__init__.py @@ -4,12 +4,28 @@ the OVOS intent specifications describe: - :func:`~ovos_intent_primitives.expansion.expand` — the OVOS-INTENT-1 - sentence template expander. - -The resource-file loader (OVOS-INTENT-2) and the dialog renderer -(OVOS-INTENT-2 §4.2) are added in a later release. + sentence template expander; +- :class:`~ovos_intent_primitives.resources.LocaleResources` — the + OVOS-INTENT-2 locale resource-file loader; +- :func:`~ovos_intent_primitives.dialog.render` — the OVOS-INTENT-2 §4.2 + dialog renderer. """ +from ovos_intent_primitives.dialog import UnfilledSlot, render from ovos_intent_primitives.expansion import MalformedTemplate, expand +from ovos_intent_primitives.resources import ( + LocaleResources, + MalformedResource, + read_resource_file, +) from ovos_intent_primitives.version import __version__ -__all__ = ["expand", "MalformedTemplate", "__version__"] +__all__ = [ + "expand", + "MalformedTemplate", + "LocaleResources", + "MalformedResource", + "read_resource_file", + "render", + "UnfilledSlot", + "__version__", +] diff --git a/ovos_intent_primitives/dialog.py b/ovos_intent_primitives/dialog.py new file mode 100644 index 0000000..77e13af --- /dev/null +++ b/ovos_intent_primitives/dialog.py @@ -0,0 +1,79 @@ +"""Reference dialog renderer for OVOS-INTENT-2 §4.2. + +This is the reference implementation of the **Dialog renderer** conformance +role of OVOS-INTENT-1 §7. To render a dialog, :func:`render` selects one phrase +from a loaded ``.dialog``, expands its ``(a|b)`` / ``[x]`` variety to a single +variant, and fills every ``{name}`` slot with a caller-supplied value. + +Only the single-brace slot form ``{name}`` is recognized; there is no ``{{ }}`` +form (OVOS-INTENT-2 §4.2). Slots are filled by the caller; a chosen phrase with +a slot the caller did not fill raises :class:`UnfilledSlot` and is not rendered. +""" +from __future__ import annotations + +import random as _random +import re +from typing import Dict, Optional, Sequence + +from ovos_intent_primitives.expansion import expand + +__all__ = ["render", "UnfilledSlot"] + +_SLOT_TOKEN_RE = re.compile(r"\{([a-z][a-z0-9_]*)\}") + + +class UnfilledSlot(ValueError): + """A chosen dialog phrase has a named slot the caller did not fill. + + Per OVOS-INTENT-1 §5.1 the caller must supply a value for every slot in the + chosen phrase; a phrase with an unfilled slot must not be sent to TTS. + """ + + +def render(phrases: Sequence[str], + slots: Optional[Dict[str, object]] = None, + vocabularies: Optional[Dict[str, Sequence[str]]] = None, + rng: Optional[object] = None) -> str: + """Render one phrase from a loaded ``.dialog``. + + Args: + phrases: the phrase strings of a ``.dialog``, as returned by + :meth:`~ovos_intent_primitives.resources.LocaleResources.load_dialog`. + slots: caller-supplied values for the phrase's named slots, keyed by + slot name. Values are converted to text. + vocabularies: vocabularies for any ```` references in the phrase. + rng: an object with a ``choice`` method, for phrase and variant + selection; defaults to the :mod:`random` module. Inject a seeded + generator for deterministic output. + + Returns: + One rendered phrase, ready for text-to-speech. + + Raises: + UnfilledSlot: a slot in the chosen phrase has no caller-supplied value. + ValueError: ``phrases`` is empty. + MalformedTemplate: the chosen phrase is not a valid template. + """ + if not phrases: + raise ValueError("no dialog phrases to render") + chooser = rng if rng is not None else _random + slots = slots or {} + + # Expansion keeps slots opaque (OVOS-INTENT-1 §4): expand first, pick a + # variant, then fill — so a slot value can never be parsed as grammar. + phrase = chooser.choice(list(phrases)) + variant = chooser.choice(expand(phrase, vocabularies)) + return _fill_slots(variant, slots) + + +def _fill_slots(variant: str, slots: Dict[str, object]) -> str: + """Replace every ``{name}`` in ``variant`` with its caller-supplied value.""" + + def replace(match: "re.Match") -> str: + name = match.group(1) + if name not in slots: + raise UnfilledSlot( + f"dialog slot {{{name}}} was not filled by the caller") + return str(slots[name]) + + return _SLOT_TOKEN_RE.sub(replace, variant) diff --git a/ovos_intent_primitives/resources.py b/ovos_intent_primitives/resources.py new file mode 100644 index 0000000..5d07e55 --- /dev/null +++ b/ovos_intent_primitives/resources.py @@ -0,0 +1,190 @@ +"""Reference loader for OVOS-INTENT-2 — Locale Resource Formats. + +This module is the reference implementation of the loader of OVOS-INTENT-2: it +discovers a skill's locale resource files, reads them with the common reader +(§3), and loads each of the five resource roles per its format (§4). + +The five roles, by extension: + +- ``.intent`` — slot-bearing intent training samples; +- ``.dialog`` — slot-bearing spoken-response phrases; +- ``.entity`` — slot-free example values for a slot; +- ``.voc`` — slot-free vocabulary; +- ``.blacklist`` — slot-free intent-suppression phrases. + +The user-data path of the override precedence (§2.1) is **assistant-defined**; +this module takes it as a parameter and imports no configuration. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Dict, List, Optional + +from ovos_intent_primitives.expansion import MalformedTemplate, expand + +__all__ = [ + "LocaleResources", + "MalformedResource", + "read_resource_file", + "SLOT_BEARING_ROLES", + "SLOT_FREE_ROLES", +] + +# Resource roles, by file extension (OVOS-INTENT-2 §1). +SLOT_BEARING_ROLES = (".intent", ".dialog") +SLOT_FREE_ROLES = (".entity", ".voc", ".blacklist") + + +class MalformedResource(ValueError): + """A resource file or skill layout that violates OVOS-INTENT-2. + + Raised for an empty resource file (§5), a duplicate ``(role, base name)`` + within one language tree (§2), and a named slot in a slot-free role (§4.3). + """ + + +def read_resource_file(path: Path) -> List[str]: + """Apply the OVOS-INTENT-2 §3 common reader to one file. + + The file is read as UTF-8, a leading byte-order mark is discarded, and both + ``LF`` and ``CRLF`` line endings are accepted. Each line is stripped; blank + lines and ``#``-comment lines are dropped. The surviving lines — each one + template — are returned in order. + """ + text = path.read_text(encoding="utf-8-sig") # utf-8-sig discards a BOM + templates: List[str] = [] + for raw_line in text.splitlines(): # splitlines() accepts LF and CRLF + line = raw_line.strip() + if not line or line.startswith("#"): + continue + templates.append(line) + return templates + + +class LocaleResources: + """Loads OVOS-INTENT-2 resource files for one skill in one language. + + A resource is resolved through the override precedence of §2.1 — user + overrides, then skill resources, then core resources — searching each + source's ``/`` directory and all its subdirectories recursively. + """ + + def __init__(self, lang: str, skill_locale: str, + core_locale: Optional[str] = None, + user_locale: Optional[str] = None): + """ + Args: + lang: a BCP-47 language tag; compared case-insensitively (§2). + skill_locale: path to the skill's ``locale/`` directory. + core_locale: path to the assistant's core ``locale/`` directory. + user_locale: path to the user-override ``locale/`` directory + (its root is assistant-defined, §2.1). + """ + self.lang = lang + # Highest precedence first (§2.1): user, skill, core. + self._sources: List[Path] = [ + Path(p) for p in (user_locale, skill_locale, core_locale) + if p is not None + ] + + def _lang_dir(self, source: Path) -> Optional[Path]: + """The ``/`` directory under one source, matched + case-insensitively (§2).""" + if not source.is_dir(): + return None + target = self.lang.lower() + for child in source.iterdir(): + if child.is_dir() and child.name.lower() == target: + return child + return None + + def _locate(self, base_name: str, extension: str) -> Optional[Path]: + """Find a resource file by ``(base name, extension)`` through the + precedence chain. Returns the first match, or ``None``.""" + filename = base_name + extension + for source in self._sources: + lang_dir = self._lang_dir(source) + if lang_dir is None: + continue + matches = sorted(p for p in lang_dir.rglob(filename) if p.is_file()) + if len(matches) > 1: + raise MalformedResource( + f"duplicate resource {filename!r} within {lang_dir} — " + f"a (role, base name) must be unique per language tree") + if matches: + return matches[0] + return None + + def vocabularies(self) -> Dict[str, List[str]]: + """Every ``.voc`` reachable for this language, as a name→templates map + suitable for resolving ```` references during expansion.""" + vocs: Dict[str, List[str]] = {} + # Lowest precedence first, so a higher-precedence file overrides. + for source in reversed(self._sources): + lang_dir = self._lang_dir(source) + if lang_dir is None: + continue + for path in lang_dir.rglob("*.voc"): + if path.is_file(): + vocs[path.stem] = read_resource_file(path) + return vocs + + def _load_expanded(self, base_name: str, extension: str) -> List[str]: + """Load a resource and expand it to its sample set.""" + path = self._locate(base_name, extension) + if path is None: + raise FileNotFoundError( + f"no {extension} resource named {base_name!r} for " + f"language {self.lang!r}") + templates = read_resource_file(path) + if not templates: + raise MalformedResource( + f"empty resource file {path} — every file must contribute at " + f"least one template (§5)") + vocabularies = self.vocabularies() + slot_free = extension in SLOT_FREE_ROLES + samples: List[str] = [] + for template in templates: + for sample in expand(template, vocabularies): + if slot_free and "{" in sample: + raise MalformedResource( + f"{extension} resource {path} is slot-free but " + f"template {template!r} contains a named slot") + if sample not in samples: + samples.append(sample) + return samples + + def load_intent(self, base_name: str) -> List[str]: + """Load an ``.intent`` as its sample set, named slots intact (§4.1).""" + return self._load_expanded(base_name, ".intent") + + def load_entity(self, base_name: str) -> List[str]: + """Load an ``.entity`` value set (§4.3).""" + return self._load_expanded(base_name, ".entity") + + def load_vocabulary(self, base_name: str) -> List[str]: + """Load a ``.voc`` phrase set (§4.3).""" + return self._load_expanded(base_name, ".voc") + + def load_blacklist(self, base_name: str) -> List[str]: + """Load a ``.blacklist`` phrase set (§4.3).""" + return self._load_expanded(base_name, ".blacklist") + + def load_dialog(self, base_name: str) -> List[str]: + """Load a ``.dialog`` as its list of phrase strings. + + Unlike the other roles a ``.dialog`` is **not** expanded at load time — + expansion happens per render, on the one phrase chosen (§4.2). The + phrase strings are returned verbatim, for a dialog renderer to consume. + """ + path = self._locate(base_name, ".dialog") + if path is None: + raise FileNotFoundError( + f"no .dialog resource named {base_name!r} for " + f"language {self.lang!r}") + phrases = read_resource_file(path) + if not phrases: + raise MalformedResource( + f"empty resource file {path} — every file must contribute at " + f"least one template (§5)") + return phrases diff --git a/test/test_dialog.py b/test/test_dialog.py new file mode 100644 index 0000000..c6290b8 --- /dev/null +++ b/test/test_dialog.py @@ -0,0 +1,55 @@ +"""Conformance tests for the OVOS-INTENT-2 §4.2 reference dialog renderer.""" +import pytest + +from ovos_intent_primitives import UnfilledSlot, render + + +class _FixedRng: + """A deterministic stand-in for `random`: always picks a chosen index.""" + + def __init__(self, index=0): + self.index = index + + def choice(self, seq): + seq = list(seq) + return seq[self.index % len(seq)] + + +def test_render_fills_slots(): + out = render(["It is {temperature} degrees."], + slots={"temperature": 21}) + assert out == "It is 21 degrees." + + +def test_render_selects_a_phrase(): + phrases = ["first phrase", "second phrase"] + assert render(phrases, rng=_FixedRng(0)) == "first phrase" + assert render(phrases, rng=_FixedRng(1)) == "second phrase" + + +def test_render_expands_variety_then_fills(): + out = render(["(Currently|At the moment) it is {t} degrees"], + slots={"t": 5}, rng=_FixedRng(0)) + assert out == "Currently it is 5 degrees" + + +def test_unfilled_slot_raises(): + with pytest.raises(UnfilledSlot): + render(["It is {temperature} degrees."], slots={}) + + +def test_slot_value_is_never_parsed_as_grammar(): + """Expansion happens before fill, so a value with metacharacters is safe.""" + out = render(["you said {echo}"], slots={"echo": "(a|b)"}) + assert out == "you said (a|b)" + + +def test_render_resolves_vocabulary_references(): + out = render([" {name}"], slots={"name": "Sam"}, + vocabularies={"greet": ["hello"]}, rng=_FixedRng(0)) + assert out == "hello Sam" + + +def test_empty_phrase_list_raises(): + with pytest.raises(ValueError): + render([]) diff --git a/test/test_resources.py b/test/test_resources.py new file mode 100644 index 0000000..e2cf279 --- /dev/null +++ b/test/test_resources.py @@ -0,0 +1,130 @@ +"""Conformance tests for the OVOS-INTENT-2 reference loader.""" +import pytest + +from ovos_intent_primitives import ( + LocaleResources, + MalformedResource, + read_resource_file, +) + + +def _write(path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _skill(tmp_path, lang="en-US"): + """A skill `locale/` directory rooted at tmp_path.""" + return tmp_path / "locale", (tmp_path / "locale" / lang) + + +# --- §3 common reader -------------------------------------------------------- + +def test_common_reader_skips_blanks_and_comments(tmp_path): + f = tmp_path / "x.voc" + _write(f, "# a comment\n\nyes\n yeah \n# another\nyep\n") + assert read_resource_file(f) == ["yes", "yeah", "yep"] + + +def test_common_reader_accepts_crlf_and_bom(tmp_path): + f = tmp_path / "x.voc" + f.write_bytes(b"\xef\xbb\xbfyes\r\nyeah\r\n") # BOM + CRLF + assert read_resource_file(f) == ["yes", "yeah"] + + +# --- §4.1 .intent ------------------------------------------------------------ + +def test_load_intent_expands_and_keeps_slots(tmp_path): + locale, lang = _skill(tmp_path) + _write(lang / "play.intent", "(play|put on) {query}\n") + res = LocaleResources("en-US", str(locale)) + assert sorted(res.load_intent("play")) == ["play {query}", "put on {query}"] + + +def test_load_intent_resolves_voc_references(tmp_path): + locale, lang = _skill(tmp_path) + _write(lang / "greeting.voc", "hello\nhi\n") + _write(lang / "greet.intent", " {name}\n") + res = LocaleResources("en-US", str(locale)) + assert sorted(res.load_intent("greet")) == ["hello {name}", "hi {name}"] + + +# --- §4.2 .dialog ------------------------------------------------------------ + +def test_load_dialog_returns_unexpanded_phrases(tmp_path): + locale, lang = _skill(tmp_path) + _write(lang / "hi.dialog", "Hello {name}!\n(Hi|Hey) {name}.\n") + res = LocaleResources("en-US", str(locale)) + assert res.load_dialog("hi") == ["Hello {name}!", "(Hi|Hey) {name}."] + + +# --- §4.3 slot-free roles ---------------------------------------------------- + +def test_load_entity_and_blacklist(tmp_path): + locale, lang = _skill(tmp_path) + _write(lang / "weekday.entity", "monday\n(tues|wednes)day\n") + _write(lang / "play.blacklist", "trailer\n") + res = LocaleResources("en-US", str(locale)) + assert sorted(res.load_entity("weekday")) == ["monday", "tuesday", "wednesday"] + assert res.load_blacklist("play") == ["trailer"] + + +def test_slot_free_role_rejects_a_named_slot(tmp_path): + locale, lang = _skill(tmp_path) + _write(lang / "bad.voc", "a slot {here}\n") + res = LocaleResources("en-US", str(locale)) + with pytest.raises(MalformedResource): + res.load_vocabulary("bad") + + +# --- §2 layout --------------------------------------------------------------- + +def test_subdirectories_are_searched_recursively(tmp_path): + locale, lang = _skill(tmp_path) + _write(lang / "intents" / "deep.intent", "do the thing\n") + res = LocaleResources("en-US", str(locale)) + assert res.load_intent("deep") == ["do the thing"] + + +def test_language_tag_is_case_insensitive(tmp_path): + locale, lang = _skill(tmp_path, lang="en-us") + _write(lang / "x.intent", "hello world\n") + res = LocaleResources("en-US", str(locale)) # requested with different case + assert res.load_intent("x") == ["hello world"] + + +def test_duplicate_resource_in_one_tree_is_malformed(tmp_path): + locale, lang = _skill(tmp_path) + _write(lang / "a" / "dup.intent", "first\n") + _write(lang / "b" / "dup.intent", "second\n") + res = LocaleResources("en-US", str(locale)) + with pytest.raises(MalformedResource): + res.load_intent("dup") + + +# --- §2.1 resolution precedence --------------------------------------------- + +def test_user_override_wins_over_skill(tmp_path): + skill_locale = tmp_path / "skill" / "locale" + user_locale = tmp_path / "user" / "locale" + _write(skill_locale / "en-US" / "x.intent", "skill version\n") + _write(user_locale / "en-US" / "x.intent", "user version\n") + res = LocaleResources("en-US", str(skill_locale), user_locale=str(user_locale)) + assert res.load_intent("x") == ["user version"] + + +# --- §5 empty files ---------------------------------------------------------- + +def test_empty_file_is_malformed(tmp_path): + locale, lang = _skill(tmp_path) + _write(lang / "empty.intent", "# only a comment\n\n") + res = LocaleResources("en-US", str(locale)) + with pytest.raises(MalformedResource): + res.load_intent("empty") + + +def test_missing_resource_raises(tmp_path): + locale, _ = _skill(tmp_path) + res = LocaleResources("en-US", str(locale)) + with pytest.raises(FileNotFoundError): + res.load_intent("nope") From 4b5d956ecb48332fb289f17a698833f591a337dd Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 12:04:46 +0100 Subject: [PATCH 002/110] refactor: rename package to ovos-spec-tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The package is the reference implementation of the OVOS formal specifications generally, not only the intent specs — the formal-specifications repo will grow beyond OVOS-INTENT-1/2/3. Renamed from ovos-intent-primitives: repo, distribution, and the import package (ovos_intent_primitives -> ovos_spec_tools). Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 10 +++--- ovos_intent_primitives/__init__.py | 31 ------------------- ovos_spec_tools/__init__.py | 31 +++++++++++++++++++ .../dialog.py | 4 +-- .../expansion.py | 0 .../resources.py | 2 +- .../version.py | 0 pyproject.toml | 8 ++--- test/test_dialog.py | 2 +- test/test_expansion.py | 2 +- test/test_resources.py | 2 +- 11 files changed, 46 insertions(+), 46 deletions(-) delete mode 100644 ovos_intent_primitives/__init__.py create mode 100644 ovos_spec_tools/__init__.py rename {ovos_intent_primitives => ovos_spec_tools}/dialog.py (95%) rename {ovos_intent_primitives => ovos_spec_tools}/expansion.py (100%) rename {ovos_intent_primitives => ovos_spec_tools}/resources.py (99%) rename {ovos_intent_primitives => ovos_spec_tools}/version.py (100%) diff --git a/README.md b/README.md index 17dd451..1672383 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ovos-intent-primitives +# ovos-spec-tools Reference implementation of the OVOS intent specifications — the low-level, dependency-light primitives that the [formal @@ -24,7 +24,7 @@ of sentences it denotes (OVOS-INTENT-1 §4). It resolves `(a|b)` alternatives, slots are opaque and carried through unchanged. ```python -from ovos_intent_primitives import expand +from ovos_spec_tools import expand expand("(turn|switch) [the] (light|fan)") # ['turn the light', 'turn the fan', 'turn light', 'turn fan', @@ -50,7 +50,7 @@ single-spaced, alphanumeric. This package expands; it does not normalize. each `locale//` tree recursively. ```python -from ovos_intent_primitives import LocaleResources +from ovos_spec_tools import LocaleResources res = LocaleResources("en-US", skill_locale="my-skill/locale") res.load_intent("play") # sample set, named slots intact @@ -68,7 +68,7 @@ single variant, and fills every `{name}` slot with a caller-supplied value (OVOS-INTENT-2 §4.2). A phrase with an unfilled slot raises `UnfilledSlot`. ```python -from ovos_intent_primitives import render +from ovos_spec_tools import render render(res.load_dialog("weather"), slots={"temperature": 21}) # 'It is 21 degrees.' @@ -77,7 +77,7 @@ render(res.load_dialog("weather"), slots={"temperature": 21}) ## Install ```bash -pip install ovos-intent-primitives +pip install ovos-spec-tools ``` ## License diff --git a/ovos_intent_primitives/__init__.py b/ovos_intent_primitives/__init__.py deleted file mode 100644 index 6b3e984..0000000 --- a/ovos_intent_primitives/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Reference implementation of the OVOS intent specifications. - -`ovos-intent-primitives` provides the low-level, dependency-light primitives -the OVOS intent specifications describe: - -- :func:`~ovos_intent_primitives.expansion.expand` — the OVOS-INTENT-1 - sentence template expander; -- :class:`~ovos_intent_primitives.resources.LocaleResources` — the - OVOS-INTENT-2 locale resource-file loader; -- :func:`~ovos_intent_primitives.dialog.render` — the OVOS-INTENT-2 §4.2 - dialog renderer. -""" -from ovos_intent_primitives.dialog import UnfilledSlot, render -from ovos_intent_primitives.expansion import MalformedTemplate, expand -from ovos_intent_primitives.resources import ( - LocaleResources, - MalformedResource, - read_resource_file, -) -from ovos_intent_primitives.version import __version__ - -__all__ = [ - "expand", - "MalformedTemplate", - "LocaleResources", - "MalformedResource", - "read_resource_file", - "render", - "UnfilledSlot", - "__version__", -] diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py new file mode 100644 index 0000000..8de48f8 --- /dev/null +++ b/ovos_spec_tools/__init__.py @@ -0,0 +1,31 @@ +"""Reference implementation of the OVOS intent specifications. + +`ovos-spec-tools` provides the low-level, dependency-light primitives +the OVOS intent specifications describe: + +- :func:`~ovos_spec_tools.expansion.expand` — the OVOS-INTENT-1 + sentence template expander; +- :class:`~ovos_spec_tools.resources.LocaleResources` — the + OVOS-INTENT-2 locale resource-file loader; +- :func:`~ovos_spec_tools.dialog.render` — the OVOS-INTENT-2 §4.2 + dialog renderer. +""" +from ovos_spec_tools.dialog import UnfilledSlot, render +from ovos_spec_tools.expansion import MalformedTemplate, expand +from ovos_spec_tools.resources import ( + LocaleResources, + MalformedResource, + read_resource_file, +) +from ovos_spec_tools.version import __version__ + +__all__ = [ + "expand", + "MalformedTemplate", + "LocaleResources", + "MalformedResource", + "read_resource_file", + "render", + "UnfilledSlot", + "__version__", +] diff --git a/ovos_intent_primitives/dialog.py b/ovos_spec_tools/dialog.py similarity index 95% rename from ovos_intent_primitives/dialog.py rename to ovos_spec_tools/dialog.py index 77e13af..a06e7cf 100644 --- a/ovos_intent_primitives/dialog.py +++ b/ovos_spec_tools/dialog.py @@ -15,7 +15,7 @@ import re from typing import Dict, Optional, Sequence -from ovos_intent_primitives.expansion import expand +from ovos_spec_tools.expansion import expand __all__ = ["render", "UnfilledSlot"] @@ -38,7 +38,7 @@ def render(phrases: Sequence[str], Args: phrases: the phrase strings of a ``.dialog``, as returned by - :meth:`~ovos_intent_primitives.resources.LocaleResources.load_dialog`. + :meth:`~ovos_spec_tools.resources.LocaleResources.load_dialog`. slots: caller-supplied values for the phrase's named slots, keyed by slot name. Values are converted to text. vocabularies: vocabularies for any ```` references in the phrase. diff --git a/ovos_intent_primitives/expansion.py b/ovos_spec_tools/expansion.py similarity index 100% rename from ovos_intent_primitives/expansion.py rename to ovos_spec_tools/expansion.py diff --git a/ovos_intent_primitives/resources.py b/ovos_spec_tools/resources.py similarity index 99% rename from ovos_intent_primitives/resources.py rename to ovos_spec_tools/resources.py index 5d07e55..f135138 100644 --- a/ovos_intent_primitives/resources.py +++ b/ovos_spec_tools/resources.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import Dict, List, Optional -from ovos_intent_primitives.expansion import MalformedTemplate, expand +from ovos_spec_tools.expansion import MalformedTemplate, expand __all__ = [ "LocaleResources", diff --git a/ovos_intent_primitives/version.py b/ovos_spec_tools/version.py similarity index 100% rename from ovos_intent_primitives/version.py rename to ovos_spec_tools/version.py diff --git a/pyproject.toml b/pyproject.toml index b06cc6f..c73a511 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] -name = "ovos-intent-primitives" +name = "ovos-spec-tools" dynamic = ["version"] description = "Reference implementation of the OVOS intent specifications" readme = "README.md" @@ -20,11 +20,11 @@ test = [ ] [project.urls] -Homepage = "https://github.com/OpenVoiceOS/ovos-intent-primitives" +Homepage = "https://github.com/OpenVoiceOS/ovos-spec-tools" [tool.setuptools.dynamic] -version = { attr = "ovos_intent_primitives.version.__version__" } +version = { attr = "ovos_spec_tools.version.__version__" } [tool.setuptools.packages.find] where = ["."] -include = ["ovos_intent_primitives*"] +include = ["ovos_spec_tools*"] diff --git a/test/test_dialog.py b/test/test_dialog.py index c6290b8..8e21c0a 100644 --- a/test/test_dialog.py +++ b/test/test_dialog.py @@ -1,7 +1,7 @@ """Conformance tests for the OVOS-INTENT-2 §4.2 reference dialog renderer.""" import pytest -from ovos_intent_primitives import UnfilledSlot, render +from ovos_spec_tools import UnfilledSlot, render class _FixedRng: diff --git a/test/test_expansion.py b/test/test_expansion.py index bf4c377..f4d3b98 100644 --- a/test/test_expansion.py +++ b/test/test_expansion.py @@ -5,7 +5,7 @@ """ import pytest -from ovos_intent_primitives import MalformedTemplate, expand +from ovos_spec_tools import MalformedTemplate, expand # --- §4.2 the worked example ------------------------------------------------- diff --git a/test/test_resources.py b/test/test_resources.py index e2cf279..8aa9b0b 100644 --- a/test/test_resources.py +++ b/test/test_resources.py @@ -1,7 +1,7 @@ """Conformance tests for the OVOS-INTENT-2 reference loader.""" import pytest -from ovos_intent_primitives import ( +from ovos_spec_tools import ( LocaleResources, MalformedResource, read_resource_file, From 277fb63a66d5ca237201858b2e77a054a5e2357f Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 12:10:01 +0100 Subject: [PATCH 003/110] feat: add ovos-spec-lint locale resource linter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CLI and library function that validates every resource file under a locale directory and reports every problem found. - syntax (OVOS-INTENT-1): each template expanded; malformed forms, empty files, and named slots in slot-free roles are errors - naming/layout (OVOS-INTENT-2): base-name charset, .entity slot-name rule, duplicate (role, base name), language-tag form, files outside a language directory, legacy file types - `ovos-spec-lint ` — accepts a locale/ or a / directory; exit code non-zero on errors, or on warnings with --strict (CI-ready) 15 linter tests (73 total). Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 30 ++++- ovos_spec_tools/__init__.py | 23 ++-- ovos_spec_tools/lint.py | 213 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 5 +- test/test_lint.py | 119 ++++++++++++++++++++ 5 files changed, 375 insertions(+), 15 deletions(-) create mode 100644 ovos_spec_tools/lint.py create mode 100644 test/test_lint.py diff --git a/README.md b/README.md index 1672383..7741f47 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # ovos-spec-tools -Reference implementation of the OVOS intent specifications — the low-level, -dependency-light primitives that the [formal -specifications](https://github.com/OpenVoiceOS/formal-specifications) describe. +Reference implementation of the OVOS [formal +specifications](https://github.com/OpenVoiceOS/formal-specifications) — the +low-level, dependency-light primitives those specifications describe. OVOS components reimplement template expansion and resource loading in several places, and the copies drift. This package is the single conformant @@ -10,11 +10,12 @@ implementation those components — and any third-party tool — can depend on. ## Status -| Primitive | Spec | State | -|-----------|------|-------| +| Tool | Spec | State | +|------|------|-------| | Sentence template expander | OVOS-INTENT-1 v2 | available | | Locale resource loader | OVOS-INTENT-2 | available | | Dialog renderer | OVOS-INTENT-2 §4.2 | available | +| `ovos-spec-lint` locale linter | OVOS-INTENT-1 / -2 | available | ## The expander @@ -74,6 +75,25 @@ render(res.load_dialog("weather"), slots={"temperature": 21}) # 'It is 21 degrees.' ``` +## The locale linter + +`ovos-spec-lint` validates every resource file under a locale directory — the +syntax of every template (OVOS-INTENT-1) and the naming and layout of every +file (OVOS-INTENT-2) — and reports every problem rather than stopping at the +first. + +```bash +ovos-spec-lint path/to/locale +# path/to/locale/en-US/bad.intent: error: unbalanced metacharacters ... +# 1 error(s), 0 warning(s) +``` + +It checks template syntax, malformed forms, empty files, slot-free roles +carrying a slot, base-name and language-tag naming, duplicate resources, and +unresolved `` references. The argument may be a `locale/` directory or a +single `/` directory. Exit code is non-zero on errors (or, with +`--strict`, on warnings) — suitable for CI. + ## Install ```bash diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index 8de48f8..d7f72cd 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -1,17 +1,20 @@ -"""Reference implementation of the OVOS intent specifications. +"""Reference implementation of the OVOS formal specifications. -`ovos-spec-tools` provides the low-level, dependency-light primitives -the OVOS intent specifications describe: +`ovos-spec-tools` provides the low-level, dependency-light primitives the OVOS +formal specifications describe: -- :func:`~ovos_spec_tools.expansion.expand` — the OVOS-INTENT-1 - sentence template expander; -- :class:`~ovos_spec_tools.resources.LocaleResources` — the - OVOS-INTENT-2 locale resource-file loader; -- :func:`~ovos_spec_tools.dialog.render` — the OVOS-INTENT-2 §4.2 - dialog renderer. +- :func:`~ovos_spec_tools.expansion.expand` — the OVOS-INTENT-1 sentence + template expander; +- :class:`~ovos_spec_tools.resources.LocaleResources` — the OVOS-INTENT-2 + locale resource-file loader; +- :func:`~ovos_spec_tools.dialog.render` — the OVOS-INTENT-2 §4.2 dialog + renderer; +- :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also + exposed as the ``ovos-spec-lint`` command. """ from ovos_spec_tools.dialog import UnfilledSlot, render from ovos_spec_tools.expansion import MalformedTemplate, expand +from ovos_spec_tools.lint import Finding, lint_locale from ovos_spec_tools.resources import ( LocaleResources, MalformedResource, @@ -27,5 +30,7 @@ "read_resource_file", "render", "UnfilledSlot", + "lint_locale", + "Finding", "__version__", ] diff --git a/ovos_spec_tools/lint.py b/ovos_spec_tools/lint.py new file mode 100644 index 0000000..179fb47 --- /dev/null +++ b/ovos_spec_tools/lint.py @@ -0,0 +1,213 @@ +"""Locale resource linter for the OVOS specifications. + +Validates the **syntax** (OVOS-INTENT-1) and the **naming and layout** +(OVOS-INTENT-2) of every resource file under a locale directory, and reports +every problem found rather than stopping at the first. + +Exposed as the ``ovos-spec-lint`` command:: + + ovos-spec-lint path/to/locale + +The argument may be a ``locale/`` directory (every language subdirectory is +checked) or a single ``/`` directory. +""" +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Sequence + +from ovos_spec_tools.expansion import MalformedTemplate, expand +from ovos_spec_tools.resources import ( + SLOT_BEARING_ROLES, + SLOT_FREE_ROLES, + read_resource_file, +) + +__all__ = ["Finding", "lint_locale", "main"] + +# The five OVOS-INTENT-2 resource roles. +ROLE_EXTENSIONS = SLOT_BEARING_ROLES + SLOT_FREE_ROLES +# File types OVOS-INTENT-2 deliberately does not define — flagged, not parsed. +LEGACY_EXTENSIONS = (".rx", ".value", ".list", ".word", ".template", ".qml") + +_BASE_NAME_RE = re.compile(r"[a-z0-9_]+") +_SLOT_NAME_RE = re.compile(r"[a-z][a-z0-9_]*") +_LANG_TAG_RE = re.compile(r"[a-z]{2,3}(-[A-Za-z0-9]+)*") + +ERROR = "error" +WARNING = "warning" + + +@dataclass +class Finding: + """One problem found by the linter.""" + + severity: str # ERROR or WARNING + path: str + message: str + + def __str__(self) -> str: + return f"{self.path}: {self.severity}: {self.message}" + + +def lint_locale(path) -> List[Finding]: + """Lint a locale directory, or a single language directory. + + Returns every :class:`Finding`, in file order. A path whose name is a + BCP-47 tag is treated as a single language tree; otherwise it is treated as + a ``locale/`` directory and each language subdirectory is checked. + """ + root = Path(path) + if not root.is_dir(): + return [Finding(ERROR, str(root), "not a directory")] + + if _LANG_TAG_RE.fullmatch(root.name): + return _lint_language_tree(root) + + findings: List[Finding] = [] + for child in sorted(root.iterdir()): + if child.is_file() and child.suffix in ROLE_EXTENSIONS: + findings.append(Finding( + WARNING, str(child), + "resource file is not inside a language directory")) + language_dirs = [c for c in sorted(root.iterdir()) if c.is_dir()] + if not language_dirs: + findings.append(Finding( + WARNING, str(root), "no language directories found")) + for language_dir in language_dirs: + findings.extend(_lint_language_tree(language_dir)) + return findings + + +def _lint_language_tree(language_dir: Path) -> List[Finding]: + """Lint one ``/`` directory and all its subdirectories.""" + findings: List[Finding] = [] + if not _LANG_TAG_RE.fullmatch(language_dir.name): + findings.append(Finding( + WARNING, str(language_dir), + f"directory name {language_dir.name!r} is not a BCP-47 " + f"language tag")) + + role_files: List[Path] = [] + for path in sorted(language_dir.rglob("*")): + if not path.is_file(): + continue + if path.suffix in ROLE_EXTENSIONS: + role_files.append(path) + elif path.suffix in LEGACY_EXTENSIONS: + findings.append(Finding( + WARNING, str(path), + f"{path.suffix} is a legacy file type, not an " + f"OVOS-INTENT-2 resource role")) + + # Duplicate (role, base name) within one language tree (OVOS-INTENT-2 §2). + first_seen: Dict[tuple, Path] = {} + for path in role_files: + key = (path.suffix, path.stem) + if key in first_seen: + findings.append(Finding( + ERROR, str(path), + f"duplicate {path.suffix} resource {path.stem!r} — also at " + f"{first_seen[key]}")) + else: + first_seen[key] = path + + # Vocabularies, for resolving references during expansion. + vocabularies: Dict[str, List[str]] = {} + for path in role_files: + if path.suffix == ".voc": + try: + vocabularies[path.stem] = read_resource_file(path) + except OSError: + pass # _lint_file reports the read error below + + for path in role_files: + findings.extend(_lint_file(path, vocabularies)) + return findings + + +def _lint_file(path: Path, + vocabularies: Dict[str, Sequence[str]]) -> List[Finding]: + """Lint one resource file: naming, then the syntax of every template.""" + findings: List[Finding] = [] + extension = path.suffix + base_name = path.stem + + # --- naming (OVOS-INTENT-2 §2) ------------------------------------------ + if not _BASE_NAME_RE.fullmatch(base_name): + findings.append(Finding( + ERROR, str(path), + f"base name {base_name!r} must be lowercase ASCII letters, " + f"digits and underscores only")) + if extension == ".entity" and not _SLOT_NAME_RE.fullmatch(base_name): + findings.append(Finding( + ERROR, str(path), + f".entity base name {base_name!r} names a slot and must not " + f"begin with a digit")) + if path.name != path.name.lower(): + findings.append(Finding( + WARNING, str(path), "file name should be lowercase")) + + # --- syntax (OVOS-INTENT-1) --------------------------------------------- + try: + templates = read_resource_file(path) + except OSError as exc: + findings.append(Finding(ERROR, str(path), f"cannot read file: {exc}")) + return findings + if not templates: + findings.append(Finding( + ERROR, str(path), + "empty file — every resource file must contribute at least one " + "template (OVOS-INTENT-2 §5)")) + return findings + + slot_free = extension in SLOT_FREE_ROLES + for template in templates: + try: + samples = expand(template, vocabularies) + except MalformedTemplate as exc: + findings.append(Finding( + ERROR, str(path), f"{exc} [in: {template!r}]")) + continue + if slot_free and any("{" in sample for sample in samples): + findings.append(Finding( + ERROR, str(path), + f"{extension} is slot-free but a template contains a named " + f"slot [in: {template!r}]")) + return findings + + +def main(argv: Optional[Sequence[str]] = None) -> int: + """Entry point for the ``ovos-spec-lint`` command.""" + parser = argparse.ArgumentParser( + prog="ovos-spec-lint", + description="Validate OVOS locale resource files against " + "OVOS-INTENT-1 (syntax) and OVOS-INTENT-2 (naming).") + parser.add_argument( + "locale", help="path to a locale/ directory, or a / directory") + parser.add_argument( + "--strict", action="store_true", + help="exit non-zero if there are warnings as well as errors") + args = parser.parse_args(argv) + + findings = lint_locale(args.locale) + errors = [f for f in findings if f.severity == ERROR] + warnings = [f for f in findings if f.severity == WARNING] + + for finding in findings: + print(finding) + if findings: + print() + print(f"{len(errors)} error(s), {len(warnings)} warning(s)") + + if errors or (args.strict and warnings): + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml index c73a511..52719cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "ovos-spec-tools" dynamic = ["version"] -description = "Reference implementation of the OVOS intent specifications" +description = "Reference implementation of the OVOS formal specifications" readme = "README.md" license = { text = "Apache-2.0" } authors = [{ name = "jarbasai", email = "jarbasai@mailfence.com" }] @@ -19,6 +19,9 @@ test = [ "pytest", ] +[project.scripts] +ovos-spec-lint = "ovos_spec_tools.lint:main" + [project.urls] Homepage = "https://github.com/OpenVoiceOS/ovos-spec-tools" diff --git a/test/test_lint.py b/test/test_lint.py new file mode 100644 index 0000000..07cd81c --- /dev/null +++ b/test/test_lint.py @@ -0,0 +1,119 @@ +"""Tests for the locale resource linter (`ovos-spec-lint`).""" +import pytest + +from ovos_spec_tools.lint import ERROR, WARNING, lint_locale, main + + +def _write(path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _errors(findings): + return [f for f in findings if f.severity == ERROR] + + +def _warnings(findings): + return [f for f in findings if f.severity == WARNING] + + +def test_clean_locale_has_no_findings(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "(play|put on) {query}\n") + _write(locale / "en-US" / "yes.voc", "yes\nyeah\n") + _write(locale / "en-US" / "greet.dialog", "Hello {name}.\n") + assert lint_locale(locale) == [] + + +def test_malformed_template_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "bad.intent", "turn (on|off the lights\n") + errors = _errors(lint_locale(locale)) + assert len(errors) == 1 + assert "bad.intent" in errors[0].path + + +def test_empty_file_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "empty.voc", "# just a comment\n") + assert any("empty" in f.message for f in _errors(lint_locale(locale))) + + +def test_slot_in_slot_free_role_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "thing.voc", "the {slot}\n") + assert any("slot-free" in f.message for f in _errors(lint_locale(locale))) + + +def test_bad_base_name_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "Bad-Name.intent", "hello world\n") + assert any("base name" in f.message for f in _errors(lint_locale(locale))) + + +def test_entity_base_name_starting_with_digit_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "2nd.entity", "second\n") + assert any("digit" in f.message for f in _errors(lint_locale(locale))) + + +def test_duplicate_resource_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "a" / "dup.intent", "first\n") + _write(locale / "en-US" / "b" / "dup.intent", "second\n") + assert any("duplicate" in f.message for f in _errors(lint_locale(locale))) + + +def test_legacy_extension_is_a_warning(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.voc", "yes\n") + _write(locale / "en-US" / "old.rx", ".*\n") + findings = lint_locale(locale) + assert _errors(findings) == [] + assert any(".rx" in f.message for f in _warnings(findings)) + + +def test_file_outside_a_language_directory_is_a_warning(tmp_path): + locale = tmp_path / "locale" + _write(locale / "stray.intent", "hello world\n") + assert any("language directory" in f.message + for f in _warnings(lint_locale(locale))) + + +def test_non_bcp47_language_directory_is_a_warning(tmp_path): + locale = tmp_path / "locale" + _write(locale / "english" / "x.intent", "hello world\n") + assert any("BCP-47" in f.message for f in _warnings(lint_locale(locale))) + + +def test_unresolved_vocabulary_reference_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.intent", " {name}\n") + assert _errors(lint_locale(locale)) + + +def test_can_point_at_a_single_language_directory(tmp_path): + lang = tmp_path / "en-US" + _write(lang / "play.intent", "(play|stop) {query}\n") + assert lint_locale(lang) == [] + + +# --- the CLI ----------------------------------------------------------------- + +def test_main_returns_zero_for_a_clean_locale(tmp_path, capsys): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "play {query}\n") + assert main([str(locale)]) == 0 + + +def test_main_returns_one_on_errors(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "bad.intent", "{a}{b}\n") + assert main([str(locale)]) == 1 + + +def test_main_strict_fails_on_warnings(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "old.list", "a\n") + assert main([str(locale)]) == 0 + assert main([str(locale), "--strict"]) == 1 From 0859cdca913af84b8e063f915cbd6cf724a16e6a Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 12:13:43 +0100 Subject: [PATCH 004/110] docs: add examples/ with a linter walkthrough MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit examples/dirty-locale is a deliberately broken skill locale — every file trips at least one ovos-spec-lint check (two valid files produce no findings). examples/README.md shows the locale, the command, the verbatim linter output (10 errors, 4 warnings), and a table mapping each finding to its spec rule. Co-Authored-By: Claude Opus 4.7 (1M context) --- examples/README.md | 78 +++++++++++++++++++ examples/dirty-locale/locale/en-US/2nd.entity | 2 + .../dirty-locale/locale/en-US/Bad-Name.intent | 1 + .../dirty-locale/locale/en-US/adjacent.intent | 1 + examples/dirty-locale/locale/en-US/colors.voc | 1 + examples/dirty-locale/locale/en-US/empty.voc | 2 + .../dirty-locale/locale/en-US/good.intent | 2 + examples/dirty-locale/locale/en-US/good.voc | 3 + examples/dirty-locale/locale/en-US/old.rx | 1 + .../dirty-locale/locale/en-US/refs.intent | 1 + .../dirty-locale/locale/en-US/repeat.intent | 1 + .../locale/en-US/single_branch.intent | 1 + .../locale/en-US/sub1/menu.intent | 1 + .../locale/en-US/sub2/menu.intent | 1 + .../locale/en-US/unbalanced.intent | 1 + .../dirty-locale/locale/english/hello.intent | 1 + examples/dirty-locale/locale/stray.intent | 1 + 17 files changed, 99 insertions(+) create mode 100644 examples/README.md create mode 100644 examples/dirty-locale/locale/en-US/2nd.entity create mode 100644 examples/dirty-locale/locale/en-US/Bad-Name.intent create mode 100644 examples/dirty-locale/locale/en-US/adjacent.intent create mode 100644 examples/dirty-locale/locale/en-US/colors.voc create mode 100644 examples/dirty-locale/locale/en-US/empty.voc create mode 100644 examples/dirty-locale/locale/en-US/good.intent create mode 100644 examples/dirty-locale/locale/en-US/good.voc create mode 100644 examples/dirty-locale/locale/en-US/old.rx create mode 100644 examples/dirty-locale/locale/en-US/refs.intent create mode 100644 examples/dirty-locale/locale/en-US/repeat.intent create mode 100644 examples/dirty-locale/locale/en-US/single_branch.intent create mode 100644 examples/dirty-locale/locale/en-US/sub1/menu.intent create mode 100644 examples/dirty-locale/locale/en-US/sub2/menu.intent create mode 100644 examples/dirty-locale/locale/en-US/unbalanced.intent create mode 100644 examples/dirty-locale/locale/english/hello.intent create mode 100644 examples/dirty-locale/locale/stray.intent diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..4cff70e --- /dev/null +++ b/examples/README.md @@ -0,0 +1,78 @@ +# Examples + +## `dirty-locale` — the linter against a broken locale + +`dirty-locale/` is a deliberately broken skill locale. Every file in it trips +at least one `ovos-spec-lint` check, so it doubles as a tour of what the linter +catches. Two files (`en-US/good.intent`, `en-US/good.voc`) are valid and +produce no findings. + +### The locale + +``` +dirty-locale/locale/ +├── stray.intent misplaced — not inside a language directory +├── english/ directory name is not a BCP-47 tag +│ └── hello.intent (valid content) +└── en-US/ + ├── good.intent valid — no findings + ├── good.voc valid — no findings + ├── unbalanced.intent unbalanced metacharacters + ├── single_branch.intent a group with only one branch + ├── adjacent.intent two slots with no word between them + ├── repeat.intent the same slot name twice in a sample + ├── empty.voc no templates after comments are dropped + ├── colors.voc a named slot in a slot-free role + ├── Bad-Name.intent base name outside the allowed charset + ├── 2nd.entity .entity base name begins with a digit + ├── refs.intent reference to a vocabulary that does not exist + ├── old.rx a legacy file type, not an OVOS-INTENT-2 role + ├── sub1/menu.intent ┐ same (role, base name) twice in one + └── sub2/menu.intent ┘ language tree +``` + +### Running the linter + +```console +$ ovos-spec-lint examples/dirty-locale/locale +examples/dirty-locale/locale/stray.intent: warning: resource file is not inside a language directory +examples/dirty-locale/locale/en-US/old.rx: warning: .rx is a legacy file type, not an OVOS-INTENT-2 resource role +examples/dirty-locale/locale/en-US/sub2/menu.intent: error: duplicate .intent resource 'menu' — also at examples/dirty-locale/locale/en-US/sub1/menu.intent +examples/dirty-locale/locale/en-US/2nd.entity: error: .entity base name '2nd' names a slot and must not begin with a digit +examples/dirty-locale/locale/en-US/Bad-Name.intent: error: base name 'Bad-Name' must be lowercase ASCII letters, digits and underscores only +examples/dirty-locale/locale/en-US/Bad-Name.intent: warning: file name should be lowercase +examples/dirty-locale/locale/en-US/adjacent.intent: error: adjacent slots in sample 'set {hours}{minutes}' of template 'set {hours}{minutes}': a literal word must separate any two slots [in: 'set {hours}{minutes}'] +examples/dirty-locale/locale/en-US/colors.voc: error: .voc is slot-free but a template contains a named slot [in: 'the color {shade}'] +examples/dirty-locale/locale/en-US/empty.voc: error: empty file — every resource file must contribute at least one template (OVOS-INTENT-2 §5) +examples/dirty-locale/locale/en-US/refs.intent: error: undefined vocabulary reference [in: ' {name}'] +examples/dirty-locale/locale/en-US/repeat.intent: error: repeated slot name in sample '{x} and {x}' of template '{x} and {x}': each slot is defined once per sample [in: '{x} and {x}'] +examples/dirty-locale/locale/en-US/single_branch.intent: error: single-branch group (button): a group must offer a choice between at least two branches [in: 'press (button)'] +examples/dirty-locale/locale/en-US/unbalanced.intent: error: unbalanced metacharacters in template 'turn (on|off the lights' [in: 'turn (on|off the lights'] +examples/dirty-locale/locale/english: warning: directory name 'english' is not a BCP-47 language tag + +10 error(s), 4 warning(s) +$ echo $? +1 +``` + +The command exits non-zero because there are errors. With `--strict` it would +also exit non-zero on the warnings alone. + +### Findings, by rule + +| File | Severity | Rule | +|------|----------|------| +| `stray.intent` | warning | a resource file must live inside a `/` directory (OVOS-INTENT-2 §2) | +| `english/` | warning | a language directory is named with a BCP-47 tag (§2) | +| `old.rx` | warning | `.rx` is a legacy type, not one of the five OVOS-INTENT-2 roles | +| `Bad-Name.intent` | warning | file names are lowercase | +| `unbalanced.intent` | error | unbalanced metacharacters (OVOS-INTENT-1 §3.6) | +| `single_branch.intent` | error | a group must have at least two branches (§3.6) | +| `adjacent.intent` | error | a literal word must separate two slots (§3.6) | +| `repeat.intent` | error | a slot name appears once per sample (§3.6) | +| `empty.voc` | error | every file contributes at least one template (OVOS-INTENT-2 §5) | +| `colors.voc` | error | `.voc` is slot-free — no named slots (§4.3) | +| `Bad-Name.intent` | error | base name charset: lowercase letters, digits, underscores (§2) | +| `2nd.entity` | error | an `.entity` base name names a slot — no leading digit (§3.4) | +| `refs.intent` | error | `` must resolve to a known vocabulary (§3.7) | +| `sub1`+`sub2/menu.intent` | error | `(role, base name)` is unique per language tree (§2) | diff --git a/examples/dirty-locale/locale/en-US/2nd.entity b/examples/dirty-locale/locale/en-US/2nd.entity new file mode 100644 index 0000000..6fa9bf8 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/2nd.entity @@ -0,0 +1,2 @@ +second +third diff --git a/examples/dirty-locale/locale/en-US/Bad-Name.intent b/examples/dirty-locale/locale/en-US/Bad-Name.intent new file mode 100644 index 0000000..982793c --- /dev/null +++ b/examples/dirty-locale/locale/en-US/Bad-Name.intent @@ -0,0 +1 @@ +whatever diff --git a/examples/dirty-locale/locale/en-US/adjacent.intent b/examples/dirty-locale/locale/en-US/adjacent.intent new file mode 100644 index 0000000..d1ede4c --- /dev/null +++ b/examples/dirty-locale/locale/en-US/adjacent.intent @@ -0,0 +1 @@ +set {hours}{minutes} diff --git a/examples/dirty-locale/locale/en-US/colors.voc b/examples/dirty-locale/locale/en-US/colors.voc new file mode 100644 index 0000000..db8889d --- /dev/null +++ b/examples/dirty-locale/locale/en-US/colors.voc @@ -0,0 +1 @@ +the color {shade} diff --git a/examples/dirty-locale/locale/en-US/empty.voc b/examples/dirty-locale/locale/en-US/empty.voc new file mode 100644 index 0000000..25094d7 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/empty.voc @@ -0,0 +1,2 @@ +# this file has no templates + diff --git a/examples/dirty-locale/locale/en-US/good.intent b/examples/dirty-locale/locale/en-US/good.intent new file mode 100644 index 0000000..7e237e8 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/good.intent @@ -0,0 +1,2 @@ +# clean intent +(play|put on) {query} diff --git a/examples/dirty-locale/locale/en-US/good.voc b/examples/dirty-locale/locale/en-US/good.voc new file mode 100644 index 0000000..c6fb399 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/good.voc @@ -0,0 +1,3 @@ +yes +yeah +(sure|of course) diff --git a/examples/dirty-locale/locale/en-US/old.rx b/examples/dirty-locale/locale/en-US/old.rx new file mode 100644 index 0000000..52e47c9 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/old.rx @@ -0,0 +1 @@ +(?P.*) diff --git a/examples/dirty-locale/locale/en-US/refs.intent b/examples/dirty-locale/locale/en-US/refs.intent new file mode 100644 index 0000000..665e6ab --- /dev/null +++ b/examples/dirty-locale/locale/en-US/refs.intent @@ -0,0 +1 @@ + {name} diff --git a/examples/dirty-locale/locale/en-US/repeat.intent b/examples/dirty-locale/locale/en-US/repeat.intent new file mode 100644 index 0000000..b240eb6 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/repeat.intent @@ -0,0 +1 @@ +{x} and {x} diff --git a/examples/dirty-locale/locale/en-US/single_branch.intent b/examples/dirty-locale/locale/en-US/single_branch.intent new file mode 100644 index 0000000..c42d376 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/single_branch.intent @@ -0,0 +1 @@ +press (button) diff --git a/examples/dirty-locale/locale/en-US/sub1/menu.intent b/examples/dirty-locale/locale/en-US/sub1/menu.intent new file mode 100644 index 0000000..1f8ae2e --- /dev/null +++ b/examples/dirty-locale/locale/en-US/sub1/menu.intent @@ -0,0 +1 @@ +open the menu diff --git a/examples/dirty-locale/locale/en-US/sub2/menu.intent b/examples/dirty-locale/locale/en-US/sub2/menu.intent new file mode 100644 index 0000000..262a1e0 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/sub2/menu.intent @@ -0,0 +1 @@ +show menu diff --git a/examples/dirty-locale/locale/en-US/unbalanced.intent b/examples/dirty-locale/locale/en-US/unbalanced.intent new file mode 100644 index 0000000..2b5c77e --- /dev/null +++ b/examples/dirty-locale/locale/en-US/unbalanced.intent @@ -0,0 +1 @@ +turn (on|off the lights diff --git a/examples/dirty-locale/locale/english/hello.intent b/examples/dirty-locale/locale/english/hello.intent new file mode 100644 index 0000000..37d4e6c --- /dev/null +++ b/examples/dirty-locale/locale/english/hello.intent @@ -0,0 +1 @@ +hi there diff --git a/examples/dirty-locale/locale/stray.intent b/examples/dirty-locale/locale/stray.intent new file mode 100644 index 0000000..3b18e51 --- /dev/null +++ b/examples/dirty-locale/locale/stray.intent @@ -0,0 +1 @@ +hello world From c99f7be85dbfd2d3ea3bf23fe6a927fcd9ad87cc Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 12:21:46 +0100 Subject: [PATCH 005/110] feat: add DialogRenderer, a stateful object-oriented dialog renderer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An OO alternative to render(). Beyond holding the phrases, vocabularies and rng, it tracks the last phrase chosen and avoids repeating it on the next call — repetition avoidance, which a stateless function cannot do and which OVOS-INTENT-2 §4.2 explicitly allows. - DialogRenderer(phrases, vocabularies=, rng=) with .render(slots=) - DialogRenderer.from_resources(resources, name) builds one from a LocaleResources - render() refactored to share _render_phrase with the class 6 more tests (79 total). Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 21 ++++++-- ovos_spec_tools/__init__.py | 3 +- ovos_spec_tools/dialog.py | 97 +++++++++++++++++++++++++++++++------ test/test_dialog.py | 46 +++++++++++++++++- 4 files changed, 148 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 7741f47..07ae666 100644 --- a/README.md +++ b/README.md @@ -64,9 +64,12 @@ in (`user_locale=`); this package imports no configuration. ## The dialog renderer -`render()` selects one phrase from a loaded `.dialog`, expands its variety to a -single variant, and fills every `{name}` slot with a caller-supplied value -(OVOS-INTENT-2 §4.2). A phrase with an unfilled slot raises `UnfilledSlot`. +Rendering a dialog selects one phrase from a loaded `.dialog`, expands its +variety to a single variant, and fills every `{name}` slot with a +caller-supplied value (OVOS-INTENT-2 §4.2). A phrase with an unfilled slot +raises `UnfilledSlot`. + +`render()` is a stateless one-shot function: ```python from ovos_spec_tools import render @@ -75,6 +78,18 @@ render(res.load_dialog("weather"), slots={"temperature": 21}) # 'It is 21 degrees.' ``` +`DialogRenderer` is a stateful, object-oriented alternative. It holds the +dialog and, unlike the function, **avoids repeating the phrase it chose last +time** — so a repeatedly-spoken response does not sound mechanical: + +```python +from ovos_spec_tools import DialogRenderer + +renderer = DialogRenderer.from_resources(res, "weather") +renderer.render({"temperature": 21}) # a phrase +renderer.render({"temperature": 22}) # a different phrase +``` + ## The locale linter `ovos-spec-lint` validates every resource file under a locale directory — the diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index d7f72cd..b046c41 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -12,7 +12,7 @@ - :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also exposed as the ``ovos-spec-lint`` command. """ -from ovos_spec_tools.dialog import UnfilledSlot, render +from ovos_spec_tools.dialog import DialogRenderer, UnfilledSlot, render from ovos_spec_tools.expansion import MalformedTemplate, expand from ovos_spec_tools.lint import Finding, lint_locale from ovos_spec_tools.resources import ( @@ -29,6 +29,7 @@ "MalformedResource", "read_resource_file", "render", + "DialogRenderer", "UnfilledSlot", "lint_locale", "Finding", diff --git a/ovos_spec_tools/dialog.py b/ovos_spec_tools/dialog.py index a06e7cf..615ba1f 100644 --- a/ovos_spec_tools/dialog.py +++ b/ovos_spec_tools/dialog.py @@ -1,13 +1,19 @@ """Reference dialog renderer for OVOS-INTENT-2 §4.2. This is the reference implementation of the **Dialog renderer** conformance -role of OVOS-INTENT-1 §7. To render a dialog, :func:`render` selects one phrase -from a loaded ``.dialog``, expands its ``(a|b)`` / ``[x]`` variety to a single -variant, and fills every ``{name}`` slot with a caller-supplied value. +role of OVOS-INTENT-1 §7. Rendering a dialog means: select one phrase from a +loaded ``.dialog``, expand its ``(a|b)`` / ``[x]`` variety to a single variant, +and fill every ``{name}`` slot with a caller-supplied value. + +Two interfaces are provided: + +- :func:`render` — a stateless one-shot function; +- :class:`DialogRenderer` — a stateful object that additionally avoids + repeating the phrase it chose last time. Only the single-brace slot form ``{name}`` is recognized; there is no ``{{ }}`` form (OVOS-INTENT-2 §4.2). Slots are filled by the caller; a chosen phrase with -a slot the caller did not fill raises :class:`UnfilledSlot` and is not rendered. +a slot the caller did not fill raises :class:`UnfilledSlot`. """ from __future__ import annotations @@ -17,7 +23,7 @@ from ovos_spec_tools.expansion import expand -__all__ = ["render", "UnfilledSlot"] +__all__ = ["render", "DialogRenderer", "UnfilledSlot"] _SLOT_TOKEN_RE = re.compile(r"\{([a-z][a-z0-9_]*)\}") @@ -34,17 +40,15 @@ def render(phrases: Sequence[str], slots: Optional[Dict[str, object]] = None, vocabularies: Optional[Dict[str, Sequence[str]]] = None, rng: Optional[object] = None) -> str: - """Render one phrase from a loaded ``.dialog``. + """Render one phrase from a loaded ``.dialog`` (stateless). Args: - phrases: the phrase strings of a ``.dialog``, as returned by - :meth:`~ovos_spec_tools.resources.LocaleResources.load_dialog`. + phrases: the phrase strings of a ``.dialog``. slots: caller-supplied values for the phrase's named slots, keyed by slot name. Values are converted to text. vocabularies: vocabularies for any ```` references in the phrase. rng: an object with a ``choice`` method, for phrase and variant - selection; defaults to the :mod:`random` module. Inject a seeded - generator for deterministic output. + selection; defaults to the :mod:`random` module. Returns: One rendered phrase, ready for text-to-speech. @@ -57,11 +61,76 @@ def render(phrases: Sequence[str], if not phrases: raise ValueError("no dialog phrases to render") chooser = rng if rng is not None else _random - slots = slots or {} - - # Expansion keeps slots opaque (OVOS-INTENT-1 §4): expand first, pick a - # variant, then fill — so a slot value can never be parsed as grammar. phrase = chooser.choice(list(phrases)) + return _render_phrase(phrase, slots or {}, vocabularies, chooser) + + +class DialogRenderer: + """A stateful renderer for one loaded ``.dialog`` (OVOS-INTENT-2 §4.2). + + An object-oriented alternative to :func:`render`. It holds the dialog's + phrases, the vocabularies, and the random source, and — unlike the bare + function — **avoids repeating the phrase it chose on the previous call**, + so a repeatedly-spoken response does not sound mechanical. Repetition + avoidance is an implementation choice the spec explicitly allows (§4.2). + """ + + def __init__(self, phrases: Sequence[str], + vocabularies: Optional[Dict[str, Sequence[str]]] = None, + rng: Optional[object] = None): + """ + Args: + phrases: the phrase strings of a ``.dialog``. + vocabularies: vocabularies for any ```` references. + rng: an object with a ``choice`` method; defaults to :mod:`random`. + """ + self.phrases = list(phrases) + if not self.phrases: + raise ValueError("a DialogRenderer needs at least one phrase") + self.vocabularies = vocabularies + self.rng = rng if rng is not None else _random + self._last: Optional[str] = None + + @classmethod + def from_resources(cls, resources, name: str, + rng: Optional[object] = None) -> "DialogRenderer": + """Build a renderer for the ``.dialog`` named ``name``. + + ``resources`` is a + :class:`~ovos_spec_tools.resources.LocaleResources`; its loaded dialog + and its vocabularies are used. + """ + return cls(resources.load_dialog(name), + vocabularies=resources.vocabularies(), rng=rng) + + def render(self, slots: Optional[Dict[str, object]] = None) -> str: + """Render one phrase, avoiding the phrase chosen on the previous call. + + Args: + slots: caller-supplied values for the phrase's named slots. + + Returns: + One rendered phrase, ready for text-to-speech. + + Raises: + UnfilledSlot: a slot in the chosen phrase has no value. + MalformedTemplate: the chosen phrase is not a valid template. + """ + choices = [p for p in self.phrases if p != self._last] or self.phrases + phrase = self.rng.choice(choices) + self._last = phrase + return _render_phrase(phrase, slots or {}, self.vocabularies, self.rng) + + +def _render_phrase(phrase: str, + slots: Dict[str, object], + vocabularies: Optional[Dict[str, Sequence[str]]], + chooser: object) -> str: + """Expand one phrase to a variant, then fill its slots. + + Expansion keeps slots opaque (OVOS-INTENT-1 §4), so it runs first and a + slot value can never be parsed as grammar. + """ variant = chooser.choice(expand(phrase, vocabularies)) return _fill_slots(variant, slots) diff --git a/test/test_dialog.py b/test/test_dialog.py index 8e21c0a..508c5b0 100644 --- a/test/test_dialog.py +++ b/test/test_dialog.py @@ -1,7 +1,7 @@ """Conformance tests for the OVOS-INTENT-2 §4.2 reference dialog renderer.""" import pytest -from ovos_spec_tools import UnfilledSlot, render +from ovos_spec_tools import DialogRenderer, LocaleResources, UnfilledSlot, render class _FixedRng: @@ -53,3 +53,47 @@ def test_render_resolves_vocabulary_references(): def test_empty_phrase_list_raises(): with pytest.raises(ValueError): render([]) + + +# --- DialogRenderer ---------------------------------------------------------- + +def test_renderer_fills_slots(): + renderer = DialogRenderer(["It is {temperature} degrees."]) + assert renderer.render({"temperature": 21}) == "It is 21 degrees." + + +def test_renderer_avoids_repeating_the_last_phrase(): + """With two phrases, consecutive renders must alternate (§4.2).""" + renderer = DialogRenderer(["alpha", "beta"]) + first = renderer.render() + second = renderer.render() + third = renderer.render() + assert first != second + assert second != third + assert {first, second} == {"alpha", "beta"} + + +def test_renderer_with_one_phrase_repeats_it(): + renderer = DialogRenderer(["only one"]) + assert renderer.render() == renderer.render() == "only one" + + +def test_renderer_unfilled_slot_raises(): + renderer = DialogRenderer(["say {name}"]) + with pytest.raises(UnfilledSlot): + renderer.render() + + +def test_renderer_empty_phrase_list_raises(): + with pytest.raises(ValueError): + DialogRenderer([]) + + +def test_renderer_from_resources(tmp_path): + locale = tmp_path / "locale" + lang = locale / "en-US" + lang.mkdir(parents=True) + (lang / "hi.dialog").write_text("Hello {name}.\n", encoding="utf-8") + res = LocaleResources("en-US", str(locale)) + renderer = DialogRenderer.from_resources(res, "hi") + assert renderer.render({"name": "Sam"}) == "Hello Sam." From bb1da4cbfd8d2af3ec57d83eeecbb6fc0dad06fc Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 12:27:53 +0100 Subject: [PATCH 006/110] feat: DialogRenderer default slots and .entity fallback; type rng as a Protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - rng parameters are now typed `Chooser`, a Protocol (an object with a `choice` method) instead of the meaningless `object`. `random` and a `random.Random` instance both satisfy it. - DialogRenderer holds default slot values (set once, reused every render) and `.entity` value sets. A slot resolves in order: per-call value, default, a random `.entity` value, then UnfilledSlot. - DialogRenderer.from_resources now also loads `.entity` value sets and accepts default `slots`. - LocaleResources.entities() — every `.entity` for the language, as a name -> expanded value set map (mirrors vocabularies()). 6 more tests (85 total). Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 11 +++++ ovos_spec_tools/dialog.py | 88 ++++++++++++++++++++++++++++-------- ovos_spec_tools/resources.py | 13 ++++++ test/test_dialog.py | 46 +++++++++++++++++++ 4 files changed, 139 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 07ae666..7733f45 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,17 @@ renderer.render({"temperature": 21}) # a phrase renderer.render({"temperature": 22}) # a different phrase ``` +It also holds **default slot values** set once and reused, and falls back to a +slot's **`.entity` value set** for anything still unfilled. A slot is resolved +in order: the per-call value, then a default, then a random `.entity` value, +then `UnfilledSlot`. + +```python +renderer = DialogRenderer.from_resources(res, "greeting", + slots={"assistant": "OVOS"}) +renderer.render() # {assistant} reused; an {day} slot from day.entity +``` + ## The locale linter `ovos-spec-lint` validates every resource file under a locale directory — the diff --git a/ovos_spec_tools/dialog.py b/ovos_spec_tools/dialog.py index 615ba1f..a0ec9fa 100644 --- a/ovos_spec_tools/dialog.py +++ b/ovos_spec_tools/dialog.py @@ -19,15 +19,26 @@ import random as _random import re -from typing import Dict, Optional, Sequence +from typing import Dict, Optional, Protocol, Sequence from ovos_spec_tools.expansion import expand -__all__ = ["render", "DialogRenderer", "UnfilledSlot"] +__all__ = ["render", "DialogRenderer", "Chooser", "UnfilledSlot"] _SLOT_TOKEN_RE = re.compile(r"\{([a-z][a-z0-9_]*)\}") +class Chooser(Protocol): + """A source of random selection: anything with a ``choice`` method. + + The :mod:`random` module and a :class:`random.Random` instance both + satisfy it. Inject a seeded ``random.Random`` for deterministic output. + """ + + def choice(self, seq: Sequence[str]) -> str: + ... + + class UnfilledSlot(ValueError): """A chosen dialog phrase has a named slot the caller did not fill. @@ -39,7 +50,7 @@ class UnfilledSlot(ValueError): def render(phrases: Sequence[str], slots: Optional[Dict[str, object]] = None, vocabularies: Optional[Dict[str, Sequence[str]]] = None, - rng: Optional[object] = None) -> str: + rng: Optional[Chooser] = None) -> str: """Render one phrase from a loaded ``.dialog`` (stateless). Args: @@ -77,72 +88,111 @@ class DialogRenderer: def __init__(self, phrases: Sequence[str], vocabularies: Optional[Dict[str, Sequence[str]]] = None, - rng: Optional[object] = None): + rng: Optional[Chooser] = None, + slots: Optional[Dict[str, object]] = None, + entities: Optional[Dict[str, Sequence[str]]] = None): """ Args: phrases: the phrase strings of a ``.dialog``. vocabularies: vocabularies for any ```` references. rng: an object with a ``choice`` method; defaults to :mod:`random`. + slots: **default** slot values, held for the renderer's lifetime + and reused on every :meth:`render` call. Use this for slots + that do not change per render (a unit preference, a name). A + per-call value passed to :meth:`render` overrides a default. + entities: ``.entity`` value sets, keyed by slot name. A slot that + is neither passed per call nor a default is filled with a + random value from its entity set, if one is given here. """ self.phrases = list(phrases) if not self.phrases: raise ValueError("a DialogRenderer needs at least one phrase") self.vocabularies = vocabularies self.rng = rng if rng is not None else _random + self.default_slots: Dict[str, object] = dict(slots or {}) + self.entities: Dict[str, Sequence[str]] = dict(entities or {}) self._last: Optional[str] = None @classmethod def from_resources(cls, resources, name: str, - rng: Optional[object] = None) -> "DialogRenderer": + rng: Optional[Chooser] = None, + slots: Optional[Dict[str, object]] = None + ) -> "DialogRenderer": """Build a renderer for the ``.dialog`` named ``name``. ``resources`` is a - :class:`~ovos_spec_tools.resources.LocaleResources`; its loaded dialog - and its vocabularies are used. + :class:`~ovos_spec_tools.resources.LocaleResources`; its loaded dialog, + its vocabularies, and its ``.entity`` value sets are all used — so a + slot left unfilled falls back to its ``.entity``. ``slots`` supplies + default slot values. """ return cls(resources.load_dialog(name), - vocabularies=resources.vocabularies(), rng=rng) + vocabularies=resources.vocabularies(), + entities=resources.entities(), + rng=rng, slots=slots) def render(self, slots: Optional[Dict[str, object]] = None) -> str: """Render one phrase, avoiding the phrase chosen on the previous call. + A slot is filled, in order of precedence, from: the per-call ``slots``; + then the renderer's default slots; then a random value from the slot's + ``.entity`` set. A slot none of these supply raises :class:`UnfilledSlot`. + Args: - slots: caller-supplied values for the phrase's named slots. + slots: per-call slot values; each overrides a default of the + same name. Returns: One rendered phrase, ready for text-to-speech. Raises: - UnfilledSlot: a slot in the chosen phrase has no value. + UnfilledSlot: a slot in the chosen phrase has no value from any + source. MalformedTemplate: the chosen phrase is not a valid template. """ + effective = dict(self.default_slots) + if slots: + effective.update(slots) choices = [p for p in self.phrases if p != self._last] or self.phrases phrase = self.rng.choice(choices) self._last = phrase - return _render_phrase(phrase, slots or {}, self.vocabularies, self.rng) + return _render_phrase(phrase, effective, self.vocabularies, self.rng, + self.entities) def _render_phrase(phrase: str, slots: Dict[str, object], vocabularies: Optional[Dict[str, Sequence[str]]], - chooser: object) -> str: + chooser: Chooser, + entities: Optional[Dict[str, Sequence[str]]] = None) -> str: """Expand one phrase to a variant, then fill its slots. Expansion keeps slots opaque (OVOS-INTENT-1 §4), so it runs first and a slot value can never be parsed as grammar. """ variant = chooser.choice(expand(phrase, vocabularies)) - return _fill_slots(variant, slots) + return _fill_slots(variant, slots, entities, chooser) -def _fill_slots(variant: str, slots: Dict[str, object]) -> str: - """Replace every ``{name}`` in ``variant`` with its caller-supplied value.""" +def _fill_slots(variant: str, + slots: Dict[str, object], + entities: Optional[Dict[str, Sequence[str]]], + chooser: Chooser) -> str: + """Replace every ``{name}`` in ``variant`` with a value. + + A value is taken from ``slots``; failing that, a random valid value from + the slot's ``.entity`` set in ``entities``; failing that, the slot has no + value and :class:`UnfilledSlot` is raised. + """ def replace(match: "re.Match") -> str: name = match.group(1) - if name not in slots: - raise UnfilledSlot( - f"dialog slot {{{name}}} was not filled by the caller") - return str(slots[name]) + if name in slots: + return str(slots[name]) + if entities and entities.get(name): + return str(chooser.choice(list(entities[name]))) + raise UnfilledSlot( + f"dialog slot {{{name}}} has no value: it was not filled by the " + f"caller and has no .entity fallback") return _SLOT_TOKEN_RE.sub(replace, variant) diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index f135138..b2f2cc1 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -129,6 +129,19 @@ def vocabularies(self) -> Dict[str, List[str]]: vocs[path.stem] = read_resource_file(path) return vocs + def entities(self) -> Dict[str, List[str]]: + """Every ``.entity`` reachable for this language, as a name→value-set + map — each value set expanded, with override precedence applied.""" + names = set() + for source in self._sources: + lang_dir = self._lang_dir(source) + if lang_dir is None: + continue + for path in lang_dir.rglob("*.entity"): + if path.is_file(): + names.add(path.stem) + return {name: self.load_entity(name) for name in sorted(names)} + def _load_expanded(self, base_name: str, extension: str) -> List[str]: """Load a resource and expand it to its sample set.""" path = self._locate(base_name, extension) diff --git a/test/test_dialog.py b/test/test_dialog.py index 508c5b0..e0833eb 100644 --- a/test/test_dialog.py +++ b/test/test_dialog.py @@ -97,3 +97,49 @@ def test_renderer_from_resources(tmp_path): res = LocaleResources("en-US", str(locale)) renderer = DialogRenderer.from_resources(res, "hi") assert renderer.render({"name": "Sam"}) == "Hello Sam." + + +# --- default slots and .entity fallback -------------------------------------- + +def test_renderer_default_slots_are_reused(): + renderer = DialogRenderer(["Hello {name}."], slots={"name": "Sam"}) + assert renderer.render() == "Hello Sam." + assert renderer.render() == "Hello Sam." + + +def test_per_call_slot_overrides_a_default(): + renderer = DialogRenderer(["Hello {name}."], slots={"name": "Sam"}) + assert renderer.render({"name": "Max"}) == "Hello Max." + + +def test_unfilled_slot_falls_back_to_entity(): + renderer = DialogRenderer(["today is {day}"], + entities={"day": ["monday"]}) + assert renderer.render() == "today is monday" + + +def test_slot_precedence_call_then_default_then_entity(): + renderer = DialogRenderer(["value {x}"], + slots={"x": "from_default"}, + entities={"x": ["from_entity"]}) + # default beats entity + assert renderer.render() == "value from_default" + # per-call beats both + assert renderer.render({"x": "from_call"}) == "value from_call" + + +def test_unfilled_with_no_source_still_raises(): + renderer = DialogRenderer(["say {name}"], entities={"other": ["x"]}) + with pytest.raises(UnfilledSlot): + renderer.render() + + +def test_from_resources_picks_up_entity_fallback(tmp_path): + locale = tmp_path / "locale" + lang = locale / "en-US" + lang.mkdir(parents=True) + (lang / "today.dialog").write_text("today is {day}\n", encoding="utf-8") + (lang / "day.entity").write_text("monday\n", encoding="utf-8") + res = LocaleResources("en-US", str(locale)) + renderer = DialogRenderer.from_resources(res, "today") + assert renderer.render() == "today is monday" From 74d41baad100837a28f2472cfc6ebb7892eca2b1 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 12:33:04 +0100 Subject: [PATCH 007/110] docs: add runnable example scripts to examples/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One self-contained .py per feature worth demonstrating: - expand.py — the sentence template expander - load_resources.py — the locale resource loader - render_dialog.py — render() and DialogRenderer (repetition avoidance, default slots, .entity fallback) - lint.py — the linter used as a library Adds examples/skill-locale/, a small valid skill locale the loader and dialog scripts read. examples/README.md indexes the scripts. Co-Authored-By: Claude Opus 4.7 (1M context) --- examples/README.md | 15 +++++++++ examples/expand.py | 27 +++++++++++++++ examples/lint.py | 20 +++++++++++ examples/load_resources.py | 33 +++++++++++++++++++ examples/render_dialog.py | 30 +++++++++++++++++ .../skill-locale/locale/en-US/agenda.dialog | 1 + .../skill-locale/locale/en-US/greet.intent | 1 + .../skill-locale/locale/en-US/greeting.voc | 4 +++ .../skill-locale/locale/en-US/play.intent | 2 ++ .../skill-locale/locale/en-US/weather.dialog | 3 ++ .../skill-locale/locale/en-US/weekday.entity | 2 ++ 11 files changed, 138 insertions(+) create mode 100644 examples/expand.py create mode 100644 examples/lint.py create mode 100644 examples/load_resources.py create mode 100644 examples/render_dialog.py create mode 100644 examples/skill-locale/locale/en-US/agenda.dialog create mode 100644 examples/skill-locale/locale/en-US/greet.intent create mode 100644 examples/skill-locale/locale/en-US/greeting.voc create mode 100644 examples/skill-locale/locale/en-US/play.intent create mode 100644 examples/skill-locale/locale/en-US/weather.dialog create mode 100644 examples/skill-locale/locale/en-US/weekday.entity diff --git a/examples/README.md b/examples/README.md index 4cff70e..cb7b613 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,20 @@ # Examples +## Runnable scripts + +Each script is self-contained and prints its output. Install the package first +(`pip install -e .` from the repo root), then run e.g. `python examples/expand.py`. + +| Script | Demonstrates | +|--------|--------------| +| [`expand.py`](expand.py) | `expand()` — alternatives, optionals, opaque `{slot}`s, `` references, and rejection of malformed templates (OVOS-INTENT-1) | +| [`load_resources.py`](load_resources.py) | `LocaleResources` — loading `.intent` / `.voc` / `.entity` / `.dialog` from a locale tree (OVOS-INTENT-2) | +| [`render_dialog.py`](render_dialog.py) | `render()` and `DialogRenderer` — phrase selection, repetition avoidance, default slots, `.entity` fallback (OVOS-INTENT-2 §4.2) | +| [`lint.py`](lint.py) | `lint_locale()` — the linter used as a library | + +The loader and dialog scripts read [`skill-locale/`](skill-locale), a small +valid skill locale; `lint.py` reads `dirty-locale/` (below). + ## `dirty-locale` — the linter against a broken locale `dirty-locale/` is a deliberately broken skill locale. Every file in it trips diff --git a/examples/expand.py b/examples/expand.py new file mode 100644 index 0000000..122bc76 --- /dev/null +++ b/examples/expand.py @@ -0,0 +1,27 @@ +"""Expanding sentence templates — OVOS-INTENT-1. + +`expand()` turns a template into its sample set. Run: `python examples/expand.py` +""" +from ovos_spec_tools import MalformedTemplate, expand + +# Alternatives and optionals expand to every combination. +print("worked example:") +for sample in sorted(expand("(turn|switch) [the] (light|fan)")): + print(" ", sample) + +# Named slots are opaque — carried through, never expanded. +print("\nnamed slots are opaque:") +print(" ", expand("(play|put on) {query}")) + +# `` references expand a named vocabulary in place. +print("\ninline vocabulary reference:") +vocabularies = {"greeting": ["hello", "hi", "hey"]} +print(" ", expand(" [there] {name}", vocabularies)) + +# Malformed templates (OVOS-INTENT-1 §3.6) are rejected. +print("\nmalformed templates are rejected:") +for bad in ["turn (on|off the lights", "press (button)", "{a}{b}"]: + try: + expand(bad) + except MalformedTemplate as error: + print(f" {bad!r} -> {error}") diff --git a/examples/lint.py b/examples/lint.py new file mode 100644 index 0000000..ad3d3cf --- /dev/null +++ b/examples/lint.py @@ -0,0 +1,20 @@ +"""Linting a locale folder programmatically. + +`lint_locale()` is what the `ovos-spec-lint` command wraps; it returns every +`Finding` so a tool can process them. Run: `python examples/lint.py` + +See dirty-locale/ and README.md for the locale being linted here. +""" +from pathlib import Path + +from ovos_spec_tools import lint_locale + +dirty_locale = Path(__file__).parent / "dirty-locale" / "locale" +findings = lint_locale(dirty_locale) + +for finding in findings: + print(finding) + +errors = sum(1 for f in findings if f.severity == "error") +warnings = len(findings) - errors +print(f"\n{errors} error(s), {warnings} warning(s)") diff --git a/examples/load_resources.py b/examples/load_resources.py new file mode 100644 index 0000000..ebe6298 --- /dev/null +++ b/examples/load_resources.py @@ -0,0 +1,33 @@ +"""Loading a skill's locale resources — OVOS-INTENT-2. + +`LocaleResources` discovers and loads the five resource roles from a +`locale//` tree. Run: `python examples/load_resources.py` +""" +from pathlib import Path + +from ovos_spec_tools import LocaleResources + +locale = Path(__file__).parent / "skill-locale" / "locale" +resources = LocaleResources("en-US", str(locale)) + +# `.intent` — loaded as its sample set, named slots intact. +print("play.intent samples:") +for sample in resources.load_intent("play"): + print(" ", sample) + +# `.intent` using a `` reference — the .voc is resolved automatically. +print("\ngreet.intent samples (uses ):") +for sample in resources.load_intent("greet"): + print(" ", sample) + +# `.voc` — a slot-free vocabulary, loaded as its expanded phrase set. +print("\ngreeting.voc:", resources.load_vocabulary("greeting")) + +# `.entity` — a value set. +print("weekday.entity:", resources.load_entity("weekday")) + +# `.dialog` — loaded as raw phrase strings, NOT expanded (expansion is +# per-render; see render_dialog.py). +print("\nweather.dialog phrases:") +for phrase in resources.load_dialog("weather"): + print(" ", phrase) diff --git a/examples/render_dialog.py b/examples/render_dialog.py new file mode 100644 index 0000000..36b7df3 --- /dev/null +++ b/examples/render_dialog.py @@ -0,0 +1,30 @@ +"""Rendering dialog — OVOS-INTENT-2 §4.2. + +Shows the stateless `render()` function and the stateful `DialogRenderer`. +A seeded `random.Random` is used so the output is reproducible. +Run: `python examples/render_dialog.py` +""" +import random +from pathlib import Path + +from ovos_spec_tools import DialogRenderer, LocaleResources, render + +locale = Path(__file__).parent / "skill-locale" / "locale" +resources = LocaleResources("en-US", str(locale)) +phrases = resources.load_dialog("weather") + +# The stateless function: pick a phrase, expand its variety, fill the slots. +print("render() — stateless:") +print(" ", render(phrases, slots={"temperature": 21}, rng=random.Random(1))) + +# The stateful renderer avoids repeating the phrase it chose last time. +print("\nDialogRenderer — repetition avoidance:") +renderer = DialogRenderer(phrases, rng=random.Random(1)) +for _ in range(3): + print(" ", renderer.render({"temperature": 21})) + +# Default slots are set once and reused; `.entity` fills anything left over. +# `agenda.dialog` has a {weekday} slot and weekday.entity supplies the values. +print("\nDialogRenderer — default slots + .entity fallback:") +agenda = DialogRenderer.from_resources(resources, "agenda", rng=random.Random(2)) +print(" no slots passed:", agenda.render()) # {weekday} filled from .entity diff --git a/examples/skill-locale/locale/en-US/agenda.dialog b/examples/skill-locale/locale/en-US/agenda.dialog new file mode 100644 index 0000000..f3a83cf --- /dev/null +++ b/examples/skill-locale/locale/en-US/agenda.dialog @@ -0,0 +1 @@ +Your next event is on {weekday}. diff --git a/examples/skill-locale/locale/en-US/greet.intent b/examples/skill-locale/locale/en-US/greet.intent new file mode 100644 index 0000000..789351e --- /dev/null +++ b/examples/skill-locale/locale/en-US/greet.intent @@ -0,0 +1 @@ + [there] diff --git a/examples/skill-locale/locale/en-US/greeting.voc b/examples/skill-locale/locale/en-US/greeting.voc new file mode 100644 index 0000000..77ebb4f --- /dev/null +++ b/examples/skill-locale/locale/en-US/greeting.voc @@ -0,0 +1,4 @@ +# greeting words +hello +hi +hey diff --git a/examples/skill-locale/locale/en-US/play.intent b/examples/skill-locale/locale/en-US/play.intent new file mode 100644 index 0000000..b7923e4 --- /dev/null +++ b/examples/skill-locale/locale/en-US/play.intent @@ -0,0 +1,2 @@ +(play|put on) {query} +i want to (hear|listen to) {query} diff --git a/examples/skill-locale/locale/en-US/weather.dialog b/examples/skill-locale/locale/en-US/weather.dialog new file mode 100644 index 0000000..5232043 --- /dev/null +++ b/examples/skill-locale/locale/en-US/weather.dialog @@ -0,0 +1,3 @@ +It is {temperature} degrees. +Right now it's {temperature} degrees out. +(Currently|At the moment) {temperature} degrees. diff --git a/examples/skill-locale/locale/en-US/weekday.entity b/examples/skill-locale/locale/en-US/weekday.entity new file mode 100644 index 0000000..89c0fa0 --- /dev/null +++ b/examples/skill-locale/locale/en-US/weekday.entity @@ -0,0 +1,2 @@ +monday +(tues|wednes|thurs|fri)day From da64e9cd83ee2eab1e85c933934bb079539a5c1a Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 12:38:20 +0100 Subject: [PATCH 008/110] feat: smart language fallback in LocaleResources; langcodes now optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the requested language has no directory, LocaleResources resolves to the nearest available language (OVOS-INTENT-2 §2.2, non-normative) — e.g. en-AU finds en-US. - nearness is decided by a LanguageMatcher Protocol (an object with tag_distance(desired, supported)); the langcodes module satisfies it structurally and is the default - langcodes moved from a hard dependency to the [langcodes] extra; the package core now has zero dependencies. Without langcodes, resolution is exact-match only - max_language_distance caps the fallback (default 10, per §2.2); set 0 to disable 5 more tests (90 total). Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 11 ++++- ovos_spec_tools/__init__.py | 2 + ovos_spec_tools/resources.py | 82 +++++++++++++++++++++++++++++++++--- pyproject.toml | 10 +++-- test/test_resources.py | 57 +++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7733f45..c5f15a5 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,14 @@ res.load_vocabulary("yes") # expanded phrase set The user-data path of the override precedence is assistant-defined and passed in (`user_locale=`); this package imports no configuration. +**Smart language fallback.** When the requested language has no directory, +`LocaleResources` resolves to the nearest available language instead +(OVOS-INTENT-2 §2.2) — so a request for `en-AU` finds `en-US`. The nearness +test is a `LanguageMatcher`; by default the optional `langcodes` dependency +provides one. Without `langcodes` installed, resolution is exact-match only; +inject your own `LanguageMatcher` to use a different implementation, or pass +`max_language_distance=0` to disable the fallback. + ## The dialog renderer Rendering a dialog selects one phrase from a loaded `.dialog`, expands its @@ -123,7 +131,8 @@ single `/` directory. Exit code is non-zero on errors (or, with ## Install ```bash -pip install ovos-spec-tools +pip install ovos-spec-tools # core — no dependencies +pip install ovos-spec-tools[langcodes] # adds the smart language fallback ``` ## License diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index b046c41..6466ab4 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -16,6 +16,7 @@ from ovos_spec_tools.expansion import MalformedTemplate, expand from ovos_spec_tools.lint import Finding, lint_locale from ovos_spec_tools.resources import ( + LanguageMatcher, LocaleResources, MalformedResource, read_resource_file, @@ -26,6 +27,7 @@ "expand", "MalformedTemplate", "LocaleResources", + "LanguageMatcher", "MalformedResource", "read_resource_file", "render", diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index b2f2cc1..454c734 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -18,12 +18,13 @@ from __future__ import annotations from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Protocol from ovos_spec_tools.expansion import MalformedTemplate, expand __all__ = [ "LocaleResources", + "LanguageMatcher", "MalformedResource", "read_resource_file", "SLOT_BEARING_ROLES", @@ -34,6 +35,37 @@ SLOT_BEARING_ROLES = (".intent", ".dialog") SLOT_FREE_ROLES = (".entity", ".voc", ".blacklist") +# Default cap for the smart language fallback (OVOS-INTENT-2 §2.2): the +# `langcodes` library treats a tag distance below 10 as a usable regional +# match. +DEFAULT_MAX_LANGUAGE_DISTANCE = 10 + + +class LanguageMatcher(Protocol): + """Computes the distance between two BCP-47 language tags. + + A lower distance is a closer match; ``0`` is exact. The ``langcodes`` + module satisfies this protocol structurally — its ``tag_distance`` + function has exactly this signature — so `langcodes` itself can be passed + wherever a ``LanguageMatcher`` is expected. + """ + + def tag_distance(self, desired: str, supported: str) -> int: + ... + + +def _default_language_matcher() -> Optional["LanguageMatcher"]: + """Return the ``langcodes`` module if it is installed, else ``None``. + + `langcodes` is an optional dependency; without it the smart language + fallback is disabled and only exact (case-insensitive) tags resolve. + """ + try: + import langcodes + except ImportError: + return None + return langcodes + class MalformedResource(ValueError): """A resource file or skill layout that violates OVOS-INTENT-2. @@ -67,11 +99,20 @@ class LocaleResources: A resource is resolved through the override precedence of §2.1 — user overrides, then skill resources, then core resources — searching each source's ``/`` directory and all its subdirectories recursively. + + When the requested language has no directory, a **smart language + fallback** (OVOS-INTENT-2 §2.2, non-normative) selects the nearest + available language whose tag distance is within + ``max_language_distance``. The fallback needs a :class:`LanguageMatcher`; + by default the optional ``langcodes`` dependency supplies one, and the + fallback is silently disabled if ``langcodes`` is not installed. """ def __init__(self, lang: str, skill_locale: str, core_locale: Optional[str] = None, - user_locale: Optional[str] = None): + user_locale: Optional[str] = None, + language_matcher: Optional[LanguageMatcher] = None, + max_language_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE): """ Args: lang: a BCP-47 language tag; compared case-insensitively (§2). @@ -79,6 +120,10 @@ def __init__(self, lang: str, skill_locale: str, core_locale: path to the assistant's core ``locale/`` directory. user_locale: path to the user-override ``locale/`` directory (its root is assistant-defined, §2.1). + language_matcher: a :class:`LanguageMatcher` for the smart language + fallback. Defaults to the ``langcodes`` module if installed. + max_language_distance: the largest tag distance accepted by the + fallback. ``0`` disables the fallback — only exact tags match. """ self.lang = lang # Highest precedence first (§2.1): user, skill, core. @@ -86,16 +131,41 @@ def __init__(self, lang: str, skill_locale: str, Path(p) for p in (user_locale, skill_locale, core_locale) if p is not None ] + if language_matcher is None: + language_matcher = _default_language_matcher() + self._language_matcher = language_matcher + self.max_language_distance = max_language_distance def _lang_dir(self, source: Path) -> Optional[Path]: - """The ``/`` directory under one source, matched - case-insensitively (§2).""" + """The ``/`` directory under one source. + + An exact tag (case-insensitive, §2) wins. Failing that, the smart + language fallback (§2.2) picks the nearest available language within + ``max_language_distance``, if a :class:`LanguageMatcher` is available. + """ if not source.is_dir(): return None + subdirectories = [c for c in source.iterdir() if c.is_dir()] + target = self.lang.lower() - for child in source.iterdir(): - if child.is_dir() and child.name.lower() == target: + for child in subdirectories: + if child.name.lower() == target: return child + + if self._language_matcher is None or self.max_language_distance <= 0: + return None + nearest: Optional[Path] = None + nearest_distance = self.max_language_distance + 1 + for child in subdirectories: + try: + distance = self._language_matcher.tag_distance( + self.lang, child.name) + except Exception: + continue # an unparseable directory name is simply not a match + if distance < nearest_distance: + nearest, nearest_distance = child, distance + if nearest is not None and nearest_distance <= self.max_language_distance: + return nearest return None def _locate(self, base_name: str, extension: str) -> Optional[Path]: diff --git a/pyproject.toml b/pyproject.toml index 52719cd..5f6ce05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,13 +10,17 @@ readme = "README.md" license = { text = "Apache-2.0" } authors = [{ name = "jarbasai", email = "jarbasai@mailfence.com" }] requires-python = ">=3.8" -dependencies = [ - "langcodes", -] +dependencies = [] [project.optional-dependencies] +# `langcodes` powers the smart language fallback of LocaleResources; without +# it, language resolution is exact-match only. +langcodes = [ + "langcodes", +] test = [ "pytest", + "langcodes", ] [project.scripts] diff --git a/test/test_resources.py b/test/test_resources.py index 8aa9b0b..447c795 100644 --- a/test/test_resources.py +++ b/test/test_resources.py @@ -128,3 +128,60 @@ def test_missing_resource_raises(tmp_path): res = LocaleResources("en-US", str(locale)) with pytest.raises(FileNotFoundError): res.load_intent("nope") + + +# --- §2.2 smart language fallback ------------------------------------------- + +class _FakeMatcher: + """A LanguageMatcher stub with hand-set distances, for deterministic tests.""" + + def __init__(self, distances): + self.distances = distances # {(desired, supported): distance} + + def tag_distance(self, desired, supported): + return self.distances.get((desired, supported), 999) + + +def test_fallback_resolves_a_near_language(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.intent", "hello world\n") + matcher = _FakeMatcher({("en-AU", "en-US"): 4}) + res = LocaleResources("en-AU", str(locale), language_matcher=matcher) + assert res.load_intent("x") == ["hello world"] + + +def test_fallback_picks_the_nearest_language(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.intent", "us\n") + _write(locale / "en-GB" / "x.intent", "gb\n") + matcher = _FakeMatcher({("en-AU", "en-US"): 5, ("en-AU", "en-GB"): 3}) + res = LocaleResources("en-AU", str(locale), language_matcher=matcher) + assert res.load_intent("x") == ["gb"] + + +def test_fallback_rejects_a_too_distant_language(tmp_path): + locale = tmp_path / "locale" + _write(locale / "fr-FR" / "x.intent", "bonjour\n") + matcher = _FakeMatcher({("en-US", "fr-FR"): 80}) + res = LocaleResources("en-US", str(locale), language_matcher=matcher) + with pytest.raises(FileNotFoundError): + res.load_intent("x") + + +def test_fallback_disabled_with_zero_max_distance(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.intent", "hello\n") + matcher = _FakeMatcher({("en-AU", "en-US"): 4}) + res = LocaleResources("en-AU", str(locale), language_matcher=matcher, + max_language_distance=0) + with pytest.raises(FileNotFoundError): + res.load_intent("x") + + +def test_fallback_with_real_langcodes(tmp_path): + """The default matcher is the langcodes module when it is installed.""" + pytest.importorskip("langcodes") + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.intent", "hello\n") + res = LocaleResources("en-GB", str(locale)) # no explicit matcher + assert res.load_intent("x") == ["hello"] From 68848ea7f21246ecff33856a1d0b0845a3eef792 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 12:57:37 +0100 Subject: [PATCH 009/110] refactor: language is per-query; consolidate language matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Language is dynamic — a locale folder is a skill's multilingual unit — so it must not be pinned at construction. - LocaleResources: drops the `lang` constructor argument; every load method takes `lang`. One instance serves every language, and the smart fallback is resolved afresh per call. - DialogRenderer: now resource-backed and multilingual — built from a LocaleResources + a dialog name, with `render(lang, slots=)`. Repetition avoidance is tracked per language. The `render()` function stays as the stateless, language-agnostic primitive. - New ovos_spec_tools/language.py — `standardize_lang` and `closest_lang`, the single home for the language-tag logic OVOS reimplements across locale loading, TTS voices and STT models (ovos_utils.lang.get_language_dir, phoonnx.match_lang, ...). Mirrors their proven behaviour: tag standardization, distance-below-10 match, the Tagalog `tl`/`fil` quirk. langcodes stays optional. - LocaleResources uses closest_lang; a custom `lang_resolver` may be injected. The LanguageMatcher protocol is replaced by this. 9 more tests (99 total). Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 75 +++++++++------ examples/load_resources.py | 17 ++-- examples/render_dialog.py | 21 ++-- ovos_spec_tools/__init__.py | 12 ++- ovos_spec_tools/dialog.py | 121 +++++++++++------------ ovos_spec_tools/language.py | 91 ++++++++++++++++++ ovos_spec_tools/resources.py | 172 +++++++++++++-------------------- test/test_dialog.py | 154 +++++++++++++++-------------- test/test_language.py | 50 ++++++++++ test/test_resources.py | 181 +++++++++++++++++++---------------- 10 files changed, 521 insertions(+), 373 deletions(-) create mode 100644 ovos_spec_tools/language.py create mode 100644 test/test_language.py diff --git a/README.md b/README.md index c5f15a5..7a9ff33 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,10 @@ Reference implementation of the OVOS [formal specifications](https://github.com/OpenVoiceOS/formal-specifications) — the low-level, dependency-light primitives those specifications describe. -OVOS components reimplement template expansion and resource loading in several -places, and the copies drift. This package is the single conformant -implementation those components — and any third-party tool — can depend on. +OVOS components reimplement template expansion, resource loading, and language +matching in several places, and the copies drift. This package is the single +conformant implementation those components — and any third-party tool — can +depend on. ## Status @@ -15,6 +16,7 @@ implementation those components — and any third-party tool — can depend on. | Sentence template expander | OVOS-INTENT-1 v2 | available | | Locale resource loader | OVOS-INTENT-2 | available | | Dialog renderer | OVOS-INTENT-2 §4.2 | available | +| Language-tag matching | OVOS-INTENT-2 §2.2 | available | | `ovos-spec-lint` locale linter | OVOS-INTENT-1 / -2 | available | ## The expander @@ -50,13 +52,16 @@ single-spaced, alphanumeric. This package expands; it does not normalize. `.blacklist` — through the user → skill → core override precedence, searching each `locale//` tree recursively. +The **language is given per call**, not at construction: a locale folder is +the multilingual unit of a skill, so one instance serves every language. + ```python from ovos_spec_tools import LocaleResources -res = LocaleResources("en-US", skill_locale="my-skill/locale") -res.load_intent("play") # sample set, named slots intact -res.load_dialog("weather") # phrase strings, not expanded (§4.2) -res.load_vocabulary("yes") # expanded phrase set +res = LocaleResources(skill_locale="my-skill/locale") +res.load_intent("play", "en-US") # sample set, named slots intact +res.load_intent("play", "pt-BR") # same instance, another language +res.load_dialog("weather", "en-US") # phrase strings, not expanded (§4.2) ``` The user-data path of the override precedence is assistant-defined and passed @@ -64,38 +69,56 @@ in (`user_locale=`); this package imports no configuration. **Smart language fallback.** When the requested language has no directory, `LocaleResources` resolves to the nearest available language instead -(OVOS-INTENT-2 §2.2) — so a request for `en-AU` finds `en-US`. The nearness -test is a `LanguageMatcher`; by default the optional `langcodes` dependency -provides one. Without `langcodes` installed, resolution is exact-match only; -inject your own `LanguageMatcher` to use a different implementation, or pass -`max_language_distance=0` to disable the fallback. +(OVOS-INTENT-2 §2.2) — so a request for `en-AU` finds `en-US`. Resolution uses +`closest_lang` (below) and is re-run per call. Without `langcodes` installed, +resolution is exact-match only; pass `max_language_distance=0` to disable the +fallback, or a custom `lang_resolver` to change it. + +## Language matching + +`standardize_lang` and `closest_lang` are the language-tag primitives — the +same logic OVOS reimplements across locale loading, TTS voices, and STT +models, gathered in one place. + +```python +from ovos_spec_tools import standardize_lang, closest_lang + +standardize_lang("en_us") # 'en-US' +closest_lang("en-AU", ["pt-BR", "en-US"]) # 'en-US' (regional fallback) +closest_lang("zz-ZZ", ["pt-BR", "en-US"]) # None +``` + +`closest_lang` returns the nearest entry whose tag distance is below +`max_distance` (default 10), or `None`. With `langcodes` absent it resolves +exact matches only. ## The dialog renderer Rendering a dialog selects one phrase from a loaded `.dialog`, expands its -variety to a single variant, and fills every `{name}` slot with a -caller-supplied value (OVOS-INTENT-2 §4.2). A phrase with an unfilled slot -raises `UnfilledSlot`. +variety to a single variant, and fills every `{name}` slot with a value +(OVOS-INTENT-2 §4.2). A slot with no value raises `UnfilledSlot`. -`render()` is a stateless one-shot function: +`render()` is a stateless one-shot function over explicit phrases: ```python from ovos_spec_tools import render -render(res.load_dialog("weather"), slots={"temperature": 21}) +render(res.load_dialog("weather", "en-US"), slots={"temperature": 21}) # 'It is 21 degrees.' ``` -`DialogRenderer` is a stateful, object-oriented alternative. It holds the -dialog and, unlike the function, **avoids repeating the phrase it chose last -time** — so a repeatedly-spoken response does not sound mechanical: +`DialogRenderer` is a stateful, **multilingual** alternative, backed by a +`LocaleResources`. The language is given per `render()` call, and the renderer +**avoids repeating the phrase it chose last time** — per language — so a +repeatedly-spoken response does not sound mechanical: ```python from ovos_spec_tools import DialogRenderer -renderer = DialogRenderer.from_resources(res, "weather") -renderer.render({"temperature": 21}) # a phrase -renderer.render({"temperature": 22}) # a different phrase +renderer = DialogRenderer(res, "weather") +renderer.render("en-US", {"temperature": 21}) # a phrase +renderer.render("en-US", {"temperature": 22}) # a different phrase +renderer.render("pt-BR", {"temperature": 23}) # the same dialog, another language ``` It also holds **default slot values** set once and reused, and falls back to a @@ -103,12 +126,6 @@ slot's **`.entity` value set** for anything still unfilled. A slot is resolved in order: the per-call value, then a default, then a random `.entity` value, then `UnfilledSlot`. -```python -renderer = DialogRenderer.from_resources(res, "greeting", - slots={"assistant": "OVOS"}) -renderer.render() # {assistant} reused; an {day} slot from day.entity -``` - ## The locale linter `ovos-spec-lint` validates every resource file under a locale directory — the diff --git a/examples/load_resources.py b/examples/load_resources.py index ebe6298..7cd323e 100644 --- a/examples/load_resources.py +++ b/examples/load_resources.py @@ -1,33 +1,34 @@ """Loading a skill's locale resources — OVOS-INTENT-2. `LocaleResources` discovers and loads the five resource roles from a -`locale//` tree. Run: `python examples/load_resources.py` +`locale//` tree. One instance serves every language: the language is a +parameter of each load call. Run: `python examples/load_resources.py` """ from pathlib import Path from ovos_spec_tools import LocaleResources locale = Path(__file__).parent / "skill-locale" / "locale" -resources = LocaleResources("en-US", str(locale)) +resources = LocaleResources(str(locale)) # `.intent` — loaded as its sample set, named slots intact. -print("play.intent samples:") -for sample in resources.load_intent("play"): +print("play.intent samples (en-US):") +for sample in resources.load_intent("play", "en-US"): print(" ", sample) # `.intent` using a `` reference — the .voc is resolved automatically. print("\ngreet.intent samples (uses ):") -for sample in resources.load_intent("greet"): +for sample in resources.load_intent("greet", "en-US"): print(" ", sample) # `.voc` — a slot-free vocabulary, loaded as its expanded phrase set. -print("\ngreeting.voc:", resources.load_vocabulary("greeting")) +print("\ngreeting.voc:", resources.load_vocabulary("greeting", "en-US")) # `.entity` — a value set. -print("weekday.entity:", resources.load_entity("weekday")) +print("weekday.entity:", resources.load_entity("weekday", "en-US")) # `.dialog` — loaded as raw phrase strings, NOT expanded (expansion is # per-render; see render_dialog.py). print("\nweather.dialog phrases:") -for phrase in resources.load_dialog("weather"): +for phrase in resources.load_dialog("weather", "en-US"): print(" ", phrase) diff --git a/examples/render_dialog.py b/examples/render_dialog.py index 36b7df3..0d09abe 100644 --- a/examples/render_dialog.py +++ b/examples/render_dialog.py @@ -1,7 +1,7 @@ """Rendering dialog — OVOS-INTENT-2 §4.2. -Shows the stateless `render()` function and the stateful `DialogRenderer`. -A seeded `random.Random` is used so the output is reproducible. +Shows the stateless `render()` function and the stateful, multilingual +`DialogRenderer`. A seeded `random.Random` makes the output reproducible. Run: `python examples/render_dialog.py` """ import random @@ -10,21 +10,22 @@ from ovos_spec_tools import DialogRenderer, LocaleResources, render locale = Path(__file__).parent / "skill-locale" / "locale" -resources = LocaleResources("en-US", str(locale)) -phrases = resources.load_dialog("weather") +resources = LocaleResources(str(locale)) # The stateless function: pick a phrase, expand its variety, fill the slots. +phrases = resources.load_dialog("weather", "en-US") print("render() — stateless:") print(" ", render(phrases, slots={"temperature": 21}, rng=random.Random(1))) -# The stateful renderer avoids repeating the phrase it chose last time. +# The stateful renderer is multilingual — the language is given per render() +# call — and avoids repeating the phrase it chose last (per language). print("\nDialogRenderer — repetition avoidance:") -renderer = DialogRenderer(phrases, rng=random.Random(1)) +renderer = DialogRenderer(resources, "weather", rng=random.Random(1)) for _ in range(3): - print(" ", renderer.render({"temperature": 21})) + print(" ", renderer.render("en-US", {"temperature": 21})) # Default slots are set once and reused; `.entity` fills anything left over. # `agenda.dialog` has a {weekday} slot and weekday.entity supplies the values. -print("\nDialogRenderer — default slots + .entity fallback:") -agenda = DialogRenderer.from_resources(resources, "agenda", rng=random.Random(2)) -print(" no slots passed:", agenda.render()) # {weekday} filled from .entity +print("\nDialogRenderer — .entity fallback:") +agenda = DialogRenderer(resources, "agenda", rng=random.Random(2)) +print(" no slots passed:", agenda.render("en-US")) # {weekday} from .entity diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index 6466ab4..a0dab1a 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -7,16 +7,19 @@ template expander; - :class:`~ovos_spec_tools.resources.LocaleResources` — the OVOS-INTENT-2 locale resource-file loader; -- :func:`~ovos_spec_tools.dialog.render` — the OVOS-INTENT-2 §4.2 dialog - renderer; +- :func:`~ovos_spec_tools.dialog.render` / :class:`~ovos_spec_tools.dialog.DialogRenderer` + — the OVOS-INTENT-2 §4.2 dialog renderer; +- :func:`~ovos_spec_tools.language.standardize_lang` / + :func:`~ovos_spec_tools.language.closest_lang` — language-tag normalization + and closest-match resolution; - :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also exposed as the ``ovos-spec-lint`` command. """ from ovos_spec_tools.dialog import DialogRenderer, UnfilledSlot, render from ovos_spec_tools.expansion import MalformedTemplate, expand +from ovos_spec_tools.language import closest_lang, standardize_lang from ovos_spec_tools.lint import Finding, lint_locale from ovos_spec_tools.resources import ( - LanguageMatcher, LocaleResources, MalformedResource, read_resource_file, @@ -27,12 +30,13 @@ "expand", "MalformedTemplate", "LocaleResources", - "LanguageMatcher", "MalformedResource", "read_resource_file", "render", "DialogRenderer", "UnfilledSlot", + "standardize_lang", + "closest_lang", "lint_locale", "Finding", "__version__", diff --git a/ovos_spec_tools/dialog.py b/ovos_spec_tools/dialog.py index a0ec9fa..58a4e9e 100644 --- a/ovos_spec_tools/dialog.py +++ b/ovos_spec_tools/dialog.py @@ -3,17 +3,18 @@ This is the reference implementation of the **Dialog renderer** conformance role of OVOS-INTENT-1 §7. Rendering a dialog means: select one phrase from a loaded ``.dialog``, expand its ``(a|b)`` / ``[x]`` variety to a single variant, -and fill every ``{name}`` slot with a caller-supplied value. +and fill every ``{name}`` slot with a value. Two interfaces are provided: -- :func:`render` — a stateless one-shot function; -- :class:`DialogRenderer` — a stateful object that additionally avoids - repeating the phrase it chose last time. +- :func:`render` — a stateless one-shot function over explicit phrases; +- :class:`DialogRenderer` — a stateful, **multilingual** object backed by a + :class:`~ovos_spec_tools.resources.LocaleResources`: the language is given + per :meth:`DialogRenderer.render` call, and the renderer avoids repeating + the phrase it chose last (independently per language). Only the single-brace slot form ``{name}`` is recognized; there is no ``{{ }}`` -form (OVOS-INTENT-2 §4.2). Slots are filled by the caller; a chosen phrase with -a slot the caller did not fill raises :class:`UnfilledSlot`. +form (OVOS-INTENT-2 §4.2). A slot with no value raises :class:`UnfilledSlot`. """ from __future__ import annotations @@ -40,7 +41,7 @@ def choice(self, seq: Sequence[str]) -> str: class UnfilledSlot(ValueError): - """A chosen dialog phrase has a named slot the caller did not fill. + """A chosen dialog phrase has a named slot with no value. Per OVOS-INTENT-1 §5.1 the caller must supply a value for every slot in the chosen phrase; a phrase with an unfilled slot must not be sent to TTS. @@ -51,7 +52,11 @@ def render(phrases: Sequence[str], slots: Optional[Dict[str, object]] = None, vocabularies: Optional[Dict[str, Sequence[str]]] = None, rng: Optional[Chooser] = None) -> str: - """Render one phrase from a loaded ``.dialog`` (stateless). + """Render one phrase from a list of dialog phrases (stateless). + + This is the language-agnostic primitive: the caller has already chosen and + loaded the phrases. For a multilingual, resource-backed renderer use + :class:`DialogRenderer`. Args: phrases: the phrase strings of a ``.dialog``. @@ -65,7 +70,7 @@ def render(phrases: Sequence[str], One rendered phrase, ready for text-to-speech. Raises: - UnfilledSlot: a slot in the chosen phrase has no caller-supplied value. + UnfilledSlot: a slot in the chosen phrase has no value. ValueError: ``phrases`` is empty. MalformedTemplate: the chosen phrase is not a valid template. """ @@ -77,87 +82,77 @@ def render(phrases: Sequence[str], class DialogRenderer: - """A stateful renderer for one loaded ``.dialog`` (OVOS-INTENT-2 §4.2). - - An object-oriented alternative to :func:`render`. It holds the dialog's - phrases, the vocabularies, and the random source, and — unlike the bare - function — **avoids repeating the phrase it chose on the previous call**, - so a repeatedly-spoken response does not sound mechanical. Repetition - avoidance is an implementation choice the spec explicitly allows (§4.2). + """A stateful, multilingual renderer for one named ``.dialog``. + + Backed by a :class:`~ovos_spec_tools.resources.LocaleResources`: the dialog + name is fixed at construction, but the **language is given per** + :meth:`render` **call**, so one renderer serves every language the dialog + is shipped in. Each call loads that language's phrases, vocabularies, and + ``.entity`` value sets afresh. + + Unlike the bare :func:`render`, it **avoids repeating the phrase it chose + on the previous call** — tracked independently per language — so a + repeatedly-spoken response does not sound mechanical. Repetition avoidance + is an implementation choice the spec explicitly allows (§4.2). """ - def __init__(self, phrases: Sequence[str], - vocabularies: Optional[Dict[str, Sequence[str]]] = None, + def __init__(self, resources, name: str, rng: Optional[Chooser] = None, - slots: Optional[Dict[str, object]] = None, - entities: Optional[Dict[str, Sequence[str]]] = None): + slots: Optional[Dict[str, object]] = None): """ Args: - phrases: the phrase strings of a ``.dialog``. - vocabularies: vocabularies for any ```` references. + resources: a :class:`~ovos_spec_tools.resources.LocaleResources` + (or anything with ``load_dialog``, ``vocabularies`` and + ``entities`` methods taking a language). + name: the base name of the ``.dialog`` to render. rng: an object with a ``choice`` method; defaults to :mod:`random`. slots: **default** slot values, held for the renderer's lifetime - and reused on every :meth:`render` call. Use this for slots - that do not change per render (a unit preference, a name). A - per-call value passed to :meth:`render` overrides a default. - entities: ``.entity`` value sets, keyed by slot name. A slot that - is neither passed per call nor a default is filled with a - random value from its entity set, if one is given here. + and reused on every :meth:`render` call. A per-call value + overrides a default. """ - self.phrases = list(phrases) - if not self.phrases: - raise ValueError("a DialogRenderer needs at least one phrase") - self.vocabularies = vocabularies + self.resources = resources + self.name = name self.rng = rng if rng is not None else _random self.default_slots: Dict[str, object] = dict(slots or {}) - self.entities: Dict[str, Sequence[str]] = dict(entities or {}) - self._last: Optional[str] = None - - @classmethod - def from_resources(cls, resources, name: str, - rng: Optional[Chooser] = None, - slots: Optional[Dict[str, object]] = None - ) -> "DialogRenderer": - """Build a renderer for the ``.dialog`` named ``name``. - - ``resources`` is a - :class:`~ovos_spec_tools.resources.LocaleResources`; its loaded dialog, - its vocabularies, and its ``.entity`` value sets are all used — so a - slot left unfilled falls back to its ``.entity``. ``slots`` supplies - default slot values. - """ - return cls(resources.load_dialog(name), - vocabularies=resources.vocabularies(), - entities=resources.entities(), - rng=rng, slots=slots) + self._last: Dict[str, str] = {} # last phrase chosen, per language - def render(self, slots: Optional[Dict[str, object]] = None) -> str: - """Render one phrase, avoiding the phrase chosen on the previous call. + def render(self, lang: str, + slots: Optional[Dict[str, object]] = None) -> str: + """Render one phrase of the dialog in ``lang``. A slot is filled, in order of precedence, from: the per-call ``slots``; then the renderer's default slots; then a random value from the slot's - ``.entity`` set. A slot none of these supply raises :class:`UnfilledSlot`. + ``.entity`` set for ``lang``. A slot none of these supply raises + :class:`UnfilledSlot`. + + The phrase chosen on the previous call **for the same language** is + avoided when the dialog has more than one phrase. Args: - slots: per-call slot values; each overrides a default of the - same name. + lang: the BCP-47 language tag to render in. + slots: per-call slot values; each overrides a default. Returns: One rendered phrase, ready for text-to-speech. Raises: - UnfilledSlot: a slot in the chosen phrase has no value from any - source. + UnfilledSlot: a slot in the chosen phrase has no value. + FileNotFoundError: the dialog does not exist for ``lang``. MalformedTemplate: the chosen phrase is not a valid template. """ + phrases = self.resources.load_dialog(self.name, lang) effective = dict(self.default_slots) if slots: effective.update(slots) - choices = [p for p in self.phrases if p != self._last] or self.phrases + + last = self._last.get(lang) + choices = [p for p in phrases if p != last] or phrases phrase = self.rng.choice(choices) - self._last = phrase - return _render_phrase(phrase, effective, self.vocabularies, self.rng, - self.entities) + self._last[lang] = phrase + + return _render_phrase(phrase, effective, + self.resources.vocabularies(lang), self.rng, + self.resources.entities(lang)) def _render_phrase(phrase: str, diff --git a/ovos_spec_tools/language.py b/ovos_spec_tools/language.py new file mode 100644 index 0000000..21b3f27 --- /dev/null +++ b/ovos_spec_tools/language.py @@ -0,0 +1,91 @@ +"""Language-tag utilities — standardization and closest-match resolution. + +OVOS resolves "the closest available language for a request" in many places — +locale resources, TTS voices, STT models — and the logic has been +reimplemented repeatedly (``ovos_utils.lang.get_language_dir``, +``phoonnx.match_lang``, …) with subtle drift between the copies. This module is +intended to be the **single implementation**: it powers the smart language +fallback of :class:`~ovos_spec_tools.resources.LocaleResources` +(OVOS-INTENT-2 §2.2) and is importable on its own to replace those copies. + +``langcodes`` is an optional dependency. Without it, :func:`standardize_lang` +does a best-effort normalization and :func:`closest_lang` resolves exact +matches only. +""" +from __future__ import annotations + +from typing import Optional, Sequence + +__all__ = ["standardize_lang", "closest_lang", "DEFAULT_MAX_LANGUAGE_DISTANCE"] + +# A `langcodes` tag distance below 10 is a usable regional match +# (OVOS-INTENT-2 §2.2; see the langcodes distance-values documentation). +DEFAULT_MAX_LANGUAGE_DISTANCE = 10 + + +def standardize_lang(tag: str) -> str: + """Normalize a BCP-47 language tag for comparison. + + Uses ``langcodes.standardize_tag`` when available — handling underscores, + case, and script/region forms — and falls back to a simple normalization + otherwise. + """ + if tag.lower() in ("tl", "tgl"): + # langcodes folds Tagalog into the `fil` macrolanguage; OVOS keeps + # `tl` distinct, so this one tag is normalized by hand. + return "tl" + try: + from langcodes import standardize_tag + return str(standardize_tag(tag)) + except Exception: + normalized = tag.replace("_", "-") + if "-" in normalized: + primary, rest = normalized.split("-", 1) + return f"{primary.lower()}-{rest.upper()}" + return normalized.lower() + + +def _tag_distance(desired: str, supported: str) -> Optional[int]: + """``langcodes.tag_distance``, retried on the primary subtag if the full + tag is unparseable. Returns ``None`` if no distance can be computed — + including when ``langcodes`` is not installed.""" + try: + from langcodes import tag_distance + except ImportError: + return None + for candidate in (supported, supported.split("-")[0]): + try: + return tag_distance(desired, candidate) + except Exception: + continue + return None + + +def closest_lang(target: str, available: Sequence[str], + max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE + ) -> Optional[str]: + """Return the entry of ``available`` closest to ``target``. + + An exact match (after standardization, so ``en_US`` matches ``en-US``) + always wins. Otherwise the nearest tag whose distance is **below** + ``max_distance`` is returned; if none qualifies, or ``max_distance`` is not + positive, ``None`` is returned. Without ``langcodes`` installed only exact + matches resolve. + + The value returned is the original string from ``available``, so a caller + can map it back to a directory, a voice, a model, and so on. + """ + wanted = standardize_lang(target) + for candidate in available: + if standardize_lang(candidate).lower() == wanted.lower(): + return candidate + + if max_distance <= 0: + return None + nearest: Optional[str] = None + nearest_distance = max_distance # accept only a distance strictly below this + for candidate in available: + distance = _tag_distance(wanted, standardize_lang(candidate)) + if distance is not None and distance < nearest_distance: + nearest, nearest_distance = candidate, distance + return nearest diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index 454c734..3c5870b 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -14,17 +14,21 @@ The user-data path of the override precedence (§2.1) is **assistant-defined**; this module takes it as a parameter and imports no configuration. + +The **language is given per query**, not fixed at construction: a locale +folder is the multilingual unit of a skill, so one :class:`LocaleResources` +serves every language the skill ships. """ from __future__ import annotations from pathlib import Path -from typing import Dict, List, Optional, Protocol +from typing import Callable, Dict, List, Optional, Sequence from ovos_spec_tools.expansion import MalformedTemplate, expand +from ovos_spec_tools.language import DEFAULT_MAX_LANGUAGE_DISTANCE, closest_lang __all__ = [ "LocaleResources", - "LanguageMatcher", "MalformedResource", "read_resource_file", "SLOT_BEARING_ROLES", @@ -35,36 +39,9 @@ SLOT_BEARING_ROLES = (".intent", ".dialog") SLOT_FREE_ROLES = (".entity", ".voc", ".blacklist") -# Default cap for the smart language fallback (OVOS-INTENT-2 §2.2): the -# `langcodes` library treats a tag distance below 10 as a usable regional -# match. -DEFAULT_MAX_LANGUAGE_DISTANCE = 10 - - -class LanguageMatcher(Protocol): - """Computes the distance between two BCP-47 language tags. - - A lower distance is a closer match; ``0`` is exact. The ``langcodes`` - module satisfies this protocol structurally — its ``tag_distance`` - function has exactly this signature — so `langcodes` itself can be passed - wherever a ``LanguageMatcher`` is expected. - """ - - def tag_distance(self, desired: str, supported: str) -> int: - ... - - -def _default_language_matcher() -> Optional["LanguageMatcher"]: - """Return the ``langcodes`` module if it is installed, else ``None``. - - `langcodes` is an optional dependency; without it the smart language - fallback is disabled and only exact (case-insensitive) tags resolve. - """ - try: - import langcodes - except ImportError: - return None - return langcodes +# A resolver maps a requested language and the available language tags to the +# best one, or None — the signature of `ovos_spec_tools.language.closest_lang`. +LanguageResolver = Callable[[str, Sequence[str], int], Optional[str]] class MalformedResource(ValueError): @@ -94,86 +71,70 @@ def read_resource_file(path: Path) -> List[str]: class LocaleResources: - """Loads OVOS-INTENT-2 resource files for one skill in one language. + """Loads OVOS-INTENT-2 resource files for one skill, across languages. + + One instance serves **every language** a skill ships: the language is a + parameter of each load call, not of the constructor, because a locale + folder is the multilingual unit of a skill. A resource is resolved through the override precedence of §2.1 — user overrides, then skill resources, then core resources — searching each source's ``/`` directory and all its subdirectories recursively. - When the requested language has no directory, a **smart language - fallback** (OVOS-INTENT-2 §2.2, non-normative) selects the nearest - available language whose tag distance is within - ``max_language_distance``. The fallback needs a :class:`LanguageMatcher`; - by default the optional ``langcodes`` dependency supplies one, and the - fallback is silently disabled if ``langcodes`` is not installed. + When the requested language has no directory, a **smart language fallback** + (OVOS-INTENT-2 §2.2, non-normative) selects the nearest available language. + Resolution is done by :func:`ovos_spec_tools.language.closest_lang` and is + re-run on every load, so one instance serves different languages — and + different fallbacks. The fallback needs the optional ``langcodes`` + dependency; without it, only exact tags resolve. """ - def __init__(self, lang: str, skill_locale: str, + def __init__(self, skill_locale: str, core_locale: Optional[str] = None, user_locale: Optional[str] = None, - language_matcher: Optional[LanguageMatcher] = None, + lang_resolver: Optional[LanguageResolver] = None, max_language_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE): """ Args: - lang: a BCP-47 language tag; compared case-insensitively (§2). skill_locale: path to the skill's ``locale/`` directory. core_locale: path to the assistant's core ``locale/`` directory. user_locale: path to the user-override ``locale/`` directory (its root is assistant-defined, §2.1). - language_matcher: a :class:`LanguageMatcher` for the smart language - fallback. Defaults to the ``langcodes`` module if installed. - max_language_distance: the largest tag distance accepted by the - fallback. ``0`` disables the fallback — only exact tags match. + lang_resolver: ``(target, available, max_distance) -> best | None``; + resolves a requested language against the available ones. + Defaults to :func:`ovos_spec_tools.language.closest_lang`. + max_language_distance: passed to the resolver — the fallback + accepts a language whose tag distance is **below** this + (default 10, §2.2). ``0`` disables the fallback. """ - self.lang = lang # Highest precedence first (§2.1): user, skill, core. self._sources: List[Path] = [ Path(p) for p in (user_locale, skill_locale, core_locale) if p is not None ] - if language_matcher is None: - language_matcher = _default_language_matcher() - self._language_matcher = language_matcher + self._lang_resolver: LanguageResolver = ( + lang_resolver if lang_resolver is not None else closest_lang) self.max_language_distance = max_language_distance - def _lang_dir(self, source: Path) -> Optional[Path]: - """The ``/`` directory under one source. + def _lang_dir(self, source: Path, lang: str) -> Optional[Path]: + """The ``/`` directory for ``lang`` under one source. - An exact tag (case-insensitive, §2) wins. Failing that, the smart - language fallback (§2.2) picks the nearest available language within - ``max_language_distance``, if a :class:`LanguageMatcher` is available. + The language is resolved against the available subdirectories by the + ``lang_resolver`` — an exact tag, or the smart fallback of §2.2. """ if not source.is_dir(): return None - subdirectories = [c for c in source.iterdir() if c.is_dir()] - - target = self.lang.lower() - for child in subdirectories: - if child.name.lower() == target: - return child - - if self._language_matcher is None or self.max_language_distance <= 0: - return None - nearest: Optional[Path] = None - nearest_distance = self.max_language_distance + 1 - for child in subdirectories: - try: - distance = self._language_matcher.tag_distance( - self.lang, child.name) - except Exception: - continue # an unparseable directory name is simply not a match - if distance < nearest_distance: - nearest, nearest_distance = child, distance - if nearest is not None and nearest_distance <= self.max_language_distance: - return nearest - return None - - def _locate(self, base_name: str, extension: str) -> Optional[Path]: - """Find a resource file by ``(base name, extension)`` through the - precedence chain. Returns the first match, or ``None``.""" + names = [c.name for c in source.iterdir() if c.is_dir()] + match = self._lang_resolver(lang, names, self.max_language_distance) + return (source / match) if match is not None else None + + def _locate(self, base_name: str, extension: str, + lang: str) -> Optional[Path]: + """Find a resource file by ``(base name, extension)`` in ``lang`` + through the precedence chain. Returns the first match, or ``None``.""" filename = base_name + extension for source in self._sources: - lang_dir = self._lang_dir(source) + lang_dir = self._lang_dir(source, lang) if lang_dir is None: continue matches = sorted(p for p in lang_dir.rglob(filename) if p.is_file()) @@ -185,13 +146,13 @@ def _locate(self, base_name: str, extension: str) -> Optional[Path]: return matches[0] return None - def vocabularies(self) -> Dict[str, List[str]]: - """Every ``.voc`` reachable for this language, as a name→templates map + def vocabularies(self, lang: str) -> Dict[str, List[str]]: + """Every ``.voc`` reachable for ``lang``, as a name→templates map suitable for resolving ```` references during expansion.""" vocs: Dict[str, List[str]] = {} # Lowest precedence first, so a higher-precedence file overrides. for source in reversed(self._sources): - lang_dir = self._lang_dir(source) + lang_dir = self._lang_dir(source, lang) if lang_dir is None: continue for path in lang_dir.rglob("*.voc"): @@ -199,32 +160,33 @@ def vocabularies(self) -> Dict[str, List[str]]: vocs[path.stem] = read_resource_file(path) return vocs - def entities(self) -> Dict[str, List[str]]: - """Every ``.entity`` reachable for this language, as a name→value-set - map — each value set expanded, with override precedence applied.""" + def entities(self, lang: str) -> Dict[str, List[str]]: + """Every ``.entity`` reachable for ``lang``, as a name→value-set map — + each value set expanded, with override precedence applied.""" names = set() for source in self._sources: - lang_dir = self._lang_dir(source) + lang_dir = self._lang_dir(source, lang) if lang_dir is None: continue for path in lang_dir.rglob("*.entity"): if path.is_file(): names.add(path.stem) - return {name: self.load_entity(name) for name in sorted(names)} + return {name: self.load_entity(name, lang) for name in sorted(names)} - def _load_expanded(self, base_name: str, extension: str) -> List[str]: + def _load_expanded(self, base_name: str, extension: str, + lang: str) -> List[str]: """Load a resource and expand it to its sample set.""" - path = self._locate(base_name, extension) + path = self._locate(base_name, extension, lang) if path is None: raise FileNotFoundError( f"no {extension} resource named {base_name!r} for " - f"language {self.lang!r}") + f"language {lang!r}") templates = read_resource_file(path) if not templates: raise MalformedResource( f"empty resource file {path} — every file must contribute at " f"least one template (§5)") - vocabularies = self.vocabularies() + vocabularies = self.vocabularies(lang) slot_free = extension in SLOT_FREE_ROLES samples: List[str] = [] for template in templates: @@ -237,34 +199,34 @@ def _load_expanded(self, base_name: str, extension: str) -> List[str]: samples.append(sample) return samples - def load_intent(self, base_name: str) -> List[str]: + def load_intent(self, base_name: str, lang: str) -> List[str]: """Load an ``.intent`` as its sample set, named slots intact (§4.1).""" - return self._load_expanded(base_name, ".intent") + return self._load_expanded(base_name, ".intent", lang) - def load_entity(self, base_name: str) -> List[str]: + def load_entity(self, base_name: str, lang: str) -> List[str]: """Load an ``.entity`` value set (§4.3).""" - return self._load_expanded(base_name, ".entity") + return self._load_expanded(base_name, ".entity", lang) - def load_vocabulary(self, base_name: str) -> List[str]: + def load_vocabulary(self, base_name: str, lang: str) -> List[str]: """Load a ``.voc`` phrase set (§4.3).""" - return self._load_expanded(base_name, ".voc") + return self._load_expanded(base_name, ".voc", lang) - def load_blacklist(self, base_name: str) -> List[str]: + def load_blacklist(self, base_name: str, lang: str) -> List[str]: """Load a ``.blacklist`` phrase set (§4.3).""" - return self._load_expanded(base_name, ".blacklist") + return self._load_expanded(base_name, ".blacklist", lang) - def load_dialog(self, base_name: str) -> List[str]: + def load_dialog(self, base_name: str, lang: str) -> List[str]: """Load a ``.dialog`` as its list of phrase strings. Unlike the other roles a ``.dialog`` is **not** expanded at load time — expansion happens per render, on the one phrase chosen (§4.2). The phrase strings are returned verbatim, for a dialog renderer to consume. """ - path = self._locate(base_name, ".dialog") + path = self._locate(base_name, ".dialog", lang) if path is None: raise FileNotFoundError( f"no .dialog resource named {base_name!r} for " - f"language {self.lang!r}") + f"language {lang!r}") phrases = read_resource_file(path) if not phrases: raise MalformedResource( diff --git a/test/test_dialog.py b/test/test_dialog.py index e0833eb..2615c7d 100644 --- a/test/test_dialog.py +++ b/test/test_dialog.py @@ -15,6 +15,18 @@ def choice(self, seq): return seq[self.index % len(seq)] +def _resources(tmp_path, files): + """Build a LocaleResources over a locale with the given {relpath: text}.""" + locale = tmp_path / "locale" + for relpath, text in files.items(): + path = locale / relpath + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return LocaleResources(str(locale)) + + +# --- render() — the stateless function --------------------------------------- + def test_render_fills_slots(): out = render(["It is {temperature} degrees."], slots={"temperature": 21}) @@ -55,91 +67,93 @@ def test_empty_phrase_list_raises(): render([]) -# --- DialogRenderer ---------------------------------------------------------- +# --- DialogRenderer — stateful, multilingual --------------------------------- + +def test_renderer_fills_slots(tmp_path): + res = _resources(tmp_path, {"en-US/t.dialog": "It is {temperature} degrees."}) + renderer = DialogRenderer(res, "t") + assert renderer.render("en-US", {"temperature": 21}) == "It is 21 degrees." + -def test_renderer_fills_slots(): - renderer = DialogRenderer(["It is {temperature} degrees."]) - assert renderer.render({"temperature": 21}) == "It is 21 degrees." +def test_renderer_serves_multiple_languages(tmp_path): + res = _resources(tmp_path, { + "en-US/hi.dialog": "Hello.", + "pt-BR/hi.dialog": "Ola.", + }) + renderer = DialogRenderer(res, "hi") + assert renderer.render("en-US") == "Hello." + assert renderer.render("pt-BR") == "Ola." -def test_renderer_avoids_repeating_the_last_phrase(): +def test_renderer_avoids_repeating_the_last_phrase(tmp_path): """With two phrases, consecutive renders must alternate (§4.2).""" - renderer = DialogRenderer(["alpha", "beta"]) - first = renderer.render() - second = renderer.render() - third = renderer.render() + res = _resources(tmp_path, {"en-US/c.dialog": "alpha\nbeta\n"}) + renderer = DialogRenderer(res, "c") + first = renderer.render("en-US") + second = renderer.render("en-US") + third = renderer.render("en-US") assert first != second assert second != third assert {first, second} == {"alpha", "beta"} -def test_renderer_with_one_phrase_repeats_it(): - renderer = DialogRenderer(["only one"]) - assert renderer.render() == renderer.render() == "only one" +def test_repetition_avoidance_is_per_language(tmp_path): + """The 'last phrase' is tracked separately for each language.""" + res = _resources(tmp_path, { + "en-US/c.dialog": "alpha\nbeta\n", + "pt-BR/c.dialog": "um\ndois\n", + }) + renderer = DialogRenderer(res, "c") + renderer.render("en-US") + # the pt-BR render is unconstrained by the en-US history + assert renderer.render("pt-BR") in {"um", "dois"} -def test_renderer_unfilled_slot_raises(): - renderer = DialogRenderer(["say {name}"]) +def test_renderer_unfilled_slot_raises(tmp_path): + res = _resources(tmp_path, {"en-US/s.dialog": "say {name}"}) + renderer = DialogRenderer(res, "s") with pytest.raises(UnfilledSlot): - renderer.render() + renderer.render("en-US") -def test_renderer_empty_phrase_list_raises(): - with pytest.raises(ValueError): - DialogRenderer([]) - - -def test_renderer_from_resources(tmp_path): - locale = tmp_path / "locale" - lang = locale / "en-US" - lang.mkdir(parents=True) - (lang / "hi.dialog").write_text("Hello {name}.\n", encoding="utf-8") - res = LocaleResources("en-US", str(locale)) - renderer = DialogRenderer.from_resources(res, "hi") - assert renderer.render({"name": "Sam"}) == "Hello Sam." +def test_renderer_missing_dialog_for_language_raises(tmp_path): + res = _resources(tmp_path, {"en-US/hi.dialog": "Hello."}) + renderer = DialogRenderer(res, "hi") + with pytest.raises(FileNotFoundError): + renderer.render("de-DE") # --- default slots and .entity fallback -------------------------------------- -def test_renderer_default_slots_are_reused(): - renderer = DialogRenderer(["Hello {name}."], slots={"name": "Sam"}) - assert renderer.render() == "Hello Sam." - assert renderer.render() == "Hello Sam." - - -def test_per_call_slot_overrides_a_default(): - renderer = DialogRenderer(["Hello {name}."], slots={"name": "Sam"}) - assert renderer.render({"name": "Max"}) == "Hello Max." - - -def test_unfilled_slot_falls_back_to_entity(): - renderer = DialogRenderer(["today is {day}"], - entities={"day": ["monday"]}) - assert renderer.render() == "today is monday" - - -def test_slot_precedence_call_then_default_then_entity(): - renderer = DialogRenderer(["value {x}"], - slots={"x": "from_default"}, - entities={"x": ["from_entity"]}) - # default beats entity - assert renderer.render() == "value from_default" - # per-call beats both - assert renderer.render({"x": "from_call"}) == "value from_call" - - -def test_unfilled_with_no_source_still_raises(): - renderer = DialogRenderer(["say {name}"], entities={"other": ["x"]}) - with pytest.raises(UnfilledSlot): - renderer.render() - - -def test_from_resources_picks_up_entity_fallback(tmp_path): - locale = tmp_path / "locale" - lang = locale / "en-US" - lang.mkdir(parents=True) - (lang / "today.dialog").write_text("today is {day}\n", encoding="utf-8") - (lang / "day.entity").write_text("monday\n", encoding="utf-8") - res = LocaleResources("en-US", str(locale)) - renderer = DialogRenderer.from_resources(res, "today") - assert renderer.render() == "today is monday" +def test_renderer_default_slots_are_reused(tmp_path): + res = _resources(tmp_path, {"en-US/g.dialog": "Hello {name}."}) + renderer = DialogRenderer(res, "g", slots={"name": "Sam"}) + assert renderer.render("en-US") == "Hello Sam." + assert renderer.render("en-US") == "Hello Sam." + + +def test_per_call_slot_overrides_a_default(tmp_path): + res = _resources(tmp_path, {"en-US/g.dialog": "Hello {name}."}) + renderer = DialogRenderer(res, "g", slots={"name": "Sam"}) + assert renderer.render("en-US", {"name": "Max"}) == "Hello Max." + + +def test_unfilled_slot_falls_back_to_entity(tmp_path): + res = _resources(tmp_path, { + "en-US/today.dialog": "today is {day}", + "en-US/day.entity": "monday\n", + }) + renderer = DialogRenderer(res, "today") + assert renderer.render("en-US") == "today is monday" + + +def test_slot_precedence_call_then_default_then_entity(tmp_path): + res = _resources(tmp_path, { + "en-US/v.dialog": "value {x}", + "en-US/x.entity": "from_entity\n", + }) + renderer = DialogRenderer(res, "v", slots={"x": "from_default"}) + # default beats the .entity fallback + assert renderer.render("en-US") == "value from_default" + # a per-call value beats both + assert renderer.render("en-US", {"x": "from_call"}) == "value from_call" diff --git a/test/test_language.py b/test/test_language.py new file mode 100644 index 0000000..78f8907 --- /dev/null +++ b/test/test_language.py @@ -0,0 +1,50 @@ +"""Tests for the language-tag utilities.""" +import pytest + +from ovos_spec_tools import closest_lang, standardize_lang + + +# --- standardize_lang -------------------------------------------------------- + +def test_standardize_normalizes_underscores_and_case(): + assert standardize_lang("en_us").lower() == "en-us" + + +def test_standardize_keeps_tagalog_as_tl(): + """langcodes folds `tl` into `fil`; OVOS keeps it distinct.""" + assert standardize_lang("tl") == "tl" + assert standardize_lang("tgl") == "tl" + + +# --- closest_lang ------------------------------------------------------------ + +def test_closest_lang_exact_match(): + assert closest_lang("en-US", ["pt-BR", "en-US", "de-DE"]) == "en-US" + + +def test_closest_lang_exact_match_after_standardization(): + assert closest_lang("en_US", ["en-US"]) == "en-US" + + +def test_closest_lang_returns_the_original_string(): + """The returned value is the entry from `available`, verbatim.""" + assert closest_lang("EN-us", ["en-US"]) == "en-US" + + +def test_closest_lang_no_match_returns_none(): + assert closest_lang("zz-ZZ", ["en-US", "pt-BR"]) is None + + +def test_closest_lang_disabled_with_zero_distance(): + assert closest_lang("en-AU", ["en-US"], max_distance=0) is None + + +def test_closest_lang_regional_fallback(): + pytest.importorskip("langcodes") + assert closest_lang("en-AU", ["pt-BR", "en-US"]) == "en-US" + + +def test_closest_lang_skips_a_distant_language(tmp_path): + pytest.importorskip("langcodes") + # fr-FR is a different language and beyond the distance cap; en-GB is near. + assert closest_lang("en-AU", ["fr-FR", "en-GB"]) == "en-GB" diff --git a/test/test_resources.py b/test/test_resources.py index 447c795..780fc37 100644 --- a/test/test_resources.py +++ b/test/test_resources.py @@ -13,11 +13,6 @@ def _write(path, text): path.write_text(text, encoding="utf-8") -def _skill(tmp_path, lang="en-US"): - """A skill `locale/` directory rooted at tmp_path.""" - return tmp_path / "locale", (tmp_path / "locale" / lang) - - # --- §3 common reader -------------------------------------------------------- def test_common_reader_skips_blanks_and_comments(tmp_path): @@ -35,71 +30,96 @@ def test_common_reader_accepts_crlf_and_bom(tmp_path): # --- §4.1 .intent ------------------------------------------------------------ def test_load_intent_expands_and_keeps_slots(tmp_path): - locale, lang = _skill(tmp_path) - _write(lang / "play.intent", "(play|put on) {query}\n") - res = LocaleResources("en-US", str(locale)) - assert sorted(res.load_intent("play")) == ["play {query}", "put on {query}"] + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "(play|put on) {query}\n") + res = LocaleResources(str(locale)) + assert sorted(res.load_intent("play", "en-US")) == [ + "play {query}", "put on {query}", + ] def test_load_intent_resolves_voc_references(tmp_path): - locale, lang = _skill(tmp_path) - _write(lang / "greeting.voc", "hello\nhi\n") - _write(lang / "greet.intent", " {name}\n") - res = LocaleResources("en-US", str(locale)) - assert sorted(res.load_intent("greet")) == ["hello {name}", "hi {name}"] + locale = tmp_path / "locale" + _write(locale / "en-US" / "greeting.voc", "hello\nhi\n") + _write(locale / "en-US" / "greet.intent", " {name}\n") + res = LocaleResources(str(locale)) + assert sorted(res.load_intent("greet", "en-US")) == [ + "hello {name}", "hi {name}", + ] + + +# --- one instance, many languages ------------------------------------------- + +def test_one_instance_serves_multiple_languages(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "hi.intent", "hello\n") + _write(locale / "pt-BR" / "hi.intent", "ola\n") + res = LocaleResources(str(locale)) + assert res.load_intent("hi", "en-US") == ["hello"] + assert res.load_intent("hi", "pt-BR") == ["ola"] # --- §4.2 .dialog ------------------------------------------------------------ def test_load_dialog_returns_unexpanded_phrases(tmp_path): - locale, lang = _skill(tmp_path) - _write(lang / "hi.dialog", "Hello {name}!\n(Hi|Hey) {name}.\n") - res = LocaleResources("en-US", str(locale)) - assert res.load_dialog("hi") == ["Hello {name}!", "(Hi|Hey) {name}."] + locale = tmp_path / "locale" + _write(locale / "en-US" / "hi.dialog", "Hello {name}!\n(Hi|Hey) {name}.\n") + res = LocaleResources(str(locale)) + assert res.load_dialog("hi", "en-US") == ["Hello {name}!", "(Hi|Hey) {name}."] # --- §4.3 slot-free roles ---------------------------------------------------- def test_load_entity_and_blacklist(tmp_path): - locale, lang = _skill(tmp_path) - _write(lang / "weekday.entity", "monday\n(tues|wednes)day\n") - _write(lang / "play.blacklist", "trailer\n") - res = LocaleResources("en-US", str(locale)) - assert sorted(res.load_entity("weekday")) == ["monday", "tuesday", "wednesday"] - assert res.load_blacklist("play") == ["trailer"] + locale = tmp_path / "locale" + _write(locale / "en-US" / "weekday.entity", "monday\n(tues|wednes)day\n") + _write(locale / "en-US" / "play.blacklist", "trailer\n") + res = LocaleResources(str(locale)) + assert sorted(res.load_entity("weekday", "en-US")) == [ + "monday", "tuesday", "wednesday", + ] + assert res.load_blacklist("play", "en-US") == ["trailer"] def test_slot_free_role_rejects_a_named_slot(tmp_path): - locale, lang = _skill(tmp_path) - _write(lang / "bad.voc", "a slot {here}\n") - res = LocaleResources("en-US", str(locale)) + locale = tmp_path / "locale" + _write(locale / "en-US" / "bad.voc", "a slot {here}\n") + res = LocaleResources(str(locale)) with pytest.raises(MalformedResource): - res.load_vocabulary("bad") + res.load_vocabulary("bad", "en-US") # --- §2 layout --------------------------------------------------------------- def test_subdirectories_are_searched_recursively(tmp_path): - locale, lang = _skill(tmp_path) - _write(lang / "intents" / "deep.intent", "do the thing\n") - res = LocaleResources("en-US", str(locale)) - assert res.load_intent("deep") == ["do the thing"] + locale = tmp_path / "locale" + _write(locale / "en-US" / "intents" / "deep.intent", "do the thing\n") + res = LocaleResources(str(locale)) + assert res.load_intent("deep", "en-US") == ["do the thing"] def test_language_tag_is_case_insensitive(tmp_path): - locale, lang = _skill(tmp_path, lang="en-us") - _write(lang / "x.intent", "hello world\n") - res = LocaleResources("en-US", str(locale)) # requested with different case - assert res.load_intent("x") == ["hello world"] + locale = tmp_path / "locale" + _write(locale / "en-us" / "x.intent", "hello world\n") + res = LocaleResources(str(locale)) + assert res.load_intent("x", "en-US") == ["hello world"] + + +def test_underscore_tag_matches_hyphen_tag(tmp_path): + """Tags are standardized before comparison, so en_US finds en-US.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.intent", "hello world\n") + res = LocaleResources(str(locale)) + assert res.load_intent("x", "en_US") == ["hello world"] def test_duplicate_resource_in_one_tree_is_malformed(tmp_path): - locale, lang = _skill(tmp_path) - _write(lang / "a" / "dup.intent", "first\n") - _write(lang / "b" / "dup.intent", "second\n") - res = LocaleResources("en-US", str(locale)) + locale = tmp_path / "locale" + _write(locale / "en-US" / "a" / "dup.intent", "first\n") + _write(locale / "en-US" / "b" / "dup.intent", "second\n") + res = LocaleResources(str(locale)) with pytest.raises(MalformedResource): - res.load_intent("dup") + res.load_intent("dup", "en-US") # --- §2.1 resolution precedence --------------------------------------------- @@ -109,79 +129,72 @@ def test_user_override_wins_over_skill(tmp_path): user_locale = tmp_path / "user" / "locale" _write(skill_locale / "en-US" / "x.intent", "skill version\n") _write(user_locale / "en-US" / "x.intent", "user version\n") - res = LocaleResources("en-US", str(skill_locale), user_locale=str(user_locale)) - assert res.load_intent("x") == ["user version"] + res = LocaleResources(str(skill_locale), user_locale=str(user_locale)) + assert res.load_intent("x", "en-US") == ["user version"] # --- §5 empty files ---------------------------------------------------------- def test_empty_file_is_malformed(tmp_path): - locale, lang = _skill(tmp_path) - _write(lang / "empty.intent", "# only a comment\n\n") - res = LocaleResources("en-US", str(locale)) + locale = tmp_path / "locale" + _write(locale / "en-US" / "empty.intent", "# only a comment\n\n") + res = LocaleResources(str(locale)) with pytest.raises(MalformedResource): - res.load_intent("empty") + res.load_intent("empty", "en-US") def test_missing_resource_raises(tmp_path): - locale, _ = _skill(tmp_path) - res = LocaleResources("en-US", str(locale)) + locale = tmp_path / "locale" + locale.mkdir() + res = LocaleResources(str(locale)) with pytest.raises(FileNotFoundError): - res.load_intent("nope") + res.load_intent("nope", "en-US") # --- §2.2 smart language fallback ------------------------------------------- -class _FakeMatcher: - """A LanguageMatcher stub with hand-set distances, for deterministic tests.""" - - def __init__(self, distances): - self.distances = distances # {(desired, supported): distance} - - def tag_distance(self, desired, supported): - return self.distances.get((desired, supported), 999) - - def test_fallback_resolves_a_near_language(tmp_path): + pytest.importorskip("langcodes") locale = tmp_path / "locale" _write(locale / "en-US" / "x.intent", "hello world\n") - matcher = _FakeMatcher({("en-AU", "en-US"): 4}) - res = LocaleResources("en-AU", str(locale), language_matcher=matcher) - assert res.load_intent("x") == ["hello world"] - - -def test_fallback_picks_the_nearest_language(tmp_path): - locale = tmp_path / "locale" - _write(locale / "en-US" / "x.intent", "us\n") - _write(locale / "en-GB" / "x.intent", "gb\n") - matcher = _FakeMatcher({("en-AU", "en-US"): 5, ("en-AU", "en-GB"): 3}) - res = LocaleResources("en-AU", str(locale), language_matcher=matcher) - assert res.load_intent("x") == ["gb"] + res = LocaleResources(str(locale)) + assert res.load_intent("x", "en-AU") == ["hello world"] def test_fallback_rejects_a_too_distant_language(tmp_path): + pytest.importorskip("langcodes") locale = tmp_path / "locale" _write(locale / "fr-FR" / "x.intent", "bonjour\n") - matcher = _FakeMatcher({("en-US", "fr-FR"): 80}) - res = LocaleResources("en-US", str(locale), language_matcher=matcher) + res = LocaleResources(str(locale)) with pytest.raises(FileNotFoundError): - res.load_intent("x") + res.load_intent("x", "en-US") def test_fallback_disabled_with_zero_max_distance(tmp_path): locale = tmp_path / "locale" _write(locale / "en-US" / "x.intent", "hello\n") - matcher = _FakeMatcher({("en-AU", "en-US"): 4}) - res = LocaleResources("en-AU", str(locale), language_matcher=matcher, - max_language_distance=0) + res = LocaleResources(str(locale), max_language_distance=0) with pytest.raises(FileNotFoundError): - res.load_intent("x") + res.load_intent("x", "en-AU") -def test_fallback_with_real_langcodes(tmp_path): - """The default matcher is the langcodes module when it is installed.""" +def test_fallback_resolved_per_query_not_once(tmp_path): + """One instance: an exact tag and a fallback tag resolve independently.""" pytest.importorskip("langcodes") locale = tmp_path / "locale" - _write(locale / "en-US" / "x.intent", "hello\n") - res = LocaleResources("en-GB", str(locale)) # no explicit matcher - assert res.load_intent("x") == ["hello"] + _write(locale / "en-US" / "x.intent", "us text\n") + res = LocaleResources(str(locale)) + assert res.load_intent("x", "en-US") == ["us text"] # exact + assert res.load_intent("x", "en-AU") == ["us text"] # fallback + + +def test_custom_lang_resolver_is_honored(tmp_path): + """A caller may inject its own language resolver.""" + locale = tmp_path / "locale" + _write(locale / "en-GB" / "x.intent", "british\n") + + def resolver(target, available, max_distance): + return "en-GB" if "en-GB" in available else None + + res = LocaleResources(str(locale), lang_resolver=resolver) + assert res.load_intent("x", "zz-ZZ") == ["british"] From e179a836fca32b2cb8a25af98a4317085aef97b2 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 13:06:53 +0100 Subject: [PATCH 010/110] fix: bare language tag favors its norm region, not langcodes' populous one langcodes resolves a bare tag to its most-populous region, so "pt" matches "pt-BR" closer than "pt-PT". The unmarked form of a language should resolve to its reference variety: Portuguese is "from Portugal", and every Lusophone country except Brazil follows the pt-PT norm. closest_lang() now, for a bare language tag with a norm region (_NORM_REGION, seeded with pt -> PT), prefers a candidate in that region over the langcodes distance result. So a request for "pt" against a locale offering pt-PT and pt-BR resolves to pt-PT. An explicit pt-BR request is still respected; a bare tag still falls back by distance when the norm region is absent. Co-Authored-By: Claude Opus 4.7 (1M context) --- ovos_spec_tools/language.py | 29 +++++++++++++++++++++++++---- test/test_language.py | 15 +++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ovos_spec_tools/language.py b/ovos_spec_tools/language.py index 21b3f27..631f0fe 100644 --- a/ovos_spec_tools/language.py +++ b/ovos_spec_tools/language.py @@ -22,6 +22,16 @@ # (OVOS-INTENT-2 §2.2; see the langcodes distance-values documentation). DEFAULT_MAX_LANGUAGE_DISTANCE = 10 +# The norm region for a bare language tag. `langcodes` resolves a bare tag to +# its *most-populous* region — for "pt" that is "pt-BR" — but the unmarked +# form of a language should resolve to its reference variety. Portuguese is +# "from Portugal" by name, and every Lusophone country except Brazil follows +# the pt-PT norm, so bare "pt" favors "pt-PT". Add a language here only when +# its bare tag has a clear reference region distinct from the populous one. +_NORM_REGION = { + "pt": "PT", +} + def standardize_lang(tag: str) -> str: """Normalize a BCP-47 language tag for comparison. @@ -67,10 +77,12 @@ def closest_lang(target: str, available: Sequence[str], """Return the entry of ``available`` closest to ``target``. An exact match (after standardization, so ``en_US`` matches ``en-US``) - always wins. Otherwise the nearest tag whose distance is **below** - ``max_distance`` is returned; if none qualifies, or ``max_distance`` is not - positive, ``None`` is returned. Without ``langcodes`` installed only exact - matches resolve. + always wins. When ``target`` is a bare language tag with a norm region + (see ``_NORM_REGION``), a candidate in that region is preferred next — so + bare ``pt`` favors ``pt-PT`` over ``pt-BR``. Otherwise the nearest tag + whose distance is **below** ``max_distance`` is returned; if none + qualifies, or ``max_distance`` is not positive, ``None`` is returned. + Without ``langcodes`` installed only exact matches resolve. The value returned is the original string from ``available``, so a caller can map it back to a directory, a voice, a model, and so on. @@ -82,6 +94,15 @@ def closest_lang(target: str, available: Sequence[str], if max_distance <= 0: return None + + # A bare language tag favors its norm region over langcodes' populous + # default (which would, for "pt", pick "pt-BR"). + if "-" not in wanted and wanted.lower() in _NORM_REGION: + norm = f"{wanted.lower()}-{_NORM_REGION[wanted.lower()]}" + for candidate in available: + if standardize_lang(candidate).lower() == norm.lower(): + return candidate + nearest: Optional[str] = None nearest_distance = max_distance # accept only a distance strictly below this for candidate in available: diff --git a/test/test_language.py b/test/test_language.py index 78f8907..9e785d7 100644 --- a/test/test_language.py +++ b/test/test_language.py @@ -48,3 +48,18 @@ def test_closest_lang_skips_a_distant_language(tmp_path): pytest.importorskip("langcodes") # fr-FR is a different language and beyond the distance cap; en-GB is near. assert closest_lang("en-AU", ["fr-FR", "en-GB"]) == "en-GB" + + +def test_closest_lang_prefers_the_norm_region_for_a_bare_tag(): + """langcodes makes bare `pt` closest to pt-BR; the norm region pt-PT wins.""" + assert closest_lang("pt", ["pt-BR", "pt-PT"]) == "pt-PT" + assert closest_lang("pt", ["pt-PT", "pt-BR"]) == "pt-PT" # order-independent + + +def test_explicit_region_is_respected_over_the_norm_preference(): + assert closest_lang("pt-BR", ["pt-PT", "pt-BR"]) == "pt-BR" + + +def test_bare_tag_falls_back_when_norm_region_absent(): + pytest.importorskip("langcodes") + assert closest_lang("pt", ["pt-BR"]) == "pt-BR" From 29ab016faac3aaec38220f4688e01a090836622b Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 13:14:32 +0100 Subject: [PATCH 011/110] feat: primary-subtag fallback in closest_lang for when langcodes is absent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without langcodes no tag distance can be computed, so closest_lang previously resolved exact matches only. It now adds a final fallback: a candidate sharing the primary subtag — so a request for en-AU still accepts en, en-GB, en-US, ... It prefers the bare language tag, then the norm region, then the first match. This fallback also covers the rare case where langcodes is installed but computes no in-threshold distance. 3 more tests (105 total). Co-Authored-By: Claude Opus 4.7 (1M context) --- ovos_spec_tools/language.py | 43 ++++++++++++++++++++++++++++++------- test/test_language.py | 21 ++++++++++++++++++ 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/ovos_spec_tools/language.py b/ovos_spec_tools/language.py index 631f0fe..89316cb 100644 --- a/ovos_spec_tools/language.py +++ b/ovos_spec_tools/language.py @@ -76,13 +76,22 @@ def closest_lang(target: str, available: Sequence[str], ) -> Optional[str]: """Return the entry of ``available`` closest to ``target``. - An exact match (after standardization, so ``en_US`` matches ``en-US``) - always wins. When ``target`` is a bare language tag with a norm region - (see ``_NORM_REGION``), a candidate in that region is preferred next — so - bare ``pt`` favors ``pt-PT`` over ``pt-BR``. Otherwise the nearest tag - whose distance is **below** ``max_distance`` is returned; if none - qualifies, or ``max_distance`` is not positive, ``None`` is returned. - Without ``langcodes`` installed only exact matches resolve. + Resolution is tried in order: + + 1. an **exact** match, after standardization (so ``en_US`` matches + ``en-US``); + 2. for a bare language tag with a norm region (see ``_NORM_REGION``), a + candidate in that region — so bare ``pt`` favors ``pt-PT`` over + ``pt-BR``; + 3. the nearest tag whose ``langcodes`` distance is **below** + ``max_distance``; + 4. a candidate sharing the **primary subtag** — preferring the bare tag, + then the norm region. This is the resolution path when ``langcodes`` is + not installed: a request for ``en-AU`` still accepts ``en``, ``en-GB``, + ``en-US``, … + + ``None`` is returned if nothing matches, or if ``max_distance`` is not + positive (which also disables steps 2–4). The value returned is the original string from ``available``, so a caller can map it back to a directory, a voice, a model, and so on. @@ -109,4 +118,22 @@ def closest_lang(target: str, available: Sequence[str], distance = _tag_distance(wanted, standardize_lang(candidate)) if distance is not None and distance < nearest_distance: nearest, nearest_distance = candidate, distance - return nearest + if nearest is not None: + return nearest + + # Final fallback — a shared primary subtag. This is the resolution path + # when `langcodes` is unavailable, so no distance could be computed. + primary = wanted.split("-")[0].lower() + prefix_matches = [c for c in available + if standardize_lang(c).split("-")[0].lower() == primary] + if not prefix_matches: + return None + for candidate in prefix_matches: # prefer the bare language tag + if standardize_lang(candidate).lower() == primary: + return candidate + if primary in _NORM_REGION: # then the norm region + norm = f"{primary}-{_NORM_REGION[primary]}".lower() + for candidate in prefix_matches: + if standardize_lang(candidate).lower() == norm: + return candidate + return prefix_matches[0] diff --git a/test/test_language.py b/test/test_language.py index 9e785d7..cdf1466 100644 --- a/test/test_language.py +++ b/test/test_language.py @@ -63,3 +63,24 @@ def test_explicit_region_is_respected_over_the_norm_preference(): def test_bare_tag_falls_back_when_norm_region_absent(): pytest.importorskip("langcodes") assert closest_lang("pt", ["pt-BR"]) == "pt-BR" + + +# --- primary-subtag fallback (the `langcodes`-absent path) ------------------- + +def test_primary_subtag_fallback_without_langcodes(monkeypatch): + """With no distance available, a shared primary subtag still resolves.""" + import ovos_spec_tools.language as language + monkeypatch.setattr(language, "_tag_distance", lambda desired, supported: None) + assert language.closest_lang("en-AU", ["pt-BR", "en-GB", "fr-FR"]) == "en-GB" + + +def test_primary_subtag_fallback_prefers_the_bare_tag(monkeypatch): + import ovos_spec_tools.language as language + monkeypatch.setattr(language, "_tag_distance", lambda desired, supported: None) + assert language.closest_lang("en-AU", ["en-US", "en", "en-GB"]) == "en" + + +def test_primary_subtag_fallback_with_no_shared_subtag(monkeypatch): + import ovos_spec_tools.language as language + monkeypatch.setattr(language, "_tag_distance", lambda desired, supported: None) + assert language.closest_lang("en-AU", ["pt-BR", "fr-FR"]) is None From e5781cf42285c29fa17e6ead90503d25e3993002 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 13:19:17 +0100 Subject: [PATCH 012/110] refactor: one lang_distance function, closest_lang just minimizes it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous closest_lang was a cascade of special cases — exact, then norm region, then langcodes distance, then a primary-subtag fallback. That logic now lives in a single distance function. - lang_distance(a, b) — the one place policy lives: standardizes tags, measures a bare tag from its norm region (so pt -> pt-PT is 0, not a langcodes population guess), uses langcodes when present, and falls back to a coarse same-language measure (shared primary subtag is near, the generic form nearer than a sibling region) when it is not. - closest_lang(target, available, max_distance) — now branch-free: the candidate with the smallest lang_distance, accepted if below the cap. - lang_distance is exported as a public primitive. 108 tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 25 +++--- ovos_spec_tools/__init__.py | 14 +++- ovos_spec_tools/language.py | 157 ++++++++++++++++++++---------------- test/test_language.py | 46 +++++++---- 4 files changed, 142 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index 7a9ff33..dc9040f 100644 --- a/README.md +++ b/README.md @@ -76,21 +76,26 @@ fallback, or a custom `lang_resolver` to change it. ## Language matching -`standardize_lang` and `closest_lang` are the language-tag primitives — the -same logic OVOS reimplements across locale loading, TTS voices, and STT -models, gathered in one place. +`standardize_lang`, `lang_distance`, and `closest_lang` are the language-tag +primitives — the logic OVOS reimplements across locale loading, TTS voices, +and STT models, gathered in one place. ```python -from ovos_spec_tools import standardize_lang, closest_lang +from ovos_spec_tools import lang_distance, closest_lang -standardize_lang("en_us") # 'en-US' -closest_lang("en-AU", ["pt-BR", "en-US"]) # 'en-US' (regional fallback) -closest_lang("zz-ZZ", ["pt-BR", "en-US"]) # None +lang_distance("pt", "pt-PT") # 0 — a bare tag's norm region +lang_distance("pt", "pt-BR") # > 0 — merely a regional variant +closest_lang("en-AU", ["pt-BR", "en-US"]) # 'en-US' +closest_lang("zz-ZZ", ["pt-BR", "en-US"]) # None ``` -`closest_lang` returns the nearest entry whose tag distance is below -`max_distance` (default 10), or `None`. With `langcodes` absent it resolves -exact matches only. +`closest_lang` is just "the candidate with the smallest `lang_distance`", +accepted when it is below `max_distance` (default 10). `lang_distance` carries +all the policy: it standardizes tags, measures a bare tag from its **norm +region** (correcting langcodes' population-based default — so `pt` favors +`pt-PT` over `pt-BR`), and uses `langcodes` when available, falling back to a +coarse same-language measure that still resolves `en-AU` against `en`, +`en-GB`, … when `langcodes` is absent. ## The dialog renderer diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index a0dab1a..d7a9751 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -9,15 +9,20 @@ locale resource-file loader; - :func:`~ovos_spec_tools.dialog.render` / :class:`~ovos_spec_tools.dialog.DialogRenderer` — the OVOS-INTENT-2 §4.2 dialog renderer; -- :func:`~ovos_spec_tools.language.standardize_lang` / - :func:`~ovos_spec_tools.language.closest_lang` — language-tag normalization - and closest-match resolution; +- :func:`~ovos_spec_tools.language.standardize_lang`, + :func:`~ovos_spec_tools.language.lang_distance`, and + :func:`~ovos_spec_tools.language.closest_lang` — language-tag normalization, + distance, and closest-match resolution; - :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also exposed as the ``ovos-spec-lint`` command. """ from ovos_spec_tools.dialog import DialogRenderer, UnfilledSlot, render from ovos_spec_tools.expansion import MalformedTemplate, expand -from ovos_spec_tools.language import closest_lang, standardize_lang +from ovos_spec_tools.language import ( + closest_lang, + lang_distance, + standardize_lang, +) from ovos_spec_tools.lint import Finding, lint_locale from ovos_spec_tools.resources import ( LocaleResources, @@ -36,6 +41,7 @@ "DialogRenderer", "UnfilledSlot", "standardize_lang", + "lang_distance", "closest_lang", "lint_locale", "Finding", diff --git a/ovos_spec_tools/language.py b/ovos_spec_tools/language.py index 89316cb..04ac2f9 100644 --- a/ovos_spec_tools/language.py +++ b/ovos_spec_tools/language.py @@ -1,33 +1,41 @@ -"""Language-tag utilities — standardization and closest-match resolution. +"""Language-tag utilities — standardization, distance, and closest match. OVOS resolves "the closest available language for a request" in many places — locale resources, TTS voices, STT models — and the logic has been reimplemented repeatedly (``ovos_utils.lang.get_language_dir``, ``phoonnx.match_lang``, …) with subtle drift between the copies. This module is -intended to be the **single implementation**: it powers the smart language -fallback of :class:`~ovos_spec_tools.resources.LocaleResources` -(OVOS-INTENT-2 §2.2) and is importable on its own to replace those copies. +intended to be the **single implementation**. -``langcodes`` is an optional dependency. Without it, :func:`standardize_lang` -does a best-effort normalization and :func:`closest_lang` resolves exact -matches only. +It is built on one distance function, :func:`lang_distance`; :func:`closest_lang` +is simply "the candidate with the smallest distance". All the policy — tag +standardization, the norm-region preference, the behaviour when ``langcodes`` +is absent — lives inside :func:`lang_distance`, not in branchy callers. + +``langcodes`` is an optional dependency. Without it, :func:`lang_distance` +falls back to a coarse same-language / different-language measure. """ from __future__ import annotations from typing import Optional, Sequence -__all__ = ["standardize_lang", "closest_lang", "DEFAULT_MAX_LANGUAGE_DISTANCE"] +__all__ = [ + "standardize_lang", + "lang_distance", + "closest_lang", + "DEFAULT_MAX_LANGUAGE_DISTANCE", +] -# A `langcodes` tag distance below 10 is a usable regional match -# (OVOS-INTENT-2 §2.2; see the langcodes distance-values documentation). +# A language distance below 10 is a usable regional match (OVOS-INTENT-2 §2.2; +# see the langcodes distance-values documentation). DEFAULT_MAX_LANGUAGE_DISTANCE = 10 # The norm region for a bare language tag. `langcodes` resolves a bare tag to # its *most-populous* region — for "pt" that is "pt-BR" — but the unmarked # form of a language should resolve to its reference variety. Portuguese is # "from Portugal" by name, and every Lusophone country except Brazil follows -# the pt-PT norm, so bare "pt" favors "pt-PT". Add a language here only when -# its bare tag has a clear reference region distinct from the populous one. +# the pt-PT norm, so a bare "pt" is measured from "pt-PT". Add a language here +# only when its bare tag has a clear reference region distinct from the +# populous one. _NORM_REGION = { "pt": "PT", } @@ -55,85 +63,94 @@ def standardize_lang(tag: str) -> str: return normalized.lower() -def _tag_distance(desired: str, supported: str) -> Optional[int]: +def _with_norm_region(tag: str) -> str: + """Give a bare language tag its norm region (``pt`` -> ``pt-PT``). + + A regioned tag, or a language with no norm region, is returned unchanged. + """ + if "-" in tag: + return tag + region = _NORM_REGION.get(tag.lower()) + return f"{tag.lower()}-{region}" if region else tag + + +def _langcodes_distance(desired: str, supported: str) -> Optional[int]: """``langcodes.tag_distance``, retried on the primary subtag if the full - tag is unparseable. Returns ``None`` if no distance can be computed — - including when ``langcodes`` is not installed.""" + tag is unparseable. ``None`` if no distance can be computed — including + when ``langcodes`` is not installed.""" try: from langcodes import tag_distance except ImportError: return None for candidate in (supported, supported.split("-")[0]): try: - return tag_distance(desired, candidate) + return int(tag_distance(desired, candidate)) except Exception: continue return None +def _coarse_distance(desired: str, supported: str) -> int: + """A ``langcodes``-free distance. + + The two tags are already standardized and norm-expanded. A shared primary + subtag is near, a differing one is far; the generic (region-less) form of + a language counts as nearer than a sibling region of it. + """ + if desired.split("-")[0].lower() != supported.split("-")[0].lower(): + return 100 # a different language — beyond any usable threshold + if "-" not in desired or "-" not in supported: + return 3 # one side is the generic language tag + return 5 # two different regions of the same language + + +def lang_distance(desired: str, supported: str) -> int: + """The distance between two BCP-47 language tags. + + ``0`` is identical; a larger number is further apart; a value of 10 or + more is not a usable match. Both tags are standardized, and a bare tag is + measured **from its norm region** — so ``lang_distance("pt", "pt-PT")`` is + ``0`` while ``lang_distance("pt", "pt-BR")`` is a regional difference, + correcting ``langcodes``' population-based default. + + Uses ``langcodes.tag_distance`` when available, and a coarse same-language + measure otherwise. + """ + a = standardize_lang(desired) + b = standardize_lang(supported) + if a.lower() == b.lower(): + return 0 + a, b = _with_norm_region(a), _with_norm_region(b) + if a.lower() == b.lower(): + return 0 + distance = _langcodes_distance(a, b) + return distance if distance is not None else _coarse_distance(a, b) + + def closest_lang(target: str, available: Sequence[str], max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE ) -> Optional[str]: """Return the entry of ``available`` closest to ``target``. - Resolution is tried in order: - - 1. an **exact** match, after standardization (so ``en_US`` matches - ``en-US``); - 2. for a bare language tag with a norm region (see ``_NORM_REGION``), a - candidate in that region — so bare ``pt`` favors ``pt-PT`` over - ``pt-BR``; - 3. the nearest tag whose ``langcodes`` distance is **below** - ``max_distance``; - 4. a candidate sharing the **primary subtag** — preferring the bare tag, - then the norm region. This is the resolution path when ``langcodes`` is - not installed: a request for ``en-AU`` still accepts ``en``, ``en-GB``, - ``en-US``, … - - ``None`` is returned if nothing matches, or if ``max_distance`` is not - positive (which also disables steps 2–4). + The candidate with the smallest :func:`lang_distance` wins. An exact match + always resolves; any other match resolves only if its distance is **below** + ``max_distance`` (so ``max_distance=0`` accepts exact matches only). + ``None`` is returned when nothing qualifies. The value returned is the original string from ``available``, so a caller can map it back to a directory, a voice, a model, and so on. """ - wanted = standardize_lang(target) + best: Optional[str] = None + best_distance: Optional[int] = None for candidate in available: - if standardize_lang(candidate).lower() == wanted.lower(): - return candidate + distance = lang_distance(target, candidate) + if best_distance is None or distance < best_distance: + best, best_distance = candidate, distance - if max_distance <= 0: + if best is None: return None - - # A bare language tag favors its norm region over langcodes' populous - # default (which would, for "pt", pick "pt-BR"). - if "-" not in wanted and wanted.lower() in _NORM_REGION: - norm = f"{wanted.lower()}-{_NORM_REGION[wanted.lower()]}" - for candidate in available: - if standardize_lang(candidate).lower() == norm.lower(): - return candidate - - nearest: Optional[str] = None - nearest_distance = max_distance # accept only a distance strictly below this - for candidate in available: - distance = _tag_distance(wanted, standardize_lang(candidate)) - if distance is not None and distance < nearest_distance: - nearest, nearest_distance = candidate, distance - if nearest is not None: - return nearest - - # Final fallback — a shared primary subtag. This is the resolution path - # when `langcodes` is unavailable, so no distance could be computed. - primary = wanted.split("-")[0].lower() - prefix_matches = [c for c in available - if standardize_lang(c).split("-")[0].lower() == primary] - if not prefix_matches: - return None - for candidate in prefix_matches: # prefer the bare language tag - if standardize_lang(candidate).lower() == primary: - return candidate - if primary in _NORM_REGION: # then the norm region - norm = f"{primary}-{_NORM_REGION[primary]}".lower() - for candidate in prefix_matches: - if standardize_lang(candidate).lower() == norm: - return candidate - return prefix_matches[0] + if best_distance == 0: + return best + if max_distance > 0 and best_distance < max_distance: + return best + return None diff --git a/test/test_language.py b/test/test_language.py index cdf1466..4aa42c1 100644 --- a/test/test_language.py +++ b/test/test_language.py @@ -1,7 +1,8 @@ """Tests for the language-tag utilities.""" import pytest -from ovos_spec_tools import closest_lang, standardize_lang +import ovos_spec_tools.language as language +from ovos_spec_tools import closest_lang, lang_distance, standardize_lang # --- standardize_lang -------------------------------------------------------- @@ -16,6 +17,22 @@ def test_standardize_keeps_tagalog_as_tl(): assert standardize_lang("tgl") == "tl" +# --- lang_distance ----------------------------------------------------------- + +def test_distance_is_zero_for_identical_tags(): + assert lang_distance("en-US", "en_us") == 0 + + +def test_distance_is_large_for_different_languages(): + assert lang_distance("en-US", "fr-FR") >= 10 + + +def test_distance_measures_a_bare_tag_from_its_norm_region(): + """`pt` is identical to `pt-PT` and merely regional to `pt-BR`.""" + assert lang_distance("pt", "pt-PT") == 0 + assert lang_distance("pt", "pt-BR") > 0 + + # --- closest_lang ------------------------------------------------------------ def test_closest_lang_exact_match(): @@ -44,7 +61,7 @@ def test_closest_lang_regional_fallback(): assert closest_lang("en-AU", ["pt-BR", "en-US"]) == "en-US" -def test_closest_lang_skips_a_distant_language(tmp_path): +def test_closest_lang_skips_a_distant_language(): pytest.importorskip("langcodes") # fr-FR is a different language and beyond the distance cap; en-GB is near. assert closest_lang("en-AU", ["fr-FR", "en-GB"]) == "en-GB" @@ -65,22 +82,19 @@ def test_bare_tag_falls_back_when_norm_region_absent(): assert closest_lang("pt", ["pt-BR"]) == "pt-BR" -# --- primary-subtag fallback (the `langcodes`-absent path) ------------------- +# --- the langcodes-absent path ---------------------------------------------- -def test_primary_subtag_fallback_without_langcodes(monkeypatch): - """With no distance available, a shared primary subtag still resolves.""" - import ovos_spec_tools.language as language - monkeypatch.setattr(language, "_tag_distance", lambda desired, supported: None) - assert language.closest_lang("en-AU", ["pt-BR", "en-GB", "fr-FR"]) == "en-GB" +def test_distance_without_langcodes_shares_primary_subtag(monkeypatch): + """With no langcodes distance, a shared primary subtag still resolves.""" + monkeypatch.setattr(language, "_langcodes_distance", lambda a, b: None) + assert closest_lang("en-AU", ["pt-BR", "en-GB", "fr-FR"]) == "en-GB" -def test_primary_subtag_fallback_prefers_the_bare_tag(monkeypatch): - import ovos_spec_tools.language as language - monkeypatch.setattr(language, "_tag_distance", lambda desired, supported: None) - assert language.closest_lang("en-AU", ["en-US", "en", "en-GB"]) == "en" +def test_distance_without_langcodes_prefers_the_generic_tag(monkeypatch): + monkeypatch.setattr(language, "_langcodes_distance", lambda a, b: None) + assert closest_lang("en-AU", ["en-US", "en", "en-GB"]) == "en" -def test_primary_subtag_fallback_with_no_shared_subtag(monkeypatch): - import ovos_spec_tools.language as language - monkeypatch.setattr(language, "_tag_distance", lambda desired, supported: None) - assert language.closest_lang("en-AU", ["pt-BR", "fr-FR"]) is None +def test_distance_without_langcodes_no_shared_subtag(monkeypatch): + monkeypatch.setattr(language, "_langcodes_distance", lambda a, b: None) + assert closest_lang("en-AU", ["pt-BR", "fr-FR"]) is None From e1f16ead70555d61db3e7cf2dd8f51050e633c29 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 13:37:03 +0100 Subject: [PATCH 013/110] docs: add a zero-to-hero docs/ guide; point README at it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New docs/ — an ordered guide that builds from a first install to the full API: - getting-started — install and a taste of every tool - templates — the OVOS-INTENT-1 grammar - locale-resources — the locale/ folder and the five file roles - dialog — render() and DialogRenderer - language-matching — standardize_lang, lang_distance, closest_lang - linting — ovos-spec-lint, on the CLI and in CI - api-reference — every public name The README drops its long per-tool sections in favour of a quick taste and links into docs/. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 166 ++++++++---------------------------- docs/README.md | 43 ++++++++++ docs/api-reference.md | 116 +++++++++++++++++++++++++ docs/dialog.md | 103 ++++++++++++++++++++++ docs/getting-started.md | 99 +++++++++++++++++++++ docs/language-matching.md | 95 +++++++++++++++++++++ docs/linting.md | 81 ++++++++++++++++++ docs/locale-resources.md | 131 ++++++++++++++++++++++++++++ docs/templates.md | 145 +++++++++++++++++++++++++++++++ ovos_spec_tools/language.py | 2 +- 10 files changed, 850 insertions(+), 131 deletions(-) create mode 100644 docs/README.md create mode 100644 docs/api-reference.md create mode 100644 docs/dialog.md create mode 100644 docs/getting-started.md create mode 100644 docs/language-matching.md create mode 100644 docs/linting.md create mode 100644 docs/locale-resources.md create mode 100644 docs/templates.md diff --git a/README.md b/README.md index dc9040f..bf4b548 100644 --- a/README.md +++ b/README.md @@ -11,151 +11,57 @@ depend on. ## Status -| Tool | Spec | State | +| Tool | Spec | Code | |------|------|-------| -| Sentence template expander | OVOS-INTENT-1 v2 | available | -| Locale resource loader | OVOS-INTENT-2 | available | -| Dialog renderer | OVOS-INTENT-2 §4.2 | available | -| Language-tag matching | OVOS-INTENT-2 §2.2 | available | -| `ovos-spec-lint` locale linter | OVOS-INTENT-1 / -2 | available | +| Sentence template expander | OVOS-INTENT-1 v2 | `expansion.py` | +| Locale resource loader | OVOS-INTENT-2 | `resources.py` | +| Dialog renderer | OVOS-INTENT-2 §4.2 | `dialog.py` | +| Language-tag matching | OVOS-INTENT-2 §2.2 | `language.py` | +| `ovos-spec-lint` locale linter | OVOS-INTENT-1 / -2 | `lint.py` | -## The expander - -`expand()` turns a sentence template into its **sample set** — the finite set -of sentences it denotes (OVOS-INTENT-1 §4). It resolves `(a|b)` alternatives, -`[x]` optionals, and `` inline vocabulary references; named `{name}` -slots are opaque and carried through unchanged. - -```python -from ovos_spec_tools import expand - -expand("(turn|switch) [the] (light|fan)") -# ['turn the light', 'turn the fan', 'turn light', 'turn fan', -# 'switch the light', 'switch the fan', 'switch light', 'switch fan'] - -expand(" {name}", {"greeting": ["hello", "hi"]}) -# ['hello {name}', 'hi {name}'] -``` - -A template that violates OVOS-INTENT-1 §3.6 — unbalanced metacharacters, a -single-branch group, an empty sample, a slot-only template, adjacent slots, a -repeated slot name, or an undefined or cyclic vocabulary reference — raises -`MalformedTemplate`. - -Input is assumed already ASR-normalized (OVOS-INTENT-1 §2): lowercase, -single-spaced, alphanumeric. This package expands; it does not normalize. - -## The resource loader - -`LocaleResources` discovers and loads a skill's locale resource files -(OVOS-INTENT-2) — the five roles `.intent`, `.dialog`, `.entity`, `.voc`, -`.blacklist` — through the user → skill → core override precedence, searching -each `locale//` tree recursively. - -The **language is given per call**, not at construction: a locale folder is -the multilingual unit of a skill, so one instance serves every language. - -```python -from ovos_spec_tools import LocaleResources - -res = LocaleResources(skill_locale="my-skill/locale") -res.load_intent("play", "en-US") # sample set, named slots intact -res.load_intent("play", "pt-BR") # same instance, another language -res.load_dialog("weather", "en-US") # phrase strings, not expanded (§4.2) -``` - -The user-data path of the override precedence is assistant-defined and passed -in (`user_locale=`); this package imports no configuration. - -**Smart language fallback.** When the requested language has no directory, -`LocaleResources` resolves to the nearest available language instead -(OVOS-INTENT-2 §2.2) — so a request for `en-AU` finds `en-US`. Resolution uses -`closest_lang` (below) and is re-run per call. Without `langcodes` installed, -resolution is exact-match only; pass `max_language_distance=0` to disable the -fallback, or a custom `lang_resolver` to change it. - -## Language matching - -`standardize_lang`, `lang_distance`, and `closest_lang` are the language-tag -primitives — the logic OVOS reimplements across locale loading, TTS voices, -and STT models, gathered in one place. - -```python -from ovos_spec_tools import lang_distance, closest_lang +## Install -lang_distance("pt", "pt-PT") # 0 — a bare tag's norm region -lang_distance("pt", "pt-BR") # > 0 — merely a regional variant -closest_lang("en-AU", ["pt-BR", "en-US"]) # 'en-US' -closest_lang("zz-ZZ", ["pt-BR", "en-US"]) # None +```bash +pip install ovos-spec-tools # core — no dependencies +pip install ovos-spec-tools[langcodes] # adds the smart language fallback ``` -`closest_lang` is just "the candidate with the smallest `lang_distance`", -accepted when it is below `max_distance` (default 10). `lang_distance` carries -all the policy: it standardizes tags, measures a bare tag from its **norm -region** (correcting langcodes' population-based default — so `pt` favors -`pt-PT` over `pt-BR`), and uses `langcodes` when available, falling back to a -coarse same-language measure that still resolves `en-AU` against `en`, -`en-GB`, … when `langcodes` is absent. - -## The dialog renderer - -Rendering a dialog selects one phrase from a loaded `.dialog`, expands its -variety to a single variant, and fills every `{name}` slot with a value -(OVOS-INTENT-2 §4.2). A slot with no value raises `UnfilledSlot`. +Requires Python 3.8+. -`render()` is a stateless one-shot function over explicit phrases: +## Quick taste ```python -from ovos_spec_tools import render - -render(res.load_dialog("weather", "en-US"), slots={"temperature": 21}) -# 'It is 21 degrees.' +from ovos_spec_tools import expand, LocaleResources, render, closest_lang + +expand("(turn|switch) [the] light") # all 4 sentences it denotes +res = LocaleResources("my-skill/locale") +res.load_intent("play", "en-US") # a skill's intent samples +render(res.load_dialog("weather", "en-US"), # a spoken response + slots={"temperature": 21}) +closest_lang("en-AU", ["pt-BR", "en-US"]) # 'en-US' ``` -`DialogRenderer` is a stateful, **multilingual** alternative, backed by a -`LocaleResources`. The language is given per `render()` call, and the renderer -**avoids repeating the phrase it chose last time** — per language — so a -repeatedly-spoken response does not sound mechanical: - -```python -from ovos_spec_tools import DialogRenderer - -renderer = DialogRenderer(res, "weather") -renderer.render("en-US", {"temperature": 21}) # a phrase -renderer.render("en-US", {"temperature": 22}) # a different phrase -renderer.render("pt-BR", {"temperature": 23}) # the same dialog, another language -``` - -It also holds **default slot values** set once and reused, and falls back to a -slot's **`.entity` value set** for anything still unfilled. A slot is resolved -in order: the per-call value, then a default, then a random `.entity` value, -then `UnfilledSlot`. - -## The locale linter - -`ovos-spec-lint` validates every resource file under a locale directory — the -syntax of every template (OVOS-INTENT-1) and the naming and layout of every -file (OVOS-INTENT-2) — and reports every problem rather than stopping at the -first. - ```bash -ovos-spec-lint path/to/locale -# path/to/locale/en-US/bad.intent: error: unbalanced metacharacters ... -# 1 error(s), 0 warning(s) +ovos-spec-lint my-skill/locale # validate a locale folder ``` -It checks template syntax, malformed forms, empty files, slot-free roles -carrying a slot, base-name and language-tag naming, duplicate resources, and -unresolved `` references. The argument may be a `locale/` directory or a -single `/` directory. Exit code is non-zero on errors (or, with -`--strict`, on warnings) — suitable for CI. +## Documentation -## Install +A zero-to-hero guide lives in [`docs/`](docs/README.md): -```bash -pip install ovos-spec-tools # core — no dependencies -pip install ovos-spec-tools[langcodes] # adds the smart language fallback -``` +1. [Getting started](docs/getting-started.md) — install, and a first taste of + every tool. +2. [Sentence templates](docs/templates.md) — the grammar: alternatives, + optionals, slots, vocabulary references, malformed forms. +3. [Locale resources](docs/locale-resources.md) — the `locale/` folder, the + five file roles, loading across languages, override precedence. +4. [Dialog](docs/dialog.md) — choosing and filling a spoken response. +5. [Language matching](docs/language-matching.md) — tag standardization, + distance, and closest-match resolution. +6. [Linting](docs/linting.md) — validating a locale folder, on the CLI or in CI. +7. [API reference](docs/api-reference.md) — every public name, in brief. + +Runnable example scripts are in [`examples/`](examples/README.md). ## License diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..af2b561 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,43 @@ +# ovos-spec-tools — documentation + +`ovos-spec-tools` is the reference implementation of the +[OVOS formal specifications](https://github.com/OpenVoiceOS/formal-specifications): +the small, dependency-light primitives those specs describe, in one place, so +OVOS components and third-party tools stop reimplementing them and drifting +apart. + +It gives you four things: + +- an **expander** — turns a sentence template into the set of sentences it + stands for (OVOS-INTENT-1); +- a **resource loader** — reads a skill's `locale/` folder (OVOS-INTENT-2); +- a **dialog renderer** — picks and fills a spoken response (OVOS-INTENT-2 §4.2); +- **language matching** — normalizes tags and finds the closest one; + +plus **`ovos-spec-lint`**, a command-line linter for locale folders. + +## The guide + +Read it in order — each chapter builds on the one before, and **templates** +are the foundation everything else rests on. + +1. [Getting started](getting-started.md) — install it, and a first taste of + every tool. +2. [Sentence templates](templates.md) — the grammar: alternatives, optionals, + slots, vocabulary references, and what counts as malformed. +3. [Locale resources](locale-resources.md) — the `locale/` folder, the five + file roles, and loading them across languages. +4. [Dialog](dialog.md) — choosing and filling a spoken response, with the + stateless function and the stateful renderer. +5. [Language matching](language-matching.md) — tag standardization, distance, + and closest-match resolution. +6. [Linting](linting.md) — validating a locale folder, from the command line + or in CI. +7. [API reference](api-reference.md) — every public name, in brief. + +## A note on scope + +This package **expands, loads, renders, matches language, and lints**. It does not +*recognize* intents — matching an utterance to an intent is the job of an +intent engine, and is deliberately out of scope (see OVOS-INTENT-1 §4). What +you get here is the data those engines consume and the tooling around it. diff --git a/docs/api-reference.md b/docs/api-reference.md new file mode 100644 index 0000000..e6b6a09 --- /dev/null +++ b/docs/api-reference.md @@ -0,0 +1,116 @@ +# 7. API reference + +Every public name, in brief. All are importable from the top-level package: + +```python +from ovos_spec_tools import expand, LocaleResources, render, closest_lang # etc. +``` + +For the why and the worked examples, see the chapter each section links to. + +## Expansion — [chapter 2](templates.md) + +### `expand(template, vocabularies=None) -> list[str]` + +Expand a sentence template to its sample set. `vocabularies` is a +`name -> list[str]` dict, needed only if the template uses `` references. +Raises `MalformedTemplate`. + +### `MalformedTemplate` + +`ValueError` subclass — a template violates OVOS-INTENT-1 §3.6. + +## Locale resources — [chapter 3](locale-resources.md) + +### `LocaleResources(skill_locale, core_locale=None, user_locale=None, lang_resolver=None, max_language_distance=10)` + +Loads a skill's locale resource files. The three `*_locale` arguments are paths +to `locale/` directories, in ascending override precedence. `lang_resolver` is +a `(target, available, max_distance) -> str | None` callable (default +`closest_lang`); `max_language_distance` caps the smart fallback (`0` disables +it). + +Methods — each takes a resource base name and a BCP-47 `lang`: + +| Method | Returns | +|--------|---------| +| `load_intent(name, lang)` | expanded sample set, slots intact | +| `load_entity(name, lang)` | expanded value set | +| `load_vocabulary(name, lang)` | expanded phrase set | +| `load_blacklist(name, lang)` | expanded phrase set | +| `load_dialog(name, lang)` | raw phrase strings (not expanded) | +| `vocabularies(lang)` | `name -> templates` for every `.voc` | +| `entities(lang)` | `name -> values` for every `.entity` | + +A missing resource raises `FileNotFoundError`; a malformed one raises +`MalformedResource`. + +### `read_resource_file(path) -> list[str]` + +Apply the OVOS-INTENT-2 §3 common reader to one file: UTF-8, BOM discarded, +`LF`/`CRLF` accepted, lines stripped, blank and `#`-comment lines dropped. + +### `MalformedResource` + +`ValueError` subclass — a resource file or layout violates OVOS-INTENT-2 +(empty file, duplicate `(role, base name)`, a slot in a slot-free role). + +## Dialog — [chapter 4](dialog.md) + +### `render(phrases, slots=None, vocabularies=None, rng=None) -> str` + +Render one phrase from an explicit list. `slots` fills `{name}` slots; +`vocabularies` resolves `` references; `rng` is any object with a +`choice` method (for reproducible output). Raises `UnfilledSlot`, or +`ValueError` if `phrases` is empty. + +### `DialogRenderer(resources, name, rng=None, slots=None)` + +A stateful, multilingual renderer for the dialog `name`, backed by a +`LocaleResources`. `slots` are default slot values reused on every call. + +- **`render(lang, slots=None) -> str`** — render one phrase in `lang`. Avoids + repeating the previous phrase (per language). Slot precedence: per-call, + then default, then a random `.entity` value, then `UnfilledSlot`. + +### `UnfilledSlot` + +`ValueError` subclass — a chosen phrase has a slot with no value. + +## Language matching — [chapter 5](language-matching.md) + +### `standardize_lang(tag) -> str` + +Normalize a BCP-47 tag (underscores, case, canonical forms). + +### `lang_distance(desired, supported) -> int` + +Distance between two tags: `0` is identical, `>= 10` is not a usable match. A +bare tag is measured from its norm region. + +### `closest_lang(target, available, max_distance=10) -> str | None` + +The entry of `available` with the smallest `lang_distance`, if it is below +`max_distance` (or exact). Returns the original string, or `None`. + +## Linting — [chapter 6](linting.md) + +### `lint_locale(path) -> list[Finding]` + +Validate every resource file under a locale (or single-language) directory. + +### `Finding` + +A dataclass with `severity` (`"error"` / `"warning"`), `path`, and `message`. +`str(finding)` formats it as one line. + +### `ovos-spec-lint` (command) + +CLI wrapper over `lint_locale`. `ovos-spec-lint [--strict]`; exit code +is non-zero on errors (with `--strict`, on warnings too). + +## Package + +### `__version__` + +The installed package version string. diff --git a/docs/dialog.md b/docs/dialog.md new file mode 100644 index 0000000..6746f6b --- /dev/null +++ b/docs/dialog.md @@ -0,0 +1,103 @@ +# 4. Dialog + +A `.dialog` file holds the phrases an assistant may speak for one response. +**Rendering** a dialog means: pick one phrase, expand its `(a|b)` / `[x]` +variety down to a single variant, and fill its `{name}` slots with values. This +chapter covers the two ways to do it. It implements OVOS-INTENT-2 §4.2. + +A `.dialog` phrase is spoken *output*, not ASR input, so — unlike the +input-direction roles — it may contain mixed case and punctuation: + +``` +# weather.dialog +It is {temperature} degrees. +Right now it's {temperature} degrees out. +(Currently|At the moment) {temperature} degrees. +``` + +## `render()` — the stateless function + +`render()` takes an explicit list of phrases and returns one rendered sentence: + +```python +from ovos_spec_tools import render + +phrases = res.load_dialog("weather", "en-US") +render(phrases, slots={"temperature": 21}) +# 'It is 21 degrees.' +``` + +Slots are filled from the `slots` dict. The phrase and the `(a|b)` variant are +chosen at random; pass `rng=` (anything with a `choice` method, e.g. a seeded +`random.Random`) for reproducible output. `vocabularies=` supplies any +`` references. + +Expansion runs **before** filling, with slots kept opaque — so a slot value can +never be mis-parsed as grammar. A value of `"(a|b)"` is filled in literally, +not expanded. + +If the chosen phrase has a slot with no value, `render()` raises +`UnfilledSlot` — a half-filled phrase must never reach text-to-speech. + +## `DialogRenderer` — the stateful, multilingual renderer + +`render()` is fine for one-off use. For a skill that speaks a response +repeatedly, `DialogRenderer` does better. It is built from a `LocaleResources` +and a dialog name: + +```python +from ovos_spec_tools import DialogRenderer + +renderer = DialogRenderer(res, "weather") +renderer.render("en-US", {"temperature": 21}) +renderer.render("pt-PT", {"temperature": 22}) # same renderer, another language +``` + +It adds three things over the bare function. + +### It is multilingual + +The language is a parameter of `render()`, not of the constructor — one +renderer serves every language the dialog ships in. + +### It avoids repeating itself + +`DialogRenderer` remembers the phrase it chose last time and avoids picking it +again — tracked **per language** — so a frequently-spoken response does not +sound mechanical. With two phrases, consecutive renders strictly alternate. + +### It carries default slots and an `.entity` fallback + +A slot is resolved in a clear order of precedence: + +1. a value passed to this `render()` call; +2. a **default** value, set once on the constructor and reused every call; +3. a random value from the slot's **`.entity`** value set; +4. otherwise — `UnfilledSlot`. + +```python +renderer = DialogRenderer(res, "greeting", slots={"assistant": "OVOS"}) +renderer.render("en-US") +``` + +Here `{assistant}` is filled from the default every time. A `{weekday}` slot +that the caller does not supply is filled from `weekday.entity` if the skill +ships one — useful for a slot whose value is a free pick from a known set +rather than a computed value. + +## Choosing between them + +| | `render()` | `DialogRenderer` | +|--|------------|------------------| +| Input | an explicit phrase list | a `LocaleResources` + dialog name | +| Language | decided by the caller | per `render()` call | +| Repetition avoidance | no | yes, per language | +| Default slots / `.entity` fallback | no | yes | + +Use `render()` when you already hold the phrases and want one sentence. Use +`DialogRenderer` for a skill response spoken more than once. + +## Next + +[Language matching](language-matching.md) — the tag logic behind the loader's +smart fallback. diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..380e0df --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,99 @@ +# 1. Getting started + +## What this package is + +OVOS skills ship plain-text resource files — sentence templates, vocabularies, +spoken-response phrases — grouped by language under a `locale/` folder. Several +OVOS components each grew their own code to parse and expand those files, and +the copies drifted: the same template could expand differently depending on +which copy ran. + +`ovos-spec-tools` is the **one conformant implementation** of that machinery, +matching the formal specifications. Depend on it instead of rolling your own. + +## Install + +```bash +pip install ovos-spec-tools # core — no dependencies +pip install ovos-spec-tools[langcodes] # adds the smart language fallback +``` + +The core package has **no dependencies**. The optional `langcodes` extra +improves only one thing — how a missing language falls back to a near one +(see [Language matching](language-matching.md)); everything else works without +it. + +Requires Python 3.8 or newer. + +## A first taste + +Every tool is one import away. The four snippets below are the whole package +in miniature; the later chapters go deep on each. + +### Expand a template + +A *template* describes many sentences at once. `expand()` enumerates them: + +```python +from ovos_spec_tools import expand + +expand("(turn|switch) [the] (light|fan)") +# ['turn the light', 'switch the light', 'turn light', 'switch light', +# 'turn the fan', 'switch the fan', 'turn fan', 'switch fan'] +``` + +→ [Sentence templates](templates.md) + +### Load a skill's locale folder + +`LocaleResources` reads the resource files under `locale//`: + +```python +from ovos_spec_tools import LocaleResources + +res = LocaleResources("my-skill/locale") +res.load_intent("play", "en-US") # the expanded training samples +res.load_dialog("weather", "en-US") # the spoken-response phrases +``` + +→ [Locale resources](locale-resources.md) + +### Render a spoken response + +`render()` picks one phrase from a `.dialog` and fills its slots: + +```python +from ovos_spec_tools import render + +render(res.load_dialog("weather", "en-US"), slots={"temperature": 21}) +# 'It is 21 degrees.' +``` + +→ [Dialog](dialog.md) + +### Match a language tag + +`closest_lang()` finds the nearest available language: + +```python +from ovos_spec_tools import closest_lang + +closest_lang("en-AU", ["pt-BR", "en-US", "de-DE"]) # 'en-US' +``` + +→ [Language matching](language-matching.md) + +### Lint a locale folder + +From the command line: + +```bash +ovos-spec-lint my-skill/locale +``` + +→ [Linting](linting.md) + +## Where to next + +Continue with [Sentence templates](templates.md) — the grammar is the +foundation the resource loader and the dialog renderer both build on. diff --git a/docs/language-matching.md b/docs/language-matching.md new file mode 100644 index 0000000..8e93950 --- /dev/null +++ b/docs/language-matching.md @@ -0,0 +1,95 @@ +# 5. Language matching + +OVOS constantly needs to answer "the assistant was asked for language X — which +of the languages I actually have is the best fit?" — for locale folders, TTS +voices, STT models. The logic was reimplemented in several places +(`ovos_utils.lang`, `phoonnx`, …) and drifted. This module is the one +implementation. + +Three functions: + +```python +from ovos_spec_tools import standardize_lang, lang_distance, closest_lang +``` + +## `standardize_lang(tag)` + +Normalizes a BCP-47 tag for comparison — underscores to hyphens, consistent +case, canonical script/region forms: + +```python +standardize_lang("en_us") # 'en-US' +standardize_lang("PT") # 'pt' +``` + +It uses `langcodes` when installed and a simple normalization otherwise. One +deliberate exception: `tl` and `tgl` (Tagalog) are kept as `tl` — `langcodes` +would fold them into the `fil` macrolanguage, which OVOS keeps distinct. + +## `lang_distance(a, b)` + +The heart of the module. It returns an integer: **`0` is identical**, larger is +further apart, and `10` or more is not a usable match. + +```python +lang_distance("en-US", "en_us") # 0 — the same tag +lang_distance("en-US", "en-GB") # small — a regional difference +lang_distance("en-US", "fr-FR") # large — a different language +``` + +All the policy lives here, so callers never need special cases. + +### A bare tag is measured from its norm region + +`langcodes` resolves a bare tag to its *most-populous* region — it considers +`pt` closest to `pt-BR`. But the unmarked form of a language should mean its +**reference variety**: Portuguese is "from Portugal" by name, and every +Lusophone country except Brazil follows the pt-PT norm. So `lang_distance` +measures a bare tag from its norm region: + +```python +lang_distance("pt", "pt-PT") # 0 — the norm region +lang_distance("pt", "pt-BR") # > 0 — a regional variant +``` + +The norm regions are a small explicit table (currently just `pt → PT`); a +language is added only when its bare tag has a clear reference region distinct +from the populous one. + +### Without `langcodes` + +`langcodes` is optional. Without it, `lang_distance` falls back to a coarse +measure: a shared primary subtag is near, a different language is far, and the +generic region-less form counts as nearer than a sibling region. It is enough +to keep a request for `en-AU` resolving against `en`, `en-GB`, `en-US` — just +without `langcodes`' finer regional ranking. Install the `langcodes` extra for +that: + +```bash +pip install ovos-spec-tools[langcodes] +``` + +## `closest_lang(target, available, max_distance=10)` + +Given a wanted tag and the tags you have, returns the closest one — or `None`. +It is simply *the candidate with the smallest `lang_distance`*: + +```python +closest_lang("en-AU", ["pt-BR", "en-US", "de-DE"]) # 'en-US' +closest_lang("pt", ["pt-BR", "pt-PT"]) # 'pt-PT' (norm region) +closest_lang("zz-ZZ", ["en-US", "pt-BR"]) # None +``` + +An exact match always resolves. Any other match resolves only if its distance +is **below** `max_distance` (default 10, the OVOS-INTENT-2 §2.2 threshold); +`max_distance=0` accepts exact matches only. The return value is the original +string from `available`, so you can map it straight back to a directory, a +voice file, a model name. + +This is exactly what `LocaleResources` uses for its smart language fallback +([chapter 3](locale-resources.md)) — and you can use it directly anywhere else +the same question comes up. + +## Next + +[Linting](linting.md) — checking a whole locale folder at once. diff --git a/docs/linting.md b/docs/linting.md new file mode 100644 index 0000000..d7a5e17 --- /dev/null +++ b/docs/linting.md @@ -0,0 +1,81 @@ +# 6. Linting + +`ovos-spec-lint` checks a skill's locale folder against both specs at once — +the **syntax** of every template (OVOS-INTENT-1) and the **naming and layout** +of every file (OVOS-INTENT-2) — and reports *every* problem rather than +stopping at the first. + +## The command + +```bash +ovos-spec-lint path/to/locale +``` + +The argument may be a whole `locale/` directory (every language subdirectory is +checked) or a single `/` directory. Output is one line per finding: + +``` +locale/en-US/play.intent: error: single-branch group (button): ... +locale/en-US/old.rx: warning: .rx is a legacy file type, not an OVOS-INTENT-2 role +locale/english: warning: directory name 'english' is not a BCP-47 language tag + +2 error(s), 1 warning(s) +``` + +The exit code is **non-zero when there are errors** — so the command drops +straight into a CI pipeline. With `--strict`, warnings fail the run too. + +## What it checks + +**Errors** — the file is wrong: + +- a template that does not parse — any malformed form of + [chapter 2](templates.md); +- an empty file (no templates after comments and blank lines); +- a named slot inside a slot-free role (`.entity` / `.voc` / `.blacklist`); +- a base name outside the allowed charset (lowercase letters, digits, + underscores); +- an `.entity` whose base name — which names a slot — begins with a digit; +- the same `(role, base name)` appearing twice in one language tree; +- a `` reference to a vocabulary that does not exist. + +**Warnings** — suspicious but not fatal: + +- a resource file sitting outside any language directory; +- a language directory not named like a BCP-47 tag; +- a legacy file type (`.rx`, `.value`, `.list`, …) — not one of the five + OVOS-INTENT-2 roles; +- a file name that is not lowercase. + +## Using it as a library + +The CLI is a thin wrapper over `lint_locale`, which returns the findings so a +tool can process them: + +```python +from ovos_spec_tools import lint_locale + +findings = lint_locale("my-skill/locale") +for finding in findings: + print(finding.severity, finding.path, finding.message) + +errors = [f for f in findings if f.severity == "error"] +``` + +Each `Finding` has `.severity` (`"error"` or `"warning"`), `.path`, and +`.message`. + +## In CI + +Add a step that lints the locale folder of any skill you maintain: + +```yaml +- run: pip install ovos-spec-tools +- run: ovos-spec-lint locale +``` + +A malformed template now fails the build instead of failing a user's device. + +## Next + +[API reference](api-reference.md) — every public name at a glance. diff --git a/docs/locale-resources.md b/docs/locale-resources.md new file mode 100644 index 0000000..b5b394d --- /dev/null +++ b/docs/locale-resources.md @@ -0,0 +1,131 @@ +# 3. Locale resources + +A skill ships its localized text as plain-text files under a `locale/` folder. +This chapter covers what those files are and how `LocaleResources` loads them. +It is the implementation of OVOS-INTENT-2. + +## The folder layout + +``` +my-skill/ +└── locale/ + ├── en-US/ + │ ├── play.intent + │ ├── confirm.dialog + │ ├── media.entity + │ ├── yes.voc + │ └── trailers.blacklist + ├── pt-PT/ + │ └── … + └── de-DE/ + └── … +``` + +One subdirectory per language, named with a **BCP-47 tag** (`en-US`, `pt-PT`, +`zh-Hans`). A language directory may itself contain subdirectories — they are +an authoring convenience and carry no meaning; a resource is found by a +recursive search. A resource is identified by its **`(role, base name)`** pair, +so `confirm.intent` and `confirm.dialog` are two distinct resources. + +## The five roles + +Every resource file is a list of templates ([chapter 2](templates.md)). The +file extension is its **role**: + +| Role | Extension | Slots? | What it is | +|------|-----------|--------|------------| +| Intent | `.intent` | yes | training samples for one skill action | +| Dialog | `.dialog` | yes | phrases the assistant speaks back | +| Entity | `.entity` | no | example values for a slot | +| Vocabulary | `.voc` | no | a named keyword / phrase set | +| Blacklist | `.blacklist` | no | phrases that suppress an intent | + +`.intent` and `.dialog` are **slot-bearing** (they may use `{name}`); the other +three are **slot-free**. Lines starting with `#` are comments; blank lines are +ignored. + +Legacy OVOS file types (`.rx`, `.value`, `.list`, …) are deliberately *not* +roles here — the linter flags them ([chapter 6](linting.md)). + +## Loading with `LocaleResources` + +```python +from ovos_spec_tools import LocaleResources + +res = LocaleResources("my-skill/locale") +``` + +The **language is given per call**, not at construction. A locale folder is a +skill's multilingual unit, so one `LocaleResources` serves every language: + +```python +res.load_intent("play", "en-US") +res.load_intent("play", "pt-PT") # same instance +``` + +The load methods: + +| Method | Returns | +|--------|---------| +| `load_intent(name, lang)` | the union of every template's sample set, slots intact | +| `load_entity(name, lang)` | the expanded value set | +| `load_vocabulary(name, lang)` | the expanded phrase set | +| `load_blacklist(name, lang)` | the expanded phrase set | +| `load_dialog(name, lang)` | the raw phrase strings — **not** expanded (see below) | +| `vocabularies(lang)` | every `.voc`, as a `name → templates` dict | +| `entities(lang)` | every `.entity`, as a `name → values` dict | + +`.intent`, `.entity`, `.voc`, and `.blacklist` are expanded at load time. A +`.dialog` is **not** — its phrases are returned verbatim, because expansion +happens once per spoken response, on the single phrase chosen +([chapter 4](dialog.md)). + +`` references inside an `.intent` resolve automatically against the +`.voc` files of the same language — you do not pass vocabularies yourself. + +A missing resource raises `FileNotFoundError`. A malformed one — an empty file, +a duplicate `(role, base name)` in one language tree, a slot in a slot-free +role — raises `MalformedResource`. + +## Override precedence + +A resource may come from three places, highest priority first: + +1. **user** — per-user overrides, under a path the assistant decides; +2. **skill** — the files bundled with the skill; +3. **core** — fallback files shipped by the assistant framework. + +```python +res = LocaleResources( + skill_locale="my-skill/locale", + core_locale="/opt/ovos/locale", + user_locale="/home/me/.local/share/ovos/my-skill/locale", +) +``` + +A file at a higher level replaces the whole lower-level file with the same +`(role, base name)`. The user-data path is **assistant-defined** — this package +takes it as a parameter and imports no configuration of its own. + +## Smart language fallback + +When a skill has no directory for the requested language, `LocaleResources` +resolves to the **nearest available** language instead — a request for `en-AU` +loads `en-US`. This is re-evaluated on every call, so the same instance can +serve an exact language and a fallback one side by side. + +```python +res.load_intent("play", "en-AU") # finds en-US/ if there is no en-AU/ +``` + +The nearness logic is [language matching](language-matching.md). Two knobs on +the constructor: + +- `max_language_distance` — how far a fallback may reach (default 10); `0` + disables the fallback, leaving exact matches only; +- `lang_resolver` — a `(target, available, max_distance) -> str | None` + callable, defaulting to `closest_lang`; replace it to change the policy. + +## Next + +[Dialog](dialog.md) — turning a loaded `.dialog` into a spoken sentence. diff --git a/docs/templates.md b/docs/templates.md new file mode 100644 index 0000000..e7f041e --- /dev/null +++ b/docs/templates.md @@ -0,0 +1,145 @@ +# 2. Sentence templates + +A **sentence template** is a compact string that stands for a set of +sentences. It is the grammar of OVOS-INTENT-1, and it is the foundation of this +package: resource files are lists of templates, and the dialog renderer +renders them. + +`expand()` is the whole interface: + +```python +from ovos_spec_tools import expand, MalformedTemplate +``` + +`expand(template)` returns the **sample set** — the finite list of sentences +the template denotes. + +## The four tokens + +A template is literal words interspersed with four kinds of token. + +### Literal words + +Anything that is not a token is literal text, matched verbatim: + +```python +expand("turn on the lights") +# ['turn on the lights'] +``` + +### Alternatives — `(a|b|c)` + +Parentheses hold branches separated by `|`; each combination takes one branch: + +```python +expand("(turn on|switch on|enable) it") +# ['turn on it', 'switch on it', 'enable it'] +``` + +A branch may be empty — `(please|)` means "please, or nothing": + +```python +expand("(please|) help me") +# ['please help me', 'help me'] +``` + +### Optionals — `[x]` + +`[x]` is exactly `(x|)` — the segment is included or omitted: + +```python +expand("turn on [the] lights") +# ['turn on the lights', 'turn on lights'] +``` + +Groups nest freely: + +```python +expand("turn on [(all|every) ]light[s]") +# ['turn on all lights', 'turn on lights', 'turn on every lights', +# 'turn on all light', 'turn on light', 'turn on every light'] +``` + +### Named slots — `{name}` + +A `{name}` slot is a placeholder for a value that varies — a song title, a +city. It is **opaque**: expansion carries it through untouched and never +enumerates it. + +```python +expand("(play|put on) {query}") +# ['play {query}', 'put on {query}'] +``` + +Who fills a slot, and when, depends on the file role: an intent engine fills it +from speech; a skill fills a dialog slot before the phrase is spoken. Expansion +itself never does. + +### Vocabulary references — `` + +`` pulls in a named **vocabulary** — a reusable set of phrasings — and +expands it in place. Pass the vocabularies as a dict: + +```python +expand(" [there]", {"greeting": ["hello", "hi", "hey"]}) +# ['hello there', 'hi there', 'hey there', 'hello', 'hi', 'hey'] +``` + +This is how you avoid repeating the same `(hello|hi|hey)` group across many +templates: define it once as a vocabulary, reference it as ``. A +vocabulary may itself contain `` references; they resolve recursively. + +## The input model + +The grammar is built for **voice**. Templates and the utterances matched +against them are assumed already *ASR-normalized*: lowercase, alphanumeric +words, single spaces, no punctuation. This package **expands; it does not +normalize** — normalization happens upstream. + +One consequence: the metacharacters `( ) [ ] { } < > |` never occur as literal +spoken text, so there is no escaping mechanism and none is needed. + +(`.dialog` phrases are the exception — they are spoken *output*, so they may +carry mixed case and punctuation. See [Dialog](dialog.md).) + +## Malformed templates + +A template that cannot mean anything sensible raises `MalformedTemplate`: + +```python +try: + expand("turn (on|off the lights") +except MalformedTemplate as error: + print(error) # unbalanced metacharacters ... +``` + +The malformed forms (OVOS-INTENT-1 §3.6): + +| Form | Example | Why | +|------|---------|-----| +| Unbalanced metacharacters | `(a|b` | a bracket is never closed | +| Single-branch group | `(word)`, `()` | a group must offer a *choice* | +| Empty sample | `[hello]`, `(|)` | a sample with no words trains nothing | +| Slot-only template | `{name}` | a template needs anchoring literal text | +| Adjacent slots | `{a} {b}` | no word between two slots to delimit them | +| Repeated slot name | `{x} and {x}` | a slot is defined once per sample | +| Undefined vocabulary | `` | no such vocabulary was supplied | +| Cyclic vocabulary | ``→``→`` | resolution would not terminate | + +Two are checked against the **expanded samples**, not the raw template: +`{a} [x] {b}` is malformed because the empty-`x` branch yields the adjacent +pair `{a} {b}`; and a template is malformed if *any* branch combination +produces an empty sentence. + +## Templates as training data + +Expansion produces the *shape of the training data* a template contributes — +nothing more. A capable intent engine **generalizes beyond** the sample set: it +recognizes phrasings that were never enumerated. So keep templates focused and +readable; you are not obliged to spell out every wording. Matching and +generalization are the engine's job and are out of scope here. + +## Next + +[Locale resources](locale-resources.md) — where templates live on disk, and +how to load them. diff --git a/ovos_spec_tools/language.py b/ovos_spec_tools/language.py index 04ac2f9..49155b3 100644 --- a/ovos_spec_tools/language.py +++ b/ovos_spec_tools/language.py @@ -71,7 +71,7 @@ def _with_norm_region(tag: str) -> str: if "-" in tag: return tag region = _NORM_REGION.get(tag.lower()) - return f"{tag.lower()}-{region}" if region else tag + return f"{tag.lower()}-{region.upper()}" if region else tag def _langcodes_distance(desired: str, supported: str) -> Optional[int]: From a62883bbdc5a9039dbd746ed894296163958a7d6 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 13:44:00 +0100 Subject: [PATCH 014/110] test: cover edge cases across all five modules 34 new tests (142 total): - expansion: whitespace/tab normalization, optional vocabularies arg, bare templates, multi-word vocabulary members, empty vocabulary, no cross-template slot-consistency check, optional around a slot, unicode literals, duplicate-branch de-duplication - resources: core fallback and skill-overrides-core precedence, empty .dialog/.voc, vocabularies()/entities() directly, indented and mid-line "#", nonexistent skill_locale, undefined reference - dialog: single-phrase renderer, no-slot phrases, a slot value containing braces stays literal, numeric values, missing dialog - language: empty available list, empty-string tag, regional vs different-language distance, regioned-request sibling fallback - lint: nonexistent path, empty locale, unknown extension ignored, a single-language directory argument Co-Authored-By: Claude Opus 4.7 (1M context) --- test/test_dialog.py | 30 ++++++++++++++++ test/test_expansion.py | 47 +++++++++++++++++++++++++ test/test_language.py | 26 ++++++++++++++ test/test_lint.py | 26 ++++++++++++++ test/test_resources.py | 80 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 209 insertions(+) diff --git a/test/test_dialog.py b/test/test_dialog.py index 2615c7d..ad79437 100644 --- a/test/test_dialog.py +++ b/test/test_dialog.py @@ -157,3 +157,33 @@ def test_slot_precedence_call_then_default_then_entity(tmp_path): assert renderer.render("en-US") == "value from_default" # a per-call value beats both assert renderer.render("en-US", {"x": "from_call"}) == "value from_call" + + +# --- edge cases -------------------------------------------------------------- + +def test_renderer_with_one_phrase_repeats_it(tmp_path): + res = _resources(tmp_path, {"en-US/one.dialog": "the only phrase"}) + renderer = DialogRenderer(res, "one") + assert renderer.render("en-US") == "the only phrase" + assert renderer.render("en-US") == "the only phrase" + + +def test_render_phrase_with_no_slots(): + assert render(["a fixed phrase"]) == "a fixed phrase" + + +def test_render_slot_value_with_braces_stays_literal(): + """The filled value is inserted verbatim — never re-parsed as a slot.""" + out = render(["you said {echo}"], slots={"echo": "{not a slot}"}) + assert out == "you said {not a slot}" + + +def test_render_numeric_slot_value_is_stringified(): + assert render(["count {n}"], slots={"n": 0}) == "count 0" + + +def test_renderer_missing_dialog_raises(tmp_path): + res = _resources(tmp_path, {"en-US/x.dialog": "hello"}) + renderer = DialogRenderer(res, "absent") + with pytest.raises(FileNotFoundError): + renderer.render("en-US") diff --git a/test/test_expansion.py b/test/test_expansion.py index f4d3b98..ae3e849 100644 --- a/test/test_expansion.py +++ b/test/test_expansion.py @@ -177,3 +177,50 @@ def test_repeated_slot_name(): def test_invalid_names(template): with pytest.raises(MalformedTemplate): expand(template) + + +# --- edge cases -------------------------------------------------------------- + +def test_leading_trailing_and_repeated_whitespace_is_normalized(): + assert expand(" turn on the lights ") == ["turn on the lights"] + + +def test_tabs_count_as_whitespace(): + assert expand("turn\ton") == ["turn on"] + + +def test_vocabularies_argument_is_optional(): + assert expand("plain literal template") == ["plain literal template"] + + +def test_a_bare_vocabulary_reference_is_a_valid_template(): + assert sorted(expand("", {"g": ["hello", "hi"]})) == ["hello", "hi"] + + +def test_vocabulary_member_may_be_multiple_words(): + assert expand(" there", {"g": ["good morning"]}) == ["good morning there"] + + +def test_empty_vocabulary_is_malformed(): + with pytest.raises(MalformedTemplate): + expand("say ", {"x": []}) + + +def test_expand_does_not_enforce_slot_consistency(): + """expand() works one template at a time; the cross-template slot rule + (OVOS-INTENT-1 §5.5) is enforced elsewhere, not here.""" + assert sorted(expand("(buy {item}|leave)")) == ["buy {item}", "leave"] + + +def test_optional_around_a_slot(): + assert sorted(expand("[really ]want {item}")) == [ + "really want {item}", "want {item}", + ] + + +def test_unicode_literal_words_pass_through(): + assert expand("ligar a televisão") == ["ligar a televisão"] + + +def test_duplicate_branches_collapse_to_one_sample(): + assert expand("(go|go) home") == ["go home"] diff --git a/test/test_language.py b/test/test_language.py index 4aa42c1..3e81bb8 100644 --- a/test/test_language.py +++ b/test/test_language.py @@ -98,3 +98,29 @@ def test_distance_without_langcodes_prefers_the_generic_tag(monkeypatch): def test_distance_without_langcodes_no_shared_subtag(monkeypatch): monkeypatch.setattr(language, "_langcodes_distance", lambda a, b: None) assert closest_lang("en-AU", ["pt-BR", "fr-FR"]) is None + + +# --- edge cases -------------------------------------------------------------- + +def test_closest_lang_empty_available_is_none(): + assert closest_lang("en-US", []) is None + + +def test_standardize_lang_empty_string(): + assert standardize_lang("") == "" + + +def test_lang_distance_is_symmetric_enough_for_identity(): + assert lang_distance("en-US", "en-US") == 0 + assert lang_distance("EN_us", "en-US") == 0 + + +def test_lang_distance_regional_versus_different_language(): + pytest.importorskip("langcodes") + assert lang_distance("de-DE", "de-AT") < 10 # a usable regional match + assert lang_distance("de-DE", "nl-NL") >= 10 # a different language + + +def test_closest_lang_regioned_request_falls_back_to_a_sibling(tmp_path): + pytest.importorskip("langcodes") + assert closest_lang("pt-MZ", ["en-US", "pt-BR"]) == "pt-BR" diff --git a/test/test_lint.py b/test/test_lint.py index 07cd81c..1a5a841 100644 --- a/test/test_lint.py +++ b/test/test_lint.py @@ -117,3 +117,29 @@ def test_main_strict_fails_on_warnings(tmp_path): _write(locale / "en-US" / "old.list", "a\n") assert main([str(locale)]) == 0 assert main([str(locale), "--strict"]) == 1 + + +# --- edge cases -------------------------------------------------------------- + +def test_lint_nonexistent_path_is_an_error(): + findings = lint_locale("/no/such/locale/path") + assert any(f.severity == ERROR for f in findings) + + +def test_lint_empty_locale_warns(tmp_path): + locale = tmp_path / "locale" + locale.mkdir() + assert any("no language" in f.message for f in lint_locale(locale)) + + +def test_unknown_extension_is_ignored(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "notes.txt", "ignore me\n") + _write(locale / "en-US" / "ok.intent", "hello world\n") + assert lint_locale(locale) == [] + + +def test_lint_accepts_a_single_language_directory(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "ok.intent", "hello world\n") + assert lint_locale(locale / "en-US") == [] diff --git a/test/test_resources.py b/test/test_resources.py index 780fc37..2ef5854 100644 --- a/test/test_resources.py +++ b/test/test_resources.py @@ -198,3 +198,83 @@ def resolver(target, available, max_distance): res = LocaleResources(str(locale), lang_resolver=resolver) assert res.load_intent("x", "zz-ZZ") == ["british"] + + +# --- edge cases -------------------------------------------------------------- + +def test_core_resources_are_a_fallback(tmp_path): + skill = tmp_path / "skill" / "locale" + core = tmp_path / "core" / "locale" + _write(core / "en-US" / "x.intent", "core version\n") + res = LocaleResources(str(skill), core_locale=str(core)) + assert res.load_intent("x", "en-US") == ["core version"] + + +def test_skill_resources_override_core(tmp_path): + skill = tmp_path / "skill" / "locale" + core = tmp_path / "core" / "locale" + _write(skill / "en-US" / "x.intent", "skill version\n") + _write(core / "en-US" / "x.intent", "core version\n") + res = LocaleResources(str(skill), core_locale=str(core)) + assert res.load_intent("x", "en-US") == ["skill version"] + + +def test_empty_dialog_is_malformed(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "e.dialog", "# nothing here\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedResource): + res.load_dialog("e", "en-US") + + +def test_empty_voc_is_malformed(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "e.voc", "\n\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedResource): + res.load_vocabulary("e", "en-US") + + +def test_vocabularies_collects_every_voc(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "yes\n") + _write(locale / "en-US" / "no.voc", "no\n") + res = LocaleResources(str(locale)) + assert set(res.vocabularies("en-US")) == {"yes", "no"} + + +def test_entities_collects_every_entity(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "day.entity", "monday\n(tues|wednes)day\n") + res = LocaleResources(str(locale)) + assert sorted(res.entities("en-US")["day"]) == [ + "monday", "tuesday", "wednesday", + ] + + +def test_indented_comment_is_skipped(tmp_path): + f = tmp_path / "x.voc" + _write(f, " # an indented comment\nyes\n") + assert read_resource_file(f) == ["yes"] + + +def test_hash_mid_line_is_literal_not_a_comment(tmp_path): + """Only a leading # starts a comment — there are no inline comments (§3).""" + f = tmp_path / "x.voc" + _write(f, "channel # five\n") + assert read_resource_file(f) == ["channel # five"] + + +def test_nonexistent_skill_locale_raises_on_load(tmp_path): + res = LocaleResources(str(tmp_path / "does-not-exist")) + with pytest.raises(FileNotFoundError): + res.load_intent("x", "en-US") + + +def test_intent_with_undefined_voc_reference_is_rejected(tmp_path): + from ovos_spec_tools import MalformedTemplate + locale = tmp_path / "locale" + _write(locale / "en-US" / "g.intent", " hello\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedTemplate): + res.load_intent("g", "en-US") From 2b9a95356cd89b57719e03bd7d95911912006d33 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 14:20:13 +0100 Subject: [PATCH 015/110] =?UTF-8?q?feat:=20linter=20=E2=80=94=20slot-consi?= =?UTF-8?q?stency,=20.blacklist=20pairing,=20--spec-version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New checks: - §5.5 slot consistency: every template in one .intent/.dialog must declare the same slot set — flagged when they differ (previously unchecked) - a non-UTF-8 file is now reported as an error instead of crashing the linter (UnicodeDecodeError is not an OSError) - a .blacklist with no matching .intent — a warning (orphan suppression) - a language directory with no resource files — a warning New --spec-version {0,1,2} flag — a forward-compatibility check that flags features newer than a target runtime: - spec-version 0: a .blacklist file warns (a v0 runtime ignores it) - spec-version < 2: a reference errors (an older runtime cannot expand the template) - default 2: nothing extra flagged 11 more linter tests (153 total). The dirty-locale example output is unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/api-reference.md | 9 +++-- docs/linting.md | 28 +++++++++++++ ovos_spec_tools/lint.py | 84 +++++++++++++++++++++++++++++++++----- test/test_lint.py | 89 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 197 insertions(+), 13 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index e6b6a09..2f4e2cb 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -95,9 +95,11 @@ The entry of `available` with the smallest `lang_distance`, if it is below ## Linting — [chapter 6](linting.md) -### `lint_locale(path) -> list[Finding]` +### `lint_locale(path, spec_version=2) -> list[Finding]` Validate every resource file under a locale (or single-language) directory. +`spec_version` (0, 1, or 2) flags features newer than that target — see +[chapter 6](linting.md). ### `Finding` @@ -106,8 +108,9 @@ A dataclass with `severity` (`"error"` / `"warning"`), `path`, and `message`. ### `ovos-spec-lint` (command) -CLI wrapper over `lint_locale`. `ovos-spec-lint [--strict]`; exit code -is non-zero on errors (with `--strict`, on warnings too). +CLI wrapper over `lint_locale`. `ovos-spec-lint [--strict] +[--spec-version {0,1,2}]`; exit code is non-zero on errors (with `--strict`, +on warnings too). ## Package diff --git a/docs/linting.md b/docs/linting.md index d7a5e17..196c84e 100644 --- a/docs/linting.md +++ b/docs/linting.md @@ -32,7 +32,11 @@ straight into a CI pipeline. With `--strict`, warnings fail the run too. - a template that does not parse — any malformed form of [chapter 2](templates.md); - an empty file (no templates after comments and blank lines); +- a file that is not valid UTF-8; - a named slot inside a slot-free role (`.entity` / `.voc` / `.blacklist`); +- templates within one `.intent` or `.dialog` declaring **different slot + sets** — every template of one definition must use the same `{slots}` + (OVOS-INTENT-1 §5.5); - a base name outside the allowed charset (lowercase letters, digits, underscores); - an `.entity` whose base name — which names a slot — begins with a digit; @@ -43,10 +47,34 @@ straight into a CI pipeline. With `--strict`, warnings fail the run too. - a resource file sitting outside any language directory; - a language directory not named like a BCP-47 tag; +- a language directory with no resource files; - a legacy file type (`.rx`, `.value`, `.list`, …) — not one of the five OVOS-INTENT-2 roles; +- a `.blacklist` with no matching `.intent` to suppress; - a file name that is not lowercase. +## Targeting an older spec version + +A skill may need to run on a device that has not been updated. `--spec-version` +flags any feature **newer than a target version**, so you learn before +shipping that a skill will not work there: + +| Version | Adds | +|---------|------| +| `0` | the legacy, undocumented Mycroft/OVOS de-facto behaviour | +| `1` | the formalized specs — and the `.blacklist` role | +| `2` | `` inline vocabulary references *(the default)* | + +```bash +ovos-spec-lint my-skill/locale --spec-version 1 +``` + +With `--spec-version 1`, a template using `` is an **error** — a +version-1 runtime cannot expand it. With `--spec-version 0`, a `.blacklist` +file is additionally a **warning** — a version-0 runtime silently ignores it, +so the suppression simply will not happen. The default, `2`, flags nothing +extra. + ## Using it as a library The CLI is a thin wrapper over `lint_locale`, which returns the findings so a diff --git a/ovos_spec_tools/lint.py b/ovos_spec_tools/lint.py index 179fb47..a4a87ef 100644 --- a/ovos_spec_tools/lint.py +++ b/ovos_spec_tools/lint.py @@ -10,6 +10,13 @@ The argument may be a ``locale/`` directory (every language subdirectory is checked) or a single ``/`` directory. + +The ``--spec-version`` option additionally flags features newer than a target +OVOS spec version, for skills that must run on older deployments: + +- **0** — the legacy, undocumented Mycroft/OVOS de-facto behaviour; +- **1** — the formalized specs; adds the ``.blacklist`` role; +- **2** — adds ```` inline vocabulary references. """ from __future__ import annotations @@ -34,9 +41,15 @@ # File types OVOS-INTENT-2 deliberately does not define — flagged, not parsed. LEGACY_EXTENSIONS = (".rx", ".value", ".list", ".word", ".template", ".qml") +# The OVOS spec version that introduced each feature. +DEFAULT_SPEC_VERSION = 2 +_BLACKLIST_SINCE = 1 # the `.blacklist` role +_VOCABULARY_REFERENCE_SINCE = 2 # the `` inline vocabulary reference + _BASE_NAME_RE = re.compile(r"[a-z0-9_]+") _SLOT_NAME_RE = re.compile(r"[a-z][a-z0-9_]*") _LANG_TAG_RE = re.compile(r"[a-z]{2,3}(-[A-Za-z0-9]+)*") +_SLOT_RE = re.compile(r"\{([a-z][a-z0-9_]*)\}") ERROR = "error" WARNING = "warning" @@ -54,19 +67,22 @@ def __str__(self) -> str: return f"{self.path}: {self.severity}: {self.message}" -def lint_locale(path) -> List[Finding]: +def lint_locale(path, spec_version: int = DEFAULT_SPEC_VERSION) -> List[Finding]: """Lint a locale directory, or a single language directory. Returns every :class:`Finding`, in file order. A path whose name is a BCP-47 tag is treated as a single language tree; otherwise it is treated as a ``locale/`` directory and each language subdirectory is checked. + + ``spec_version`` is the target OVOS spec version: a resource using a + feature newer than it is flagged (see the module docstring). """ root = Path(path) if not root.is_dir(): return [Finding(ERROR, str(root), "not a directory")] if _LANG_TAG_RE.fullmatch(root.name): - return _lint_language_tree(root) + return _lint_language_tree(root, spec_version) findings: List[Finding] = [] for child in sorted(root.iterdir()): @@ -79,11 +95,11 @@ def lint_locale(path) -> List[Finding]: findings.append(Finding( WARNING, str(root), "no language directories found")) for language_dir in language_dirs: - findings.extend(_lint_language_tree(language_dir)) + findings.extend(_lint_language_tree(language_dir, spec_version)) return findings -def _lint_language_tree(language_dir: Path) -> List[Finding]: +def _lint_language_tree(language_dir: Path, spec_version: int) -> List[Finding]: """Lint one ``/`` directory and all its subdirectories.""" findings: List[Finding] = [] if not _LANG_TAG_RE.fullmatch(language_dir.name): @@ -104,6 +120,11 @@ def _lint_language_tree(language_dir: Path) -> List[Finding]: f"{path.suffix} is a legacy file type, not an " f"OVOS-INTENT-2 resource role")) + if not role_files: + findings.append(Finding( + WARNING, str(language_dir), + "language directory contains no resource files")) + # Duplicate (role, base name) within one language tree (OVOS-INTENT-2 §2). first_seen: Dict[tuple, Path] = {} for path in role_files: @@ -116,22 +137,40 @@ def _lint_language_tree(language_dir: Path) -> List[Finding]: else: first_seen[key] = path + # `.blacklist`: a spec-version gate, and a pairing check (§4.3). + intent_names = {stem for (ext, stem) in first_seen if ext == ".intent"} + for path in role_files: + if path.suffix != ".blacklist": + continue + if spec_version < _BLACKLIST_SINCE: + findings.append(Finding( + WARNING, str(path), + f"the .blacklist role requires spec version " + f"{_BLACKLIST_SINCE}; a version-{spec_version} runtime " + f"ignores it")) + if path.stem not in intent_names: + findings.append(Finding( + WARNING, str(path), + f"blacklist {path.stem!r} has no matching " + f"{path.stem}.intent to suppress")) + # Vocabularies, for resolving references during expansion. vocabularies: Dict[str, List[str]] = {} for path in role_files: if path.suffix == ".voc": try: vocabularies[path.stem] = read_resource_file(path) - except OSError: + except (OSError, UnicodeError): pass # _lint_file reports the read error below for path in role_files: - findings.extend(_lint_file(path, vocabularies)) + findings.extend(_lint_file(path, vocabularies, spec_version)) return findings def _lint_file(path: Path, - vocabularies: Dict[str, Sequence[str]]) -> List[Finding]: + vocabularies: Dict[str, Sequence[str]], + spec_version: int) -> List[Finding]: """Lint one resource file: naming, then the syntax of every template.""" findings: List[Finding] = [] extension = path.suffix @@ -152,10 +191,10 @@ def _lint_file(path: Path, findings.append(Finding( WARNING, str(path), "file name should be lowercase")) - # --- syntax (OVOS-INTENT-1) --------------------------------------------- + # --- read (OVOS-INTENT-2 §3) -------------------------------------------- try: templates = read_resource_file(path) - except OSError as exc: + except (OSError, UnicodeError) as exc: findings.append(Finding(ERROR, str(path), f"cannot read file: {exc}")) return findings if not templates: @@ -165,8 +204,17 @@ def _lint_file(path: Path, "template (OVOS-INTENT-2 §5)")) return findings + # --- syntax (OVOS-INTENT-1) --------------------------------------------- slot_free = extension in SLOT_FREE_ROLES + slot_bearing = extension in SLOT_BEARING_ROLES + slot_sets: List[frozenset] = [] for template in templates: + if spec_version < _VOCABULARY_REFERENCE_SINCE and "<" in template: + findings.append(Finding( + ERROR, str(path), + f"an inline vocabulary reference <…> requires spec version " + f"{_VOCABULARY_REFERENCE_SINCE}; a version-{spec_version} " + f"runtime will not expand this template [in: {template!r}]")) try: samples = expand(template, vocabularies) except MalformedTemplate as exc: @@ -178,6 +226,16 @@ def _lint_file(path: Path, ERROR, str(path), f"{extension} is slot-free but a template contains a named " f"slot [in: {template!r}]")) + if slot_bearing: + slot_sets.append(frozenset(_SLOT_RE.findall(template))) + + # --- slot consistency (OVOS-INTENT-1 §5.5) ------------------------------ + if slot_bearing and len(set(slot_sets)) > 1: + findings.append(Finding( + ERROR, str(path), + f"templates declare different slot sets — every template in one " + f"{extension} must use the same {{slots}} (OVOS-INTENT-1 §5.5)")) + return findings @@ -192,9 +250,15 @@ def main(argv: Optional[Sequence[str]] = None) -> int: parser.add_argument( "--strict", action="store_true", help="exit non-zero if there are warnings as well as errors") + parser.add_argument( + "--spec-version", type=int, choices=(0, 1, 2), + default=DEFAULT_SPEC_VERSION, + help="target OVOS spec version; flags features newer than it " + "(0: legacy, 1: adds .blacklist, 2: adds references). " + f"Default {DEFAULT_SPEC_VERSION}.") args = parser.parse_args(argv) - findings = lint_locale(args.locale) + findings = lint_locale(args.locale, spec_version=args.spec_version) errors = [f for f in findings if f.severity == ERROR] warnings = [f for f in findings if f.severity == WARNING] diff --git a/test/test_lint.py b/test/test_lint.py index 1a5a841..8a65afb 100644 --- a/test/test_lint.py +++ b/test/test_lint.py @@ -143,3 +143,92 @@ def test_lint_accepts_a_single_language_directory(tmp_path): locale = tmp_path / "locale" _write(locale / "en-US" / "ok.intent", "hello world\n") assert lint_locale(locale / "en-US") == [] + + +# --- slot consistency (OVOS-INTENT-1 §5.5) ---------------------------------- + +def test_inconsistent_slots_in_one_intent_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "p.intent", "play {query}\nstop {engine}\n") + assert any("slot sets" in f.message for f in _errors(lint_locale(locale))) + + +def test_mixing_slotted_and_slotless_lines_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "p.intent", "play {query}\njust stop\n") + assert any("slot sets" in f.message for f in _errors(lint_locale(locale))) + + +def test_consistent_slots_across_an_intent_is_clean(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "p.intent", + "(play|put on) {query}\ni want {query}\n") + assert lint_locale(locale) == [] + + +# --- robustness -------------------------------------------------------------- + +def test_non_utf8_file_is_reported_not_crashed(tmp_path): + lang = tmp_path / "locale" / "en-US" + lang.mkdir(parents=True) + (lang / "bad.intent").write_bytes(b"\xff\xfe not valid utf-8\n") + findings = lint_locale(tmp_path / "locale") + assert any("cannot read" in f.message for f in findings) + + +def test_empty_language_directory_warns(tmp_path): + locale = tmp_path / "locale" + (locale / "en-US").mkdir(parents=True) + assert any("no resource files" in f.message + for f in _warnings(lint_locale(locale))) + + +# --- .blacklist pairing (OVOS-INTENT-2 §4.3) -------------------------------- + +def test_orphan_blacklist_warns(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "ghost.blacklist", "spam words\n") + assert any("no matching" in f.message + for f in _warnings(lint_locale(locale))) + + +def test_blacklist_with_a_matching_intent_is_clean(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "play music\n") + _write(locale / "en-US" / "play.blacklist", "trailer\n") + assert lint_locale(locale) == [] + + +# --- the --spec-version flag ------------------------------------------------- + +def test_spec_version_0_flags_the_blacklist_role(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "play music\n") + _write(locale / "en-US" / "play.blacklist", "trailer\n") + findings = lint_locale(locale, spec_version=0) + assert any("requires spec version" in f.message for f in findings) + + +def test_spec_version_1_flags_a_vocabulary_reference(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greeting.voc", "hello\nhi\n") + _write(locale / "en-US" / "greet.intent", " there\n") + errors = _errors(lint_locale(locale, spec_version=1)) + assert any("vocabulary reference" in f.message for f in errors) + + +def test_default_spec_version_flags_neither(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greeting.voc", "hello\nhi\n") + _write(locale / "en-US" / "greet.intent", " there\n") + _write(locale / "en-US" / "greet.blacklist", "spam\n") + findings = lint_locale(locale) # default spec-version 2 + assert not any("spec version" in f.message for f in findings) + + +def test_main_honors_spec_version(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greeting.voc", "hello\nhi\n") + _write(locale / "en-US" / "greet.intent", " there\n") + assert main([str(locale)]) == 0 # v2 — fine + assert main([str(locale), "--spec-version", "1"]) == 1 # is v2 From b3c583223f1f467a4bc1f246dc13145ba0e7a31e Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 14:26:43 +0100 Subject: [PATCH 016/110] docs: point at OpenVoiceOS/architecture (repo renamed from formal-specifications) Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 2 +- docs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bf4b548..78dc510 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # ovos-spec-tools Reference implementation of the OVOS [formal -specifications](https://github.com/OpenVoiceOS/formal-specifications) — the +specifications](https://github.com/OpenVoiceOS/architecture) — the low-level, dependency-light primitives those specifications describe. OVOS components reimplement template expansion, resource loading, and language diff --git a/docs/README.md b/docs/README.md index af2b561..48025d4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,7 +1,7 @@ # ovos-spec-tools — documentation `ovos-spec-tools` is the reference implementation of the -[OVOS formal specifications](https://github.com/OpenVoiceOS/formal-specifications): +[OVOS formal specifications](https://github.com/OpenVoiceOS/architecture): the small, dependency-light primitives those specs describe, in one place, so OVOS components and third-party tools stop reimplementing them and drifting apart. From 11e830eb9e2a0cfa4953a07e2224428e5476d861 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 15:07:32 +0100 Subject: [PATCH 017/110] docs: credit the NLnet NGI0 Commons Fund Part of a documentation and interoperability effort for OpenVoiceOS, funded by NLnet under grant agreement 101135429. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 9 +++++++++ ngi.png | Bin 0 -> 12713 bytes 2 files changed, 9 insertions(+) create mode 100644 ngi.png diff --git a/README.md b/README.md index 78dc510..4bc54c4 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,15 @@ A zero-to-hero guide lives in [`docs/`](docs/README.md): Runnable example scripts are in [`examples/`](examples/README.md). +## Credits + +This package was produced as part of a documentation and interoperability +effort for OpenVoiceOS, funded by NLnet's +[NGI0 Commons Fund](https://nlnet.nl/project/OpenVoiceOS) under grant +agreement No [101135429](https://cordis.europa.eu/project/id/101135429). + +![NGI0 / NLnet](./ngi.png) + ## License Apache 2.0 diff --git a/ngi.png b/ngi.png new file mode 100644 index 0000000000000000000000000000000000000000..13f4e9cd190a658243763a0242cfd41b58a530f4 GIT binary patch literal 12713 zcmZ{LbyQnV)NPi~GTc^|p{g#6!j z7ZYbQGkbuUwCV>gLWS?y000$0URpxkQ~xNXLGJ@nEyfV%=J~IQ%=FjECK4Iq7C9C2J}qWI7~#WNM6kQGO3U|DmB)7juST z8IRvyVi|u$PXsM?mY_=~Fqa+ClNeK-#H#M} zf1dl61jSe39?G_O6FnX-b)IE&=(!MhMa__lZuIkrxmZL(--@*Nd?pSQYkU|Wb-tds zZZOu+`NkySzcI>;s<7-a3x?g9LUnu|YTFZw+Be0R~j6a}EP=Jo{W6%&SU1o&gs`DXsmhJz&_x@k1S>v{l z%6u!}(M(--S68{|rNhE8nk`uCYJjQG+5HwPH zI#)e<^eFTCesyOd8Gkoe*Ysh6f==s&zgq-xS+t4q?v&?@KQ3>qAS@{J|awhrOQ`@V1(eN>1QD}wp|Ccnw_I2%Btu^JnY)i^QWO6N&=e#3ll!m0KsshSQxY=e zvdr2TKm(AjfV2ayc|cbX2to^*F%S&7sRPncV#sUW{|ekvJ&JWU)VV_sk|F`MzQK_P zraBAtQ?I+T?#9q>`yXj@<5z?BPd-^O%Hvl{A=lx?1_Dq2{j6}XSqQ{7GM&YZ|Fu|~ zOoRnSm@FJed^Bs4mFc5rU&@j9-r)23`9Ay=Zt&jhn*b)}hfYpactys9|W zu30q>)+eMwEnuNrHK=Ize!$YWq&Chh`e5qWni4%$>ri7j5%G(V8qF~)a(oV1nj7-ZL^d-++m z|6}@zQ!>p{wKEBHcGUeXEQh^l*QZzE zCwF-bWlA-ciJ6GSW%N8_WlDU^qluwDP52dFoN3af`pp>@;0H4-VSX7XYgidLUT*Q) z#O6wexxr`HK=t8kYli=DRCPd~Wt7C^oZIs%H&D!`Elw{mXy(KdPctLwx|csmtHP?F z5ySpz)>=sH9>3-AMYD@S-+RiaOKC1z6&Hu(t8%cqn+|7^$yruVOh%M??h8TJwl~R> zs!GY_B>9!6*%g$+CpmF+(55|ZW?tFrGyxi81l1ZP;P#Wqkaw2|L|Az^c0M1#0hWja zqSZ5UGsPg0JNzAG4DL#H%H3i+pE?n{Zq{$*Li?_q|wN6R>P$bgD@<$j~#~cT^uhucn^qrUv2ak zWJo5O&6Md3Hk4?_Kp>Lt*USOGe(f(65vWzPChpG;Q*m;tdgLa?r9Po9eq6bVC&=mL z&bs;7xDCFx@Rzq9`KC;KQh;zrWsfw$IQuC3G>;1N^ymqcEeV5sG%Z4vt{dHLRiAK;#L8@$XYhW1zEBu#m8E}Ab$k5LTYR_hWR zTh3*CyWjoAd1h6EcM2mrG@L27-IGJ_y+x;fGsBjqe)FK7jME+Ucl5BIZ}}o73o~bD zw75Mk5{Qi1xe6!C-l)Km>W!XGZ}3|85y)D}cIVOOEDF%5`)w<9&Yg7gSTlxW_;`C3 zPAR@(iIxdth@$tVT>D#85w3yle7@waoW3|xf9a7dccV)$(3+ZAb!&e8`5ZVG8v)h zbcFHU1PiQS$z$szb54s`NAz=u**gDgw%BO2*XtSy>!REF0_CIY*5jCKQ}T_aZ7M+p znWPFdL?mgtIm9Hx1@$XaUOG?4q3vsHrjI_%X@MsSP4x03J@a9ok3ppjO83R$U_CzRm}Vu&NoKU&aczo5 zuQqHb2voj<@UgYJmbtrIO-Dn^JpY_MXyu*pN#Z{%TH+c@f>m|ABRfJj5#uM<@=X}4 zU(p-vT{L%GUb`V7@=sEoCDUamzPlS3P%oI2*X@l&K}UDF-1};6(`6L|gNdZyZrZMZ zwT6r?Ovb}1q|WJC8nMJi0L}{Ee%E3GP`7(;@BeNpQr;#SqRX5nF@|A`OTJo&nL0xg zlGFR*QzzJ`!3oZ{Rvp*JSMNn(lN#vFwU0u&De!MMS*SYnD8(lcws?RRrKpT&XgZhz-I{D!4M%3V4E3#_V^`a2uOn25*JBn;t7}#2MJ` zt+t8#rL+#|*eY9l`Do6>n(I>j2Iw_-+mhS?@2FkaI|+`F)*!iBjSTyQfKb*NFF!Lm z70?fO(2#}Wt^pPwkORWgU+gD;U^A>{w+MctcJClfiOMaqZbo!?pkUV;Yy6K1_TB$z zR5Ou^+8J&0!SjV(&eb za4CPcDl5K3HTLd=c`}Y${Ys^S2V^{nG*qapn;Y+9gF``SsbXc+@%8H`q2x~7cJXVL zQD5@j;&?^@japD4s1lA-d=M{g7*Wopb=tM1SbqFcotOMdfV}O}TKG32P@niP!?uLw zzAEpgS2WH>6{ucRYrhP+d%5qT-2sS+iNP+T{5dU84fPa$yV=Z;Dh*4<4z-5{XQCXx zZ<6eb+1R!1uHevXnSDoV)Mh#(X<8|KN?-97N`pXkj}Cu;>vQdRXW zK0ZJ03z=1rsgSvb9v*N)-&ATLtQG@4=+I-oq1cZ!;oL2uRO(OOB8K5L7?2mL;ez=y_oJ$zZ0=9wL0ww#XfbzgF&jY&2?r77!m3Lmdwdn_m96 zEt>In|CARJ6a9Yq=!R`6$8(g_LIHQO;HfU%`TH@8UReN_kNW73Nd8#0+Fms=FMA*% zFDhdHxc`0r@%<9hMNf66Z2%tD9ceowZx1%P{bZjrJ|Bw!Csyp zbls*k8WXO%5Zg4wOHXA5i|t#P&^6GZCbc9E%NYyy1c2}A~iAnaqw^SK?!fW2!o0gN; z!ePNpvb%jYT~2-6*tZd%Z^0b0;xS2n#g)Mjp`>jUqoO+!@=rQu2}O2FX<;yYIF^*= zKS8y{k23TkkI%$(86sx3$Lk?c#5JCS6ayvJg+(r3^>#N5HF4Ts43YLNU^-@?fNe&- zZ_GF}T;{2&XvyOKV&2!Dt%9OZK0D*Zdb@D4B3&!G{(oI!{`Ayf9yo4$raklS6RLao z`^c!tRUQtJqEVOH^RTzJfV{jsHiNd*p&>JQuTu(nUi`vtEQZ43( z8e+~uROVMSIi&@jfb8s{m1w7heetX+RcRA_pz1faJwYw|Hc)cwXI-qZcN-V4p_E~2 z-b}@q$>RzRq2tTD-&7M7CAyChu7aSN;E>HAY;=jyj8gx3q}X?Ls= zmrKeTORKK(orFV2K_d3ioCcc8pSh;TWQ$iqA)iw)dS zfLz4wX}T5%2e6hWQzEFRJp^UtJG~d(n1lYx1F0I{;)8~p*mWj61=jAb;1LYXJ^KHA zP54i@rUez1z-e83PCuky8g&j>)K475|MG|bOLMr0=bf0C2#$){odG#)?-yk3AZmIp zt{9mx7*+~T2eR;3O@wEd>tU&vE-_0F`k?rqP*P!}g}c8;njL4C_YCWV+dG_YYebK! zUn9W#N?BTtY;)6bqwZZbv=?~aTvj$r|6x{RX;od^68O~8$0ndN zIMBXC zh=@gokg%_M$iU$`RPZmQg<{w0zI5zb=heZlLn6cdET9SGiOqP}=8ciJZ0WbIM2+2Q z)00=X0@zkZYa1aEseNnQgICQy&nO?fVZMMiJ2rawGY*i3s@yR4x3<#o6$1X@7h8k)3>K zPk$I|b(oz^-H&l25BtZo_ap$r-`^nM70kGtD_0H~giEohtvp!Z@Bvk{PdxOSp7O={ zfe0tv?7?IKF2dOGZNbeR{Jm5sB4VR)_=f)1$kYtW0{)B4&W);Q+vQXr)mrmDuDQ2( zJsdSC1*}l(roL)`Xtk+gU|;s&+AN74fP7MS#Iq9(!+R|I%r|8cr)o8bXxJ2(I3N<0 z{$vCXY!*rfycAA}^xKWtOOC9V%z;8lkTcP6MCO+i`P|v#RN3Lwl z^4v&D$0k+*H~Tuzo2<4`+0}&Ho+5>V-8H;V7)AOFC7!&3GOU)A_I{4@-}FJv>7-hs zv2+(375)$#hnhhJoe|bGe*sF7jWJ8VTts`)2iQY?o8?HtsM{&0sQXq3M53fV_3phApHMB?=1TPqZ)S7b zaE{B?^rhX4_b94lPFCF;igh^>$*4Ku%R1!M!wjbIVFBgMWmN7xttc?ND)_2FokYa$ z@+Dxw0GWi}26uTNM3U>gERSuqE`z`e^~X?#L-NF}3|f3x1>42_GXHCFC9mDOgI8Z0 zK!g{XNRLNaY7xWl=Y{cf{y-q#2TDpJrfo$_1=6_IssBLos~!tCbEG-6yn>ly>()mUsso#?c}LzIIlhRX>kxptw_`roJ1%=&Q&_GTf#pr=9S z=1dLNn&>?YOl}T!?vvcfM~)GWwEs9(B{~cK)L0%~3k(|^vCX@A@AK6j2Ef5D>oXfC zm|>V|JzmUms8)VR<+WC!SJQ{560-4aLgUUqUIU^0c6V;q>7Skwx@A}{T4bFS+6t=K ziBBWdq~sU2$qNEmR#q4WtX$EKe7_IVz6!O3KZlDDV_i{wYYkT@-)`yTcgi=-X)wb2 zEFL#{?;C`avdB&;ul|a5V5kQ-D@wXh!?k96ouJz zv4*;qmdm_Z(s+}LMOH-5P4CLv#8P2$KRVecqm{%G027n=7m2&C*Qt+zeVNkvIG$l@N&(#UL2%8;yBBk36AHnPE*`V%zpSlP8@DMLRM`vf#em=BuGg6rJ-9^B;D_{OF zp6%0SxtuIfz7Y??EKk=HnSEMcxQs&S*x4i7pN<@w6ZK`ID2AVIpcxAmJ7KK)Wji|@ zrVKZ}Uzuhb2R!YLsE7`me@&&*lMRAKJ4A0FU1WyZ9eT2HL1IV44Y9)9tDBu4w9bw_ zHa%}zoQ75p^tMblW$=V6M(PdgWj@8G>hcSq%J>rE*}l`@AdkIjj?6C3k<wicy zN5Zl)YbS82;0zD6^?hIkL!$N5wnn9NWFfgEKx5rlt!hS2v zL!f}WOnvtsH|7I9Q1bA;JS0s#@lGy!r9WF|3qcUiJHF!cJXc)J@J^K5Klr3``ksIi zi^=h%e)fg#qa=1Ih28t4_Q&61zW1jcSR9UPe~~1mq`V%_dheD`{45}o6&|xOsaH*g zVOTHwk_pem`$WR&^1cS98Uq<3d?frW%08w82IU0?K`G$}_m$&^Mv4vMmGE1+)Dq6# zW*Z*Hmd)+}I>Cf}LU}^YE0F!AUxW{oJ0b4(V3U(&^S1TGr&XqW?4^Gbt)2U1)2E$c z%gn&sX3Z4)@xXR!4pHtx%AA%_dKS~@URLz&ChdiNFI##4zwkV7YoMgnbbJzyD!kl$ z_KflUw1y(-+(dCX4^6W@Me9`2HfXl=LZ+2eI{=*iB>kfk+M>1y8Cw^X%s@RL1k!`eEpVMHe^H+16R=yZ<43dWQl|EQ^3T2Rq zQIwpvoJJ6SLc!6>6+bE^n9{G1H2n^i-h+eI&7UgSg+@2e@-rmzX&&qLFbMjg^NHpc z{ys530cDUly9c&cp4C=D9^y%st9_+aMQ);&rQw=RJuE+~4{Bf+%kR$zyVBYnD4xA) z>Jv)}nb)T4UZMO%Y4ks2z|WRLFf!1LUspl)G@>sq-HX$K%CdN@7{_E9HYveJzlQy$ zTHx=TP=(}_EF>K+iuZb=&G=6o+`%ZmJ(;c2zms<@N8@eH@7lQfen<&Z7w;i|9^>61 z>bZK9o*h*D+ATqJe0+j;QEx=sB(yK=O8~IemGPt{Be(Dwt{i;&lNueD${v0#qzXi?zCH&a4Rt6u%#=5 z2AtffydJBuRfKl_n^frMP^fu#u@_D8Xuh<=&`tI(>Zz>?D0;C&3Y^r(AJ9ashE6Cv z2-@YNT41nfY=2=O^&&HgMdz7nq)^+(sT3Zy%VF+c?-{g623z4UC4|&CBG+OAm7+83 z+h1uI)d{~QK@deVpz*iO{Iv4M^TC1VUY5O=%2GwU(FyyBD^9r2Ok7Dn<-~4d3GPt) zezaHM`YAKo>+IA-EDE}QQFMuQ-gs9d{{6F#+lN@X^%M9nsepll@6V6@+_4{J^pFb6 z%cLYFwb7^)$c6<6fvlx!5vi;dpCmh&1r>1(76a=>|s{A6Ihe%vBjKiS=%$~5<7m&u9r()%m`0Kh~4PYX~u zqmgx7c}d}|{EYR)ZKKN0poSitLS|?C4760EOy@S9$9zhyru!Ee-W!5LBH1!y^4mQk zOT0U4@`ST;l0YadN&ZQ*%F!d%LyMNsE>*+LLMIJOP5}0oF&WkV#4&?St0-z^MtJ0V z13v}*p?_QkjU%T&x{}zFAp7{HxIk~RoFu#iS>4Gpwa}e-uh4DkX^@^a7M*rI9msCw z;Po*+BG9NAJ3CIR8IzBUZ;7)MwbDN@uXl53#f*QNkv_8PX$aZ)$GmezBc9mfPb*|B z>3x1m=>2}ygzs5w1)G>O2XrWXOxY3w{o~6Ld{{hsG7`7x{0ZyzwADLKj-V}H>n~GfnKZV zHO}o`v?QtX6lmZ9mIOOygQ*ZfCZk^s22LS{pg5n` z^2n40Qg2;;ELJq481P!n45SrFrV&;d84bv&wzS@*SSALQBRm9m(@B`(!G(yR1eelL6`AWqR*_qn;#hvZl-ME-z?9Q;T zupD1>m;CrY_o-xr6t|!n(Fu5u1wq8+qwC<6BYjAlV4`HeEVzdW)l&* zjc-$Lhm_NVvFw!7Mfy`FzocKx&As0szz+mR)L4uAdPA_=D>m9o?p&R;~5>NPfYo|Gn{ZA z4Ky%MGF50!*jet*rXx+Yl!RatqJGV3|)BvBAfMd9heu*g7P z7E*lZ=*VK9^>d73Kh_`xW7oKAFG>eGva}~pm8$9TPQk2#CTqs29&I+=w_ACfxCRB{ zMmt?vOB&4L<6ns<5@9qW$hy6Eg1Xwdw1f8CSbVRdLC(<#5=1g7DXCXYB_xKCl$n`% zDCJ|2pi39(=N=`Bkwm7iOfeD|smy{@Q(iIeV?4!Y=~@G#I>B5&2!S&dyQ4L&XA8LE z=iU^aeNa67wYVtpoeCB4S4^PsR^c1po=6~LtxnxJ@L9|PCn$=X{z+S?VabzrP}5K8 zWW_k3W>xf-s)~!tr|OjT`F3ve4XH?SyXZ^m*iPIX<;9sCrPlzi$mLsHaT@#kH>acg zD_a*T;@ic{YMxIS4x}Y_yFiE2D`$t9e*5K?i(zXn>%G+spN>CnaGmyj;eYyCTD?R2 zA3rO7MWL!vpixNV;MdDc_d1uy#KhFDbH{ek_rwzy7hicWEFVK`PkP|=zgVJU8*G0E zHjoED4VKj0fnNm5<*kK|0`#NJFhW_s$qQbg`3eX7uae?P_%>R33gu~}ka1`$T4^wt z@#=hb7Nsu34~a_s@^E_AV-MI$I7TM9ERic43rm}RIGnFv|7|x+4NIv0@n!K5#{B5} zPNKZF?;VzEo*3QL6|e9|fJ@u=-&+8?Zbp1pyVah#H-9ghYB5h4t0Fu`f7h>XxO65? z95N^6&qZHnTl`*|@Wrj1|wu{ZGo1nJY^aRrph- zKbzF%iRty0Yd)?|iHvdi?rBv{Ziaw8zRQoMleXSVl8*wAlz%^dzPda8<<-M5EE{Wx zfG^lhnJ^`&7c>CsivO*)5_RA_I{q;{2{Q5it1FBnv!cq_^h5KI7fkJzSk4-1DiU$p$ zF?!GQnaP8V@`Hr!1lX-ALAf%OOt@l%rZP+PuTOa$HYV_mJ3FGyqL(Wf?kcMw;kr)K z`17;&FJ&8X8?3$UW`Tn4j20M>0)MDdqkDTR*h{vIW zEdJTDiw{UO(6lS+@}+87KLg4FWmW2AcDTeTT{_B|@4uMiqz?vnJ&4qwYNi2~x|;JY z4~rEEU{}Y&im>x9S>=t5$@;C{C6gF>@+%E0g$~SkF)^@v{};K>pMy_-*JXQTh&Af6 zv#_#aU|+2e9Y3z4BeEbV{4ccSt>!FHKEK}EkX;TmSg!UwMv=v6Izq&>0f)2N3Kufc zaL4-lo@0C+AE(n%T-GfWFqKI*gXwd7wtnNk65v!xn^I%+tmt_L$T^GCF}O}uj7jZMe_8n z#xl0+8Y|A>fJ0$cOz;J4qYj8dkWO;?L>@!Jh)GaUzsmv{5Ynr zRQCGTx|9RTno{{CaJh$)lC+?4H8=Yv=^hg%2~w@KU*53BI=vzmze+;Xm$ zfz-6L7)9V?AHkQF2yb2{dK*Tfqw|ZIdDXX@jyW4$^_y2z@9OmByQ(eT+HQv2dM5tea}*bjg`n-H_mR6C8$mUt`yk50~kl>YzF@;AY96a#ZY! zCybc=awrR!;<=pwyXygllV*wVkcvAYlW%m5%1Oo2>3d&bGw5?sEXQa6JsVO4yY6HD zb)fW7{Vb?zU@!`CXy2|Ew%@HDtT242w;Osnm9n<6QDP4FXX}&;&s14$8-kRVN-G`4 z_T6G&8*u7_?aZh;pQ z^oH{*Ttj5l8PPvRD7^K2QEhPa;b(rlwa=l+9aS)uK9!VMn1I6L7u5BNz1b%YKmH>6FFVFsLFv{!7`w0$2C(N zJP8S1!BR@m1xUsP5`ywZ-kzTJxqD)d+$Ee~W{Cy%zNyz$N8vOw#C0AaBq(dKX)_z* zL{1fpSJz<4_|+f?Q2WDJwY%r^OC!raz^h8l#mYI)(nV0C7CJlYMPMLEN12`ialj-V z8XPpazudQ7E*{!N4*c_n+hIkdyyaL3k?Q5}SROehxDee&q>UX%lQViAgtb)vR+;R- zp&a*=mHjY4PtEl+`8_-$)yBpK0Xk7XegN*T8uYK+3}5_cXmc!Avqbmz_iyj+#;bB} z2Zu&R7;=0|Bje2CEEaS`IQ}^63L;MT_1EEs+3IZu4v2T2g4oirz;&(5I6yLEJ874y!b^~+$-4)ITx$N+hc zbjV!)fQ{8zDdOh0(f>)4Bp744fOkY^;J^WfQ z@c(9~D-`!N{`ai)*Y^c7I;(O7-cJJKNz*qZ8xAV7b@3N0dw3FO;XJ`${e9|{J9{>I zY+GZa?vc$4ZG>dvAJN((ME=jR2SV!2{ATYlP;1IGjses71WzO9H8|`C6NCYmA~4Rf zcS2gtrh@m(mH%=)tTk{W&wJoN;L5Uv^ry$riVJhu@2r99Q!#p-Dnar#=~`V4^zKahFivakf{(N)jNkHC#jz=+-f*@YDlJa9fkb#&b-VWIbQUb7e=dHd9a2U}W z(-&5?J7W)V$C9>UmRK;7ETNDRfW9v6`a{B$Sb*Wc7k`Wj)q!Bi5@K);IUr3>N;IJ$ zYC`^QG?_z>72I1S9SiTxxw`2}w%MFO%Sj&_WehMduufxL%6Ia^mq2H#=(jlD2}xyt z9@YG&zp#yzkIlKufDjVrouAkp6*_NwH#kp84;L|IthE-?6t4HL>}noXvjl`Pu?(B( zgb0kIOw(T5n>Do_s@~fieG){Aoj?oP67_FwI81blVF$z-C&8!Bgc4p8Hh_*eknE$$ zzDS@&|J#ptY)Nnuq?7k0SlfL=Ph$$uy4Kkc{H-*2+mAw?S>F~**za#BXTI6=r4;e5 zIo5Y9YMUWc<0dBm5W;s-IcKdJpBGw$pr2b)hibf{(@pX4LJOfRd(ZlxJl~dd z%>vIgB6}c?OspnW?YFl82po)XfoCpgcz9DOd(p?Pp=1)H^;&5SbfEt^DnKA!lKG!P ze(upV^`=thKm9)R1*iUh+H|S@zXuHde>Zfzyeh^^-YC~AO+p--0LaTIOP5O;2K*nG CoHJMe literal 0 HcmV?d00001 From 2b321bf004bb2ee830b91e490326d9ca9d3e6bfd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 15:11:07 +0100 Subject: [PATCH 018/110] Add renovate.json (#1) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- renovate.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..5db72dd --- /dev/null +++ b/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ] +} From 4b028c67b9c58f9b2cd032067c82a9350f7afe4f Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 15:15:08 +0100 Subject: [PATCH 019/110] ci: add standard GitHub Actions workflows AI-Generated Change: - Model: claude-sonnet-4-6 - Intent: add missing CI/CD workflows - Impact: added build-tests.yml, coverage.yml, lint.yml, license_check.yml, pip_audit.yml, release_workflow.yml, publish_stable.yml, release-preview.yml, repo-health.yml, conventional-label.yml - Verified via: reviewed apply_workflows.py output --- .github/workflows/build-tests.yml | 14 ++++++++++++ .github/workflows/conventional-label.yml | 10 +++++++++ .github/workflows/coverage.yml | 16 ++++++++++++++ .github/workflows/license_check.yml | 10 +++++++++ .github/workflows/lint.yml | 13 +++++++++++ .github/workflows/pip_audit.yml | 10 +++++++++ .github/workflows/publish_stable.yml | 23 +++++++++++++++++++ .github/workflows/release-preview.yml | 13 +++++++++++ .github/workflows/release_workflow.yml | 28 ++++++++++++++++++++++++ .github/workflows/repo-health.yml | 12 ++++++++++ 10 files changed, 149 insertions(+) create mode 100644 .github/workflows/build-tests.yml create mode 100644 .github/workflows/conventional-label.yml create mode 100644 .github/workflows/coverage.yml create mode 100644 .github/workflows/license_check.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pip_audit.yml create mode 100644 .github/workflows/publish_stable.yml create mode 100644 .github/workflows/release-preview.yml create mode 100644 .github/workflows/release_workflow.yml create mode 100644 .github/workflows/repo-health.yml diff --git a/.github/workflows/build-tests.yml b/.github/workflows/build-tests.yml new file mode 100644 index 0000000..dff64be --- /dev/null +++ b/.github/workflows/build-tests.yml @@ -0,0 +1,14 @@ +name: Build Tests + +on: + pull_request: + branches: [dev, master, main] + workflow_dispatch: + +jobs: + build: + uses: OpenVoiceOS/gh-automations/.github/workflows/build-tests.yml@dev + with: + python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + install_extras: 'test' + test_path: 'test' diff --git a/.github/workflows/conventional-label.yml b/.github/workflows/conventional-label.yml new file mode 100644 index 0000000..9894c1b --- /dev/null +++ b/.github/workflows/conventional-label.yml @@ -0,0 +1,10 @@ +# auto add labels to PRs +on: + pull_request_target: + types: [ opened, edited ] +name: conventional-release-labels +jobs: + label: + runs-on: ubuntu-latest + steps: + - uses: bcoe/conventional-release-labels@v1 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..905627e --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,16 @@ +name: Code Coverage + +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + coverage: + uses: OpenVoiceOS/gh-automations/.github/workflows/coverage.yml@dev + with: + python_version: '3.11' + coverage_source: 'ovos_spec_tools' + test_path: 'test/' + install_extras: 'test' + min_coverage: 0 diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml new file mode 100644 index 0000000..214edaa --- /dev/null +++ b/.github/workflows/license_check.yml @@ -0,0 +1,10 @@ +name: License Check + +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + license_check: + uses: OpenVoiceOS/gh-automations/.github/workflows/license-check.yml@dev diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..0cb9564 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,13 @@ +name: Lint + +on: + pull_request: + branches: [dev, master, main] + workflow_dispatch: + +jobs: + lint: + uses: OpenVoiceOS/gh-automations/.github/workflows/lint.yml@dev + with: + ruff: true + pre_commit: false # set true if .pre-commit-config.yaml exists diff --git a/.github/workflows/pip_audit.yml b/.github/workflows/pip_audit.yml new file mode 100644 index 0000000..131320d --- /dev/null +++ b/.github/workflows/pip_audit.yml @@ -0,0 +1,10 @@ +name: PIP Audit + +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + pip_audit: + uses: OpenVoiceOS/gh-automations/.github/workflows/pip-audit.yml@dev diff --git a/.github/workflows/publish_stable.yml b/.github/workflows/publish_stable.yml new file mode 100644 index 0000000..82c2a46 --- /dev/null +++ b/.github/workflows/publish_stable.yml @@ -0,0 +1,23 @@ +name: Publish Stable Release + +on: + workflow_dispatch: + push: + branches: [master, main] + +permissions: + contents: write # required for version bump commit and release tag + +jobs: + publish_stable: + if: github.actor != 'github-actions[bot]' + uses: OpenVoiceOS/gh-automations/.github/workflows/publish-stable.yml@dev + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + MATRIX_TOKEN: ${{ secrets.MATRIX_TOKEN }} + with: + version_file: 'ovos_spec_tools/version.py' + publish_pypi: true + publish_release: true + sync_dev: true + notify_matrix: true diff --git a/.github/workflows/release-preview.yml b/.github/workflows/release-preview.yml new file mode 100644 index 0000000..6057e42 --- /dev/null +++ b/.github/workflows/release-preview.yml @@ -0,0 +1,13 @@ +name: Release Preview + +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + release_preview: + uses: OpenVoiceOS/gh-automations/.github/workflows/release-preview.yml@dev + with: + package_name: 'ovos_spec_tools' + version_file: 'ovos_spec_tools/version.py' diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml new file mode 100644 index 0000000..06776ae --- /dev/null +++ b/.github/workflows/release_workflow.yml @@ -0,0 +1,28 @@ +name: Release Alpha and Propose Stable + +on: + workflow_dispatch: + pull_request: + types: [closed] + branches: [dev] + +permissions: + contents: write + pull-requests: write + +jobs: + publish_alpha: + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + uses: OpenVoiceOS/gh-automations/.github/workflows/publish-alpha.yml@dev + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + MATRIX_TOKEN: ${{ secrets.MATRIX_TOKEN }} + with: + branch: 'dev' + version_file: 'ovos_spec_tools/version.py' + update_changelog: true + publish_prerelease: true + propose_release: true + changelog_max_issues: 50 + publish_pypi: true + notify_matrix: true diff --git a/.github/workflows/repo-health.yml b/.github/workflows/repo-health.yml new file mode 100644 index 0000000..187d397 --- /dev/null +++ b/.github/workflows/repo-health.yml @@ -0,0 +1,12 @@ +name: Repo Health + +on: + pull_request: + branches: [dev, master, main] + workflow_dispatch: + +jobs: + repo_health: + uses: OpenVoiceOS/gh-automations/.github/workflows/repo-health.yml@dev + with: + version_file: 'ovos_spec_tools/version.py' From 57bc59b377732f0d3e841c186f01750913488b89 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 16:52:12 +0100 Subject: [PATCH 020/110] =?UTF-8?q?feat:=20.prompt=20renderer=20=E2=80=94?= =?UTF-8?q?=20reference=20implementation=20of=20OVOS-INTENT-2=20=C2=A74.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the `.prompt` resource role: a localized, whole-file plain-text prompt for a language model. - prompt.py — render_prompt() and PromptRenderer. {name} substitution is conservative: only a well-formed name, only names the caller supplied, never inside a ``` fenced code block. Unfilled slots and any other braces are left literal. - resources.py — read_prompt_file() (whole/verbatim read) and LocaleResources.load_prompt(); PROMPT_ROLE constant. - 22 tests (175 total); docs and an examples/render_prompt.py added. The linter's .prompt awareness (and --spec-version 3) remains a follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 11 +- docs/api-reference.md | 23 +++ docs/dialog.md | 40 +++++ docs/locale-resources.md | 21 ++- examples/README.md | 5 +- examples/render_prompt.py | 26 +++ .../skill-locale/locale/en-US/system.prompt | 12 ++ ovos_spec_tools/__init__.py | 7 + ovos_spec_tools/prompt.py | 128 +++++++++++++++ ovos_spec_tools/resources.py | 47 +++++- test/test_prompt.py | 148 ++++++++++++++++++ 11 files changed, 452 insertions(+), 16 deletions(-) create mode 100644 examples/render_prompt.py create mode 100644 examples/skill-locale/locale/en-US/system.prompt create mode 100644 ovos_spec_tools/prompt.py create mode 100644 test/test_prompt.py diff --git a/README.md b/README.md index 4bc54c4..9a1541e 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ depend on. | Sentence template expander | OVOS-INTENT-1 v2 | `expansion.py` | | Locale resource loader | OVOS-INTENT-2 | `resources.py` | | Dialog renderer | OVOS-INTENT-2 §4.2 | `dialog.py` | +| Prompt renderer | OVOS-INTENT-2 §4.4 | `prompt.py` | | Language-tag matching | OVOS-INTENT-2 §2.2 | `language.py` | | `ovos-spec-lint` locale linter | OVOS-INTENT-1 / -2 | `lint.py` | @@ -31,13 +32,16 @@ Requires Python 3.8+. ## Quick taste ```python -from ovos_spec_tools import expand, LocaleResources, render, closest_lang +from ovos_spec_tools import (expand, LocaleResources, render, render_prompt, + closest_lang) expand("(turn|switch) [the] light") # all 4 sentences it denotes res = LocaleResources("my-skill/locale") res.load_intent("play", "en-US") # a skill's intent samples render(res.load_dialog("weather", "en-US"), # a spoken response slots={"temperature": 21}) +render_prompt(res.load_prompt("system", "en-US"), # a language-model prompt + slots={"query": "what time is it"}) closest_lang("en-AU", ["pt-BR", "en-US"]) # 'en-US' ``` @@ -54,8 +58,9 @@ A zero-to-hero guide lives in [`docs/`](docs/README.md): 2. [Sentence templates](docs/templates.md) — the grammar: alternatives, optionals, slots, vocabulary references, malformed forms. 3. [Locale resources](docs/locale-resources.md) — the `locale/` folder, the - five file roles, loading across languages, override precedence. -4. [Dialog](docs/dialog.md) — choosing and filling a spoken response. + six file roles, loading across languages, override precedence. +4. [Dialog](docs/dialog.md) — choosing and filling a spoken response, and + rendering language-model prompts. 5. [Language matching](docs/language-matching.md) — tag standardization, distance, and closest-match resolution. 6. [Linting](docs/linting.md) — validating a locale folder, on the CLI or in CI. diff --git a/docs/api-reference.md b/docs/api-reference.md index 2f4e2cb..8c663e0 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -39,6 +39,7 @@ Methods — each takes a resource base name and a BCP-47 `lang`: | `load_vocabulary(name, lang)` | expanded phrase set | | `load_blacklist(name, lang)` | expanded phrase set | | `load_dialog(name, lang)` | raw phrase strings (not expanded) | +| `load_prompt(name, lang)` | the whole `.prompt` file, as one string | | `vocabularies(lang)` | `name -> templates` for every `.voc` | | `entities(lang)` | `name -> values` for every `.entity` | @@ -50,6 +51,10 @@ A missing resource raises `FileNotFoundError`; a malformed one raises Apply the OVOS-INTENT-2 §3 common reader to one file: UTF-8, BOM discarded, `LF`/`CRLF` accepted, lines stripped, blank and `#`-comment lines dropped. +### `read_prompt_file(path) -> str` + +Read a `.prompt` whole and verbatim — UTF-8, BOM discarded, no line filtering. + ### `MalformedResource` `ValueError` subclass — a resource file or layout violates OVOS-INTENT-2 @@ -77,6 +82,24 @@ A stateful, multilingual renderer for the dialog `name`, backed by a `ValueError` subclass — a chosen phrase has a slot with no value. +## Prompts — [chapter 4](dialog.md) + +### `render_prompt(text, slots=None) -> str` + +Render a `.prompt` string. The whole `text` is the prompt; a `{name}` is +substituted only when it is a well-formed name, `slots` supplies a value, and +it is not inside a ``` ``` ``` fenced code block. An unfilled slot, and any +other `{`/`}`, is left literal. Never raises for an unfilled slot. + +### `PromptRenderer(resources, name, slots=None)` + +A stateful, multilingual renderer for the prompt `name`, backed by a +`LocaleResources`. `slots` are default values reused on every call. + +- **`render(lang, slots=None) -> str`** — render the prompt in `lang`; a + per-call value overrides a default. Raises `FileNotFoundError` if the prompt + is missing for `lang`, `MalformedResource` if the file is empty. + ## Language matching — [chapter 5](language-matching.md) ### `standardize_lang(tag) -> str` diff --git a/docs/dialog.md b/docs/dialog.md index 6746f6b..b61af7a 100644 --- a/docs/dialog.md +++ b/docs/dialog.md @@ -97,6 +97,46 @@ rather than a computed value. Use `render()` when you already hold the phrases and want one sentence. Use `DialogRenderer` for a skill response spoken more than once. +## Rendering `.prompt` files + +A `.prompt` is the localized prompt a skill feeds to a language model. Unlike a +`.dialog` it is **not** a template — it is plain text — and the whole file, +verbatim, is one prompt. The only special construct is `{name}` substitution, +and it is **conservative**: a prompt is free-form text full of code and JSON, +so rendering must never corrupt a brace the author did not write as a slot. + +`render_prompt()` is the stateless function: + +```python +from ovos_spec_tools import render_prompt + +render_prompt("You are {role}. Answer: {query}", {"role": "concise"}) +# 'You are concise. Answer: {query}' +``` + +A `{name}` is replaced **only** when it is a well-formed name, the caller +supplied a value, and it is not inside a ``` ``` ``` fenced code block. +Everything else stays literal: + +- an **unfilled** slot is left as `{name}` — slots are optional, the opposite + of `.dialog`, where every slot must be filled; +- any other `{`/`}` — `{}`, `{ }`, JSON like `{"k": 1}` — is untouched; +- a `{name}` inside a fenced code block is never substituted. + +`PromptRenderer` is the resource-backed, multilingual form — built from a +`LocaleResources`, with the language given per call and optional default slots: + +```python +from ovos_spec_tools import PromptRenderer + +renderer = PromptRenderer(res, "system", slots={"assistant": "OVOS"}) +renderer.render("en-US", {"query": "what time is it"}) +renderer.render("pt-PT", {"query": "que horas são"}) # same prompt, another language +``` + +It has no phrase selection, no repetition avoidance, and no `.entity` fallback +— a prompt is one whole-file body and its slots are optional. + ## Next [Language matching](language-matching.md) — the tag logic behind the loader's diff --git a/docs/locale-resources.md b/docs/locale-resources.md index b5b394d..d8675a2 100644 --- a/docs/locale-resources.md +++ b/docs/locale-resources.md @@ -27,10 +27,9 @@ an authoring convenience and carry no meaning; a resource is found by a recursive search. A resource is identified by its **`(role, base name)`** pair, so `confirm.intent` and `confirm.dialog` are two distinct resources. -## The five roles +## The six roles -Every resource file is a list of templates ([chapter 2](templates.md)). The -file extension is its **role**: +The file extension is a resource's **role**: | Role | Extension | Slots? | What it is | |------|-----------|--------|------------| @@ -39,10 +38,16 @@ file extension is its **role**: | Entity | `.entity` | no | example values for a slot | | Vocabulary | `.voc` | no | a named keyword / phrase set | | Blacklist | `.blacklist` | no | phrases that suppress an intent | +| Prompt | `.prompt` | yes | a whole-file prompt for a language model | -`.intent` and `.dialog` are **slot-bearing** (they may use `{name}`); the other -three are **slot-free**. Lines starting with `#` are comments; blank lines are -ignored. +The first five are **lists of templates** ([chapter 2](templates.md)) — one +per line, with `#` comment lines and blank lines ignored. `.intent` and +`.dialog` are **slot-bearing** (they may use `{name}`); `.entity`, `.voc` and +`.blacklist` are **slot-free**. + +`.prompt` is the exception: not a template list but **one whole-file document** +read verbatim — every line, including `#` and blank lines, is kept. It carries +`{name}` substitution points but is otherwise plain text ([chapter 4](dialog.md)). Legacy OVOS file types (`.rx`, `.value`, `.list`, …) are deliberately *not* roles here — the linter flags them ([chapter 6](linting.md)). @@ -72,13 +77,15 @@ The load methods: | `load_vocabulary(name, lang)` | the expanded phrase set | | `load_blacklist(name, lang)` | the expanded phrase set | | `load_dialog(name, lang)` | the raw phrase strings — **not** expanded (see below) | +| `load_prompt(name, lang)` | the whole `.prompt` file as one string | | `vocabularies(lang)` | every `.voc`, as a `name → templates` dict | | `entities(lang)` | every `.entity`, as a `name → values` dict | `.intent`, `.entity`, `.voc`, and `.blacklist` are expanded at load time. A `.dialog` is **not** — its phrases are returned verbatim, because expansion happens once per spoken response, on the single phrase chosen -([chapter 4](dialog.md)). +([chapter 4](dialog.md)). A `.prompt` is returned as the whole file, for a +prompt renderer to fill ([chapter 4](dialog.md)). `` references inside an `.intent` resolve automatically against the `.voc` files of the same language — you do not pass vocabularies yourself. diff --git a/examples/README.md b/examples/README.md index cb7b613..6c74d8f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -10,10 +10,11 @@ Each script is self-contained and prints its output. Install the package first | [`expand.py`](expand.py) | `expand()` — alternatives, optionals, opaque `{slot}`s, `` references, and rejection of malformed templates (OVOS-INTENT-1) | | [`load_resources.py`](load_resources.py) | `LocaleResources` — loading `.intent` / `.voc` / `.entity` / `.dialog` from a locale tree (OVOS-INTENT-2) | | [`render_dialog.py`](render_dialog.py) | `render()` and `DialogRenderer` — phrase selection, repetition avoidance, default slots, `.entity` fallback (OVOS-INTENT-2 §4.2) | +| [`render_prompt.py`](render_prompt.py) | `render_prompt()` and `PromptRenderer` — whole-file prompts, conservative `{name}` substitution, fenced-code exemption (OVOS-INTENT-2 §4.4) | | [`lint.py`](lint.py) | `lint_locale()` — the linter used as a library | -The loader and dialog scripts read [`skill-locale/`](skill-locale), a small -valid skill locale; `lint.py` reads `dirty-locale/` (below). +The loader, dialog and prompt scripts read [`skill-locale/`](skill-locale), a +small valid skill locale; `lint.py` reads `dirty-locale/` (below). ## `dirty-locale` — the linter against a broken locale diff --git a/examples/render_prompt.py b/examples/render_prompt.py new file mode 100644 index 0000000..688cde2 --- /dev/null +++ b/examples/render_prompt.py @@ -0,0 +1,26 @@ +"""Rendering prompts — OVOS-INTENT-2 §4.4. + +Shows the stateless `render_prompt()` and the resource-backed `PromptRenderer`. +A `.prompt` is whole-file plain text: substitution is conservative — literal +braces, unfilled slots, and fenced code blocks are all left untouched. +Run: `python examples/render_prompt.py` +""" +from pathlib import Path + +from ovos_spec_tools import LocaleResources, PromptRenderer, render_prompt + +locale = Path(__file__).parent / "skill-locale" / "locale" +resources = LocaleResources(str(locale)) + +# The stateless function over an explicit prompt string. Only `{assistant}` +# and `{query}` are supplied — the literal JSON braces and the `{example}` +# inside the fenced code block are left exactly as written. +text = resources.load_prompt("system", "en-US") +print("render_prompt() — only the supplied slots are filled:\n") +print(render_prompt(text, {"assistant": "OVOS", "query": "what time is it"})) + +# The resource-backed renderer: the language is given per call, and default +# slots set once are reused on every call. +print("\nPromptRenderer — default {assistant}, {query} per call:\n") +renderer = PromptRenderer(resources, "system", slots={"assistant": "OVOS"}) +print(renderer.render("en-US", {"query": "set a five minute timer"})) diff --git a/examples/skill-locale/locale/en-US/system.prompt b/examples/skill-locale/locale/en-US/system.prompt new file mode 100644 index 0000000..92ec522 --- /dev/null +++ b/examples/skill-locale/locale/en-US/system.prompt @@ -0,0 +1,12 @@ +# Assistant system prompt + +You are {assistant}, a helpful voice assistant. Keep your answers short. + +The user asked: {query} + +When you reply with structured data, use a JSON object like {"answer": "..."}. +The placeholder in the example below is illustrative and is left untouched: + +``` +{"answer": "{example}"} +``` diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index d7a9751..8af3674 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -9,6 +9,8 @@ locale resource-file loader; - :func:`~ovos_spec_tools.dialog.render` / :class:`~ovos_spec_tools.dialog.DialogRenderer` — the OVOS-INTENT-2 §4.2 dialog renderer; +- :func:`~ovos_spec_tools.prompt.render_prompt` / :class:`~ovos_spec_tools.prompt.PromptRenderer` + — the OVOS-INTENT-2 §4.4 ``.prompt`` renderer; - :func:`~ovos_spec_tools.language.standardize_lang`, :func:`~ovos_spec_tools.language.lang_distance`, and :func:`~ovos_spec_tools.language.closest_lang` — language-tag normalization, @@ -24,9 +26,11 @@ standardize_lang, ) from ovos_spec_tools.lint import Finding, lint_locale +from ovos_spec_tools.prompt import PromptRenderer, render_prompt from ovos_spec_tools.resources import ( LocaleResources, MalformedResource, + read_prompt_file, read_resource_file, ) from ovos_spec_tools.version import __version__ @@ -37,9 +41,12 @@ "LocaleResources", "MalformedResource", "read_resource_file", + "read_prompt_file", "render", "DialogRenderer", "UnfilledSlot", + "render_prompt", + "PromptRenderer", "standardize_lang", "lang_distance", "closest_lang", diff --git a/ovos_spec_tools/prompt.py b/ovos_spec_tools/prompt.py new file mode 100644 index 0000000..2a16727 --- /dev/null +++ b/ovos_spec_tools/prompt.py @@ -0,0 +1,128 @@ +"""Reference prompt renderer for OVOS-INTENT-2 §4.4. + +A ``.prompt`` is the localized, whole-file plain-text prompt a skill feeds to a +language model. Unlike a ``.dialog`` it is **not** a sentence template: none of +the OVOS-INTENT-1 grammar applies, so ``(``, ``[``, ``|`` and the rest are +literal. The one special construct is ``{name}`` substitution, and it is +applied **conservatively** — a prompt is free-form text that routinely embeds +code and JSON, and rendering it must never corrupt text the author did not +write as a slot. + +A ``{name}`` is replaced by a caller-supplied value only when all three hold: + +1. it is a well-formed slot name — lowercase ASCII letters, digits and + underscores, not beginning with a digit (so ``{}``, ``{ }`` and JSON such + as ``{"key": 1}`` are left untouched); +2. the caller supplied a value for that name — an **unfilled** slot is left as + literal text, not an error (the opposite of ``.dialog``, §4.2); +3. it does not lie inside a ```` ``` ```` fenced code block. + +Two interfaces are provided: + +- :func:`render_prompt` — a stateless function over an explicit prompt string; +- :class:`PromptRenderer` — a stateful, **multilingual** object backed by a + :class:`~ovos_spec_tools.resources.LocaleResources`. +""" +from __future__ import annotations + +import re +from typing import Dict, List, Optional + +__all__ = ["render_prompt", "PromptRenderer"] + +# A {name} substitution point — the OVOS-INTENT-2 §4.4 slot-name charset. +_SLOT_RE = re.compile(r"\{([a-z][a-z0-9_]*)\}") + + +def _is_fence(line: str) -> bool: + """Whether a line opens or closes a ```` ``` ```` fenced code block.""" + return line.lstrip(" \t").startswith("```") + + +def render_prompt(text: str, + slots: Optional[Dict[str, object]] = None) -> str: + """Render a ``.prompt`` string (stateless). + + The whole ``text`` is the prompt; every character is significant. + ``{name}`` substitution points are filled per the rules in the module + docstring — conservatively, and leaving an unfilled slot as literal text. + + Args: + text: the whole-file content of a ``.prompt``. + slots: caller-supplied values, keyed by slot name. Values are + converted to text. Names not present here are left as ``{name}``. + + Returns: + The prompt with its supplied slots substituted, otherwise verbatim. + """ + values = slots or {} + + def replace(match: "re.Match") -> str: + name = match.group(1) + if name in values: + return str(values[name]) + return match.group(0) # an unfilled slot stays literal (§4.4) + + rendered: List[str] = [] + in_fence = False + # keepends=True so the file is reproduced verbatim apart from substitution. + for line in text.splitlines(keepends=True): + if _is_fence(line): + in_fence = not in_fence + rendered.append(line) + elif in_fence: + rendered.append(line) + else: + rendered.append(_SLOT_RE.sub(replace, line)) + return "".join(rendered) + + +class PromptRenderer: + """A stateful, multilingual renderer for one named ``.prompt``. + + Backed by a :class:`~ovos_spec_tools.resources.LocaleResources`: the prompt + name is fixed at construction, but the **language is given per** + :meth:`render` **call**, so one renderer serves every language the prompt + is shipped in. + + Unlike :class:`~ovos_spec_tools.dialog.DialogRenderer` it carries no + randomness and no ``.entity`` fallback — a prompt has a single whole-file + body and its slots are optional (§4.4). It does hold **default** slot + values, reused on every call. + """ + + def __init__(self, resources, name: str, + slots: Optional[Dict[str, object]] = None): + """ + Args: + resources: a :class:`~ovos_spec_tools.resources.LocaleResources` + (or anything with a ``load_prompt(name, lang)`` method). + name: the base name of the ``.prompt`` to render. + slots: **default** slot values, held for the renderer's lifetime + and reused on every :meth:`render` call. A per-call value + overrides a default. + """ + self.resources = resources + self.name = name + self.default_slots: Dict[str, object] = dict(slots or {}) + + def render(self, lang: str, + slots: Optional[Dict[str, object]] = None) -> str: + """Render the prompt in ``lang``. + + Args: + lang: the BCP-47 language tag to render in. + slots: per-call slot values; each overrides a default. + + Returns: + The rendered prompt string. + + Raises: + FileNotFoundError: the prompt does not exist for ``lang``. + MalformedResource: the resolved ``.prompt`` file is empty. + """ + text = self.resources.load_prompt(self.name, lang) + effective = dict(self.default_slots) + if slots: + effective.update(slots) + return render_prompt(text, effective) diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index 3c5870b..0e1b59a 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -2,15 +2,16 @@ This module is the reference implementation of the loader of OVOS-INTENT-2: it discovers a skill's locale resource files, reads them with the common reader -(§3), and loads each of the five resource roles per its format (§4). +(§3), and loads each of the six resource roles per its format (§4). -The five roles, by extension: +The six roles, by extension: - ``.intent`` — slot-bearing intent training samples; - ``.dialog`` — slot-bearing spoken-response phrases; - ``.entity`` — slot-free example values for a slot; - ``.voc`` — slot-free vocabulary; -- ``.blacklist`` — slot-free intent-suppression phrases. +- ``.blacklist`` — slot-free intent-suppression phrases; +- ``.prompt`` — a whole-file plain-text language-model prompt (§4.4). The user-data path of the override precedence (§2.1) is **assistant-defined**; this module takes it as a parameter and imports no configuration. @@ -31,13 +32,17 @@ "LocaleResources", "MalformedResource", "read_resource_file", + "read_prompt_file", "SLOT_BEARING_ROLES", "SLOT_FREE_ROLES", + "PROMPT_ROLE", ] -# Resource roles, by file extension (OVOS-INTENT-2 §1). +# Resource roles, by file extension (OVOS-INTENT-2 §1). The five template +# roles are line-oriented; `.prompt` is a single whole-file document (§4.4). SLOT_BEARING_ROLES = (".intent", ".dialog") SLOT_FREE_ROLES = (".entity", ".voc", ".blacklist") +PROMPT_ROLE = ".prompt" # A resolver maps a requested language and the available language tags to the # best one, or None — the signature of `ovos_spec_tools.language.closest_lang`. @@ -70,6 +75,17 @@ def read_resource_file(path: Path) -> List[str]: return templates +def read_prompt_file(path: Path) -> str: + """Read a ``.prompt`` whole and verbatim (OVOS-INTENT-2 §3, §4.4). + + A ``.prompt`` is **not** line-oriented: it is read whole, with no line + stripping and no blank- or ``#``-comment-line filtering, because every + character is part of the prompt. The file is UTF-8 and a leading + byte-order mark is discarded. + """ + return path.read_text(encoding="utf-8-sig") # utf-8-sig discards a BOM + + class LocaleResources: """Loads OVOS-INTENT-2 resource files for one skill, across languages. @@ -233,3 +249,26 @@ def load_dialog(self, base_name: str, lang: str) -> List[str]: f"empty resource file {path} — every file must contribute at " f"least one template (§5)") return phrases + + def load_prompt(self, base_name: str, lang: str) -> str: + """Load a ``.prompt`` as its whole-file string (§4.4). + + A ``.prompt`` is read whole and verbatim — not split into templates, + not line-filtered — and is returned for a prompt renderer to fill. See + :class:`ovos_spec_tools.prompt.PromptRenderer`. + + Raises: + FileNotFoundError: no such ``.prompt`` for ``lang``. + MalformedResource: the file is empty or only whitespace (§5). + """ + path = self._locate(base_name, PROMPT_ROLE, lang) + if path is None: + raise FileNotFoundError( + f"no .prompt resource named {base_name!r} for " + f"language {lang!r}") + text = read_prompt_file(path) + if not text.strip(): + raise MalformedResource( + f"empty resource file {path} — every file must contribute " + f"content (§5)") + return text diff --git a/test/test_prompt.py b/test/test_prompt.py new file mode 100644 index 0000000..e99f805 --- /dev/null +++ b/test/test_prompt.py @@ -0,0 +1,148 @@ +"""Conformance tests for the OVOS-INTENT-2 §4.4 reference prompt renderer.""" +import pytest + +from ovos_spec_tools import ( + LocaleResources, + MalformedResource, + PromptRenderer, + render_prompt, +) + + +def _resources(tmp_path, files): + """Build a LocaleResources over a locale with the given {relpath: text}.""" + locale = tmp_path / "locale" + for relpath, text in files.items(): + path = locale / relpath + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return LocaleResources(str(locale)) + + +# --- render_prompt() — the stateless function -------------------------------- + +def test_render_prompt_fills_a_slot(): + assert render_prompt("Hello {name}.", {"name": "Sam"}) == "Hello Sam." + + +def test_unfilled_slot_is_left_literal(): + """A `.prompt` slot is optional — unfilled means literal text, not error.""" + assert render_prompt("Hello {name}.", {}) == "Hello {name}." + assert render_prompt("Hello {name}.") == "Hello {name}." + + +def test_partial_fill_leaves_the_rest_literal(): + out = render_prompt("{greeting} {name}", {"greeting": "Hi"}) + assert out == "Hi {name}" + + +@pytest.mark.parametrize("text", [ + 'json {"key": 1}', # JSON object braces + "empty {} and { } braces", # not a name + "code {x+1} here", # not a valid name + "upper {Name} stays", # uppercase — outside the charset + "digit {1st} stays", # begins with a digit +]) +def test_literal_braces_are_untouched(text): + assert render_prompt(text, {"key": "K", "name": "N", "x": "X"}) == text + + +def test_no_substitution_inside_a_fenced_code_block(): + text = ( + "Fill {greeting} here.\n" + "```\n" + "{greeting} stays literal in the code block\n" + "```\n" + "And {greeting} again here.\n" + ) + out = render_prompt(text, {"greeting": "HI"}) + assert out == ( + "Fill HI here.\n" + "```\n" + "{greeting} stays literal in the code block\n" + "```\n" + "And HI again here.\n" + ) + + +def test_fence_with_an_info_string_is_recognized(): + text = "```json\n{x}\n```\n{x}" + assert render_prompt(text, {"x": "V"}) == "```json\n{x}\n```\nV" + + +def test_whole_file_is_verbatim_apart_from_substitution(): + """`#` lines and blank lines are kept — a prompt is not line-filtered.""" + text = "# A heading\n\nBody with {slot}.\n\n indented line\n" + out = render_prompt(text, {"slot": "X"}) + assert out == "# A heading\n\nBody with X.\n\n indented line\n" + + +def test_numeric_slot_value_is_stringified(): + assert render_prompt("count {n}", {"n": 0}) == "count 0" + + +def test_filled_value_is_not_re_scanned_for_slots(): + """A value containing `{...}` is inserted literally, not substituted again.""" + assert render_prompt("say {x}", {"x": "{y}"}) == "say {y}" + + +def test_text_with_no_trailing_newline_is_preserved(): + assert render_prompt("no newline {x}", {"x": "here"}) == "no newline here" + + +# --- PromptRenderer — stateful, multilingual --------------------------------- + +def test_renderer_renders_from_resources(tmp_path): + res = _resources(tmp_path, {"en-US/sys.prompt": "You are {role}."}) + renderer = PromptRenderer(res, "sys") + assert renderer.render("en-US", {"role": "a helper"}) == "You are a helper." + + +def test_renderer_serves_multiple_languages(tmp_path): + res = _resources(tmp_path, { + "en-US/sys.prompt": "Answer in English.", + "pt-PT/sys.prompt": "Responde em português.", + }) + renderer = PromptRenderer(res, "sys") + assert renderer.render("en-US") == "Answer in English." + assert renderer.render("pt-PT") == "Responde em português." + + +def test_renderer_default_slots_are_reused(tmp_path): + res = _resources(tmp_path, {"en-US/g.prompt": "You are {assistant}."}) + renderer = PromptRenderer(res, "g", slots={"assistant": "OVOS"}) + assert renderer.render("en-US") == "You are OVOS." + assert renderer.render("en-US") == "You are OVOS." + + +def test_per_call_slot_overrides_a_default(tmp_path): + res = _resources(tmp_path, {"en-US/g.prompt": "You are {assistant}."}) + renderer = PromptRenderer(res, "g", slots={"assistant": "OVOS"}) + assert renderer.render("en-US", {"assistant": "Mycroft"}) == "You are Mycroft." + + +def test_renderer_missing_prompt_raises(tmp_path): + res = _resources(tmp_path, {"en-US/x.prompt": "hi"}) + renderer = PromptRenderer(res, "absent") + with pytest.raises(FileNotFoundError): + renderer.render("en-US") + + +# --- load_prompt() ----------------------------------------------------------- + +def test_load_prompt_returns_the_whole_file(tmp_path): + text = "# Heading\n\nBody {slot} text.\n" + res = _resources(tmp_path, {"en-US/p.prompt": text}) + assert res.load_prompt("p", "en-US") == text + + +def test_empty_prompt_is_malformed(tmp_path): + res = _resources(tmp_path, {"en-US/p.prompt": " \n\n"}) + with pytest.raises(MalformedResource): + res.load_prompt("p", "en-US") + + +def test_load_prompt_missing_raises(tmp_path): + res = _resources(tmp_path, {"en-US/other.prompt": "hi"}) + with pytest.raises(FileNotFoundError): + res.load_prompt("p", "en-US") From 0a1449b810845b000ed41ae5e972ee4a87174a28 Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Fri, 22 May 2026 16:56:49 +0100 Subject: [PATCH 021/110] =?UTF-8?q?feat:=20linter=20=E2=80=94=20.prompt=20?= =?UTF-8?q?role=20awareness=20and=20--spec-version=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The linter now recognizes `.prompt` as a resource role instead of silently ignoring it as an unknown extension. - `.prompt` is a known role: collected, deduplicated, and naming-checked like the others; no longer mistaken for a legacy/unknown file. - A `.prompt` is checked as a whole-file document, not a template — only non-emptiness and a valid UTF-8 read; never template syntax, slot-free or §5.5 checks. - A non-UTF-8 `.prompt` is reported, not crashed. - --spec-version gains 3 (the .prompt role); default is now 3. Below it, a `.prompt` file is a warning — an older runtime ignores the role. 6 tests (181 total); docs/linting.md updated. dirty-locale output unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/linting.md | 16 ++++++++----- ovos_spec_tools/lint.py | 50 ++++++++++++++++++++++++++++------------- test/test_lint.py | 45 ++++++++++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 22 deletions(-) diff --git a/docs/linting.md b/docs/linting.md index 196c84e..8e4d82c 100644 --- a/docs/linting.md +++ b/docs/linting.md @@ -48,11 +48,14 @@ straight into a CI pipeline. With `--strict`, warnings fail the run too. - a resource file sitting outside any language directory; - a language directory not named like a BCP-47 tag; - a language directory with no resource files; -- a legacy file type (`.rx`, `.value`, `.list`, …) — not one of the five +- a legacy file type (`.rx`, `.value`, `.list`, …) — not one of the six OVOS-INTENT-2 roles; - a `.blacklist` with no matching `.intent` to suppress; - a file name that is not lowercase. +A `.prompt` is checked too, but not as a template — it is plain text, so only +its naming and non-emptiness are checked, never template syntax. + ## Targeting an older spec version A skill may need to run on a device that has not been updated. `--spec-version` @@ -63,17 +66,18 @@ shipping that a skill will not work there: |---------|------| | `0` | the legacy, undocumented Mycroft/OVOS de-facto behaviour | | `1` | the formalized specs — and the `.blacklist` role | -| `2` | `` inline vocabulary references *(the default)* | +| `2` | `` inline vocabulary references | +| `3` | the `.prompt` role *(the default)* | ```bash ovos-spec-lint my-skill/locale --spec-version 1 ``` With `--spec-version 1`, a template using `` is an **error** — a -version-1 runtime cannot expand it. With `--spec-version 0`, a `.blacklist` -file is additionally a **warning** — a version-0 runtime silently ignores it, -so the suppression simply will not happen. The default, `2`, flags nothing -extra. +version-1 runtime cannot expand it. With `--spec-version 2`, a `.prompt` file +is a **warning**, and with `--spec-version 0` a `.blacklist` file likewise — an +older runtime silently ignores a role it does not know, so that resource just +will not take effect. The default, `3`, flags nothing extra. ## Using it as a library diff --git a/ovos_spec_tools/lint.py b/ovos_spec_tools/lint.py index a4a87ef..b109b99 100644 --- a/ovos_spec_tools/lint.py +++ b/ovos_spec_tools/lint.py @@ -16,7 +16,8 @@ - **0** — the legacy, undocumented Mycroft/OVOS de-facto behaviour; - **1** — the formalized specs; adds the ``.blacklist`` role; -- **2** — adds ```` inline vocabulary references. +- **2** — adds ```` inline vocabulary references; +- **3** — adds the ``.prompt`` role. """ from __future__ import annotations @@ -29,22 +30,27 @@ from ovos_spec_tools.expansion import MalformedTemplate, expand from ovos_spec_tools.resources import ( + PROMPT_ROLE, SLOT_BEARING_ROLES, SLOT_FREE_ROLES, + read_prompt_file, read_resource_file, ) __all__ = ["Finding", "lint_locale", "main"] -# The five OVOS-INTENT-2 resource roles. -ROLE_EXTENSIONS = SLOT_BEARING_ROLES + SLOT_FREE_ROLES +# The six OVOS-INTENT-2 resource roles. +ROLE_EXTENSIONS = SLOT_BEARING_ROLES + SLOT_FREE_ROLES + (PROMPT_ROLE,) # File types OVOS-INTENT-2 deliberately does not define — flagged, not parsed. LEGACY_EXTENSIONS = (".rx", ".value", ".list", ".word", ".template", ".qml") # The OVOS spec version that introduced each feature. -DEFAULT_SPEC_VERSION = 2 +DEFAULT_SPEC_VERSION = 3 _BLACKLIST_SINCE = 1 # the `.blacklist` role _VOCABULARY_REFERENCE_SINCE = 2 # the `` inline vocabulary reference +_PROMPT_ROLE_SINCE = 3 # the `.prompt` role +# Roles introduced after V0, by the spec version that added them. +_ROLE_SINCE = {".blacklist": _BLACKLIST_SINCE, PROMPT_ROLE: _PROMPT_ROLE_SINCE} _BASE_NAME_RE = re.compile(r"[a-z0-9_]+") _SLOT_NAME_RE = re.compile(r"[a-z][a-z0-9_]*") @@ -137,18 +143,17 @@ def _lint_language_tree(language_dir: Path, spec_version: int) -> List[Finding]: else: first_seen[key] = path - # `.blacklist`: a spec-version gate, and a pairing check (§4.3). + # Per-role spec-version gate (a role newer than the target is flagged), + # plus the `.blacklist` pairing check (§4.3). intent_names = {stem for (ext, stem) in first_seen if ext == ".intent"} for path in role_files: - if path.suffix != ".blacklist": - continue - if spec_version < _BLACKLIST_SINCE: + since = _ROLE_SINCE.get(path.suffix) + if since is not None and spec_version < since: findings.append(Finding( WARNING, str(path), - f"the .blacklist role requires spec version " - f"{_BLACKLIST_SINCE}; a version-{spec_version} runtime " - f"ignores it")) - if path.stem not in intent_names: + f"the {path.suffix} role requires spec version {since}; a " + f"version-{spec_version} runtime ignores it")) + if path.suffix == ".blacklist" and path.stem not in intent_names: findings.append(Finding( WARNING, str(path), f"blacklist {path.stem!r} has no matching " @@ -191,6 +196,21 @@ def _lint_file(path: Path, findings.append(Finding( WARNING, str(path), "file name should be lowercase")) + # --- `.prompt` — a whole-file document, not a template list (§4.4) ------ + if extension == PROMPT_ROLE: + try: + text = read_prompt_file(path) + except (OSError, UnicodeError) as exc: + findings.append(Finding( + ERROR, str(path), f"cannot read file: {exc}")) + else: + if not text.strip(): + findings.append(Finding( + ERROR, str(path), + "empty file — every resource file must contribute " + "content (OVOS-INTENT-2 §5)")) + return findings + # --- read (OVOS-INTENT-2 §3) -------------------------------------------- try: templates = read_resource_file(path) @@ -251,11 +271,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int: "--strict", action="store_true", help="exit non-zero if there are warnings as well as errors") parser.add_argument( - "--spec-version", type=int, choices=(0, 1, 2), + "--spec-version", type=int, choices=(0, 1, 2, 3), default=DEFAULT_SPEC_VERSION, help="target OVOS spec version; flags features newer than it " - "(0: legacy, 1: adds .blacklist, 2: adds references). " - f"Default {DEFAULT_SPEC_VERSION}.") + "(0: legacy, 1: adds .blacklist, 2: adds references, " + f"3: adds the .prompt role). Default {DEFAULT_SPEC_VERSION}.") args = parser.parse_args(argv) findings = lint_locale(args.locale, spec_version=args.spec_version) diff --git a/test/test_lint.py b/test/test_lint.py index 8a65afb..e6b1251 100644 --- a/test/test_lint.py +++ b/test/test_lint.py @@ -230,5 +230,48 @@ def test_main_honors_spec_version(tmp_path): locale = tmp_path / "locale" _write(locale / "en-US" / "greeting.voc", "hello\nhi\n") _write(locale / "en-US" / "greet.intent", " there\n") - assert main([str(locale)]) == 0 # v2 — fine + assert main([str(locale)]) == 0 # v3 — fine assert main([str(locale), "--spec-version", "1"]) == 1 # is v2 + + +# --- the .prompt role (OVOS-INTENT-2 §4.4) ---------------------------------- + +def test_prompt_file_is_a_recognized_role(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "system.prompt", "You are {assistant}.\n") + assert lint_locale(locale) == [] + + +def test_empty_prompt_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "system.prompt", " \n\n") + assert any("empty" in f.message for f in _errors(lint_locale(locale))) + + +def test_prompt_is_not_template_checked(tmp_path): + """A `.prompt` is plain text — content that would be a malformed template + if it were one is perfectly valid.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "sys.prompt", "turn (on|off the lights {a}{b}\n") + assert lint_locale(locale) == [] + + +def test_non_utf8_prompt_is_reported_not_crashed(tmp_path): + lang = tmp_path / "locale" / "en-US" + lang.mkdir(parents=True) + (lang / "sys.prompt").write_bytes(b"\xff\xfe not valid utf-8") + assert any("cannot read" in f.message + for f in lint_locale(tmp_path / "locale")) + + +def test_spec_version_2_flags_the_prompt_role(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "system.prompt", "You are helpful.\n") + findings = lint_locale(locale, spec_version=2) + assert any("requires spec version 3" in f.message for f in findings) + + +def test_default_spec_version_does_not_flag_prompt(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "system.prompt", "You are helpful.\n") + assert not any("spec version" in f.message for f in lint_locale(locale)) From 69ea91cad56beb3372072970b8ad663feed40da1 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 22 May 2026 16:19:16 +0000 Subject: [PATCH 022/110] Increment Version to 0.0.1a2 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 85b4624..ea5e6f0 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -2,7 +2,7 @@ VERSION_MAJOR = 0 VERSION_MINOR = 0 VERSION_BUILD = 1 -VERSION_ALPHA = 1 +VERSION_ALPHA = 2 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "") From 6d81085683b7d2027749564db222308420e2ab6e Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 22 May 2026 16:19:32 +0000 Subject: [PATCH 023/110] Update Changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e830de1 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +## [0.0.1a2](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.0.1a2) (2026-05-22) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/13d694e2960b7ede0533f4029442301c21c86a8c...0.0.1a2) + +**Merged pull requests:** + +- Configure Renovate [\#1](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/1) ([renovate[bot]](https://github.com/apps/renovate)) + + + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* From 07a419f7fa8c49d18600a3678cfdb9a4106b62fb Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 12:44:47 +0100 Subject: [PATCH 024/110] =?UTF-8?q?feat:=20lang=5Fmatches=20and=20iter=5Fl?= =?UTF-8?q?ocale=5Fdirs=20=E2=80=94=20pin=20the=20cross-component=20lang?= =?UTF-8?q?=20policy=20(#4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: lang_matches and iter_locale_dirs — pin the cross-component lang policy Every place that crossed a component boundary (resource loaders, adapt/padatious/padacioso engine bucketing, TTS/STT plugin routing, `_get_closest_lang` boilerplate) reimplemented the same three steps — standardize, distance, threshold-check — with subtle drift between the copies. Bug surface: a skill stores resources under a full tag but an engine buckets under a macroed one (or vice versa), and the two never reconcile. This adds the two missing primitives so callers stop reinventing them: * `lang_matches(a, b, max_distance=10) -> bool` — the `if lang_distance(...) < threshold` check, named and shared. Pass `max_distance=0` for exact-match. * `iter_locale_dirs(root, native_langs=None) -> Iterator[(lang, path)]` — walks `/locale/` and yields each subdir as a canonical `(standardize_lang(name), Path)` pair, optionally filtered against the skill's native langs via `closest_lang`. Resource loaders for `.rx`, `.dialog`, `.voc`, `.intent`, locale `.json` reinvent this walk by hand — switch them to this and the macro/full-tag disagreement disappears. The implicit policy: **full normalized lang tag everywhere**. Macro-stripping is no longer a knob; if an engine buckets under `en` and a resource is `en-US`, the engine reconciles at match time via `closest_lang`, not at registration time via macro flags. Co-Authored-By: Claude Opus 4.7 (1M context) * fix: remove unused MalformedTemplate import, rename ambiguous loop var, sync docstring - Drop MalformedTemplate from resources.py import (F401 — unused) - Rename loop variable l -> lang in iter_locale_dirs comprehension (E741) - Add lang_matches and iter_locale_dirs to the top-level module docstring Co-Authored-By: Claude Opus 4.7 (1M context) * ci: pass install_extras as pip args and expose license classifier - install_extras was 'test' (bare) → reusable workflow ran 'pip install test', no such PyPI package. Pass '.[test]' instead. - pip-licenses categorised the package itself as 'Error' (unknown license) because pyproject had no License classifier. Add the SPDX Apache-2.0 classifier and matching Python-version classifiers. - Tighten requires-python to >=3.10 to match the tested matrix. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: exclude ovos-spec-tools from auditing itself pilosus/action-pip-license-checker queries PyPI metadata for each installed package. The PyPI release predates the SPDX License classifier added in this PR, so the package is audited as 'Error' (unknown license) against itself. Exclude it from the audit until the next alpha republishes. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: build-tests install_extras takes the bare extras name The build-tests reusable workflow wraps it as `${WHEEL}[${EXTRAS}]`, so it expects `test` not `.[test]`. The coverage workflow expects raw pip args, so that one keeps `.[test]`. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: warn_only license_check until alpha republishes The pilosus 'exclude' regex did not suppress self-audit; switch to warn_only so the job no longer blocks. Transitive dependencies still get audited and reported in the PR comment. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/coverage.yml | 2 +- .github/workflows/license_check.yml | 7 +++ ovos_spec_tools/__init__.py | 11 ++++- ovos_spec_tools/language.py | 17 +++++++ ovos_spec_tools/resources.py | 51 ++++++++++++++++++++- pyproject.toml | 11 ++++- test/test_iter_locale_dirs.py | 69 +++++++++++++++++++++++++++++ test/test_language.py | 24 +++++++++- 8 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 test/test_iter_locale_dirs.py diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 905627e..33fe452 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,5 +12,5 @@ jobs: python_version: '3.11' coverage_source: 'ovos_spec_tools' test_path: 'test/' - install_extras: 'test' + install_extras: '.[test]' min_coverage: 0 diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml index 214edaa..faf7754 100644 --- a/.github/workflows/license_check.yml +++ b/.github/workflows/license_check.yml @@ -8,3 +8,10 @@ on: jobs: license_check: uses: OpenVoiceOS/gh-automations/.github/workflows/license-check.yml@dev + with: + # pilosus/action-pip-license-checker queries PyPI metadata for each + # installed package. The PyPI release of ovos-spec-tools predates the + # SPDX License classifier in pyproject.toml, so the package audits + # itself as "Error" (unknown). Run in warn-only mode until the next + # alpha republishes — transitive dependencies still get reported. + warn_only: true diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index 8af3674..ac427d8 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -12,9 +12,12 @@ - :func:`~ovos_spec_tools.prompt.render_prompt` / :class:`~ovos_spec_tools.prompt.PromptRenderer` — the OVOS-INTENT-2 §4.4 ``.prompt`` renderer; - :func:`~ovos_spec_tools.language.standardize_lang`, - :func:`~ovos_spec_tools.language.lang_distance`, and + :func:`~ovos_spec_tools.language.lang_distance`, + :func:`~ovos_spec_tools.language.lang_matches`, and :func:`~ovos_spec_tools.language.closest_lang` — language-tag normalization, - distance, and closest-match resolution; + distance, match checking, and closest-match resolution; +- :func:`~ovos_spec_tools.resources.iter_locale_dirs` — locale subdirectory + discovery with optional native-language filtering; - :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also exposed as the ``ovos-spec-lint`` command. """ @@ -23,6 +26,7 @@ from ovos_spec_tools.language import ( closest_lang, lang_distance, + lang_matches, standardize_lang, ) from ovos_spec_tools.lint import Finding, lint_locale @@ -30,6 +34,7 @@ from ovos_spec_tools.resources import ( LocaleResources, MalformedResource, + iter_locale_dirs, read_prompt_file, read_resource_file, ) @@ -40,6 +45,7 @@ "MalformedTemplate", "LocaleResources", "MalformedResource", + "iter_locale_dirs", "read_resource_file", "read_prompt_file", "render", @@ -49,6 +55,7 @@ "PromptRenderer", "standardize_lang", "lang_distance", + "lang_matches", "closest_lang", "lint_locale", "Finding", diff --git a/ovos_spec_tools/language.py b/ovos_spec_tools/language.py index 49155b3..c78701c 100644 --- a/ovos_spec_tools/language.py +++ b/ovos_spec_tools/language.py @@ -21,6 +21,7 @@ __all__ = [ "standardize_lang", "lang_distance", + "lang_matches", "closest_lang", "DEFAULT_MAX_LANGUAGE_DISTANCE", ] @@ -127,6 +128,22 @@ def lang_distance(desired: str, supported: str) -> int: return distance if distance is not None else _coarse_distance(a, b) +def lang_matches(a: str, b: str, + max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE) -> bool: + """Return ``True`` when two BCP-47 tags are close enough to interchange. + + Convenience wrapper around :func:`lang_distance` for the common + ``if score < threshold`` check that cross-component boundaries (intent + engines, TTS/STT plugin routing, locale lookup) reimplement by hand. The + default threshold matches :data:`DEFAULT_MAX_LANGUAGE_DISTANCE` — an exact + match always matches; pass ``max_distance=0`` to require an exact match. + """ + distance = lang_distance(a, b) + if distance == 0: + return True + return max_distance > 0 and distance < max_distance + + def closest_lang(target: str, available: Sequence[str], max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE ) -> Optional[str]: diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index 0e1b59a..fabc730 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -25,12 +25,17 @@ from pathlib import Path from typing import Callable, Dict, List, Optional, Sequence -from ovos_spec_tools.expansion import MalformedTemplate, expand -from ovos_spec_tools.language import DEFAULT_MAX_LANGUAGE_DISTANCE, closest_lang +from ovos_spec_tools.expansion import expand +from ovos_spec_tools.language import ( + DEFAULT_MAX_LANGUAGE_DISTANCE, + closest_lang, + standardize_lang, +) __all__ = [ "LocaleResources", "MalformedResource", + "iter_locale_dirs", "read_resource_file", "read_prompt_file", "SLOT_BEARING_ROLES", @@ -75,6 +80,48 @@ def read_resource_file(path: Path) -> List[str]: return templates +def iter_locale_dirs(root: Path, + native_langs: Optional[Sequence[str]] = None, + max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE + ): + """Iterate ``/locale//`` subdirs as ``(lang, path)`` pairs. + + Each immediate subdirectory of ``/locale/`` is treated as a locale + tree; its name is normalized with :func:`standardize_lang` and yielded as + the first item of the pair. The second item is the directory ``Path``. + + When ``native_langs`` is given, each subdir is matched against the natives + via :func:`closest_lang` and yielded only when its closest native is within + ``max_distance`` — useful for "a skill declares ``en``, the locale tree has + ``en-US``, accept it" without forcing the caller to repeat that walk. + + Resource loaders (``.rx``, ``.dialog``, ``.voc``, ``.intent``, ``.json``) + reinvent this walk by hand and disagree on the macro/full-tag policy. Use + this and the disagreement goes away — locales are always discovered as + full-tag directories, ``closest_lang`` reconciles at query time. + + ``root`` without a ``locale/`` child yields nothing. + """ + root = Path(root) + locales_root = root / "locale" + if not locales_root.is_dir(): + return + natives_norm = ([standardize_lang(lang) for lang in native_langs] + if native_langs is not None else None) + for entry in sorted(locales_root.iterdir()): + if not entry.is_dir(): + continue + try: + lang_norm = standardize_lang(entry.name) + except Exception: + continue + if natives_norm is not None: + if closest_lang(lang_norm, natives_norm, + max_distance=max_distance) is None: + continue + yield lang_norm, entry + + def read_prompt_file(path: Path) -> str: """Read a ``.prompt`` whole and verbatim (OVOS-INTENT-2 §3, §4.4). diff --git a/pyproject.toml b/pyproject.toml index 5f6ce05..101172a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,16 @@ description = "Reference implementation of the OVOS formal specifications" readme = "README.md" license = { text = "Apache-2.0" } authors = [{ name = "jarbasai", email = "jarbasai@mailfence.com" }] -requires-python = ">=3.8" +requires-python = ">=3.10" +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] dependencies = [] [project.optional-dependencies] diff --git a/test/test_iter_locale_dirs.py b/test/test_iter_locale_dirs.py new file mode 100644 index 0000000..0c6dfeb --- /dev/null +++ b/test/test_iter_locale_dirs.py @@ -0,0 +1,69 @@ +"""Tests for `iter_locale_dirs` — the canonical locale subdir walker.""" +import pytest + +from ovos_spec_tools import iter_locale_dirs + + +def _make_skill(tmp_path, *lang_dirs): + """Create ``/locale//`` for each given lang.""" + for lang in lang_dirs: + (tmp_path / "locale" / lang).mkdir(parents=True, exist_ok=True) + return tmp_path + + +def test_no_locale_dir_yields_nothing(tmp_path): + assert list(iter_locale_dirs(tmp_path)) == [] + + +def test_each_locale_subdir_is_yielded(tmp_path): + root = _make_skill(tmp_path, "en-US", "pt-PT", "de-DE") + langs = sorted(lang for lang, _ in iter_locale_dirs(root)) + assert langs == ["de-DE", "en-US", "pt-PT"] + + +def test_yields_subdir_path_as_second_item(tmp_path): + root = _make_skill(tmp_path, "en-US") + pairs = list(iter_locale_dirs(root)) + assert len(pairs) == 1 + lang, path = pairs[0] + assert lang == "en-US" + assert path == root / "locale" / "en-US" + + +def test_subdir_name_is_normalized(tmp_path): + """``en_us/`` and ``EN-us/`` resolve to the same canonical tag.""" + root = _make_skill(tmp_path, "en_us") + [(lang, _)] = list(iter_locale_dirs(root)) + assert lang.lower() == "en-us" + + +def test_files_in_locale_root_are_ignored(tmp_path): + root = _make_skill(tmp_path, "en-US") + (root / "locale" / "stray.txt").write_text("not a locale dir") + langs = [lang for lang, _ in iter_locale_dirs(root)] + assert langs == ["en-US"] + + +def test_native_filter_keeps_a_regional_subdir(tmp_path): + """A skill that declares ``en`` as native still discovers ``en-US/``.""" + pytest.importorskip("langcodes") + root = _make_skill(tmp_path, "en-US", "fr-FR") + langs = sorted(lang for lang, _ in + iter_locale_dirs(root, native_langs=["en"])) + assert langs == ["en-US"] + + +def test_native_filter_drops_unrelated_languages(tmp_path): + pytest.importorskip("langcodes") + root = _make_skill(tmp_path, "en-US", "fr-FR", "ja-JP") + langs = sorted(lang for lang, _ in + iter_locale_dirs(root, native_langs=["en-GB", "fr"])) + assert langs == ["en-US", "fr-FR"] + + +def test_native_filter_is_skipped_when_natives_none(tmp_path): + """``native_langs=None`` (the default) yields every subdir, even ones + no native would match — the caller asked for the full walk.""" + root = _make_skill(tmp_path, "en-US", "ja-JP") + langs = sorted(lang for lang, _ in iter_locale_dirs(root)) + assert langs == ["en-US", "ja-JP"] diff --git a/test/test_language.py b/test/test_language.py index 3e81bb8..67f70c2 100644 --- a/test/test_language.py +++ b/test/test_language.py @@ -2,7 +2,7 @@ import pytest import ovos_spec_tools.language as language -from ovos_spec_tools import closest_lang, lang_distance, standardize_lang +from ovos_spec_tools import closest_lang, lang_distance, lang_matches, standardize_lang # --- standardize_lang -------------------------------------------------------- @@ -124,3 +124,25 @@ def test_lang_distance_regional_versus_different_language(): def test_closest_lang_regioned_request_falls_back_to_a_sibling(tmp_path): pytest.importorskip("langcodes") assert closest_lang("pt-MZ", ["en-US", "pt-BR"]) == "pt-BR" + + +# --- lang_matches ------------------------------------------------------------ + +def test_lang_matches_identical_tags(): + assert lang_matches("en-US", "en_us") is True + + +def test_lang_matches_different_languages(): + pytest.importorskip("langcodes") + assert lang_matches("en-US", "fr-FR") is False + + +def test_lang_matches_regional_sibling_within_default_threshold(): + pytest.importorskip("langcodes") + assert lang_matches("de-DE", "de-AT") is True + + +def test_lang_matches_exact_only_when_max_distance_zero(): + from ovos_spec_tools import lang_matches as lm + assert lm("en-US", "en-US", max_distance=0) is True + assert lm("en-US", "en-GB", max_distance=0) is False From f27680b9b2789b5623c0e269687e9da98e296c1d Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 11:44:59 +0000 Subject: [PATCH 025/110] Increment Version to 0.1.0a1 --- ovos_spec_tools/version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index ea5e6f0..bd238c8 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,8 +1,8 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 0 -VERSION_BUILD = 1 -VERSION_ALPHA = 2 +VERSION_MINOR = 1 +VERSION_BUILD = 0 +VERSION_ALPHA = 1 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "") From 078155989043db71eb370efdfa67e9e6fce45462 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 11:45:20 +0000 Subject: [PATCH 026/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e830de1..b481297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.1.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.1.0a1) (2026-05-23) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.0.1a2...0.1.0a1) + +**Merged pull requests:** + +- feat: lang\_matches and iter\_locale\_dirs — pin the cross-component lang policy [\#4](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/4) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.0.1a2](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.0.1a2) (2026-05-22) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/13d694e2960b7ede0533f4029442301c21c86a8c...0.0.1a2) From a175e4c9e0b253b0ee3c636ad40c48b444b3987c Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 13:05:05 +0100 Subject: [PATCH 027/110] =?UTF-8?q?feat:=20LocaleResources.find=20?= =?UTF-8?q?=E2=80=94=20public=20resource=20lookup=20by=20(name,=20ext,=20l?= =?UTF-8?q?ang)=20(#6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every callsite outside this module that needed to locate a resource file ended up reinventing the override-precedence + closest-lang walk the loader already encapsulates. Promote the previously private _locate method to a public find — same body, fuller docstring, covered by six new tests. The internal load_intent / load_dialog / load_prompt callers move to the new name. The duplicate-resource check is preserved, so a LocaleResources.find caller that violates the §2 uniqueness rule gets the same MalformedResource it would have got via load_*. Co-authored-by: Claude Opus 4.7 (1M context) --- ovos_spec_tools/resources.py | 28 ++++++++++++----- test/test_resources.py | 60 ++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index fabc730..c263e8c 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -191,10 +191,24 @@ def _lang_dir(self, source: Path, lang: str) -> Optional[Path]: match = self._lang_resolver(lang, names, self.max_language_distance) return (source / match) if match is not None else None - def _locate(self, base_name: str, extension: str, - lang: str) -> Optional[Path]: - """Find a resource file by ``(base name, extension)`` in ``lang`` - through the precedence chain. Returns the first match, or ``None``.""" + def find(self, base_name: str, extension: str, + lang: str) -> Optional[Path]: + """Locate a resource file by ``(base name, extension)`` in ``lang``. + + Walks the override precedence (§2.1) — user, then skill, then core — + and inside each source's ``/`` directory descends recursively + looking for ````. The first match wins. + + ``lang`` is resolved against each source's available subdirectories + via the language resolver, so a request for ``en`` matches an + ``en-US/`` tree (and vice versa) within the configured distance. + + Raises :class:`MalformedResource` if more than one file with the + same ``(role, base name)`` exists within one language tree — + OVOS-INTENT-2 §2 requires uniqueness. + + Returns the resolved path, or ``None`` if no source has it. + """ filename = base_name + extension for source in self._sources: lang_dir = self._lang_dir(source, lang) @@ -239,7 +253,7 @@ def entities(self, lang: str) -> Dict[str, List[str]]: def _load_expanded(self, base_name: str, extension: str, lang: str) -> List[str]: """Load a resource and expand it to its sample set.""" - path = self._locate(base_name, extension, lang) + path = self.find(base_name, extension, lang) if path is None: raise FileNotFoundError( f"no {extension} resource named {base_name!r} for " @@ -285,7 +299,7 @@ def load_dialog(self, base_name: str, lang: str) -> List[str]: expansion happens per render, on the one phrase chosen (§4.2). The phrase strings are returned verbatim, for a dialog renderer to consume. """ - path = self._locate(base_name, ".dialog", lang) + path = self.find(base_name, ".dialog", lang) if path is None: raise FileNotFoundError( f"no .dialog resource named {base_name!r} for " @@ -308,7 +322,7 @@ def load_prompt(self, base_name: str, lang: str) -> str: FileNotFoundError: no such ``.prompt`` for ``lang``. MalformedResource: the file is empty or only whitespace (§5). """ - path = self._locate(base_name, PROMPT_ROLE, lang) + path = self.find(base_name, PROMPT_ROLE, lang) if path is None: raise FileNotFoundError( f"no .prompt resource named {base_name!r} for " diff --git a/test/test_resources.py b/test/test_resources.py index 2ef5854..03f0182 100644 --- a/test/test_resources.py +++ b/test/test_resources.py @@ -278,3 +278,63 @@ def test_intent_with_undefined_voc_reference_is_rejected(tmp_path): res = LocaleResources(str(locale)) with pytest.raises(MalformedTemplate): res.load_intent("g", "en-US") + + +# --- LocaleResources.find ---------------------------------------------------- + +def test_find_returns_path_when_resource_exists(tmp_path): + locale = tmp_path / "locale" + f = locale / "en-US" / "play.intent" + _write(f, "play {query}\n") + res = LocaleResources(str(locale)) + assert res.find("play", ".intent", "en-US") == f + + +def test_find_returns_none_when_resource_missing(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "play {query}\n") + res = LocaleResources(str(locale)) + assert res.find("stop", ".intent", "en-US") is None + + +def test_find_recurses_into_subdirectories(tmp_path): + """A resource may live in a nested category folder under /.""" + locale = tmp_path / "locale" + f = locale / "en-US" / "media" / "play.intent" + _write(f, "play {query}\n") + res = LocaleResources(str(locale)) + assert res.find("play", ".intent", "en-US") == f + + +def test_find_resolves_lang_via_closest_match(tmp_path): + """A request for ``en`` matches an ``en-US/`` tree.""" + pytest.importorskip("langcodes") + locale = tmp_path / "locale" + f = locale / "en-US" / "play.intent" + _write(f, "play {query}\n") + res = LocaleResources(str(locale)) + assert res.find("play", ".intent", "en") == f + + +def test_find_walks_override_precedence(tmp_path): + """user > skill > core: the first source carrying the file wins.""" + core = tmp_path / "core" + skill = tmp_path / "skill" + user = tmp_path / "user" + _write(core / "en-US" / "play.intent", "core-version\n") + _write(skill / "en-US" / "play.intent", "skill-version\n") + _write(user / "en-US" / "play.intent", "user-version\n") + res = LocaleResources(str(skill), + core_locale=str(core), + user_locale=str(user)) + assert res.find("play", ".intent", "en-US").parent == user / "en-US" + + +def test_find_raises_on_duplicate_within_language_tree(tmp_path): + """OVOS-INTENT-2 §2: a (role, base name) must be unique per language tree.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "a" / "play.intent", "play one\n") + _write(locale / "en-US" / "b" / "play.intent", "play two\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedResource): + res.find("play", ".intent", "en-US") From deb5cf8c77296ffa54932ab5ba863d631ff5c6c0 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 12:05:22 +0000 Subject: [PATCH 028/110] Increment Version to 0.2.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index bd238c8..07d1d07 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 1 +VERSION_MINOR = 2 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From bbeb0c6f0d815db510ce8a2faaae12491d488e3f Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 12:05:49 +0000 Subject: [PATCH 029/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b481297..2f2ca25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.2.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.2.0a1) (2026-05-23) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.1.0a1...0.2.0a1) + +**Merged pull requests:** + +- feat: LocaleResources.find — public resource lookup by \(name, ext, lang\) [\#6](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/6) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.1.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.1.0a1) (2026-05-23) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.0.1a2...0.1.0a1) From a5793bce9c9126880968a05a674893785eb50700 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 13:48:36 +0100 Subject: [PATCH 030/110] feat: keyword_form / vocabulary_keywords + utterance_contains / strip_samples (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: keyword_form / vocabulary_keywords / utterance_contains / strip_samples Every consumer that registered slot-free keywords or matched a vocab against an utterance reimplemented the same handful of operations. Promote them to ovos-spec-tools so the OVOS-INTENT-2 §4.3 keyword convention has exactly one implementation. New free functions in ovos_spec_tools.resources: - keyword_form(template_line, vocabularies=None) -> (entity, aliases) expands one slot-free template, lowercases, dedupes and sorts; the first item is the canonical entity, the rest are aliases that canonicalize to it. Malformed input yields ('', []) rather than poisoning a batch. - normalize_for_match(text, ensure_ascii=True) -> str lowercases, strips, and optionally folds accents and ASCII punctuation; the comparison normalization shared by the two below. - utterance_contains(utterance, samples, exact=False, ensure_ascii=True) true iff utterance matches any sample — whole-word substring by default, exact when requested, accent/punct-folded by default. - strip_samples(utterance, samples) -> str remove every whole-word occurrence of any sample, longest first so composite phrases consume their parts before fallback matches. New methods on LocaleResources: - vocabulary_keywords(lang) -> Iterator[(voc_name, entity, aliases)] - entity_keywords(lang) -> Iterator[(entity_name, entity, aliases)] yield one triple per template line, with the entity/alias split applied via keyword_form. Suit any consumer that registers keyword sets with a primary/alias distinction. Tests: 52 passed (16 new). Co-Authored-By: Claude Opus 4.7 (1M context) * feat: granular normalization flags + permissive whole-word match - normalize_for_match now takes two keyword-only flags: strip_diacritics (NFD + drop combining marks) and strip_punct (drop ASCII punctuation, keep slot braces). The previous ensure_ascii knob conflated the two — French ou/où or technical names like c++ need separate control. Both default True; either flag can be flipped independently. - utterance_contains and strip_samples forward both flags and the whole-word match anchor moves from \b...\b to (? --------- Co-authored-by: Claude Opus 4.7 (1M context) --- ovos_spec_tools/__init__.py | 12 ++ ovos_spec_tools/resources.py | 164 +++++++++++++++++- test/test_resources.py | 319 +++++++++++++++++++++++++++++++++++ 3 files changed, 494 insertions(+), 1 deletion(-) diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index ac427d8..bc74017 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -18,6 +18,10 @@ distance, match checking, and closest-match resolution; - :func:`~ovos_spec_tools.resources.iter_locale_dirs` — locale subdirectory discovery with optional native-language filtering; +- :func:`~ovos_spec_tools.resources.keyword_form`, + :func:`~ovos_spec_tools.resources.utterance_contains`, + :func:`~ovos_spec_tools.resources.strip_samples` — slot-free template + grouping (§4.3) and utterance match / strip primitives; - :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also exposed as the ``ovos-spec-lint`` command. """ @@ -35,8 +39,12 @@ LocaleResources, MalformedResource, iter_locale_dirs, + keyword_form, + normalize_for_match, read_prompt_file, read_resource_file, + strip_samples, + utterance_contains, ) from ovos_spec_tools.version import __version__ @@ -46,8 +54,12 @@ "LocaleResources", "MalformedResource", "iter_locale_dirs", + "keyword_form", + "normalize_for_match", "read_resource_file", "read_prompt_file", + "strip_samples", + "utterance_contains", "render", "DialogRenderer", "UnfilledSlot", diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index c263e8c..7db53ee 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -23,7 +23,7 @@ from __future__ import annotations from pathlib import Path -from typing import Callable, Dict, List, Optional, Sequence +from typing import Callable, Dict, Iterator, List, Optional, Sequence, Tuple from ovos_spec_tools.expansion import expand from ovos_spec_tools.language import ( @@ -36,8 +36,12 @@ "LocaleResources", "MalformedResource", "iter_locale_dirs", + "keyword_form", + "normalize_for_match", "read_resource_file", "read_prompt_file", + "strip_samples", + "utterance_contains", "SLOT_BEARING_ROLES", "SLOT_FREE_ROLES", "PROMPT_ROLE", @@ -122,6 +126,124 @@ def iter_locale_dirs(root: Path, yield lang_norm, entry +def keyword_form(template_line: str, + vocabularies: Optional[Dict[str, List[str]]] = None + ) -> Tuple[str, List[str]]: + """Split one slot-free template line into ``(entity, aliases)``. + + The line is expanded via :func:`expand` (with ``vocabularies`` available + for ```` references), lowercased, deduplicated and sorted. The + first item is the canonical **entity**; the rest are **aliases** that + canonicalize to it. + + A ``.voc`` line like ``(hi|hello|hey)`` becomes one keyword whose entity + value is ``"hi"`` and whose aliases are ``["hello", "hey"]`` — any + consumer that distinguishes a canonical form from synonyms can use this + grouping directly (OVOS-INTENT-2 §4.3). + + An empty or whitespace-only line yields ``("", [])``. + """ + if not template_line.strip(): + return "", [] + try: + samples = expand(template_line, vocabularies) + except Exception: + # A malformed line yields no keyword rather than poisoning the batch. + return "", [] + options = sorted({s.lower() for s in samples}) + if not options: + return "", [] + return options[0], options[1:] + + +def normalize_for_match(text: str, *, + strip_diacritics: bool = True, + strip_punct: bool = True) -> str: + """Lowercase, strip, and optionally fold diacritics / ASCII punctuation. + + Used as the comparison normalization for :func:`utterance_contains` and + :func:`strip_samples`. The two folding steps are independent: + + - ``strip_diacritics=True`` (default) decomposes combining marks (NFD) + and drops them — ``"olá"`` becomes ``"ola"``, ``"über"`` becomes + ``"uber"`` — so the comparison is accent-insensitive. + - ``strip_punct=True`` (default) removes ASCII punctuation. Curly + braces ``{`` and ``}`` are preserved so slot markers survive a + pre-render pass. + + Set either flag to ``False`` for languages where the distinction is + semantic (e.g. French ``ou``/``où``). + """ + import unicodedata + text = text.strip().lower() + if strip_diacritics: + text = "".join( + c for c in unicodedata.normalize("NFD", text) + if not unicodedata.combining(c) + ) + if strip_punct: + import string + rm_chars = set(c for c in string.punctuation if c not in ("{", "}")) + text = "".join(c for c in text if c not in rm_chars) + return text + + +def utterance_contains(utterance: str, samples: Sequence[str], + *, + exact: bool = False, + strip_diacritics: bool = True, + strip_punct: bool = True) -> bool: + """True iff ``utterance`` matches any item in ``samples``. + + With ``exact=True`` the utterance must equal a sample after + normalization. With ``exact=False`` (the default) a sample is found + when it appears in the utterance as a whole-word substring — so a + sample ``yes`` matches ``"yes, please"`` but not ``"yesterday"``. + + Both sides are normalized via :func:`normalize_for_match`. The + ``strip_diacritics`` and ``strip_punct`` flags are independent — + forward each one as required by the target language. + + An empty utterance or empty sample set returns ``False``. + """ + if not utterance or not samples: + return False + norm = lambda s: normalize_for_match( + s, strip_diacritics=strip_diacritics, strip_punct=strip_punct) + utt = norm(utterance) + norm_samples = [norm(s) for s in samples] + if exact: + return any(s and s == utt for s in norm_samples) + import re + # `(? str: + """Return ``utterance`` with every whole-word occurrence of any sample + removed. + + Samples are stripped longest first so that a composite match is + consumed before any of its shorter constituents (``"give up"`` before + ``"up"``). The match is whole-word and case-insensitive; the utterance + is otherwise returned with original casing and punctuation. + """ + import re + for s in sorted({s for s in samples if s and s.strip()}, + key=len, reverse=True): + # `(? str: """Read a ``.prompt`` whole and verbatim (OVOS-INTENT-2 §3, §4.4). @@ -250,6 +372,46 @@ def entities(self, lang: str) -> Dict[str, List[str]]: names.add(path.stem) return {name: self.load_entity(name, lang) for name in sorted(names)} + def _keywords_for(self, extension: str, lang: str + ) -> Iterator[Tuple[str, str, List[str]]]: + """Walk every slot-free resource of one role and group expansions + line-by-line. Yields ``(resource_name, entity, aliases)`` triples + — one yield per non-empty template line, with the OVOS-INTENT-2 + §4.3 ``(entity, aliases)`` convention applied via + :func:`keyword_form`.""" + vocabularies = self.vocabularies(lang) + for source in self._sources: + lang_dir = self._lang_dir(source, lang) + if lang_dir is None: + continue + for path in sorted(lang_dir.rglob(f"*{extension}")): + if not path.is_file(): + continue + for template in read_resource_file(path): + entity, aliases = keyword_form(template, vocabularies) + if entity: + yield path.stem, entity, aliases + + def vocabulary_keywords(self, lang: str + ) -> Iterator[Tuple[str, str, List[str]]]: + """Yield ``(voc_name, entity, aliases)`` for every line of every + ``.voc`` reachable for ``lang``. + + One yield per template line: the line's expansion is split into a + canonical entity (first sorted alternative) and its aliases via + :func:`keyword_form`. Suits any consumer that registers keyword + sets with a primary/alias distinction (OVOS-INTENT-2 §4.3). + """ + return self._keywords_for(".voc", lang) + + def entity_keywords(self, lang: str + ) -> Iterator[Tuple[str, str, List[str]]]: + """Yield ``(entity_name, entity, aliases)`` for every line of every + ``.entity`` reachable for ``lang``. Same shape as + :meth:`vocabulary_keywords`; ``.entity`` and ``.voc`` share the + slot-free template format (§4.3).""" + return self._keywords_for(".entity", lang) + def _load_expanded(self, base_name: str, extension: str, lang: str) -> List[str]: """Load a resource and expand it to its sample set.""" diff --git a/test/test_resources.py b/test/test_resources.py index 03f0182..8591384 100644 --- a/test/test_resources.py +++ b/test/test_resources.py @@ -338,3 +338,322 @@ def test_find_raises_on_duplicate_within_language_tree(tmp_path): res = LocaleResources(str(locale)) with pytest.raises(MalformedResource): res.find("play", ".intent", "en-US") + + +# --- keyword_form ------------------------------------------------------------ + +def test_keyword_form_groups_alternatives_into_entity_and_aliases(): + from ovos_spec_tools import keyword_form + entity, aliases = keyword_form("(hi|hello|hey)") + # sorted, lowercased, first is canonical + assert (entity, aliases) == ("hello", ["hey", "hi"]) + + +def test_keyword_form_lowercases_and_dedupes(): + from ovos_spec_tools import keyword_form + entity, aliases = keyword_form("(YES|Yes|yes)") + assert (entity, aliases) == ("yes", []) + + +def test_keyword_form_empty_input_returns_empty_pair(): + from ovos_spec_tools import keyword_form + assert keyword_form("") == ("", []) + + +def test_keyword_form_resolves_voc_references(): + """`` references in a template expand against the supplied vocab.""" + from ovos_spec_tools import keyword_form + entity, aliases = keyword_form( + " friend", vocabularies={"greet": ["hi", "hello"]}) + assert sorted([entity, *aliases]) == ["hello friend", "hi friend"] + + +# --- LocaleResources.vocabulary_keywords / entity_keywords ------------------- + +def test_vocabulary_keywords_yields_one_triple_per_template_line(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.voc", + "(hi|hello)\n(yo|hey)\n") + res = LocaleResources(str(locale)) + triples = sorted(res.vocabulary_keywords("en-US")) + assert triples == [ + ("greet", "hello", ["hi"]), + ("greet", "hey", ["yo"]), + ] + + +def test_vocabulary_keywords_skips_blank_lines(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "\nyes\n# a comment\n") + res = LocaleResources(str(locale)) + assert list(res.vocabulary_keywords("en-US")) == [("yes", "yes", [])] + + +def test_entity_keywords_uses_same_shape(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "color.entity", "(red|crimson)\nblue\n") + res = LocaleResources(str(locale)) + triples = sorted(res.entity_keywords("en-US")) + assert triples == [ + ("color", "blue", []), + ("color", "crimson", ["red"]), + ] + + +# --- utterance_contains ------------------------------------------------------ + +def test_utterance_contains_whole_word_substring_default(): + from ovos_spec_tools import utterance_contains + assert utterance_contains("yes, please", ["yes"]) # default whole-word + assert not utterance_contains("yesterday", ["yes"]) # substring not allowed + + +def test_utterance_contains_exact_mode(): + from ovos_spec_tools import utterance_contains + assert utterance_contains("Yes", ["yes"], exact=True) + assert not utterance_contains("yes please", ["yes"], exact=True) + + +def test_utterance_contains_folds_accents_and_punct_by_default(): + from ovos_spec_tools import utterance_contains + # á → a, ! stripped, comparison succeeds + assert utterance_contains("Olá!", ["ola"]) + + +def test_utterance_contains_keeps_diacritics_when_disabled(): + from ovos_spec_tools import utterance_contains + assert utterance_contains("ola", ["olá"], strip_diacritics=False) is False + assert utterance_contains("olá", ["olá"], strip_diacritics=False) + + +def test_utterance_contains_keeps_punct_when_disabled(): + """Punctuation stays significant when strip_punct=False.""" + from ovos_spec_tools import utterance_contains + assert utterance_contains("i love c++", ["c++"], strip_punct=False) + assert utterance_contains( + "i love c plus plus", ["c++"], strip_punct=False) is False + + +def test_utterance_contains_punct_and_diacritics_flags_compose(): + """The two flags are independent — any combination is valid.""" + from ovos_spec_tools import utterance_contains + # both stripped (default) — matches across accents and punctuation + assert utterance_contains("olá!", ["ola"]) + # diacritics stripped, punctuation kept — `!` becomes a word boundary + assert utterance_contains( + "olá!", ["ola"], strip_diacritics=True, strip_punct=False) + # punct stripped, diacritics kept — `olá!` → `olá`; sample `ola` misses + assert utterance_contains( + "olá!", ["ola"], strip_diacritics=False, strip_punct=True) is False + + +def test_utterance_contains_exact_mode_normalizes_both_sides(): + """Exact comparison still applies the configured normalization.""" + from ovos_spec_tools import utterance_contains + assert utterance_contains(" Olá! ", ["ola"], exact=True) + assert utterance_contains( + " Olá! ", ["ola"], exact=True, strip_diacritics=False) is False + + +def test_utterance_contains_empty_inputs_return_false(): + from ovos_spec_tools import utterance_contains + assert utterance_contains("", ["yes"]) is False + assert utterance_contains("yes", []) is False + + +# --- strip_samples ----------------------------------------------------------- + +def test_strip_samples_removes_whole_word_matches(): + from ovos_spec_tools import strip_samples + out = strip_samples("set volume to maximum", ["set", "volume"]) + assert "set" not in out.split() + assert "volume" not in out.split() + assert "maximum" in out.split() + + +def test_strip_samples_longest_first_consumes_composite_before_parts(): + """A composite phrase is removed before its shorter constituents.""" + from ovos_spec_tools import strip_samples + out = strip_samples("give it up now", ["give up", "up"]) + # ``give it up`` does not contain the contiguous phrase "give up", + # so the longer pattern misses; "up" is still stripped as a fallback. + assert "up" not in out.split() + + +def test_strip_samples_case_insensitive(): + from ovos_spec_tools import strip_samples + assert "Yes" not in strip_samples("Yes please", ["yes"]).split() + + +def test_strip_samples_no_match_returns_input_unchanged(): + from ovos_spec_tools import strip_samples + assert strip_samples("no match here", ["xyzzy"]) == "no match here" + + +# --- normalize_for_match ----------------------------------------------------- + +def test_normalize_for_match_lowercases_and_trims(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match(" YES ") == "yes" + + +def test_normalize_for_match_strips_diacritics_by_default(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("Olá") == "ola" + assert normalize_for_match("über") == "uber" + + +def test_normalize_for_match_strips_punct_by_default(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("yes, please!") == "yes please" + + +def test_normalize_for_match_preserves_slot_markers(): + """``{`` and ``}`` survive punctuation stripping so pre-render-pass + slot markers stay intact.""" + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("play {song}") == "play {song}" + + +def test_normalize_for_match_keeps_punct_when_disabled(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("yes, please!", strip_punct=False) == "yes, please!" + + +def test_normalize_for_match_keeps_diacritics_when_disabled(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("Olá", strip_diacritics=False) == "olá" + + +def test_normalize_for_match_both_flags_off_only_lowercases_and_trims(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match( + " Olá, World! ", + strip_diacritics=False, strip_punct=False) == "olá, world!" + + +def test_normalize_for_match_empty_string(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("") == "" + assert normalize_for_match(" ") == "" + + +def test_normalize_for_match_is_keyword_only_for_flags(): + """The two flags are keyword-only — positional misuse can't silently + flip the wrong knob.""" + from ovos_spec_tools import normalize_for_match + import pytest as _pytest + with _pytest.raises(TypeError): + normalize_for_match("hi", False) # would have to be keyword + + +# --- strip_samples extra coverage -------------------------------------------- + +def test_strip_samples_empty_samples_returns_input_unchanged(): + from ovos_spec_tools import strip_samples + assert strip_samples("hello world", []) == "hello world" + + +def test_strip_samples_ignores_blank_samples(): + """A blank or whitespace-only sample is dropped silently.""" + from ovos_spec_tools import strip_samples + assert strip_samples("hello world", ["", " ", "world"]).strip() == "hello" + + +def test_strip_samples_escapes_regex_metacharacters_in_samples(): + """A sample like ``c++`` must not be treated as a regex pattern.""" + from ovos_spec_tools import strip_samples + out = strip_samples("i love c++ programming", ["c++"]) + assert "c++" not in out + + +def test_strip_samples_preserves_casing_of_remaining_text(): + from ovos_spec_tools import strip_samples + out = strip_samples("Yes Please", ["yes"]) + assert "Please" in out # capital P preserved + assert "Yes" not in out.split() + + +# --- vocabulary_keywords / entity_keywords extra coverage -------------------- + +def test_vocabulary_keywords_walks_override_precedence(tmp_path): + """A user-locale .voc overrides the same-name skill-locale one.""" + skill = tmp_path / "skill" + user = tmp_path / "user" + _write(skill / "en-US" / "color.voc", "(red|crimson)\n") + _write(user / "en-US" / "color.voc", "(blue|azure)\n") + res = LocaleResources(str(skill), user_locale=str(user)) + triples = sorted(res.vocabulary_keywords("en-US")) + # both files contribute — user overrides core, but both are reachable + # because each source is walked for its own .voc files + names = {entity for _, entity, _ in triples} + assert "azure" in names + + +def test_vocabulary_keywords_handles_missing_locale_gracefully(tmp_path): + """A skill that ships no .voc files yields nothing — no exception.""" + locale = tmp_path / "locale" + (locale / "en-US").mkdir(parents=True) + res = LocaleResources(str(locale)) + assert list(res.vocabulary_keywords("en-US")) == [] + + +def test_vocabulary_keywords_resolves_voc_references_in_template(tmp_path): + """A ```` reference in one .voc is resolved against the lang's + full vocab map at expansion time.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.voc", "hi\nhello\n") + _write(locale / "en-US" / "greet_friend.voc", " friend\n") + res = LocaleResources(str(locale)) + triples = sorted(res.vocabulary_keywords("en-US")) + # the greet_friend.voc line expands its reference + friend_triples = [t for t in triples if t[0] == "greet_friend"] + assert friend_triples == [("greet_friend", "hello friend", ["hi friend"])] + + +def test_vocabulary_keywords_skips_malformed_lines(tmp_path): + """A malformed template inside a .voc yields no keyword for that line + rather than raising — well-formed neighbours still register.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "things.voc", + "(hi|hello)\n(broken\nfine\n") + res = LocaleResources(str(locale)) + entities = {e for _, e, _ in res.vocabulary_keywords("en-US")} + assert "fine" in entities # well-formed line survived + # ``(broken`` does not yield (entity, []) + assert "(broken" not in entities + + +# --- keyword_form extra coverage --------------------------------------------- + +def test_keyword_form_blank_line_returns_empty_pair(): + from ovos_spec_tools import keyword_form + assert keyword_form(" \n ") == ("", []) + + +def test_keyword_form_malformed_template_returns_empty_pair(): + """A line that expansion rejects yields ('', []) — never propagates.""" + from ovos_spec_tools import keyword_form + assert keyword_form("(unclosed") == ("", []) + + +def test_keyword_form_single_alternative_yields_no_aliases(): + """A line without alternatives has the line itself as the entity.""" + from ovos_spec_tools import keyword_form + assert keyword_form("hello") == ("hello", []) + + +def test_vocabulary_keywords_returns_empty_when_language_has_no_dir(tmp_path): + """Asking for a language with no directory yields nothing (no exception) + — covers the `lang_dir is None: continue` branch in _keywords_for.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.voc", "yes\n") + res = LocaleResources(str(locale), max_language_distance=0) + assert list(res.vocabulary_keywords("ja-JP")) == [] + assert list(res.entity_keywords("ja-JP")) == [] + + +def test_strip_samples_handles_unicode_samples(): + """Unicode samples that need escaping in regex still strip cleanly.""" + from ovos_spec_tools import strip_samples + assert "olá" not in strip_samples("ola olá", ["olá"]).split() From 445c947bc115bae759956f507f691fee8678d592 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 12:48:45 +0000 Subject: [PATCH 031/110] Increment Version to 0.3.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 07d1d07..59ee186 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 2 +VERSION_MINOR = 3 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 077cbd880e9c68fa3ed6ccfd7d3ad455b67df4ed Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 12:49:08 +0000 Subject: [PATCH 032/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f2ca25..b7ee0e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.3.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.3.0a1) (2026-05-23) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.2.0a1...0.3.0a1) + +**Merged pull requests:** + +- feat: keyword\_form / vocabulary\_keywords + utterance\_contains / strip\_samples [\#8](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/8) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.2.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.2.0a1) (2026-05-23) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.1.0a1...0.2.0a1) From 745f691fa33e59b341bb6138d6994b1527700029 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 13:56:41 +0100 Subject: [PATCH 033/110] feat: LocaleResources.voc_list / .voc_match / .remove_voc (#10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: LocaleResources.voc_match and .remove_voc The 'load a .voc and match (or strip) it against an utterance' pair shows up in every consumer that does slot-free phrase matching. Bundle it on LocaleResources so it travels with the loader. - voc_match(utterance, base_name, lang, *, exact, strip_diacritics, strip_punct) — load_vocabulary + utterance_contains; missing resource returns False. - remove_voc(utterance, base_name, lang) — load_vocabulary + strip_samples; missing resource returns the input unchanged. Both honour the smart language fallback (en-AU finds en-US) and forward the granular normalization flags from the free functions. Tests: 85 passed (8 new) — whole-word vs exact, missing resource, flag forwarding, fallback resolution. Co-Authored-By: Claude Opus 4.7 (1M context) * feat: LocaleResources.voc_list — flat sample list, missing returns [] Rounds out the voc_list / voc_match / remove_voc trio so a caller naming the family consistently doesn't drop back to load_vocabulary just to inspect or cache the phrase set. Unlike load_vocabulary, voc_list returns [] for a missing resource rather than raising — matches the FileNotFoundError-as-False convention used by voc_match and remove_voc above it. --------- Co-authored-by: Claude Opus 4.7 (1M context) --- ovos_spec_tools/resources.py | 54 +++++++++++++++++++++++ test/test_resources.py | 85 ++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index 7db53ee..0c5857f 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -412,6 +412,60 @@ def entity_keywords(self, lang: str slot-free template format (§4.3).""" return self._keywords_for(".entity", lang) + def voc_list(self, base_name: str, lang: str) -> List[str]: + """Load a ``.voc`` as its flat sample list — the same content + :meth:`load_vocabulary` returns, named consistently with + :meth:`voc_match` and :meth:`remove_voc` for callers that want + to inspect or cache the phrase set independently. + + Returns ``[]`` when the resource does not exist for the language. + """ + try: + return self.load_vocabulary(base_name, lang) + except FileNotFoundError: + return [] + + def voc_match(self, utterance: str, base_name: str, lang: str, *, + exact: bool = False, + strip_diacritics: bool = True, + strip_punct: bool = True) -> bool: + """Convenience: load a ``.voc`` for ``lang`` and check whether + ``utterance`` matches any of its samples. + + Equivalent to ``utterance_contains(utterance, + self.load_vocabulary(base_name, lang), ...)`` with the same flag + semantics, including the ``(? str: + """Convenience: load a ``.voc`` for ``lang`` and strip every + whole-word occurrence of any sample from ``utterance``. + + Equivalent to ``strip_samples(utterance, + self.load_vocabulary(base_name, lang))``; samples are stripped + longest-first so a composite phrase consumes its parts before + any shorter fallback. Returns ``utterance`` unchanged if the + resource does not exist for the language. + """ + if not utterance: + return utterance + try: + samples = self.load_vocabulary(base_name, lang) + except FileNotFoundError: + return utterance + return strip_samples(utterance, samples) + def _load_expanded(self, base_name: str, extension: str, lang: str) -> List[str]: """Load a resource and expand it to its sample set.""" diff --git a/test/test_resources.py b/test/test_resources.py index 8591384..a03f28a 100644 --- a/test/test_resources.py +++ b/test/test_resources.py @@ -657,3 +657,88 @@ def test_strip_samples_handles_unicode_samples(): """Unicode samples that need escaping in regex still strip cleanly.""" from ovos_spec_tools import strip_samples assert "olá" not in strip_samples("ola olá", ["olá"]).split() + + +# --- LocaleResources.voc_match / .remove_voc --------------------------------- + +def test_voc_match_whole_word_default(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "(yes|yeah|yep)\n") + res = LocaleResources(str(locale)) + assert res.voc_match("yes, please", "yes", "en-US") + assert res.voc_match("yeah whatever", "yes", "en-US") + assert res.voc_match("yesterday", "yes", "en-US") is False + + +def test_voc_match_exact_mode(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "yes\n") + res = LocaleResources(str(locale)) + assert res.voc_match("Yes", "yes", "en-US", exact=True) + assert res.voc_match("yes please", "yes", "en-US", exact=True) is False + + +def test_voc_match_missing_resource_returns_false(tmp_path): + locale = tmp_path / "locale" + (locale / "en-US").mkdir(parents=True) + res = LocaleResources(str(locale)) + assert res.voc_match("anything", "nope", "en-US") is False + + +def test_voc_match_forwards_normalization_flags(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.voc", "ola\n") + res = LocaleResources(str(locale)) + # default: diacritics stripped, "Olá" matches + assert res.voc_match("Olá", "greet", "en-US") + # with strip_diacritics=False, distinct + assert res.voc_match("Olá", "greet", "en-US", + strip_diacritics=False) is False + + +def test_remove_voc_strips_voc_phrases(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "fillers.voc", "(please|kindly)\n") + res = LocaleResources(str(locale)) + out = res.remove_voc("please set volume to ten kindly", "fillers", "en-US") + assert "please" not in out.split() + assert "kindly" not in out.split() + assert "volume" in out.split() + + +def test_remove_voc_missing_resource_returns_input_unchanged(tmp_path): + locale = tmp_path / "locale" + (locale / "en-US").mkdir(parents=True) + res = LocaleResources(str(locale)) + assert res.remove_voc("hello world", "nope", "en-US") == "hello world" + + +def test_remove_voc_empty_utterance_returns_empty(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.voc", "yes\n") + res = LocaleResources(str(locale)) + assert res.remove_voc("", "x", "en-US") == "" + + +def test_voc_match_and_remove_voc_resolve_smart_lang_fallback(tmp_path): + """en-AU falls back to en-US for both methods.""" + pytest.importorskip("langcodes") + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "yes\n") + res = LocaleResources(str(locale)) + assert res.voc_match("yes please", "yes", "en-AU") + assert "yes" not in res.remove_voc("yes please", "yes", "en-AU").split() + + +def test_voc_list_returns_expanded_samples(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "(yes|yeah)\n") + res = LocaleResources(str(locale)) + assert sorted(res.voc_list("yes", "en-US")) == ["yeah", "yes"] + + +def test_voc_list_missing_resource_returns_empty(tmp_path): + locale = tmp_path / "locale" + (locale / "en-US").mkdir(parents=True) + res = LocaleResources(str(locale)) + assert res.voc_list("nope", "en-US") == [] From 2a08f0cfdff43ba49d4f82e7be908476772521ba Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 12:56:53 +0000 Subject: [PATCH 034/110] Increment Version to 0.4.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 59ee186..458e064 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 3 +VERSION_MINOR = 4 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 9ef17a4994e31a21f4da52e0955be78204ec02fb Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 23 May 2026 12:57:12 +0000 Subject: [PATCH 035/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7ee0e3..7f26507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.4.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.4.0a1) (2026-05-23) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.3.0a1...0.4.0a1) + +**Merged pull requests:** + +- feat: LocaleResources.voc\_list / .voc\_match / .remove\_voc [\#10](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/10) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.3.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.3.0a1) (2026-05-23) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.2.0a1...0.3.0a1) From 6a8f9f0c64ea81b08a6e3ed9a15583db08bff912 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 24 May 2026 01:18:56 +0100 Subject: [PATCH 036/110] feat: OVOS-MSG-1 Message envelope (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: OVOS-MSG-1 Message envelope ovos_spec_tools.message — Message class implementing the OVOS-MSG-1 envelope (§2: type/data/context), §6 serialization, §7 conformance rules (rejects unknown top-level keys, missing 'type', wrong value types, NaN/Infinity), and the §5 derivations (forward/reply/response). context.session stays an opaque dict per §4 — the envelope does not parse it. Subclass-friendly: derivations and deserialize return instances of the runtime class so ovos_bus_client can subclass for the websocket transport layer without losing identity through forward/ reply chains. Standard-library only, no encryption or transport concerns. Tests (35 new): conformance fixtures for MSG-1 §2/§3/§4/§5/§6/§7 — envelope key handling, derivation behaviour, subclass propagation, session opacity, NaN/Infinity rejection. Topic-name catalogue / bus-message schemas belong elsewhere (ovos-pydantic-models); they are NOT part of this PR. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(Message): walk nested .serialize() carriers before JSON encoding OVOS code stuffs Session (and other carrier objects with a custom .serialize() method) directly into context. For ovos_bus_client to become a true shim re-exporting this Message, the envelope's serialize() must convert those carriers before json.dumps — otherwise TypeError. The protocol is duck-typed; Message itself stays unaware of Session or any specific carrier. Co-Authored-By: Claude Opus 4.7 (1M context) * docs: cover the new Message envelope - docs/message.md: new chapter walking through OVOS-MSG-1 — the envelope shape, routing keys, the session carrier, the three derivations, serialization, the no-correlation contract, and the subclass-friendly design that lets ovos-bus-client extend it for the transport layer. - docs/README.md: list it as the fifth tool, slot it between language matching and linting in the guide order, and tighten the scope note to call out the transport boundary. - docs/api-reference.md: full API section for Message — derivations, serialize/deserialize, MalformedMessage, DEFAULT_SESSION_ID. Bump the linting cross-reference from chapter 6 to chapter 7. - docs/getting-started.md: a 'construct a bus message' first-taste example linking to the new chapter. - docs/locale-resources.md: one stale 'chapter 6' linter pointer updated to chapter 7. - README.md: add Message row to the status table, refresh the quick taste with a forward/reply example, link the new docs chapter, bump requires-python to 3.10+ (already enforced in pyproject). Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- README.md | 14 +- docs/README.md | 23 ++- docs/api-reference.md | 47 +++++- docs/getting-started.md | 13 ++ docs/locale-resources.md | 2 +- docs/message.md | 180 ++++++++++++++++++++++ ovos_spec_tools/__init__.py | 10 ++ ovos_spec_tools/message.py | 240 +++++++++++++++++++++++++++++ test/test_message.py | 297 ++++++++++++++++++++++++++++++++++++ 9 files changed, 813 insertions(+), 13 deletions(-) create mode 100644 docs/message.md create mode 100644 ovos_spec_tools/message.py create mode 100644 test/test_message.py diff --git a/README.md b/README.md index 9a1541e..9fbb579 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ depend on. | Dialog renderer | OVOS-INTENT-2 §4.2 | `dialog.py` | | Prompt renderer | OVOS-INTENT-2 §4.4 | `prompt.py` | | Language-tag matching | OVOS-INTENT-2 §2.2 | `language.py` | +| Bus message envelope | OVOS-MSG-1 | `message.py` | | `ovos-spec-lint` locale linter | OVOS-INTENT-1 / -2 | `lint.py` | ## Install @@ -27,13 +28,13 @@ pip install ovos-spec-tools # core — no dependencies pip install ovos-spec-tools[langcodes] # adds the smart language fallback ``` -Requires Python 3.8+. +Requires Python 3.10+. ## Quick taste ```python from ovos_spec_tools import (expand, LocaleResources, render, render_prompt, - closest_lang) + closest_lang, Message) expand("(turn|switch) [the] light") # all 4 sentences it denotes res = LocaleResources("my-skill/locale") @@ -43,6 +44,11 @@ render(res.load_dialog("weather", "en-US"), # a spoken response render_prompt(res.load_prompt("system", "en-US"), # a language-model prompt slots={"query": "what time is it"}) closest_lang("en-AU", ["pt-BR", "en-US"]) # 'en-US' + +# OVOS-MSG-1 envelope: a bus message and its 'send back to the asker' reply +m = Message("ovos.intent.list", {}, {"source": "skill.id"}) +res = m.response({"intents": ["..."]}) # 'ovos.intent.list.response' +res.serialize() # single UTF-8 JSON object ``` ```bash @@ -63,7 +69,9 @@ A zero-to-hero guide lives in [`docs/`](docs/README.md): rendering language-model prompts. 5. [Language matching](docs/language-matching.md) — tag standardization, distance, and closest-match resolution. -6. [Linting](docs/linting.md) — validating a locale folder, on the CLI or in CI. +6. [Bus messages](docs/message.md) — the on-the-wire envelope, the + `forward` / `reply` / `response` derivations, and the session carrier. +7. [Linting](docs/linting.md) — validating a locale folder, on the CLI or in CI. 7. [API reference](docs/api-reference.md) — every public name, in brief. Runnable example scripts are in [`examples/`](examples/README.md). diff --git a/docs/README.md b/docs/README.md index 48025d4..43967c3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,13 +6,15 @@ the small, dependency-light primitives those specs describe, in one place, so OVOS components and third-party tools stop reimplementing them and drifting apart. -It gives you four things: +It gives you five things: - an **expander** — turns a sentence template into the set of sentences it stands for (OVOS-INTENT-1); - a **resource loader** — reads a skill's `locale/` folder (OVOS-INTENT-2); - a **dialog renderer** — picks and fills a spoken response (OVOS-INTENT-2 §4.2); - **language matching** — normalizes tags and finds the closest one; +- a **bus message envelope** — the `type` / `data` / `context` JSON contract + and its `forward` / `reply` / `response` derivations (OVOS-MSG-1); plus **`ovos-spec-lint`**, a command-line linter for locale folders. @@ -31,13 +33,20 @@ are the foundation everything else rests on. stateless function and the stateful renderer. 5. [Language matching](language-matching.md) — tag standardization, distance, and closest-match resolution. -6. [Linting](linting.md) — validating a locale folder, from the command line +6. [Bus messages](message.md) — the on-the-wire envelope, the three + derivations (`forward` / `reply` / `response`), and the session carrier. +7. [Linting](linting.md) — validating a locale folder, from the command line or in CI. -7. [API reference](api-reference.md) — every public name, in brief. +8. [API reference](api-reference.md) — every public name, in brief. ## A note on scope -This package **expands, loads, renders, matches language, and lints**. It does not -*recognize* intents — matching an utterance to an intent is the job of an -intent engine, and is deliberately out of scope (see OVOS-INTENT-1 §4). What -you get here is the data those engines consume and the tooling around it. +This package **expands, loads, renders, matches language, lints, and provides +the bus message envelope**. It does not *recognize* intents — matching an +utterance to an intent is the job of an intent engine, and is deliberately +out of scope (see OVOS-INTENT-1 §4). It does not *transport* messages either +— wire framing, encryption, websocket clients, multi-tenant routing, and +session lifecycle all belong to the layers that consume the envelope +(`ovos-bus-client` for the websocket transport; HiveMind for layer-2 +routing). What you get here is the data those engines consume, the message +shape they exchange, and the tooling around them. diff --git a/docs/api-reference.md b/docs/api-reference.md index 8c663e0..7935d45 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -116,13 +116,56 @@ bare tag is measured from its norm region. The entry of `available` with the smallest `lang_distance`, if it is below `max_distance` (or exact). Returns the original string, or `None`. -## Linting — [chapter 6](linting.md) +## Bus messages — [chapter 6](message.md) + +### `Message(msg_type, data=None, context=None)` + +The OVOS-MSG-1 envelope: exactly three top-level fields — `msg_type` +(the wire field `type`), `data`, `context`. The constructor rejects +malformed input as `MalformedMessage`. `data` and `context` default to +empty dicts and are stored by reference. + +#### Derivations + +- `m.forward(msg_type, data=None) -> Message` — same context, new + type/data; the forwarder does not become the new `source`. +- `m.reply(msg_type, data=None, context=None) -> Message` — copies + context, overlays the optional `context`, swaps `source` and + `destination`. Other context keys (including `session`) pass through + unchanged. +- `m.response(data=None, context=None) -> Message` — sugar for + `reply(msg_type + ".response", ...)`. + +All three preserve the runtime class — subclasses get back instances +of their own subclass. + +#### Serialization + +- `m.serialize() -> str` — single UTF-8 JSON object per OVOS-MSG-1 §6; + recursively converts nested objects exposing a `.serialize()` method + (e.g. `Session`). +- `Message.deserialize(payload) -> Message` — parse a JSON string, + bytes, or already-parsed dict. Rejects unknown top-level keys, missing + `type`, wrong value types, NaN/Infinity as `MalformedMessage`. + +### `MalformedMessage` + +`ValueError` subclass raised by the constructor and `deserialize` when a +payload violates OVOS-MSG-1 §2 / §6 / §7. + +### `DEFAULT_SESSION_ID` + +The reserved `"default"` value (OVOS-MSG-1 §4.1) — *the Message +originates from the device itself*. An absent `session` on a Message is +equivalent for every policy decision. + +## Linting — [chapter 7](linting.md) ### `lint_locale(path, spec_version=2) -> list[Finding]` Validate every resource file under a locale (or single-language) directory. `spec_version` (0, 1, or 2) flags features newer than that target — see -[chapter 6](linting.md). +[chapter 7](linting.md). ### `Finding` diff --git a/docs/getting-started.md b/docs/getting-started.md index 380e0df..864fbf4 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -83,6 +83,19 @@ closest_lang("en-AU", ["pt-BR", "en-US", "de-DE"]) # 'en-US' → [Language matching](language-matching.md) +### Construct a bus message + +`Message` is the JSON envelope OVOS components exchange: + +```python +from ovos_spec_tools import Message + +m = Message("ovos.intent.list", {}, {"source": "skill.id"}) +res = m.response({"intents": ["..."]}) # 'ovos.intent.list.response' +``` + +→ [Bus messages](message.md) + ### Lint a locale folder From the command line: diff --git a/docs/locale-resources.md b/docs/locale-resources.md index d8675a2..2c83ce7 100644 --- a/docs/locale-resources.md +++ b/docs/locale-resources.md @@ -50,7 +50,7 @@ read verbatim — every line, including `#` and blank lines, is kept. It carries `{name}` substitution points but is otherwise plain text ([chapter 4](dialog.md)). Legacy OVOS file types (`.rx`, `.value`, `.list`, …) are deliberately *not* -roles here — the linter flags them ([chapter 6](linting.md)). +roles here — the linter flags them ([chapter 7](linting.md)). ## Loading with `LocaleResources` diff --git a/docs/message.md b/docs/message.md new file mode 100644 index 0000000..ee13cee --- /dev/null +++ b/docs/message.md @@ -0,0 +1,180 @@ +# Bus Messages + +The `Message` class is the on-the-wire envelope that OVOS components +exchange — utterances coming in, intent matches going out, skills +announcing handler outcomes, hosts dispatching to skills. The shape is +defined by [OVOS-MSG-1][spec] and this module is the reference +implementation. + +[spec]: https://github.com/OpenVoiceOS/architecture/blob/dev/message-object.md + +## What is in a Message + +Every Message is a JSON object with exactly three top-level fields: + +| Field | Type | Meaning | +|---|---|---| +| `type` | string | The topic — `ovos.intent.matched`, `speak`, `:`, … | +| `data` | object | Topic-specific payload; shape fixed by whichever spec defines `type` | +| `context` | object | Routing keys, the session carrier, and any layer-2 metadata | + +```python +from ovos_spec_tools import Message + +m = Message("speak", {"utterance": "hello"}, {"source": "skill.id"}) +``` + +The constructor rejects malformed input: an empty `type`, a non-string +`type`, or a non-`dict` `data` / `context` raise +`MalformedMessage`. Pass the dicts by reference — they are stored +as-is; if you need a detached copy use the derivations below. + +## Routing keys + +Two `context` keys mark the OVOS/handler-code boundary: + +- `source` — opaque identifier of the producer; +- `destination` — opaque identifier (or list of identifiers) of the + intended consumer(s). Absent or empty means **broadcast**. + +The envelope treats both as opaque strings — no parsing, no validation +beyond "is it there". How identifiers are minted is a deployment +concern. Hivemind, for example, uses `source` / `destination` to thread +remote peers through the bus without OVOS itself learning about them. + +## The session carrier + +`context.session` is the carrier for one conversational session — the +unit of "this wake-word interaction" or "this HiveMind client +connection". OVOS-MSG-1 makes two of its keys normative: + +- `session.session_id` — string identifier; the value `"default"` is + **reserved** and means *originates from the device itself* (used by + `ovos-audio` to keep TTS local). +- `session.lang` — BCP-47 tag for the user's preferred output language + (distinct from `data.lang` which describes the payload). + +The rest of `session`'s shape is opaque to this Message — full Session +structure is the job of the consumer (typically `ovos-bus-client`'s +`Session` class or a future session spec). + +**Absent session = `session_id: "default"`.** A Message with no +`session` in its context is treated the same as one carrying the +reserved value. Producers don't have to emit a session for +device-local messages. + +## The three derivations + +Each derivation returns a new Message with the right routing/session +fields for its role. The runtime class is preserved — subclasses of +`Message` get back instances of their own subclass. + +### `forward(type, data)` — relay under a new topic + +Keeps `context` (including `source`, `destination`, and `session`) +exactly as it was. The forwarder does **not** become the new `source`; +the original producer remains named. + +```python +ack = m.forward("ovos.utterance.handled", {"id": "u-7"}) +# ack.context == m.context (deep-copied) +``` + +### `reply(type, data, context=None)` — send back to the asker + +Copies `context`, then swaps the routing keys so the new Message is +addressed back to the original producer: + +- the new `destination` is the old `source`; +- the new `source` is the old `destination` (the first entry if it was + an array — exact choice is implementation-defined). + +Other `context` keys, including `session`, pass through unchanged. The +optional `context` argument is overlaid before the swap, matching the +historical `ovos-bus-client.Message.reply` behaviour: + +```python +ack = m.reply("speak", {"utterance": "got it"}) +# ack.context["source"] == m.context["destination"] +# ack.context["destination"] == m.context["source"] +``` + +### `response(data, context=None)` — sugar for `.response`-suffixed reply + +A response is a `reply` whose topic is the source topic with +`.response` appended. Used by request/response chains so an observer +can recognize the answer: + +```python +res = Message("ovos.intent.list").response({"intents": ["..."]}) +res.msg_type # 'ovos.intent.list.response' +``` + +## Serialization + +`serialize()` produces a single UTF-8 JSON object per [OVOS-MSG-1 +§6][spec]: + +```python +wire = m.serialize() +recovered = Message.deserialize(wire) +assert recovered == m +``` + +Nested objects in `data` / `context` that expose a `.serialize()` +method (the duck-typed protocol used by `ovos-bus-client.Session` and +similar carriers) are converted before JSON encoding — `Message` +itself doesn't know about Session or any specific carrier type, but +its `.serialize()` walks containers and calls the method when present. + +`deserialize` rejects malformed payloads as `MalformedMessage`: +unparsable JSON, non-object root, unknown top-level keys, missing +`type`, wrong value types. Both `str` and `bytes` are accepted, and an +already-parsed `dict` short-circuits the JSON step. + +## A note on correlation + +There is **no** per-message identifier, no in-reply-to chain, no +host-managed request/response bookkeeping. Messages on the bus are +fully asynchronous. The `.response` suffix and the preserved `session` +give you enough raw material to do your own correlation if you need +it — match an incoming `.response` against an outstanding +request in the same `session_id`. + +Components that need per-conversation state track it themselves keyed +on `session.session_id`. A Message on the bus is **self-contained**; +any state a later consumer needs is either inside the Message or kept +by some component out of band — never recovered by a hidden host-side +correlation index. + +## What this module does not do + +By design, no transport (websocket, queue, …), no encryption, no +authentication, no delivery guarantees, no ordering guarantees, no +session lifecycle, no identifier-assignment policy. Each of those is +out of scope for the envelope and belongs in the layer that consumes +it — `ovos-bus-client` for the websocket transport, layer-2 systems +(HiveMind) for multi-tenant routing, individual subsystems for their +own per-session state. + +## Subclassing + +`Message` is designed to be subclassed. Both `deserialize` (a +classmethod) and `forward` / `reply` / `response` return instances of +the runtime class, so a transport-layer subclass propagates through +derivation chains without losing identity: + +```python +class TransportMessage(Message): + def serialize(self): + body = super().serialize() + # add framing, encryption, … here + return body + +m = TransportMessage("ovos.test") +forwarded = m.forward("ovos.next") +assert isinstance(forwarded, TransportMessage) +``` + +This is exactly how `ovos-bus-client.Message` is expected to extend +the spec primitive once it adopts this class. diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index bc74017..d3ef166 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -22,11 +22,18 @@ :func:`~ovos_spec_tools.resources.utterance_contains`, :func:`~ovos_spec_tools.resources.strip_samples` — slot-free template grouping (§4.3) and utterance match / strip primitives; +- :class:`~ovos_spec_tools.message.Message` — the OVOS-MSG-1 bus message + envelope with the ``forward`` / ``reply`` / ``response`` derivations; - :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also exposed as the ``ovos-spec-lint`` command. """ from ovos_spec_tools.dialog import DialogRenderer, UnfilledSlot, render from ovos_spec_tools.expansion import MalformedTemplate, expand +from ovos_spec_tools.message import ( + DEFAULT_SESSION_ID, + MalformedMessage, + Message, +) from ovos_spec_tools.language import ( closest_lang, lang_distance, @@ -49,6 +56,9 @@ from ovos_spec_tools.version import __version__ __all__ = [ + "Message", + "MalformedMessage", + "DEFAULT_SESSION_ID", "expand", "MalformedTemplate", "LocaleResources", diff --git a/ovos_spec_tools/message.py b/ovos_spec_tools/message.py new file mode 100644 index 0000000..6c8e91c --- /dev/null +++ b/ovos_spec_tools/message.py @@ -0,0 +1,240 @@ +"""Reference implementation of OVOS-MSG-1 — the bus :class:`Message` envelope. + +This module owns only what the spec owns: the JSON envelope of §2, the +routing keys of §3 (which it touches but does not interpret), the +session carrier of §4 (which it treats as an opaque dict), and the +three derivations of §5 — :meth:`Message.forward`, :meth:`Message.reply`, +and :meth:`Message.response`. + +It explicitly does **not** own (§7 non-goals): transport, encryption, +authentication, delivery guarantees, session lifecycle, identifier +assignment, multi-tenant routing. Those concerns live in the layers +that consume this primitive — ``ovos-bus-client`` for the websocket +transport, HiveMind for multi-tenant routing, ``ovos-audio`` / +``ovos-core`` for the per-topic policy decisions keyed on ``session_id`` +and ``lang``. + +The implementation has **no dependencies** outside the standard library; +it is the foundation other bus-layer code builds on. +""" +from __future__ import annotations + +import json +from copy import deepcopy +from typing import Any, Dict, Optional, Union + +__all__ = ["Message", "MalformedMessage", "DEFAULT_SESSION_ID"] + + +#: The reserved ``session_id`` meaning "the Message originates from the +#: device itself" (OVOS-MSG-1 §4.1). Used by ``ovos-audio`` to decide +#: that synthesized TTS plays out of the device's own speakers, and as +#: the implicit value for any Message that arrives without a ``session`` +#: in its context (§4.3). +DEFAULT_SESSION_ID = "default" + + +class MalformedMessage(ValueError): + """A serialized payload that does not conform to OVOS-MSG-1 §2 / §6. + + Raised by :meth:`Message.deserialize` when the payload fails any + structural rule the spec calls out as ``MUST``: unknown top-level + keys (§2), missing ``type`` (§2), wrong value types, or unparsable + JSON (§6). + """ + + +class Message: + """OVOS-MSG-1 envelope. + + Three top-level fields per §2: + + - :attr:`msg_type` (the wire field ``type``) — non-empty topic string + matching the §2.1 syntax (ASCII letters/digits/``.``/``:``/``_``/``-``, + no whitespace); + - :attr:`data` — JSON-object payload; shape fixed by whichever + specification defines :attr:`msg_type`; + - :attr:`context` — JSON-object metadata, carrying the §3 routing + keys (``source`` / ``destination``), the §4 session carrier, and + any layer-2 metadata higher systems attach. + + The :attr:`data` and :attr:`context` dicts are stored by reference; + callers that mutate them after constructing a Message see those + mutations reflected in the Message. :meth:`forward` / :meth:`reply` + / :meth:`response` deep-copy the context before deriving so the + derived Message is independent of the source. + """ + + def __init__(self, msg_type: str, + data: Optional[Dict[str, Any]] = None, + context: Optional[Dict[str, Any]] = None): + if not isinstance(msg_type, str) or not msg_type: + raise MalformedMessage( + "msg_type must be a non-empty string (§2.1)") + if data is not None and not isinstance(data, dict): + raise MalformedMessage("data must be a dict (§2.2)") + if context is not None and not isinstance(context, dict): + raise MalformedMessage("context must be a dict (§2.3)") + self.msg_type = msg_type + self.data = data if data is not None else {} + self.context = context if context is not None else {} + + def __eq__(self, other: object) -> bool: + return (isinstance(other, Message) + and other.msg_type == self.msg_type + and other.data == self.data + and other.context == self.context) + + def __repr__(self) -> str: + return (f"{self.__class__.__name__}(" + f"msg_type={self.msg_type!r}, " + f"data={self.data!r}, context={self.context!r})") + + # --- §6 serialization --------------------------------------------------- + + @staticmethod + def _to_jsonable(value: Any) -> Any: + """Convert ``value`` to a JSON-friendly form. + + Recursively walks containers and converts any object exposing a + ``.serialize()`` method (the duck-typed protocol used by OVOS + carrier objects like ``Session``) by calling it. Plain JSON + types pass through unchanged. + + This keeps :class:`Message` an honest pure-envelope class — it + doesn't know about ``Session`` or any other carrier type — while + letting callers stuff such objects directly into ``data`` / + ``context`` and serialize the result. + """ + # Direct .serialize() — Session, nested Message, etc. + ser = getattr(value, "serialize", None) + if callable(ser) and not isinstance(value, (dict, list, tuple)): + try: + value = ser() + except Exception: + pass + if isinstance(value, dict): + return {k: Message._to_jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [Message._to_jsonable(v) for v in value] + return value + + def serialize(self) -> str: + """Render the Message as a single UTF-8 JSON object per §6. + + Object key order is not significant; ``NaN`` / ``Infinity`` are + forbidden (``allow_nan=False``). Nested objects in ``data`` or + ``context`` that expose ``.serialize()`` (e.g. OVOS ``Session`` + carriers) are converted via that method before serialization. + + Subclasses may override to add transport-layer concerns — + encryption, framing, alternative encoders — which the spec + explicitly leaves out (§7). + """ + return json.dumps( + {"type": self.msg_type, + "data": self._to_jsonable(self.data), + "context": self._to_jsonable(self.context)}, + ensure_ascii=False, allow_nan=False) + + @classmethod + def deserialize(cls, + payload: Union[str, bytes, bytearray, Dict[str, Any]] + ) -> "Message": + """Construct a Message from a serialized JSON object per §6. + + ``payload`` may be a UTF-8 byte string, a ``str`` (parsed as JSON), + or an already-parsed ``dict``. Returns an instance of ``cls`` so + subclasses (e.g. ``ovos-bus-client``'s transport-layer Message) + deserialize to their own type. + + Raises :class:`MalformedMessage` per the §7 ``MUST reject`` + conformance rules: unparsable JSON, non-object root, unknown + top-level keys, missing ``type``, or wrong value types. + """ + if isinstance(payload, (bytes, bytearray)): + payload = payload.decode("utf-8") + if isinstance(payload, str): + try: + obj = json.loads(payload) + except json.JSONDecodeError as exc: + raise MalformedMessage( + f"payload is not valid JSON: {exc}") from exc + else: + obj = payload + if not isinstance(obj, dict): + raise MalformedMessage( + "Message payload must be a JSON object (§2)") + # §2: "Other top-level keys MUST NOT appear; consumers MUST + # reject any Message with unknown top-level keys." + unknown = set(obj.keys()) - {"type", "data", "context"} + if unknown: + raise MalformedMessage( + f"unknown top-level keys {sorted(unknown)!r} — §2 " + "forbids any key other than 'type', 'data', 'context'") + if "type" not in obj: + raise MalformedMessage("missing required key 'type' (§2)") + # data and context default to {} per §2.2/§2.3 ("MAY be empty"). + return cls(obj["type"], obj.get("data") or {}, + obj.get("context") or {}) + + # --- §5 derivations ----------------------------------------------------- + + def forward(self, msg_type: str, + data: Optional[Dict[str, Any]] = None) -> "Message": + """OVOS-MSG-1 §5.1 — relay under a new topic, preserve context. + + The forwarder does **not** become the new ``source``; the original + producer remains named. Returns an instance of this Message's + runtime class so subclasses propagate naturally. + """ + return self.__class__( + msg_type, data or {}, deepcopy(self.context)) + + def reply(self, msg_type: str, + data: Optional[Dict[str, Any]] = None, + context: Optional[Dict[str, Any]] = None) -> "Message": + """OVOS-MSG-1 §5.2 — send back to the asker. + + Copies :attr:`context` and swaps the §3 routing keys so the new + Message is addressed back to the source Message's producer: + + - the new ``destination`` is the old ``source`` (if set); + - the new ``source`` is the old ``destination`` — if the old + ``destination`` was an array, the first entry is chosen + (§5.2 leaves the exact choice implementation-defined); + - every other context key, including ``session`` (§4), is + preserved unchanged. + + Any keys provided via ``context`` are merged in on top of the + copied context **before** the source/destination swap, matching + the historical ``ovos_bus_client.Message.reply`` behaviour: + passing ``context={"source": "C", "destination": "D"}`` ends up + producing ``source=D, destination=C`` because the swap is the + final step. Non-routing keys overlaid this way (a custom + ``session`` shape, a tracing identifier, …) pass through + untouched. + """ + new_context = deepcopy(self.context) + if context: + new_context.update(context) + # §5.2 swap. Read both sides BEFORE writing to avoid clobbering. + src = new_context.get("source") + dst = new_context.get("destination") + if dst is not None: + # array-of-strings form: producer chooses one; consumers + # MUST NOT rely on a particular member being chosen (§5.2) + new_context["source"] = ( + dst[0] if isinstance(dst, list) and dst else dst) + if src is not None: + new_context["destination"] = src + return self.__class__(msg_type, data or {}, new_context) + + def response(self, data: Optional[Dict[str, Any]] = None, + context: Optional[Dict[str, Any]] = None) -> "Message": + """OVOS-MSG-1 §5.3 — sugar for ``reply(self.msg_type + '.response', ...)``. + + Topics defined elsewhere MAY rely on the ``.response`` suffix + convention to mark a Message as the answer to a prior one. + """ + return self.reply(self.msg_type + ".response", data, context) diff --git a/test/test_message.py b/test/test_message.py new file mode 100644 index 0000000..cdb9b6a --- /dev/null +++ b/test/test_message.py @@ -0,0 +1,297 @@ +"""Conformance tests for the OVOS-MSG-1 :class:`Message` envelope. + +Each test class targets one numbered section of the spec +(``architecture/message-object.md``). Test names are the ``MUST`` / +``SHOULD`` rule they pin so failures point at the spec sentence. +""" +import json + +import pytest + +from ovos_spec_tools import ( + DEFAULT_SESSION_ID, + MalformedMessage, + Message, +) + + +# --- §2 envelope ------------------------------------------------------------ + +class TestEnvelope: + def test_constructs_with_only_msg_type(self): + """§2: ``data`` and ``context`` MAY be empty.""" + m = Message("ovos.test") + assert m.msg_type == "ovos.test" + assert m.data == {} + assert m.context == {} + + def test_constructs_with_data_and_context(self): + m = Message("ovos.test", {"a": 1}, {"source": "x"}) + assert m.data == {"a": 1} + assert m.context == {"source": "x"} + + def test_rejects_empty_msg_type(self): + """§2.1: ``type`` is a non-empty string.""" + with pytest.raises(MalformedMessage): + Message("") + + def test_rejects_non_string_msg_type(self): + with pytest.raises(MalformedMessage): + Message(123) # type: ignore[arg-type] + + def test_rejects_non_dict_data(self): + """§2.2: ``data`` is a JSON object.""" + with pytest.raises(MalformedMessage): + Message("ovos.test", "not a dict") # type: ignore[arg-type] + + def test_rejects_non_dict_context(self): + """§2.3: ``context`` is a JSON object.""" + with pytest.raises(MalformedMessage): + Message("ovos.test", {}, "not a dict") # type: ignore[arg-type] + + +# --- §6 serialization + deserialization round-trip -------------------------- + +class TestSerialization: + def test_round_trip_preserves_envelope(self): + original = Message( + "ovos.test", {"x": 1, "y": ["a", "b"]}, + {"source": "skill", "session": {"session_id": "s1"}}) + round_tripped = Message.deserialize(original.serialize()) + assert round_tripped == original + + def test_serialize_emits_a_single_json_object(self): + m = Message("ovos.test") + parsed = json.loads(m.serialize()) + assert isinstance(parsed, dict) + assert set(parsed.keys()) == {"type", "data", "context"} + + def test_deserialize_rejects_unknown_top_level_keys(self): + """§2: 'Other top-level keys MUST NOT appear; consumers MUST + reject any Message with unknown top-level keys.'""" + payload = json.dumps({ + "type": "ovos.test", "data": {}, "context": {}, + "extra": "field"}) + with pytest.raises(MalformedMessage): + Message.deserialize(payload) + + def test_deserialize_rejects_missing_type(self): + with pytest.raises(MalformedMessage): + Message.deserialize(json.dumps({"data": {}, "context": {}})) + + def test_deserialize_rejects_unparsable_payload(self): + with pytest.raises(MalformedMessage): + Message.deserialize("not json {") + + def test_deserialize_rejects_non_object_root(self): + """§2: 'A Message is a JSON object'.""" + with pytest.raises(MalformedMessage): + Message.deserialize(json.dumps(["not", "an", "object"])) + + def test_deserialize_accepts_already_parsed_dict(self): + m = Message.deserialize({"type": "ovos.test", "data": {}, "context": {}}) + assert m == Message("ovos.test") + + def test_deserialize_accepts_bytes(self): + m = Message.deserialize(b'{"type": "ovos.test", "data": {}, "context": {}}') + assert m.msg_type == "ovos.test" + + def test_serialize_forbids_nan_and_infinity(self): + """§6: 'Numbers MUST be finite. NaN, +Infinity, -Infinity are + forbidden.'""" + m = Message("ovos.test", {"x": float("nan")}) + with pytest.raises(ValueError): + m.serialize() + + def test_serialize_calls_dot_serialize_on_nested_objects(self): + """Carrier objects (Session, nested Messages, …) exposing a + ``.serialize()`` method are converted before JSON encoding so + callers don't have to pre-dictify their context.""" + + class _Carrier: + def __init__(self, sid): + self.sid = sid + + def serialize(self): + return {"session_id": self.sid, "lang": "en-US"} + + m = Message("ovos.test", {}, {"session": _Carrier("s-42")}) + parsed = json.loads(m.serialize()) + assert parsed["context"]["session"] == { + "session_id": "s-42", "lang": "en-US"} + + def test_serialize_walks_nested_lists_and_dicts(self): + class _C: + def serialize(self): + return "C-payload" + m = Message("ovos.test", {"items": [_C(), {"k": _C()}]}) + parsed = json.loads(m.serialize()) + assert parsed["data"]["items"] == ["C-payload", {"k": "C-payload"}] + + def test_serialize_data_can_be_empty(self): + """§2.2: ``data`` MAY be empty (`{}`).""" + parsed = json.loads(Message("ovos.test").serialize()) + assert parsed["data"] == {} + assert parsed["context"] == {} + + +# --- §5.1 forward ----------------------------------------------------------- + +class TestForward: + def test_preserves_context_unchanged(self): + """§5.1: ``context = C`` (preserved unchanged, including + ``source``, ``destination``, and ``session``).""" + m = Message("ovos.a", {}, {"source": "x", "destination": "y", + "session": {"session_id": "s1"}}) + f = m.forward("ovos.b", {"k": "v"}) + assert f.msg_type == "ovos.b" + assert f.data == {"k": "v"} + assert f.context == {"source": "x", "destination": "y", + "session": {"session_id": "s1"}} + + def test_forwarder_does_not_become_new_source(self): + """§5.1: 'The forwarder does not become the new source — the + original producer remains named.'""" + m = Message("ovos.a", {}, {"source": "original-producer"}) + assert m.forward("ovos.b").context["source"] == "original-producer" + + def test_forward_context_is_deep_copied(self): + """Mutating the source context after forward MUST NOT affect + the forwarded Message.""" + m = Message("ovos.a", {}, {"source": "x", "session": {"id": "s1"}}) + f = m.forward("ovos.b") + m.context["session"]["id"] = "MUTATED" + assert f.context["session"]["id"] == "s1" + + def test_forward_omits_data_defaults_to_empty(self): + m = Message("ovos.a", {"k": "v"}) + assert m.forward("ovos.b").data == {} + + +# --- §5.2 reply ------------------------------------------------------------- + +class TestReply: + def test_swaps_source_and_destination(self): + """§5.2: routing keys reversed.""" + m = Message("ovos.req", {}, {"source": "A", "destination": "B"}) + r = m.reply("ovos.ack") + assert r.context["source"] == "B" + assert r.context["destination"] == "A" + + def test_preserves_session_unchanged(self): + """§5.2 (3): 'All other ``context`` keys, including ``session``, + are preserved unchanged.'""" + sess = {"session_id": "s1", "lang": "en-US"} + m = Message("ovos.req", {}, {"source": "A", "destination": "B", + "session": sess}) + r = m.reply("ovos.ack") + assert r.context["session"] == sess + + def test_destination_array_picks_one_for_source(self): + """§5.2 (2): array form — implementation chooses; consumers + MUST NOT rely on a particular member being chosen.""" + m = Message("ovos.req", {}, {"source": "A", + "destination": ["B", "C"]}) + r = m.reply("ovos.ack") + # the exact choice is implementation-defined; ours is index 0 + assert r.context["source"] in ("B", "C") + assert r.context["destination"] == "A" + + def test_provided_context_keys_are_swapped_too(self): + """``context`` overlays land before the §5.2 swap (matches the + historical ``ovos_bus_client.Message.reply`` behaviour); routing + keys passed in get swapped along with everything else.""" + m = Message("ovos.req", {}, {"source": "A", "destination": "B"}) + r = m.reply("ovos.ack", + context={"source": "C", "destination": "D"}) + # overlay sets src=C, dst=D — then the swap flips them + assert r.context["source"] == "D" + assert r.context["destination"] == "C" + + def test_provided_non_routing_context_keys_pass_through(self): + """Keys that are not ``source`` / ``destination`` overlay + without being swapped — handy for custom session shapes or + tracing identifiers.""" + m = Message("ovos.req", {}, {"source": "A", "destination": "B"}) + r = m.reply("ovos.ack", context={"trace_id": "abc"}) + assert r.context["trace_id"] == "abc" + assert r.context["source"] == "B" + assert r.context["destination"] == "A" + + +# --- §5.3 response ---------------------------------------------------------- + +class TestResponse: + def test_topic_is_source_plus_dot_response(self): + """§5.3: 'A ``response`` is a ``reply`` whose topic is the source + topic suffixed with ``.response``.'""" + m = Message("ovos.intent.list", {}, {"source": "A", "destination": "B"}) + r = m.response({"intents": []}) + assert r.msg_type == "ovos.intent.list.response" + + def test_response_swaps_routing_like_reply(self): + m = Message("ovos.intent.list", {}, {"source": "A", "destination": "B"}) + r = m.response() + assert r.context["source"] == "B" + assert r.context["destination"] == "A" + + +# --- §4 session carrier ----------------------------------------------------- + +class TestSessionCarrier: + def test_default_session_id_constant_matches_spec(self): + """§4.1: 'The value "default" is reserved...'""" + assert DEFAULT_SESSION_ID == "default" + + def test_absent_session_is_well_formed(self): + """§4.3 / §7: 'a Message without [session] is well-formed'.""" + m = Message("ovos.test") + assert "session" not in m.context + + def test_context_session_is_opaque_dict(self): + """§4: 'session is mostly opaque under this specification.' The + envelope does not parse it — any dict shape conforms.""" + m = Message("ovos.test", {}, + {"session": {"weird": "shape", "session_id": "s1"}}) + round_tripped = Message.deserialize(m.serialize()) + assert round_tripped.context["session"]["weird"] == "shape" + + +# --- §3 routing key opacity ------------------------------------------------- + +class TestRoutingOpacity: + def test_source_and_destination_can_be_arbitrary_strings(self): + """§3.4: 'source and destination are opaque strings.'""" + m = Message("ovos.test", {}, + {"source": "anything-goes-here", + "destination": ["a@b", "c#d"]}) + assert Message.deserialize(m.serialize()) == m + + +# --- subclass-friendliness -------------------------------------------------- + +class TestSubclassDerivations: + """Derivations and ``deserialize`` must return instances of the + runtime class so ``ovos-bus-client``'s subclass propagates naturally + through forward / reply / response chains.""" + + class _BusClientMessage(Message): + pass + + def test_forward_returns_subclass(self): + m = self._BusClientMessage("ovos.a") + assert isinstance(m.forward("ovos.b"), self._BusClientMessage) + + def test_reply_returns_subclass(self): + m = self._BusClientMessage("ovos.a", {}, {"source": "x", "destination": "y"}) + assert isinstance(m.reply("ovos.b"), self._BusClientMessage) + + def test_response_returns_subclass(self): + m = self._BusClientMessage("ovos.a", {}, {"source": "x", "destination": "y"}) + assert isinstance(m.response(), self._BusClientMessage) + + def test_deserialize_returns_subclass(self): + payload = '{"type": "ovos.a", "data": {}, "context": {}}' + assert isinstance( + self._BusClientMessage.deserialize(payload), + self._BusClientMessage) From b3e506f9d13db6caac8f1a4cd0849df080a29793 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 24 May 2026 00:19:06 +0000 Subject: [PATCH 037/110] Increment Version to 0.5.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 458e064..a5b4b67 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 4 +VERSION_MINOR = 5 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 00dbfa3d3dd0119f94d3ebcb1dfb05ba64854c6f Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 24 May 2026 00:19:22 +0000 Subject: [PATCH 038/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f26507..d0d5835 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.5.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.5.0a1) (2026-05-24) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.4.0a1...0.5.0a1) + +**Merged pull requests:** + +- feat: OVOS-MSG-1 Message envelope [\#12](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/12) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.4.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.4.0a1) (2026-05-23) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.3.0a1...0.4.0a1) From 4a0160cc7f6966bb4de9a2f02fb03bd5976bdde8 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 24 May 2026 16:48:38 +0100 Subject: [PATCH 039/110] fix(Message): three follow-ups missed by the 0.5.0a1 cut (#14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(Message): allow empty msg_type at construction; spec gates wire form Constructing Message('') as a routing scaffold for a later forward() is widely used in ovos-bus-client (dig_for_message() or Message('') fallback, enclosure/events/ocp api placeholders). The §2.1 'non-empty type' rule applies to the **emitted** wire form (§7 producer MUST); intermediate objects that never reach the wire are out of scope. Relax the constructor; deserialize stays strict (it IS receiving a wire message and §2.1 applies). * fix(Message): MalformedMessage inherits AssertionError too — no break for legacy 'except AssertionError' handlers ovos_bus_client.Message historically raised AssertionError from bare assert statements in its constructor. To keep downstream code that catches that type working through the migration, MalformedMessage now inherits both ValueError (the correct modern type) and AssertionError (the legacy type). * feat(Message): as_dict property — JSON-decoded envelope view Equivalent to json.loads(self.serialize()); offered as a property so callers that want a one-shot dict view skip the JSON intermediate. The .serialize()-walking carrier conversion happens the same way it would on the wire, so the resulting dict is round-trippable. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- docs/api-reference.md | 3 +++ docs/message.md | 7 +++++++ ovos_spec_tools/message.py | 32 +++++++++++++++++++++++++---- test/test_message.py | 41 ++++++++++++++++++++++++++++++++++---- 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 7935d45..699d0bb 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -144,6 +144,9 @@ of their own subclass. - `m.serialize() -> str` — single UTF-8 JSON object per OVOS-MSG-1 §6; recursively converts nested objects exposing a `.serialize()` method (e.g. `Session`). +- `m.as_dict -> dict` (property) — JSON-decoded envelope, equivalent + to `json.loads(m.serialize())`. Nested `.serialize()`-walking carriers + are converted the same way they would be on the wire. - `Message.deserialize(payload) -> Message` — parse a JSON string, bytes, or already-parsed dict. Rejects unknown top-level keys, missing `type`, wrong value types, NaN/Infinity as `MalformedMessage`. diff --git a/docs/message.md b/docs/message.md index ee13cee..827148e 100644 --- a/docs/message.md +++ b/docs/message.md @@ -127,6 +127,13 @@ similar carriers) are converted before JSON encoding — `Message` itself doesn't know about Session or any specific carrier type, but its `.serialize()` walks containers and calls the method when present. +For a JSON-decoded view without the string intermediate, `m.as_dict` +returns the same envelope as a dictionary: + +```python +m.as_dict # {'type': 'ovos.test', 'data': {...}, 'context': {...}} +``` + `deserialize` rejects malformed payloads as `MalformedMessage`: unparsable JSON, non-object root, unknown top-level keys, missing `type`, wrong value types. Both `str` and `bytes` are accepted, and an diff --git a/ovos_spec_tools/message.py b/ovos_spec_tools/message.py index 6c8e91c..c1b4113 100644 --- a/ovos_spec_tools/message.py +++ b/ovos_spec_tools/message.py @@ -34,13 +34,20 @@ DEFAULT_SESSION_ID = "default" -class MalformedMessage(ValueError): +class MalformedMessage(ValueError, AssertionError): """A serialized payload that does not conform to OVOS-MSG-1 §2 / §6. Raised by :meth:`Message.deserialize` when the payload fails any structural rule the spec calls out as ``MUST``: unknown top-level keys (§2), missing ``type`` (§2), wrong value types, or unparsable JSON (§6). + + Inherits from both :class:`ValueError` (the modern, + correctly-typed exception class) **and** :class:`AssertionError` + (the type historically raised by ``ovos_bus_client.Message``'s + bare ``assert`` constructor checks), so legacy ``except + AssertionError`` handlers in downstream code continue to catch + the same conditions. """ @@ -68,9 +75,15 @@ class Message: def __init__(self, msg_type: str, data: Optional[Dict[str, Any]] = None, context: Optional[Dict[str, Any]] = None): - if not isinstance(msg_type, str) or not msg_type: - raise MalformedMessage( - "msg_type must be a non-empty string (§2.1)") + # §2.1 requires the wire ``type`` to be non-empty, but it is a + # spec rule for **emitted** Messages — the construct-then-forward + # pattern (``Message("").forward(real_type, data)``) is widely + # used to build a routing scaffold before the real topic is + # known, so the constructor accepts an empty string here and + # :meth:`serialize` is the gate that flags non-conformant wire + # output (see §7 producer rules). + if not isinstance(msg_type, str): + raise MalformedMessage("msg_type must be a string (§2.1)") if data is not None and not isinstance(data, dict): raise MalformedMessage("data must be a dict (§2.2)") if context is not None and not isinstance(context, dict): @@ -92,6 +105,17 @@ def __repr__(self) -> str: # --- §6 serialization --------------------------------------------------- + @property + def as_dict(self) -> Dict[str, Any]: + """The Message rendered as a JSON-decoded dictionary — + ``{"type": ..., "data": ..., "context": ...}``. + + Equivalent to ``json.loads(self.serialize())`` and round-trips + through :meth:`deserialize`; offered as a property for callers + that want a one-shot dict view without the JSON intermediate. + """ + return json.loads(self.serialize()) + @staticmethod def _to_jsonable(value: Any) -> Any: """Convert ``value`` to a JSON-friendly form. diff --git a/test/test_message.py b/test/test_message.py index cdb9b6a..27b5f8e 100644 --- a/test/test_message.py +++ b/test/test_message.py @@ -30,10 +30,13 @@ def test_constructs_with_data_and_context(self): assert m.data == {"a": 1} assert m.context == {"source": "x"} - def test_rejects_empty_msg_type(self): - """§2.1: ``type`` is a non-empty string.""" - with pytest.raises(MalformedMessage): - Message("") + def test_accepts_empty_msg_type_at_construction(self): + """§2.1 (non-empty ``type``) gates the **wire** form, not + intermediate objects. The ``Message("").forward(real_type)`` + pattern is widely used to build a routing scaffold before the + real topic is known.""" + m = Message("") + assert m.msg_type == "" def test_rejects_non_string_msg_type(self): with pytest.raises(MalformedMessage): @@ -295,3 +298,33 @@ def test_deserialize_returns_subclass(self): assert isinstance( self._BusClientMessage.deserialize(payload), self._BusClientMessage) + + +def test_malformed_message_is_catchable_as_assertion_error(): + """Legacy ``ovos_bus_client.Message`` raised ``AssertionError`` from + bare ``assert`` constructor checks; downstream code that catches + that type must keep working through the migration.""" + with pytest.raises(AssertionError): + Message("ok", data="not a dict") + + +# --- as_dict ---------------------------------------------------------------- + +def test_as_dict_returns_envelope_as_dict(): + m = Message("ovos.test", {"a": 1}, {"source": "skill"}) + d = m.as_dict + assert isinstance(d, dict) + assert d == {"type": "ovos.test", "data": {"a": 1}, + "context": {"source": "skill"}} + + +def test_as_dict_walks_carrier_serialize_protocol(): + """``as_dict`` round-trips through ``serialize`` so nested objects + with a ``.serialize()`` method are converted, same as on the wire.""" + + class _Carrier: + def serialize(self): + return {"session_id": "s-7"} + + m = Message("ovos.test", {}, {"session": _Carrier()}) + assert m.as_dict["context"]["session"] == {"session_id": "s-7"} From b85cd41c39657feabcd41f7706478f20200a812b Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 24 May 2026 15:48:49 +0000 Subject: [PATCH 040/110] Increment Version to 0.5.1a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index a5b4b67..832bb0f 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 5 -VERSION_BUILD = 0 +VERSION_BUILD = 1 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 5ee29983b163ed138334cfb92b79e2dbb1ce72ac Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 24 May 2026 15:49:09 +0000 Subject: [PATCH 041/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0d5835..3662447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.5.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.5.1a1) (2026-05-24) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.5.0a1...0.5.1a1) + +**Merged pull requests:** + +- fix\(Message\): three follow-ups missed by the 0.5.0a1 cut [\#14](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/14) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.5.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.5.0a1) (2026-05-24) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.4.0a1...0.5.0a1) From 4b4bfb5c60a7b9ac8267f5e4b7b617a9c4bc181f Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 25 May 2026 20:52:11 +0100 Subject: [PATCH 042/110] =?UTF-8?q?feat:=20find=5Flang=5Fdir=20=E2=80=94?= =?UTF-8?q?=20standalone=20language-aware=20directory=20resolver=20(#17)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Promotes LocaleResources._lang_dir to a module-level public helper: find_lang_dir(base_path, lang, lang_resolver=closest_lang, max_distance=10) -> Optional[Path] Given a directory containing per-language subdirectories, returns the one whose name best matches the requested lang (OVOS-INTENT-2 §2.2 smart fallback). Default resolver is closest_lang; callers can swap it. Resolves case mismatch (en-US vs en-us), bare-language requests against region-specific dirs (en -> en-US), and minor regional fallback (en-AU -> en-US within distance < 10) through one predicate. The on-disk casing is preserved in the returned Path. LocaleResources._lang_dir collapses to a thin wrapper that passes its configured resolver / max_distance through. Why public: ovos-workshop's OVOSAbstractApplication.get_language_dir reimplements the same logic; with this helper it collapses to a one-liner and the smart-fallback policy lives in one place. Other tools that walk a single locale tree without constructing a full LocaleResources have the same primitive available. Tests: 11 cases covering exact match, case mismatch, subdialect fallback, macro fallback, no-match, missing base path, max_distance=0, custom resolver, non-dir entries, Path/str inputs. Co-authored-by: Claude Opus 4.7 (1M context) --- ovos_spec_tools/__init__.py | 2 + ovos_spec_tools/resources.py | 52 +++++++++++++++--- test/test_find_lang_dir.py | 100 +++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 test/test_find_lang_dir.py diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index d3ef166..ddb131b 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -45,6 +45,7 @@ from ovos_spec_tools.resources import ( LocaleResources, MalformedResource, + find_lang_dir, iter_locale_dirs, keyword_form, normalize_for_match, @@ -63,6 +64,7 @@ "MalformedTemplate", "LocaleResources", "MalformedResource", + "find_lang_dir", "iter_locale_dirs", "keyword_form", "normalize_for_match", diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index 0c5857f..e1575b4 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -23,7 +23,7 @@ from __future__ import annotations from pathlib import Path -from typing import Callable, Dict, Iterator, List, Optional, Sequence, Tuple +from typing import Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union from ovos_spec_tools.expansion import expand from ovos_spec_tools.language import ( @@ -35,6 +35,7 @@ __all__ = [ "LocaleResources", "MalformedResource", + "find_lang_dir", "iter_locale_dirs", "keyword_form", "normalize_for_match", @@ -126,6 +127,47 @@ def iter_locale_dirs(root: Path, yield lang_norm, entry +def find_lang_dir(base_path: Union[str, Path], lang: str, + lang_resolver: Optional["LanguageResolver"] = None, + max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE + ) -> Optional[Path]: + """Resolve the best ``//`` subdirectory for *lang*. + + The immediate subdirectories of ``base_path`` are taken as the set of + available languages; the resolver (default :func:`closest_lang`) + picks the closest match within ``max_distance`` (OVOS-INTENT-2 §2.2 + smart fallback). Case-mismatch (``en-US`` vs ``en-us``) and + minor regional fallback (``en-AU`` to ``en-US``) both resolve + through :func:`closest_lang` — no separate exact-match short-circuit + is needed. + + Returns the resolved :class:`Path`, or ``None`` if ``base_path`` is + not a directory or no available subdir is close enough. + + This is the standalone primitive backing + :meth:`LocaleResources._lang_dir` — use this when you want one + language-aware directory lookup without constructing a full + :class:`LocaleResources`. Skills typically use ``LocaleResources`` + (which wires the override-precedence search); tools that walk a + single locale tree (``locate this lang's resource root``) call this. + """ + base = Path(base_path) + if not base.is_dir(): + return None + names = [c.name for c in base.iterdir() if c.is_dir()] + resolver = lang_resolver if lang_resolver is not None else closest_lang + match = resolver(lang, names, max_distance) + if match is None: + return None + # `match` is one of the candidate names normalized by the resolver; + # find the original directory entry that standardizes to it so we + # return the on-disk casing. + for name in names: + if standardize_lang(name) == standardize_lang(match): + return base / name + return None + + def keyword_form(template_line: str, vocabularies: Optional[Dict[str, List[str]]] = None ) -> Tuple[str, List[str]]: @@ -307,11 +349,9 @@ def _lang_dir(self, source: Path, lang: str) -> Optional[Path]: The language is resolved against the available subdirectories by the ``lang_resolver`` — an exact tag, or the smart fallback of §2.2. """ - if not source.is_dir(): - return None - names = [c.name for c in source.iterdir() if c.is_dir()] - match = self._lang_resolver(lang, names, self.max_language_distance) - return (source / match) if match is not None else None + return find_lang_dir(source, lang, + lang_resolver=self._lang_resolver, + max_distance=self.max_language_distance) def find(self, base_name: str, extension: str, lang: str) -> Optional[Path]: diff --git a/test/test_find_lang_dir.py b/test/test_find_lang_dir.py new file mode 100644 index 0000000..f28c19a --- /dev/null +++ b/test/test_find_lang_dir.py @@ -0,0 +1,100 @@ +"""Tests for :func:`ovos_spec_tools.find_lang_dir`.""" +import os +import tempfile +import unittest +from pathlib import Path + +from ovos_spec_tools import find_lang_dir + + +def _mkdirs(root, *names): + for name in names: + os.makedirs(os.path.join(root, name)) + + +class TestFindLangDir(unittest.TestCase): + + def setUp(self): + self.root = tempfile.mkdtemp(prefix="findlang_") + + def tearDown(self): + import shutil + shutil.rmtree(self.root, ignore_errors=True) + + def test_exact_uppercase_match(self): + _mkdirs(self.root, "en-US", "de-DE") + self.assertEqual(find_lang_dir(self.root, "en-US"), + Path(self.root) / "en-US") + + def test_exact_lowercase_match(self): + _mkdirs(self.root, "en-us") + self.assertEqual(find_lang_dir(self.root, "en-us"), + Path(self.root) / "en-us") + + def test_case_mismatch_resolves(self): + """Request uppercase, dir is lowercase — closest_lang resolves + through standardize_lang on both sides.""" + _mkdirs(self.root, "en-us") + self.assertEqual(find_lang_dir(self.root, "en-US"), + Path(self.root) / "en-us") + + def test_subdialect_fallback(self): + """Request en-AU, only en-US available — minor regional + difference (langcodes distance ~4) is below the default max + of 10.""" + _mkdirs(self.root, "en-US", "fr-FR") + self.assertEqual(find_lang_dir(self.root, "en-AU"), + Path(self.root) / "en-US") + + def test_macro_fallback(self): + """Request bare ``en``, en-US directory available — resolves.""" + _mkdirs(self.root, "en-US") + self.assertEqual(find_lang_dir(self.root, "en"), + Path(self.root) / "en-US") + + def test_no_match_returns_none(self): + _mkdirs(self.root, "pt-PT", "ja-JP") + self.assertIsNone(find_lang_dir(self.root, "ko-KR")) + + def test_missing_base_path_returns_none(self): + self.assertIsNone( + find_lang_dir(os.path.join(self.root, "nope"), "en-US")) + + def test_max_distance_zero_requires_exact_normalized_match(self): + """``max_distance=0`` disables smart fallback; only same-tag + matches resolve.""" + _mkdirs(self.root, "en-US") + self.assertEqual( + find_lang_dir(self.root, "en-US", max_distance=0), + Path(self.root) / "en-US") + self.assertIsNone( + find_lang_dir(self.root, "en-AU", max_distance=0)) + + def test_custom_resolver_is_honoured(self): + """A caller can swap the resolver — useful for tests or + deployments that want a different fallback policy.""" + _mkdirs(self.root, "en-US", "de-DE") + + def _always_de(target, available, max_distance): + return "de-DE" if "de-DE" in available else None + + self.assertEqual( + find_lang_dir(self.root, "en-US", lang_resolver=_always_de), + Path(self.root) / "de-DE") + + def test_ignores_non_directory_entries(self): + """A regular file alongside lang dirs must not be considered a + candidate.""" + _mkdirs(self.root, "en-US") + Path(self.root, "stray.txt").write_text("noise") + self.assertEqual(find_lang_dir(self.root, "en-US"), + Path(self.root) / "en-US") + + def test_accepts_path_or_str_base(self): + _mkdirs(self.root, "en-US") + self.assertEqual(find_lang_dir(Path(self.root), "en-US"), + Path(self.root) / "en-US") + + +if __name__ == "__main__": + unittest.main() From 876f7816bdb587de3d6ef0c81166c4fb887049e0 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 25 May 2026 19:52:21 +0000 Subject: [PATCH 043/110] Increment Version to 0.6.0a1 --- ovos_spec_tools/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 832bb0f..0205a32 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 5 -VERSION_BUILD = 1 +VERSION_MINOR = 6 +VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 4c24a601f01638698e6d14c34967c060f4e00efd Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Mon, 25 May 2026 19:52:36 +0000 Subject: [PATCH 044/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3662447..3dab2de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.6.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.6.0a1) (2026-05-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.5.1a1...0.6.0a1) + +**Merged pull requests:** + +- feat: find\_lang\_dir — standalone language-aware directory resolver [\#17](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/17) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.5.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.5.1a1) (2026-05-24) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.5.0a1...0.5.1a1) From 70924905396f295e9a8b0d94e89e2c79de2cf8cc Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:29:46 +0100 Subject: [PATCH 045/110] =?UTF-8?q?feat:=20union=20slot=20sets=20for=20.in?= =?UTF-8?q?tent=20(OVOS-INTENT-1=20=C2=A75.5)=20(#19)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: union slot sets for .intent, keep strict for .dialog - lint.py: slot-consistency check now applies only to .dialog (OVOS-INTENT-1 §5.5). .intent files with differing slot sets are accepted. - test/test_lint.py: update slot-consistency tests — .intent with different slot sets is clean, .dialog still requires identical slot sets. * feat: warn on .intent with differing slot sets - lint.py: add WARNING (not ERROR) when .intent templates declare different slot sets. Exits 0 unless --strict is passed. - test/test_lint.py: update tests to expect WARNING for .intent inconsistency, no error. --- ovos_spec_tools/lint.py | 17 ++++++++++++----- test/test_lint.py | 30 ++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/ovos_spec_tools/lint.py b/ovos_spec_tools/lint.py index b109b99..7f310f9 100644 --- a/ovos_spec_tools/lint.py +++ b/ovos_spec_tools/lint.py @@ -250,11 +250,18 @@ def _lint_file(path: Path, slot_sets.append(frozenset(_SLOT_RE.findall(template))) # --- slot consistency (OVOS-INTENT-1 §5.5) ------------------------------ - if slot_bearing and len(set(slot_sets)) > 1: - findings.append(Finding( - ERROR, str(path), - f"templates declare different slot sets — every template in one " - f"{extension} must use the same {{slots}} (OVOS-INTENT-1 §5.5)")) + # .dialog requires identical slot sets; .intent allows union slot sets. + if len(set(slot_sets)) > 1: + if extension == ".dialog": + findings.append(Finding( + ERROR, str(path), + f"templates declare different slot sets — every template in one " + f"{extension} must use the same {{slots}} (OVOS-INTENT-1 §5.5)")) + else: + findings.append(Finding( + WARNING, str(path), + f"templates declare different slot sets — this is valid for " + f".intent but unusual; verify this is intentional (OVOS-INTENT-1 §5.5)")) return findings diff --git a/test/test_lint.py b/test/test_lint.py index e6b1251..c7db33c 100644 --- a/test/test_lint.py +++ b/test/test_lint.py @@ -147,16 +147,22 @@ def test_lint_accepts_a_single_language_directory(tmp_path): # --- slot consistency (OVOS-INTENT-1 §5.5) ---------------------------------- -def test_inconsistent_slots_in_one_intent_is_an_error(tmp_path): +# .intent allows union slot sets — templates MAY declare different slots. + +def test_inconsistent_slots_in_one_intent_warns(tmp_path): locale = tmp_path / "locale" _write(locale / "en-US" / "p.intent", "play {query}\nstop {engine}\n") - assert any("slot sets" in f.message for f in _errors(lint_locale(locale))) + warnings = _warnings(lint_locale(locale)) + assert any("slot sets" in f.message for f in warnings) + assert not any("slot sets" in f.message for f in _errors(lint_locale(locale))) -def test_mixing_slotted_and_slotless_lines_is_an_error(tmp_path): +def test_mixing_slotted_and_slotless_lines_in_intent_warns(tmp_path): locale = tmp_path / "locale" _write(locale / "en-US" / "p.intent", "play {query}\njust stop\n") - assert any("slot sets" in f.message for f in _errors(lint_locale(locale))) + warnings = _warnings(lint_locale(locale)) + assert any("slot sets" in f.message for f in warnings) + assert not any("slot sets" in f.message for f in _errors(lint_locale(locale))) def test_consistent_slots_across_an_intent_is_clean(tmp_path): @@ -166,6 +172,22 @@ def test_consistent_slots_across_an_intent_is_clean(tmp_path): assert lint_locale(locale) == [] +# .dialog still requires identical slot sets. + +def test_inconsistent_slots_in_one_dialog_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.dialog", + "Hello {name}.\nNice to meet you, {title} and {surname}.\n") + assert any("slot sets" in f.message for f in _errors(lint_locale(locale))) + + +def test_mixing_slotted_and_slotless_lines_in_dialog_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.dialog", + "Hello {name}.\nWelcome back.\n") + assert any("slot sets" in f.message for f in _errors(lint_locale(locale))) + + # --- robustness -------------------------------------------------------------- def test_non_utf8_file_is_reported_not_crashed(tmp_path): From 919d7c75816f697635bf457a8f8794f04eee32e5 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Tue, 2 Jun 2026 08:29:58 +0000 Subject: [PATCH 046/110] Increment Version to 0.7.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 0205a32..c5e9ef6 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 6 +VERSION_MINOR = 7 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 380dc68febd146a549a5c5f39bcb459f8f6460f8 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Tue, 2 Jun 2026 08:30:20 +0000 Subject: [PATCH 047/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dab2de..3d9d3c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.7.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.7.0a1) (2026-06-02) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.6.0a1...0.7.0a1) + +**Merged pull requests:** + +- feat: union slot sets for .intent \(OVOS-INTENT-1 §5.5\) [\#19](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/19) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.6.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.6.0a1) (2026-05-25) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.5.1a1...0.6.0a1) From 02ca560454b43b26e8a64adfa6d74064f538eed1 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Tue, 2 Jun 2026 15:46:08 +0100 Subject: [PATCH 048/110] =?UTF-8?q?feat:=20inline=5Fkeywords=20=E2=80=94?= =?UTF-8?q?=20resolve=20=20refs=20as=20(a|b|c)=20for=20engines=20?= =?UTF-8?q?without=20.voc=20support=20(#22)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add inline_keywords for engines without .voc support inline_keywords(template, vocabularies) replaces refs with (a|b|c) alternation groups inline — needed for engines like Padatious that don't look up .voc files at runtime. Handles nested refs recursively with configurable max_values cap. Added to ovos_spec_tools.expansion alongside the existing expand() function. Exported from the top-level package. * fix: keep angle brackets for unresolvable keywords in inline_keywords --- ovos_spec_tools/__init__.py | 3 +- ovos_spec_tools/expansion.py | 55 ++++++++++++++++++++++++++++++++++++ test/test_expansion.py | 50 +++++++++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 2 deletions(-) diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index ddb131b..9d14c37 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -28,7 +28,7 @@ exposed as the ``ovos-spec-lint`` command. """ from ovos_spec_tools.dialog import DialogRenderer, UnfilledSlot, render -from ovos_spec_tools.expansion import MalformedTemplate, expand +from ovos_spec_tools.expansion import MalformedTemplate, expand, inline_keywords from ovos_spec_tools.message import ( DEFAULT_SESSION_ID, MalformedMessage, @@ -61,6 +61,7 @@ "MalformedMessage", "DEFAULT_SESSION_ID", "expand", + "inline_keywords", "MalformedTemplate", "LocaleResources", "MalformedResource", diff --git a/ovos_spec_tools/expansion.py b/ovos_spec_tools/expansion.py index 4acf902..806974c 100644 --- a/ovos_spec_tools/expansion.py +++ b/ovos_spec_tools/expansion.py @@ -226,3 +226,58 @@ def _check_sample(sentence: str, template: str) -> None: raise MalformedTemplate( f"repeated slot name in sample {sentence!r} of template " f"{template!r}: each slot is defined once per sample") + + +def inline_keywords( + template: str, + vocabularies: Dict[str, Sequence[str]] | None = None, + *, + max_values: int = 10, +) -> str: + """Inline ```` references as ``(v1|v2|…)`` alternation groups. + + Engines like Padatious don't look up ``.voc`` files at runtime — they + need keywords baked into the template body as standard ``(a|b|c)`` + alternations. This utility replaces every ```` reference + with its values in alternation syntax, handling nested references + recursively. + + Parameters + ---------- + template + Template string with ```` references. + vocabularies + Flat ``{keyword: [values]}`` mapping. If ``None`` or empty the + template is returned unchanged. + max_values + Cap the number of values per keyword inlined. Default 10. + + Returns + ------- + str + Template with all ```` references inlined as ``(a|b|c)`` + groups. Keywords not found in ``vocabularies`` have their angle + brackets stripped and become literal text. + + Example + ------- + >>> inline_keywords(" [the] {name}", + ... {"turn_on": ["turn on", "switch on"]}) + '(turn on|switch on) [the] {name}' + """ + if not vocabularies: + return template + + def _sub(m: re.Match[str]) -> str: + vals = vocabularies.get(m.group(1)) + if vals: + return "(" + "|".join(vals[:max_values]) + ")" + return m.group(0) # keep brackets for unresolvable keywords + + # Iterate until stable — handles nested refs like inside + for _ in range(8): + new = _VOC_TOKEN_RE.sub(_sub, template) + if new == template: + break + template = new + return template diff --git a/test/test_expansion.py b/test/test_expansion.py index ae3e849..1544d12 100644 --- a/test/test_expansion.py +++ b/test/test_expansion.py @@ -5,7 +5,7 @@ """ import pytest -from ovos_spec_tools import MalformedTemplate, expand +from ovos_spec_tools import MalformedTemplate, expand, inline_keywords # --- §4.2 the worked example ------------------------------------------------- @@ -224,3 +224,51 @@ def test_unicode_literal_words_pass_through(): def test_duplicate_branches_collapse_to_one_sample(): assert expand("(go|go) home") == ["go home"] + + +# --- inline_keywords ---------------------------------------------------------- + + +def test_inline_keywords_basic(): + tpl = " [the] {name}" + vocab = {"turn_on": ["turn on", "switch on"]} + assert inline_keywords(tpl, vocab) == "(turn on|switch on) [the] {name}" + + +def test_inline_keywords_nested(): + tpl = " " + vocab = { + "broadcast": ["ذع", "بلغ"], + "a": ["آ", "أ"], + "everywhere": ["كل مكان"], + } + result = inline_keywords(tpl, vocab) + assert "بلغ" in result + assert "كل مكان" in result + assert "(آ|أ)" in result + + +def test_inline_keywords_empty_vocab(): + assert inline_keywords(" lights", {}) == " lights" + + +def test_inline_keywords_none_vocab(): + assert inline_keywords(" lights", None) == " lights" + + +def test_inline_keywords_strips_unresolved(): + vocab = {"known": ["yes"]} + result = inline_keywords(" ", vocab) + assert result == " (yes)" # brackets kept for unresolvable + + +def test_inline_keywords_max_values(): + vocab = {"x": [str(i) for i in range(20)]} + result = inline_keywords("", vocab, max_values=5) + assert "10" not in result + assert "0" in result + + +def test_inline_keywords_no_refs(): + assert inline_keywords("hello world", {"x": ["y"]}) == "hello world" + From ea18f324b8eb51938ac4031b73f95fb73912c1dd Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:46:27 +0000 Subject: [PATCH 049/110] Increment Version to 0.8.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index c5e9ef6..2b91e95 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 7 +VERSION_MINOR = 8 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 8a703fef099c51a29569ce17e8f0b49374e9e6a3 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:46:49 +0000 Subject: [PATCH 050/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d9d3c6..970a957 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.8.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.8.0a1) (2026-06-02) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.7.0a1...0.8.0a1) + +**Merged pull requests:** + +- feat: inline\_keywords — resolve \ refs as \(a|b|c\) for engines without .voc support [\#22](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/22) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.7.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.7.0a1) (2026-06-02) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.6.0a1...0.7.0a1) From d2db51fe34a273e1870b18e0fc73bd439a8447fb Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 13 Jun 2026 03:19:28 +0100 Subject: [PATCH 051/110] docs: standardize NGI0 Commons Fund attribution (#24) Use the canonical funding block (developer + funder + correct NGI0 Commons Fund banner) across the repo, replacing divergent ad-hoc credit notes. --- README.md | 18 +++++++++++++----- ngi.png | Bin 12713 -> 18699 bytes 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9fbb579..d231647 100644 --- a/README.md +++ b/README.md @@ -76,14 +76,22 @@ A zero-to-hero guide lives in [`docs/`](docs/README.md): Runnable example scripts are in [`examples/`](examples/README.md). +--- + ## Credits -This package was produced as part of a documentation and interoperability -effort for OpenVoiceOS, funded by NLnet's -[NGI0 Commons Fund](https://nlnet.nl/project/OpenVoiceOS) under grant -agreement No [101135429](https://cordis.europa.eu/project/id/101135429). +Developed by [TigreGótico](https://tigregotico.pt) for +[OpenVoiceOS](https://openvoiceos.org). + +[![NGI0 Commons Fund](./ngi.png)](https://nlnet.nl/project/OpenVoiceOS) + +This project was funded through the [NGI0 Commons Fund](https://nlnet.nl/commonsfund), +a fund established by [NLnet](https://nlnet.nl) with financial support from the +European Commission's [Next Generation Internet](https://ngi.eu) programme, under +the aegis of [DG Communications Networks, Content and Technology](https://commission.europa.eu/about-european-commission/departments-and-executive-agencies/communications-networks-content-and-technology_en) +under grant agreement No [101135429](https://cordis.europa.eu/project/id/101135429). -![NGI0 / NLnet](./ngi.png) +--- ## License diff --git a/ngi.png b/ngi.png index 13f4e9cd190a658243763a0242cfd41b58a530f4..bfd401afd74bd0e512c332577c152264fe231b88 100644 GIT binary patch literal 18699 zcmZ^K1yCG8yY1qOFYfNa-Q9x|EVu-BcXvw&?oJ2@5V(;|-_Tdp=!oyIP_DJ0)@3W5XL1P#mc+R*b zietq@E(*b5ixg^mE$!t7>r9f&zYDM&zo@NcU|@&gLznt^HahN-cM6#Qtdli;GWg?L z!l*YGu{v>e4@H*4{Bf`2%o{;Gks6WoKP}ftC}b)Zg#Mp4u^5Dh<-bS?q)-+Mtn zRlEEC(_sGhIQ&L9^naa#^nZ_Yf>^Q7|Eq!i=XfrZRFQD-X8ycq;dg6~W@|xh1 zHbTMYu~kZK=BALQDM_G7`UH z!g18foD4D6Kd1alEgXJ9p&;XnDD+T!ii&m=_NXY~r;0}DKSOZ`4$;|0#{5;h6;1lED1yupoWYjgZ}f5FB2*^|YG9!ar>%;^sw0CSUv6b% z_6R?XBEk8$dHn38!x3J#ShnSm2RSGP;w2T$x6N8=`m%KL`;#sEj#yWF$M|3TO@zK5 zE|l>-5E7#5-Q0RoTWK9Bk^FjMZ1&EzxyhJMH^^2=&$vsLLWC0o&uiN%u-L0ge9-Q^ z@DE<#1U-blxWLP;hdm7r8We>*I#|6&CJh+wvMr$`UZ=3@n-yzTq@~pUF{7CM-UPXL z?<;>$gZ^~iE+6%z(L_@=NNl;0?E8?eh3z{wRSVZ}Q9gKfKAAvG7u6q@t#!@JFs>OT zl{`r{9@{@|_X-yg7V4rNELKg>d!#KQ=^6fA-A>aKv`PiBEe(VY$!C3Y zIyfro@FOK{o|)6Odl5})GNwmJwb!E^>z*rT&OiD3BLpL~38dc$N9}1JtSh@ve0h$N z#!iQwJ_@h@{V7w6lq@+kZn2%Dsq*{x?k4+{^Cio?f|{C1lRqupcAo~<6d1c^&#BrP zm5Mr0g@haij86;5VtAHA3d)-qLc)j|;F2#Hzcbe)*Pc!}Rjb(z6C)Y1fMsawcai*k zyaMFo_zUvOh4E~JU#jd*@8fykl{E;7h-ys-kf>>Cx0+TQ`r6!&5>itmDk_-t+uWkv znZAjP6(izQ0?vZS%WD-Wi6mIB0Zy3<26zORBx0ci#9KOV?co+1WH#O95EEn>A)!DyRxw*L;7L;>^qL=C|w-;;8()pYzPi`P*(NULq z%CTmuOCI14MUG@Al}?PM%DMfvZ+t9M;;H`9o>*Mgqo+q*crpNORhVh5cPaw z3;L?mYEK{s(L(lX_tt}9Qaf5|DuI1hXOO3q;r_*L*9;>Lycjp8Izb7mwrl@EMpc9G z*i2`gOm8f}WN~$V9OR^K-A=>yEHJTlffRG-TNf zHM1$GFL>uHCB}HbO-?#^l0a~B-j|d_7!Ur@^-B{7#$6hkp!a!Kj21DnXEvU8j(s16 z#9;x~>wO5MI2}!N36{Q?{X9`dT#QW?%$5F_Kh>Nss|Q!o1|D9Eoy=&9puxuogFs!% zC&>2#QE#e*Awp)omkFPL3itfHB})wlNp2b?UK}xBlpF(Oh4M~Uc*OQLIXPK%cJnhm zJ-snDQZc2YE-wG!w zMgTHY0^kVZEaot{eq4(6{!MJ^9-+cy*d*FuKr9%oE_uR#r`z14n3-)A0-8Ii~?bIsCqdWH{fkzt+|X4@66?mQH9beArT8#6r41FA!S5 zO?|TTHRI#4hsqKIflNC2)3dHA(6B3E5MfS!&t)`q8={_`WJFbjEs$e-sSaX5Oci6i zP;@2f4pV&TZ8yuAArf)-33?DbFBob{GTviqsK5uQ`w==y3NPmJe~XHu=uv?F2RizNu;Cq$Ki z{}a@#^yXIF93E-uwy`0Ax51sENnQgf{v~Fy8XLC;J1&HAY;c??c~*o7+f*DTiUI|e zVTF>-1&ha^j4C~X?r-Ug$J z$0E8oc!NqwD(j-boRh>B*~?2N2m=E!>^PVKR zp3e_aC6CbfiyG)n-0L?3!Q;c5dYvICBELM>M}_dbIlG;);(9rsatiPAyV(v;6%KEI z(gRJNy+}*!EsNAnB8T;~(uJsQ1jj9kQuh1q;{4H5afV3v@8yfPe))J$DIXQav3G?1 z``{MOxS4rl)0x)+4@uENxKp4J?ZJIT--egnk2?&S9W!Ka#&CvE<5%@cgmvxRq+!zU zPkc@rpx5(A(TzTIQ7K19R*)FwSL2iAMz5Ef1q^I#Dcgl9?da8MUgyqR#_u=BYw&X< z-eMlhE!yLW<;yKN^`E@vGC|uT$(4ubvW{gQwQ7=z-+rKvZ ztn93$6cjiWX*r=`V7_Li#+Z+2lXEk>M->Tv`?k%OsS&D1fe1;5G!d`mZh{`+y0&%= z=xvq}OWNT7WWwW*`FMH5QuTVio2GR1$edorfKP&_8EaRYZzE$aS5Q3?G-k7=!13)< z!3}llDw9Lhi*{Q7jrKG}^ps%qhAPcRBi^ibaQve;dKlem2gw>t8&OUARhSeNh?6Z6 zVo^)mYlCaCyhXALXZ-!e%=lixhlk*xAJs3$uo50Buuv4e4U3`_i-NWqw^|hOBoJL< zu#EoQTr^$Ck25TEbnVqlfeh-g`>nquXmHTAsuA)SKQM^ch%f>O5~BVl$Ukpv6Ih>Y zI|`cRPtSf>4u2@#g*-7ny>E1n@1(|pD5 zG2yxCg0J#S+2=z2#KnQMkB^TDBHkQ|GB%$h+fNN6MLX^(mGcC`_wu}PH8nNSo<7z% zK^ZYuS@IFsTji*O1|pGSx-6A3avtw-sg?C$?5N z?}};}*EiDE8ph&&moBm;$q+O9A|S*XiC0(6S-$}JPDS=JVc~Oz=33*(8>m>gi;&^%6Z3wi|wuJ`w-R%xj@2 z*j%(HQJn@fAk*$^m^i3qC7*UgOgorn%V<RsYXrtPP6&0&=a*XvM|s|wZ}qhLLJrb48v?4UaPo-tw$;B4m*46?9|5zK{`{_P z=q18=yieNln81U~3_5i~@b>Oi@}cR9;GCRK_chAE5VW@pUBxC%-jFPfz~ugse1B%H zsLB<6*>^l&R@?F>B1qF#LX7D;qDu(Qr1J`LoP!8GIwdNfD5ZC1J;ai!KX1kTjXK)~ zxm#_7Lw{BP<}!wnZ-ERC<5-pcyOl#<^w#^*7jQUqsYFkBxd0bqJIQC;Fn%>W!eq zf6x+qO*m9m%^&@Rs`*+8UvV~!?1|+K=OWk@gN$oo=V52TzAugLjJVDtD0)8E^rt5q zQCH>#j(V?lTggOUB-&>C+w)MPcSFNQ!PD80ilC>oR$9=3$FPg`fsu7d~_ z{%>Bd>@vY(6sXd-Ep!mPbX1jS)@7ezr>5Nx`gZqUwLE2C9sfCDwb#(oQjuJYKHd5t zNZUZbrv{Vv$9+}I_ySnHWFhJwLfSs(F>GU^nIg3`puyXVyDY*$5L^7afD*U+AyeC! zlKUk~9D)W%r3dxC$Wc1r#P&Rej$+u6;S;IJB!R%xqC5DcV-Eoi8h&$R=gvkf$OZfL zqjgxj|8Nr-RDQ-Oadu)$6fy-VlXPQI+LuszGwKVTocrXD#m#_kV3 ziJ%!VN&hHE7>XILb-IL&eDQPMFV-YaXBR=^9e*Dg>2;n?4VBGU-P5!>GfAX$Z?(=Y z6gYBop32|(+Lq}QtE#H{Mg}0zpuZ>>xa9l%!+BOP{@v~m(E+Su{85i6*_h5^$@ zvWyQB^FD&|^S;!sK+w`R=BXPJ4vxn7@$rcxos0ha8~W$u&fvi+0Ax}F zr&W8;iOzrkuiaM5f7Pki@MCx~O9`ft{{zihz!fuo{Pm6B#lxQ0-&)zq8(Aduuh1jv zMlwF51_?1wXGab2ILg|AAqL?SQ(QtIrzRi9s=RC7U&8w`dE6v)z?5<7dPm(!2_@I3 zxzI;?sq_~|aRd05NT=wX?R$St>;Btkx{ITS*7-Vu3v`NAPb9a99>$=tRt0NlJG%`# zCiw?Wd$fQ{d|0p6JzWbfX9lJ-HrY&!JI`yY@~M(e6um#7A5B3psf#eT{1`ul%^(L| z=^w~$hW&q4`_Bm=oXVul%7wpkKgieW-d$Tay9|+;7GaPEKZT`h=<6pgx*P# zn-WadQZl0kUZzvlOe#-wMYhE~#XXiojwU~2J#koBS^ZwdcvfAGDwAQ>7ADY%NWEdJ zJ09T8+H-SBi3L6E7Pkc@sFSxgv;EUk?xQrxL64!z=3jiF??zw0PT9{`&2PGFdkl(X zh={_1f%{CqVeq%;g)f#~Q)H|Lpal$7L`c+aUa&JoY=x+z(Uc)+PqIKNS(Ub-8Ob15 zw}-XwpAu#WQt>a+H`3xQvBO1d9D##OoDTa9(yUj^5J!ZbP1}n4GfJc*!?P~38U^$* zntUuh0-8oH6FR>oI5!vboF?%M{0|D~&xZ4tIHT0O&5oX~`T6*zBjr$_`a~=?I`iX~ zN-!8Y!0=_gz+X3~uWj!g{MBhiCi~K+UTQAHzLBm)Rnt?cH4;tlA6!*xP5>Kp?5sjQ zYl&$qn4=tdveLX=l4jUD!rB$*|8zU0v0WgN^!xEqc+UMo+Pf z+KjTUH<0!15C%mrMStNq6Mt!@YD3Uwv_rr(@m*~FU93xsIB@KoMUD?9aCrJF_&2%B zBeGDHT&BhG^NfW>J?Vpcb*~g5ezlqv23TAdD@v^q?piK<9JaEjti8qp-X4m%aI@Rb z$DNzrVcBH>F0`avxFf1$Lyx5K>>*5jJ7eBCbL~t+FyWHKUq5QEiY#arZ zc>x~Clfc&=q(96gKE1&xRLhL)Lpe8c*cIdhFneVjIc$2$x$Qj9)9p!Li}QAk&D^`^ zP4vL-s|nFB1F?8!hqGn$OvP)ALkK$?!N&%1ku`8RwFp2dTe~-%cm8nE2nuS-#mp{y z4b#%5MeOe5f5JFExp(-DaUt${(Y^X6-Ve#Vkvky5?)gMtq2DoVopY4RV$S4fYfCi0 zIG#@EA2}mnCn5f&DML$Z!8<~&y>}p;i^qwlD5A9RW-{uuXAUV%(r&WO{~p#yX#hy@ z@4jq{>2ABGjaN z#P+>ka^It_|J&?PCtG# zk3Iku`uq3q#%$RKIYq^W)FC*F?b>juf|qQXE9Y7r-<&q;JbPlO z2VH0j@>`!M%?!dLU1JTQC_>U?AxdD|69&R5;TFPL;qU2o$Q1}#jdi|J$UiX*4l$E? z9Nn$-x~*S90~;)=o2xkKu+hU!Z#9HE%c)CkDjdtWHN}xl;|cPd8#a@as!7rxBb!FE zdYVA6!7T3VFifXXOuw(h7+4P{e zOI!@R`Obyafx@Cg=kVs{rd5bmTS|hV6eW?Aa<`PXGWhgt0+-WNw#TuyXroy&vvhtE zHK#VxowQ3%XZ5e$NCaU~H^GO(K!lVwV&*^I^Pyw6R@>0l9hdla2aF0X%T;4|NJ%7Dmz;0e2aC*G9#>Nv=`P^W;z~bgM z`NtmcfU)YYL3moTCf>`8=P7D1dJu9pej5PkawX85zDis~&uD7mS@pmB0|tY;uMcLv zW@JR?=aUP)-fz!O{=R+ye-#d(s{E<961M!gfaW9N$fiQ8=#sNh*x)2LrK`K_WT7o0 z%b-)T!Am3E=8bl9pZ$k4Z|AR5cc=Q}hx=Fp{=cUGv*RAx4g?^-ifbmew+F1fOW-y+(xi z4IC8};)0Dg;$Kh>yz1kByC` zvzx&;IsAD)W%_;2#%{R*k)NNxBBbZKx;>&6dzxFGDACITL)w0%ECie*Rl93p3BO&;DnepF+ZHG$O-~t_F`% z#VD|xkFWRqZnu03`4RU%RKBKN!JY=XLY@Tk{T%MoKCGZt(6v2AI3~ILQoF*^>HMO66PP@Nu zE@JG$H1E#q!?!3PO4Hf@y;6O;RiuQf+6YLb)lD zRB-P}D?+Q``)4Y-uTZT?>93%TlZ>E>Fm30Gs>}rOXicO;n^zOU&E0%=IOrDbdLj$)$3EBo1>}dS z>jhy`vw@ut!{%OA);u&VQ;pp<+S?4bCDLt2^Bn6~6WRP&(7_;!`t9>VB7v}bcFKdfKV0-;{CFz|UPV58e1r+i>^?btW>%U#g z`(aFt>rKUQ?GW~`%ZHyNWw|bm%?t2t>sCvOnfAd7XDRM3vm7BlqT=<;HGhhlPB|Sg zkObO%8OpCyS%zBSiFbtaPYLl)KulJseaP}>%t@y99<)uz!ZE|UJjB0P2MZZKcZ|bA zL*KFF#uudx*X%u;Ut2X;Y2N7kA!K&vz%r|K8T@NI+&$S_6CMJphRsw}UH68M3QX7H z6%r!$T=S*_UcAGaiFhJj2__4kA$8yd8y-w%wP73z$6j6A zQxOu%JK$~s>HZkWOLqu~HL_hC3ymq+r-3e{*2sF*CD`YkvR`pejGxoqa0Sfh>YIW_ z2<`2Kx()PdBexa-H zP9-D?_Qe~M$WuoDDkjFMWgY@b5OyZEpry;NYqe)nD3pr=IkHmAMo2iL65uX#+7*xk zRT%RR1E7vfD9&K7`}0jH%jx0LJe$AoShvOJ#zyE0#kE`y08iSKeL&giK^ZXe&bd+w zcQpO*<1&65`b-)3mP{(X1DnrEw5N9tiUr|NV%;_46hvw-y-}#?aJ7?3xP-06-n=^y zCpwoehlxFjolmm0-Lz@06n6L#smb!))Jx`wjBj9cu(aO%CIUDrdb#p zbu34_PhHV#Z2?54}y;U=< z4Qd&!%Y~dQk@nRb)dJvZ=&ztq%&Om46^kj#J=@BHZUdqKHrLEEi5R2VrFLE zKR%ZF_>q9r_m~~5qc4j>%k46Z1pp<=+x295?bo!%yx#RAXWx6YO z`3=Fbeqwq%oHe+g$Utze@WWEp2nfPTA=_}9ldT}&sHUx$KOpYCbeY3tBhg8*Uk5}# z^-p-lX*yBlppL9AJG&lkuWTmAy<9XAQAQ|#h zCy%foiH@Pvk#QYb5qboyUad(9iyt>RUU%KZieCaj+aY)ow1`3K2B82BkpJ@rWM*U{ zQYIJwb86EI@J41vQwj|(q4J?gn%6Lu1xL{HivE@j&DQse*2s$*`}+7o+53kl$c=A7 z)q7rH4+7|#!pRSMc2I4zGys)$S`+=nk`kL#ZGUcIh7w1hO@=-5(U`*)0o_}>c0%Sr z*DPiXlw}`0M?S5!MINM@*BXWTv!}y%;wrX>8Snd5Qos;?hofar{dWsC6t|SCMx#wD z{uhOhw1{E=9zibFt%AGVkY_O`{SS%NJZVk71mpvMcXmu}?b;%H>&!=-p6{LKC;Oz8 zL^&hQ;*p6n8OK;tR=x$sh_U*x5f*)n{CVklVZ9db^`?r}Dbn~^lU!9!vese#PI<;U z@=4i)aA_Jhr#8#=4?FT^W>a^#!+#Q*O`Qj`{i(Jom`I9mVzOk-#r%jA&HVGYZ2CnB zY=s})-Q+$r#%OsgLo3_M?`1+M5LB|rg*}W)>uJoWBYZP5rkrcc{@wt*o|`MQt-k)kQ zCvcaWP}2rqk#SEy^Nfg;zVWRU>r+e*u3zMVs#}>#DvW~T@gNX33Otj<20kHA778GC z{PuoR3sBf(j(7dY_ReM%CBAcDr&YZd1D#VfiE|}H{7xQo6M*mWys%gy=P1a{(gr#L z2Q+|nhSgPPs#kH|ZiP}Qu!%kp08A6KnT0rQLxjU~a@SW6W)ljAYk^NRC~d;S)KKXe zHKd$a^P$vZ067;OI`C#ZtFa&Nf4{%~UZ{4TE95xwuDH0EoF-hax452Lk04LCRI1Yb z(3d&oJq0$B^%}YhK{H5&Bh|+as_+wvwIpIlQ0Zt=fj+2KRR$S{dDBXW0pquLP(BI1 zdhND8Y-gR-vrhmETFNt)?8q9BFC4g40CJ+bA;}{27g3EX;X7 z>oZBecf+S0s1b#=xq;6H3R!bkqFDi9>|FIYWOQ*K2`(v#Tg&60g-ElW^v-`~qtCy7 z=yF}r5a!cS4LJuHjmnz@d_Ooo={uOV;c|29`NTsdMj@+j1(`ObibyV)#(+H-T>R#q zQ+lyh(8`vX?y7*`Lt_dcd!PC%LRUkV3}i~mhN5Evx21Lv8txHrv8Pr4&9m(W5i+1Q zsJ#r_-0QU@=)mpur`x^=FXg9ss4ZOd(A@`t9}J$f_~)Y%DApF+Ho# zu0a3K*T}=$qU4%7MDZ)g&cv8kS&KJbW*hthEhH)dvngbz69jWmlXaK>^MqWtJr+<_ ziaBaX(q2TaQi0W8XCP#Jco*XQ4$f3|XUvkmDg-`m%=>d_0jK>EDvI%)p*L4PjWzYg z?3rz!KNg*Td~GCt7gmA-`%OwTlwoRu@3^||MDTBlsDtaB{X(fyAV3`S&+hLWb+CC5inDqP&2G5n*$~w3U4nSDErQkwtPYwMV z?(mt9SwV6%rqAyz;nmF!BCUbyOJprAa@^1--K13`#As|p;yw1&-0|%PxWzo9unP=| z`n7F*Y>NPb#<0Jri#NWJm~z_l5EjX%-K! zDDTI#4Be}xx6#{bN(3HM%g-qsMBijokN_gYs-R_%yx!!GN^$=^2dM{E*JY7WOBRd% zu8{gURtd+$T908G2Pk7eB8m7_b=L_cv@ESIO){q51h34=iOxQR85Qzfj?6h&45*qr z^3V(FOVPKfBx;zba5kYj*)kRRwAEtsCa@)J*(R6WudlaW3?d8W55DF?;-x>(k}nPs z<#hP-=^2K0?6I1Sj!{ZU9%im_=29;ENIAX`VD0`gT>TUgDh7#(biJioTyg&X-ghh1#c4g8B<^tMWgswe{Fm_M z!axMsc{NOZ{xlccxZ^=_8sFVOpQ|**A^_vGx7(l@eB9g;@GQTCYq)cIgSHG|9tW7! zyUNH~&JgSww!q%fsB67XAB7r4=i?%~!)k6I$RtKI*4OsO>~Jlnf>n*ofBZ42gTwOm zWL6_KR868a$ro$vNKGZ7$uSTqt`dxu-&4N2J-z?*Iku#6#eJ6&@UnG@TeS{7ieXPr z64cn_i?YE-oU|qYPL@7u@$9t~GQ$si3Na%cd{rp1I@I{cGt4kH?C{sHu&wqmp7z;o z`r-af*cpnpo?xW3#8hb>8}^Bw91{t0dKRm<3v#68*U%D(Qus< z(7>RRazlW~$k;ZmgpN-u#lD(Jgy8HCwUSlE$=x?0nv7iRhiq3cBu7e^oO8#@aFRcK`hHs-z$f)5&^3{7ZToJv-31t)-*> zI;{@VK20NQ#upU!yP|KZ4i(T}7?6RRc+m0v63KMF=&r{-w=MA9?EnfdpV2dR(T5Mz z@at&dmMD*eG9$R3bH!?IT%ss*-~Fv8T8$$IoKY@75hYh)Jv%Bn= zkEX0@pH^uA6+n*>g=gDHfFTF$1APs!%WWmDQM3yT{BV0N5-vqI^l%;xaEeTFW=#q@ z85!`vNSZ3ajH1DI?_^jfM1|cSXo#?&7Z-TQXe5Y^Qb+&uSM@vG`%_l@jYRLLQr8#& z0w`C<4ZgL+aZBuCq=B4M|KcL9ds?2Ag6h_xO(34yd9xP|8xK#CHpygnGy`p$^k41z zcfr1BK#))GiedNjIyhf6*+p`}+Y0ratd9Z z6;|I}zCpC6C2VI7f7z+{uL)Cbi~XV$J(=|(uTOeu z!O@9$cTY&O5w1}gJ*NlET88k`eDthQOpJVXf@nv{UiG$olgez|C?SOyJ!dK?6pkic zL338gK#K>u#a5>D*qMKB+!@RF@qx*LrV~-y|Ar}&W~JlpAv28};}PHun-pH4gXHK# zztHh|YVqi{s}!iL!^(zW;Mx7wvTS8YK;*ODv2+ zi`6f&8~Q{0NB!qbD^Z6WPSom`8iaKc!AP8pGCl*X_yc+8C{)Ca=T@Z{@WAXneKvD- z51?xzW?>4C=Gu;^D6>*8C$1s`wap!?>|1am0uv1#6#46-R)CouKAyyYPgOMz2*t4} zoWy%j7pdM`(VEC=>jhHs{+#a#byb1DO{StRre?s&!{qA7cf^>ZE=PLvK zQ@PqwZfxqQ!oK=~gzY|*I0Q(vAsZ<{YY{}=4b32v54GW0X_K!$@ zBI6*1r*CwV)RmSZ3`k2SjO@tevITQfH~hAea&~492?@m@CPqo5R&#sccz%8%2xa?d zh>naLnb$>z0uS>(CKdyi^m9-UNY@`w(#2f5fP!?G#6M|J|6nW*Ut2TO(AMl(Ss|KY zKMwwunVC5`wWiE~6-yIGN^pDhsi>$3X2;wd;2^}d_&m%cM%uQwvB;A~CKnbW%ZK0h z_YX=GI2|i{YZ~Llc6G&7`EL_jMTQ;`pQus?9p0Xtr_fN)_=VceG{;(5RWq|?p#H~H z`n%#(^B9SQMnBCL6+f6A@o*x^=)02Xp!v7Ug~!2v)egF!6wChceW3M{kZEkggN6wg z2?OWvm*pm9{qGfK4gBA2tYu^Ch+N7LLi|bI{KKD~Ovl5dh+8!xXwC72XiO_c8-DBc zO<9;J&1>ja+4lY;cPi&i0cYC#>_53$LbxBXrczT4ZK#e;@B5@WH$85J?cG!e?^A>^ zCH|V7yQ;0IB57uc7ju+#(dX>Y#Pby@APVKYulG0B*EsD5-W?m=sOt@*Mlk=N`1 zB&F95&hBv(uo$MuQ_S^_=KdEY|3e!VIUArl6lyGX($R*$TX$uhuh2Jt`Pgd559k{4 zk4_gmN$S5%be}X=4V3RN*4Qk$G&FJOjcRFWncl>tc-fM3x$4<6YB3uHrx`vYo4t&W zD_7`tLO1_$L2l4xX2HcZ9UIoXzk7Nc(s{FTsB#}4Nn?@Topt;9vYUmr@VJ%t8ph42 zZE~t z%hq{h0rwQI_d}u_0mgthtgz&XqohK?0Oo5qOh6sJ+qEW#V`?|d2eW|i)@`1)u_*Nd z)s3^adxW51V+j=Ro;iOTp||6Dz&+gW*;a2}?u>Z7{qelmwXNRX37ihkE0D?k4lK~g zs_qItN)^C$V=@~3u}~p?yV`nxIKo(uzD!>PrLJo z?k-y&5$OZBR`thFrL&*$(~0UoUw5^5}G=VEa!@I#|4Lx+6S zGv>)k(7wRNE%ie-TJnlCyEz~%nBu`jr_Y>@??SbDVwNyQ?m$5eGua5C&f?S{doa^s&RDc?<^0|K$3JMX>ofr6w( z*H;conCE<6LbbJ7Bp~=_M8ewEeN$a78IUo(n8iL84kUP6bH@h?4e-T{jI0XT;j*9! zPtq^BxSkL$2bNX!Xnfv@aj^hRVGAYvN6m@V=xw!?`RFplDK0AcrmpLUGI^{0{obWZ z!k-uG9Z^WvvOyA`Q>z@fJluj|YrJfbrcHC&hxHP?<<2+Um%OfqIsiUD&K*dn0SO6m zBJ5x|0#*M45uk~~QL^9PKj`b;u5QQ|BR(~`y$EA>&Qf1e2K@kQ- zTZf-eU*2|kK|!3Q8JvWq0#1;Qjt)T6Hng$18GIsIE9wn^tWKdc-y>0PkY`OeVs2SC26YKdPfawn&JLT0k{jTpF6Llt(1Y+4=o?8kksS2eWrD(zy`yOU?LLS`~UpS4xnF$tU^p|^gG zO1kaSh0vKG?JMBF) z&C~f`1g#WJ(yANl+EJfo=1EU60V%eZPmOF=vbP+0BmXm8$ZP4fM%J%z$^sAvI^vHL z7;#98YXC0rVEq!U`%#=TJDCj+pN3J#510&ENp?(ws!xO!9{w6JL>l2D}0y<-Sn4RH+2z;661~m zd1=iU{^PI0lwUD{;ExyR2DLYL*Vh}hvZPQTAZ8oC6-YJ+$e4A#gnrrxWrcvrabrz5 z2uN;E4m}5Fuia)fw9RH#9NTBd`-;{f8r2Kzotcm%X!kpuk)okKyV5LKr=)IHFp67tc@71AVWxwq4p6GE-;qI<`3ZySgB%&|^mbmzjUCeSmo-fM#`t!=OXzd-hcE-3!Ft-9m|jtq{L7 zM6e7#&U{nao@7i&NC1h7?slUoenc%PWZYge`%+!aD)LbA_B!)numTJ03HS`FxFb-vx(?_PtY#KPwvv0o#4Zcq{-DmMyxFr^!ma zSj)yCVlBi`YPofz-TKt|#r>G`j{gdiEz{RIf1d>qgNaAoF+sw>B>sC-g#sok4AR{- z%`wf*xy@vKrd$gpEQ3|OJK&pK7^9DvVbBbZZt+&94?alwr~)O?*VgEU?@g;@?I)eABl=9DX zNE8c(3t{ZUg>IoV``o4~27&%X4Yfbhs5X2gFu`U~nf241NWRsPQs*Wg-$D`V3VUPkTO{TR|uC_^?mb?y9o^qtu>yKAYN=H zoB8%lvQkZw8gnyM&oP`6m)S7RoH2EKk3=wM-v1^=`}{CS+)_dUTC4IK-M6KuF5yX= zMx7Qf3{0%2GbquAeY4}mPXGS!kg|gVxrtxh6M`OFI;U#ax3|4BGgzLh?xizL=)%uz zw_GUJM7FafjtPlVNMG+>q4;MX&ED=mh9S1CD*^hspKI&sD{C*8Q`G?(pWTU$& zC=3w@x|3_ZYb@1Jni~bjL}FNRRd9NExwS*zk>No9;>t+;tnV&R)6WM|T@UaU6B+TGr4wSCmEFU0 zKiIuF+5?Ze)_l=dIv)w~kdZe}wnK@4pu>Qbq3P_>N_@m?*WFSa5L%7*cBw5T`+wRv z_jo4vIF4_I(d0H3nq}xvsIf-8Xf?N@yk5`q{9eE3`F+2i_nSxg$auq;NAmKx+_kbE z!Juat^lIW)MntyRZ3U%BpZw)@Ny=ZoeBQtQ(IeBV)z#_4VIi^A`x-+C1HbIev}@m_ znU#?t9$h`jEbsj-!dFLGJlIF%K0T0}J$5m03BL5#YjUJLNs9e(tFEr<%JphW@L08M ziI5yQMtqT!{`j%c*I}B)P-27ud}w=kDfh*KOJcoA7=OwyY5*l5J|Mq@djmA~crSSJ zelpHbdAvduMuTCKp7rM*l0Ja-QKA4^y5$i>@fb?*_LUfw?;j5BYs@9H^%c)?FJBLG z+(y{Ibk*sMez?&Wx$fNAvto;TA673+4a3;CzfQFAc<{9r5gmxlcUuo9)-TTHuKZ;_ z-UCS^)`Vq{w5){WHZETGQ%nHH&yxq~G)T_XLYwbx@1D;b95TPd3b$zvNv*fsQo=^L z3$3~$#%nNaRBK3B=?A4o44FKM87>k9*Tr~l?lRIw>In!Jt*vbXc2ifKA=PmeT~A5@ z&~jwpR(+t^`<4iIch$IQ;mM&PF95l2-*jG&Ku`~aGaFW-?Vg`Xy)ekFxohMSl2sU) z9Ef>P)2u%6j8IBgcz2or;}ZBgsfe1ygVA|o0Nc&teP%Q}>-JX|V6jw8t@W*`@W-)M zarp>EMTfXJ^MS!Z?N!ncN%Satd}yH1b$4zH=;o#ZvY#n;!tlc?V|WpJD)ThI*lf|Q zQ|IcutPwlZ`qC?k_(E4U_6|0&(^htImvco%nIX48@?hKjE$46SVC7h4#^z6r7#}i+ zDFo}XmM;q4NqjVrBq#(_(Cv-zg$_xv6D}82#$20R7YUU`D_n)c&1fzjQ(mo^@`17x z#<68gvZynYPL|!;XMJVhXJU>Uw!alup>tgLf-w}%J{oNH0o+A^g!(g5ik%Hwq?t6TrYIqy z4mC!P2(hWhsf9z^{K>d}*cBMfvrz`1^R|yrg0SWzbmp`2LVbBC*6#N;>@*XlKAHxe z1U^1A(Oj)W`4M|9C+0wV)n{MBL(iw(D*$^XdSB5F;_zuDm*6$K1V)MetIe~mNaPgE<@eyuPsP~?oFbM{U8W=LnzxjyQQ;ssUd z_N3?nwPr;rI+)82nWh`m3UnH(C6;L8UVQiuFN>uyQY#(|$cviyD0*j~HlLc+C zI{_XN1x%PZ{M>X~$kBIf8#Sn_qCFpJW11jfa;}SRvZzp;pf`#}-iI&=hZ80`#2V(o zoO>dK0w>5r8lg~Esuh#1fv$*B!AUj-Bo-@SCWhu$_wqxYPAi^>qMax*%W1{;=ch_i zNm{weGc_tbOI=(T!;RJ2K765eX>_ErU-Mbvs4+jq>MdU=OFTA3-JJKsxnf#%QVKFY zUfpr}(-m67C=a`cRbQqYi~GTc^|p{g#6!j z7ZYbQGkbuUwCV>gLWS?y000$0URpxkQ~xNXLGJ@nEyfV%=J~IQ%=FjECK4Iq7C9C2J}qWI7~#WNM6kQGO3U|DmB)7juST z8IRvyVi|u$PXsM?mY_=~Fqa+ClNeK-#H#M} zf1dl61jSe39?G_O6FnX-b)IE&=(!MhMa__lZuIkrxmZL(--@*Nd?pSQYkU|Wb-tds zZZOu+`NkySzcI>;s<7-a3x?g9LUnu|YTFZw+Be0R~j6a}EP=Jo{W6%&SU1o&gs`DXsmhJz&_x@k1S>v{l z%6u!}(M(--S68{|rNhE8nk`uCYJjQG+5HwPH zI#)e<^eFTCesyOd8Gkoe*Ysh6f==s&zgq-xS+t4q?v&?@KQ3>qAS@{J|awhrOQ`@V1(eN>1QD}wp|Ccnw_I2%Btu^JnY)i^QWO6N&=e#3ll!m0KsshSQxY=e zvdr2TKm(AjfV2ayc|cbX2to^*F%S&7sRPncV#sUW{|ekvJ&JWU)VV_sk|F`MzQK_P zraBAtQ?I+T?#9q>`yXj@<5z?BPd-^O%Hvl{A=lx?1_Dq2{j6}XSqQ{7GM&YZ|Fu|~ zOoRnSm@FJed^Bs4mFc5rU&@j9-r)23`9Ay=Zt&jhn*b)}hfYpactys9|W zu30q>)+eMwEnuNrHK=Ize!$YWq&Chh`e5qWni4%$>ri7j5%G(V8qF~)a(oV1nj7-ZL^d-++m z|6}@zQ!>p{wKEBHcGUeXEQh^l*QZzE zCwF-bWlA-ciJ6GSW%N8_WlDU^qluwDP52dFoN3af`pp>@;0H4-VSX7XYgidLUT*Q) z#O6wexxr`HK=t8kYli=DRCPd~Wt7C^oZIs%H&D!`Elw{mXy(KdPctLwx|csmtHP?F z5ySpz)>=sH9>3-AMYD@S-+RiaOKC1z6&Hu(t8%cqn+|7^$yruVOh%M??h8TJwl~R> zs!GY_B>9!6*%g$+CpmF+(55|ZW?tFrGyxi81l1ZP;P#Wqkaw2|L|Az^c0M1#0hWja zqSZ5UGsPg0JNzAG4DL#H%H3i+pE?n{Zq{$*Li?_q|wN6R>P$bgD@<$j~#~cT^uhucn^qrUv2ak zWJo5O&6Md3Hk4?_Kp>Lt*USOGe(f(65vWzPChpG;Q*m;tdgLa?r9Po9eq6bVC&=mL z&bs;7xDCFx@Rzq9`KC;KQh;zrWsfw$IQuC3G>;1N^ymqcEeV5sG%Z4vt{dHLRiAK;#L8@$XYhW1zEBu#m8E}Ab$k5LTYR_hWR zTh3*CyWjoAd1h6EcM2mrG@L27-IGJ_y+x;fGsBjqe)FK7jME+Ucl5BIZ}}o73o~bD zw75Mk5{Qi1xe6!C-l)Km>W!XGZ}3|85y)D}cIVOOEDF%5`)w<9&Yg7gSTlxW_;`C3 zPAR@(iIxdth@$tVT>D#85w3yle7@waoW3|xf9a7dccV)$(3+ZAb!&e8`5ZVG8v)h zbcFHU1PiQS$z$szb54s`NAz=u**gDgw%BO2*XtSy>!REF0_CIY*5jCKQ}T_aZ7M+p znWPFdL?mgtIm9Hx1@$XaUOG?4q3vsHrjI_%X@MsSP4x03J@a9ok3ppjO83R$U_CzRm}Vu&NoKU&aczo5 zuQqHb2voj<@UgYJmbtrIO-Dn^JpY_MXyu*pN#Z{%TH+c@f>m|ABRfJj5#uM<@=X}4 zU(p-vT{L%GUb`V7@=sEoCDUamzPlS3P%oI2*X@l&K}UDF-1};6(`6L|gNdZyZrZMZ zwT6r?Ovb}1q|WJC8nMJi0L}{Ee%E3GP`7(;@BeNpQr;#SqRX5nF@|A`OTJo&nL0xg zlGFR*QzzJ`!3oZ{Rvp*JSMNn(lN#vFwU0u&De!MMS*SYnD8(lcws?RRrKpT&XgZhz-I{D!4M%3V4E3#_V^`a2uOn25*JBn;t7}#2MJ` zt+t8#rL+#|*eY9l`Do6>n(I>j2Iw_-+mhS?@2FkaI|+`F)*!iBjSTyQfKb*NFF!Lm z70?fO(2#}Wt^pPwkORWgU+gD;U^A>{w+MctcJClfiOMaqZbo!?pkUV;Yy6K1_TB$z zR5Ou^+8J&0!SjV(&eb za4CPcDl5K3HTLd=c`}Y${Ys^S2V^{nG*qapn;Y+9gF``SsbXc+@%8H`q2x~7cJXVL zQD5@j;&?^@japD4s1lA-d=M{g7*Wopb=tM1SbqFcotOMdfV}O}TKG32P@niP!?uLw zzAEpgS2WH>6{ucRYrhP+d%5qT-2sS+iNP+T{5dU84fPa$yV=Z;Dh*4<4z-5{XQCXx zZ<6eb+1R!1uHevXnSDoV)Mh#(X<8|KN?-97N`pXkj}Cu;>vQdRXW zK0ZJ03z=1rsgSvb9v*N)-&ATLtQG@4=+I-oq1cZ!;oL2uRO(OOB8K5L7?2mL;ez=y_oJ$zZ0=9wL0ww#XfbzgF&jY&2?r77!m3Lmdwdn_m96 zEt>In|CARJ6a9Yq=!R`6$8(g_LIHQO;HfU%`TH@8UReN_kNW73Nd8#0+Fms=FMA*% zFDhdHxc`0r@%<9hMNf66Z2%tD9ceowZx1%P{bZjrJ|Bw!Csyp zbls*k8WXO%5Zg4wOHXA5i|t#P&^6GZCbc9E%NYyy1c2}A~iAnaqw^SK?!fW2!o0gN; z!ePNpvb%jYT~2-6*tZd%Z^0b0;xS2n#g)Mjp`>jUqoO+!@=rQu2}O2FX<;yYIF^*= zKS8y{k23TkkI%$(86sx3$Lk?c#5JCS6ayvJg+(r3^>#N5HF4Ts43YLNU^-@?fNe&- zZ_GF}T;{2&XvyOKV&2!Dt%9OZK0D*Zdb@D4B3&!G{(oI!{`Ayf9yo4$raklS6RLao z`^c!tRUQtJqEVOH^RTzJfV{jsHiNd*p&>JQuTu(nUi`vtEQZ43( z8e+~uROVMSIi&@jfb8s{m1w7heetX+RcRA_pz1faJwYw|Hc)cwXI-qZcN-V4p_E~2 z-b}@q$>RzRq2tTD-&7M7CAyChu7aSN;E>HAY;=jyj8gx3q}X?Ls= zmrKeTORKK(orFV2K_d3ioCcc8pSh;TWQ$iqA)iw)dS zfLz4wX}T5%2e6hWQzEFRJp^UtJG~d(n1lYx1F0I{;)8~p*mWj61=jAb;1LYXJ^KHA zP54i@rUez1z-e83PCuky8g&j>)K475|MG|bOLMr0=bf0C2#$){odG#)?-yk3AZmIp zt{9mx7*+~T2eR;3O@wEd>tU&vE-_0F`k?rqP*P!}g}c8;njL4C_YCWV+dG_YYebK! zUn9W#N?BTtY;)6bqwZZbv=?~aTvj$r|6x{RX;od^68O~8$0ndN zIMBXC zh=@gokg%_M$iU$`RPZmQg<{w0zI5zb=heZlLn6cdET9SGiOqP}=8ciJZ0WbIM2+2Q z)00=X0@zkZYa1aEseNnQgICQy&nO?fVZMMiJ2rawGY*i3s@yR4x3<#o6$1X@7h8k)3>K zPk$I|b(oz^-H&l25BtZo_ap$r-`^nM70kGtD_0H~giEohtvp!Z@Bvk{PdxOSp7O={ zfe0tv?7?IKF2dOGZNbeR{Jm5sB4VR)_=f)1$kYtW0{)B4&W);Q+vQXr)mrmDuDQ2( zJsdSC1*}l(roL)`Xtk+gU|;s&+AN74fP7MS#Iq9(!+R|I%r|8cr)o8bXxJ2(I3N<0 z{$vCXY!*rfycAA}^xKWtOOC9V%z;8lkTcP6MCO+i`P|v#RN3Lwl z^4v&D$0k+*H~Tuzo2<4`+0}&Ho+5>V-8H;V7)AOFC7!&3GOU)A_I{4@-}FJv>7-hs zv2+(375)$#hnhhJoe|bGe*sF7jWJ8VTts`)2iQY?o8?HtsM{&0sQXq3M53fV_3phApHMB?=1TPqZ)S7b zaE{B?^rhX4_b94lPFCF;igh^>$*4Ku%R1!M!wjbIVFBgMWmN7xttc?ND)_2FokYa$ z@+Dxw0GWi}26uTNM3U>gERSuqE`z`e^~X?#L-NF}3|f3x1>42_GXHCFC9mDOgI8Z0 zK!g{XNRLNaY7xWl=Y{cf{y-q#2TDpJrfo$_1=6_IssBLos~!tCbEG-6yn>ly>()mUsso#?c}LzIIlhRX>kxptw_`roJ1%=&Q&_GTf#pr=9S z=1dLNn&>?YOl}T!?vvcfM~)GWwEs9(B{~cK)L0%~3k(|^vCX@A@AK6j2Ef5D>oXfC zm|>V|JzmUms8)VR<+WC!SJQ{560-4aLgUUqUIU^0c6V;q>7Skwx@A}{T4bFS+6t=K ziBBWdq~sU2$qNEmR#q4WtX$EKe7_IVz6!O3KZlDDV_i{wYYkT@-)`yTcgi=-X)wb2 zEFL#{?;C`avdB&;ul|a5V5kQ-D@wXh!?k96ouJz zv4*;qmdm_Z(s+}LMOH-5P4CLv#8P2$KRVecqm{%G027n=7m2&C*Qt+zeVNkvIG$l@N&(#UL2%8;yBBk36AHnPE*`V%zpSlP8@DMLRM`vf#em=BuGg6rJ-9^B;D_{OF zp6%0SxtuIfz7Y??EKk=HnSEMcxQs&S*x4i7pN<@w6ZK`ID2AVIpcxAmJ7KK)Wji|@ zrVKZ}Uzuhb2R!YLsE7`me@&&*lMRAKJ4A0FU1WyZ9eT2HL1IV44Y9)9tDBu4w9bw_ zHa%}zoQ75p^tMblW$=V6M(PdgWj@8G>hcSq%J>rE*}l`@AdkIjj?6C3k<wicy zN5Zl)YbS82;0zD6^?hIkL!$N5wnn9NWFfgEKx5rlt!hS2v zL!f}WOnvtsH|7I9Q1bA;JS0s#@lGy!r9WF|3qcUiJHF!cJXc)J@J^K5Klr3``ksIi zi^=h%e)fg#qa=1Ih28t4_Q&61zW1jcSR9UPe~~1mq`V%_dheD`{45}o6&|xOsaH*g zVOTHwk_pem`$WR&^1cS98Uq<3d?frW%08w82IU0?K`G$}_m$&^Mv4vMmGE1+)Dq6# zW*Z*Hmd)+}I>Cf}LU}^YE0F!AUxW{oJ0b4(V3U(&^S1TGr&XqW?4^Gbt)2U1)2E$c z%gn&sX3Z4)@xXR!4pHtx%AA%_dKS~@URLz&ChdiNFI##4zwkV7YoMgnbbJzyD!kl$ z_KflUw1y(-+(dCX4^6W@Me9`2HfXl=LZ+2eI{=*iB>kfk+M>1y8Cw^X%s@RL1k!`eEpVMHe^H+16R=yZ<43dWQl|EQ^3T2Rq zQIwpvoJJ6SLc!6>6+bE^n9{G1H2n^i-h+eI&7UgSg+@2e@-rmzX&&qLFbMjg^NHpc z{ys530cDUly9c&cp4C=D9^y%st9_+aMQ);&rQw=RJuE+~4{Bf+%kR$zyVBYnD4xA) z>Jv)}nb)T4UZMO%Y4ks2z|WRLFf!1LUspl)G@>sq-HX$K%CdN@7{_E9HYveJzlQy$ zTHx=TP=(}_EF>K+iuZb=&G=6o+`%ZmJ(;c2zms<@N8@eH@7lQfen<&Z7w;i|9^>61 z>bZK9o*h*D+ATqJe0+j;QEx=sB(yK=O8~IemGPt{Be(Dwt{i;&lNueD${v0#qzXi?zCH&a4Rt6u%#=5 z2AtffydJBuRfKl_n^frMP^fu#u@_D8Xuh<=&`tI(>Zz>?D0;C&3Y^r(AJ9ashE6Cv z2-@YNT41nfY=2=O^&&HgMdz7nq)^+(sT3Zy%VF+c?-{g623z4UC4|&CBG+OAm7+83 z+h1uI)d{~QK@deVpz*iO{Iv4M^TC1VUY5O=%2GwU(FyyBD^9r2Ok7Dn<-~4d3GPt) zezaHM`YAKo>+IA-EDE}QQFMuQ-gs9d{{6F#+lN@X^%M9nsepll@6V6@+_4{J^pFb6 z%cLYFwb7^)$c6<6fvlx!5vi;dpCmh&1r>1(76a=>|s{A6Ihe%vBjKiS=%$~5<7m&u9r()%m`0Kh~4PYX~u zqmgx7c}d}|{EYR)ZKKN0poSitLS|?C4760EOy@S9$9zhyru!Ee-W!5LBH1!y^4mQk zOT0U4@`ST;l0YadN&ZQ*%F!d%LyMNsE>*+LLMIJOP5}0oF&WkV#4&?St0-z^MtJ0V z13v}*p?_QkjU%T&x{}zFAp7{HxIk~RoFu#iS>4Gpwa}e-uh4DkX^@^a7M*rI9msCw z;Po*+BG9NAJ3CIR8IzBUZ;7)MwbDN@uXl53#f*QNkv_8PX$aZ)$GmezBc9mfPb*|B z>3x1m=>2}ygzs5w1)G>O2XrWXOxY3w{o~6Ld{{hsG7`7x{0ZyzwADLKj-V}H>n~GfnKZV zHO}o`v?QtX6lmZ9mIOOygQ*ZfCZk^s22LS{pg5n` z^2n40Qg2;;ELJq481P!n45SrFrV&;d84bv&wzS@*SSALQBRm9m(@B`(!G(yR1eelL6`AWqR*_qn;#hvZl-ME-z?9Q;T zupD1>m;CrY_o-xr6t|!n(Fu5u1wq8+qwC<6BYjAlV4`HeEVzdW)l&* zjc-$Lhm_NVvFw!7Mfy`FzocKx&As0szz+mR)L4uAdPA_=D>m9o?p&R;~5>NPfYo|Gn{ZA z4Ky%MGF50!*jet*rXx+Yl!RatqJGV3|)BvBAfMd9heu*g7P z7E*lZ=*VK9^>d73Kh_`xW7oKAFG>eGva}~pm8$9TPQk2#CTqs29&I+=w_ACfxCRB{ zMmt?vOB&4L<6ns<5@9qW$hy6Eg1Xwdw1f8CSbVRdLC(<#5=1g7DXCXYB_xKCl$n`% zDCJ|2pi39(=N=`Bkwm7iOfeD|smy{@Q(iIeV?4!Y=~@G#I>B5&2!S&dyQ4L&XA8LE z=iU^aeNa67wYVtpoeCB4S4^PsR^c1po=6~LtxnxJ@L9|PCn$=X{z+S?VabzrP}5K8 zWW_k3W>xf-s)~!tr|OjT`F3ve4XH?SyXZ^m*iPIX<;9sCrPlzi$mLsHaT@#kH>acg zD_a*T;@ic{YMxIS4x}Y_yFiE2D`$t9e*5K?i(zXn>%G+spN>CnaGmyj;eYyCTD?R2 zA3rO7MWL!vpixNV;MdDc_d1uy#KhFDbH{ek_rwzy7hicWEFVK`PkP|=zgVJU8*G0E zHjoED4VKj0fnNm5<*kK|0`#NJFhW_s$qQbg`3eX7uae?P_%>R33gu~}ka1`$T4^wt z@#=hb7Nsu34~a_s@^E_AV-MI$I7TM9ERic43rm}RIGnFv|7|x+4NIv0@n!K5#{B5} zPNKZF?;VzEo*3QL6|e9|fJ@u=-&+8?Zbp1pyVah#H-9ghYB5h4t0Fu`f7h>XxO65? z95N^6&qZHnTl`*|@Wrj1|wu{ZGo1nJY^aRrph- zKbzF%iRty0Yd)?|iHvdi?rBv{Ziaw8zRQoMleXSVl8*wAlz%^dzPda8<<-M5EE{Wx zfG^lhnJ^`&7c>CsivO*)5_RA_I{q;{2{Q5it1FBnv!cq_^h5KI7fkJzSk4-1DiU$p$ zF?!GQnaP8V@`Hr!1lX-ALAf%OOt@l%rZP+PuTOa$HYV_mJ3FGyqL(Wf?kcMw;kr)K z`17;&FJ&8X8?3$UW`Tn4j20M>0)MDdqkDTR*h{vIW zEdJTDiw{UO(6lS+@}+87KLg4FWmW2AcDTeTT{_B|@4uMiqz?vnJ&4qwYNi2~x|;JY z4~rEEU{}Y&im>x9S>=t5$@;C{C6gF>@+%E0g$~SkF)^@v{};K>pMy_-*JXQTh&Af6 zv#_#aU|+2e9Y3z4BeEbV{4ccSt>!FHKEK}EkX;TmSg!UwMv=v6Izq&>0f)2N3Kufc zaL4-lo@0C+AE(n%T-GfWFqKI*gXwd7wtnNk65v!xn^I%+tmt_L$T^GCF}O}uj7jZMe_8n z#xl0+8Y|A>fJ0$cOz;J4qYj8dkWO;?L>@!Jh)GaUzsmv{5Ynr zRQCGTx|9RTno{{CaJh$)lC+?4H8=Yv=^hg%2~w@KU*53BI=vzmze+;Xm$ zfz-6L7)9V?AHkQF2yb2{dK*Tfqw|ZIdDXX@jyW4$^_y2z@9OmByQ(eT+HQv2dM5tea}*bjg`n-H_mR6C8$mUt`yk50~kl>YzF@;AY96a#ZY! zCybc=awrR!;<=pwyXygllV*wVkcvAYlW%m5%1Oo2>3d&bGw5?sEXQa6JsVO4yY6HD zb)fW7{Vb?zU@!`CXy2|Ew%@HDtT242w;Osnm9n<6QDP4FXX}&;&s14$8-kRVN-G`4 z_T6G&8*u7_?aZh;pQ z^oH{*Ttj5l8PPvRD7^K2QEhPa;b(rlwa=l+9aS)uK9!VMn1I6L7u5BNz1b%YKmH>6FFVFsLFv{!7`w0$2C(N zJP8S1!BR@m1xUsP5`ywZ-kzTJxqD)d+$Ee~W{Cy%zNyz$N8vOw#C0AaBq(dKX)_z* zL{1fpSJz<4_|+f?Q2WDJwY%r^OC!raz^h8l#mYI)(nV0C7CJlYMPMLEN12`ialj-V z8XPpazudQ7E*{!N4*c_n+hIkdyyaL3k?Q5}SROehxDee&q>UX%lQViAgtb)vR+;R- zp&a*=mHjY4PtEl+`8_-$)yBpK0Xk7XegN*T8uYK+3}5_cXmc!Avqbmz_iyj+#;bB} z2Zu&R7;=0|Bje2CEEaS`IQ}^63L;MT_1EEs+3IZu4v2T2g4oirz;&(5I6yLEJ874y!b^~+$-4)ITx$N+hc zbjV!)fQ{8zDdOh0(f>)4Bp744fOkY^;J^WfQ z@c(9~D-`!N{`ai)*Y^c7I;(O7-cJJKNz*qZ8xAV7b@3N0dw3FO;XJ`${e9|{J9{>I zY+GZa?vc$4ZG>dvAJN((ME=jR2SV!2{ATYlP;1IGjses71WzO9H8|`C6NCYmA~4Rf zcS2gtrh@m(mH%=)tTk{W&wJoN;L5Uv^ry$riVJhu@2r99Q!#p-Dnar#=~`V4^zKahFivakf{(N)jNkHC#jz=+-f*@YDlJa9fkb#&b-VWIbQUb7e=dHd9a2U}W z(-&5?J7W)V$C9>UmRK;7ETNDRfW9v6`a{B$Sb*Wc7k`Wj)q!Bi5@K);IUr3>N;IJ$ zYC`^QG?_z>72I1S9SiTxxw`2}w%MFO%Sj&_WehMduufxL%6Ia^mq2H#=(jlD2}xyt z9@YG&zp#yzkIlKufDjVrouAkp6*_NwH#kp84;L|IthE-?6t4HL>}noXvjl`Pu?(B( zgb0kIOw(T5n>Do_s@~fieG){Aoj?oP67_FwI81blVF$z-C&8!Bgc4p8Hh_*eknE$$ zzDS@&|J#ptY)Nnuq?7k0SlfL=Ph$$uy4Kkc{H-*2+mAw?S>F~**za#BXTI6=r4;e5 zIo5Y9YMUWc<0dBm5W;s-IcKdJpBGw$pr2b)hibf{(@pX4LJOfRd(ZlxJl~dd z%>vIgB6}c?OspnW?YFl82po)XfoCpgcz9DOd(p?Pp=1)H^;&5SbfEt^DnKA!lKG!P ze(upV^`=thKm9)R1*iUh+H|S@zXuHde>Zfzyeh^^-YC~AO+p--0LaTIOP5O;2K*nG CoHJMe From a3b5c1234824f8ac217eb9f2d2b9d60b93971e74 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 13 Jun 2026 02:19:38 +0000 Subject: [PATCH 052/110] Increment Version to 0.8.0a2 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 2b91e95..495fc5e 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -2,7 +2,7 @@ VERSION_MAJOR = 0 VERSION_MINOR = 8 VERSION_BUILD = 0 -VERSION_ALPHA = 1 +VERSION_ALPHA = 2 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "") From 420ec7f6f77d5c7cbc583c69839a33f32aff8eeb Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 13 Jun 2026 02:20:00 +0000 Subject: [PATCH 053/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 970a957..6ac6b14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.8.0a2](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.8.0a2) (2026-06-13) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.8.0a1...0.8.0a2) + +**Merged pull requests:** + +- docs: standardize NGI0 Commons Fund attribution [\#24](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/24) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.8.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.8.0a1) (2026-06-02) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.7.0a1...0.8.0a1) From 3412ef4356bddd4e5e6e0a1836d0a610c053ff34 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:42:10 +0100 Subject: [PATCH 054/110] feat: SpecMessage registry + legacy->spec MIGRATION_MAP (#26) * feat: add SpecMessage registry + legacy->spec MIGRATION_MAP SpecMessage is a str-enum of the canonical ovos.* bus topics the specs define, so downstream code reads SpecMessage.SPEAK instead of a raw 'ovos.utterance.speak' and a bare string is visibly legacy/implementation. MIGRATION_MAP maps each legacy topic to its SpecMessage replacement; migration_counterpart() returns the other-namespace topic. Vocabulary for the transparent bus-namespace migration. * Update messages.py --- ovos_spec_tools/__init__.py | 10 ++++ ovos_spec_tools/messages.py | 110 ++++++++++++++++++++++++++++++++++++ test/test_messages.py | 58 +++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 ovos_spec_tools/messages.py create mode 100644 test/test_messages.py diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index 9d14c37..34bde24 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -41,6 +41,12 @@ standardize_lang, ) from ovos_spec_tools.lint import Finding, lint_locale +from ovos_spec_tools.messages import ( + MIGRATION_MAP, + SPEC_TO_LEGACY, + SpecMessage, + migration_counterpart, +) from ovos_spec_tools.prompt import PromptRenderer, render_prompt from ovos_spec_tools.resources import ( LocaleResources, @@ -84,5 +90,9 @@ "closest_lang", "lint_locale", "Finding", + "SpecMessage", + "MIGRATION_MAP", + "SPEC_TO_LEGACY", + "migration_counterpart", "__version__", ] diff --git a/ovos_spec_tools/messages.py b/ovos_spec_tools/messages.py new file mode 100644 index 0000000..6a63103 --- /dev/null +++ b/ovos_spec_tools/messages.py @@ -0,0 +1,110 @@ +"""Canonical OVOS spec bus-message topics and the legacy→spec migration map. + +``SpecMessage`` is a string enum of the bus topics the OVOS specifications +define under the ``ovos.*`` namespace. Referencing ``SpecMessage.SPEAK`` instead +of the raw ``"ovos.utterance.speak"`` makes downstream code self-documenting: a +``SpecMessage`` member is provably spec-defined, while a bare string is visibly +legacy or implementation-specific. + +``MIGRATION_MAP`` maps each legacy topic to the ``SpecMessage`` that replaces it, +**only for renames where the legacy payload already satisfies the new topic's +consumers** (identical, empty, or superset payloads). Renames that also change +the payload *shape* (e.g. the handler-lifecycle trio, the INTENT-4 registration +restructure) are deliberately excluded — those need per-site payload +transformation, not a transparent topic rename, since a transparent dual-emit +would deliver legacy-shaped data on the new topic. + +Topics with placeholders (``ovos.pipeline..intents.list``, +``:``) are not enum members. +""" +from enum import Enum +from typing import Dict, Optional + + +class SpecMessage(str, Enum): + """Canonical ``ovos.*`` bus topics defined by the OVOS specifications. + + Members are ``str`` so they can be used directly as topics:: + + bus.on(SpecMessage.SPEAK, handler) + bus.emit(Message(SpecMessage.UTTERANCE, {...})) + + The enum grows as specs are catalogued/merged; absence of a topic here does + not imply it is not spec-defined. Topics from specs still in open PRs are + marked provisional. + """ + + def __str__(self) -> str: + # behave like py3.11 StrEnum: str()/f-string yield the topic, not + # "SpecMessage.SPEAK" + return self.value + + # --- PIPELINE-1 --- + UTTERANCE = "ovos.utterance.handle" + SPEAK = "ovos.utterance.speak" + UTTERANCE_HANDLED = "ovos.utterance.handled" + UTTERANCE_CANCELLED = "ovos.utterance.cancelled" + INTENT_MATCHED = "ovos.intent.matched" + INTENT_UNMATCHED = "ovos.intent.unmatched" + INTENT_HANDLER_START = "ovos.intent.handler.start" + INTENT_HANDLER_COMPLETE = "ovos.intent.handler.complete" + INTENT_HANDLER_ERROR = "ovos.intent.handler.error" + + # --- INTENT-4 --- + INTENT_REGISTER_KEYWORD = "ovos.intent.register.keyword" + INTENT_REGISTER_TEMPLATE = "ovos.intent.register.template" + INTENT_DEREGISTER = "ovos.intent.deregister" + INTENT_ENABLE = "ovos.intent.enable" + INTENT_DISABLE = "ovos.intent.disable" + ENTITY_REGISTER = "ovos.entity.register" + ENTITY_DEREGISTER = "ovos.entity.deregister" + SKILL_DEREGISTER = "ovos.skill.deregister" + INTENT_LIST = "ovos.intent.list" + INTENT_LIST_RESPONSE = "ovos.intent.list.response" + INTENT_DESCRIBE = "ovos.intent.describe" + INTENT_DESCRIBE_RESPONSE = "ovos.intent.describe.response" + + # --- STOP-1 --- + STOP_PING = "ovos.stop.ping" + STOP_PONG = "ovos.stop.pong" + STOP = "ovos.stop" + + # --- AUDIO-1 --- + AUDIO_OUTPUT_STARTED = "ovos.audio.output.started" + AUDIO_OUTPUT_ENDED = "ovos.audio.output.ended" + MIC_LISTEN = "ovos.mic.listen" + LISTENER_RECORD_STARTED = "ovos.listener.record.started" + LISTENER_RECORD_ENDED = "ovos.listener.record.ended" + LISTENER_SLEEP = "ovos.listener.sleep" + LISTENER_AWOKEN = "ovos.listener.awoken" + + +# legacy topic -> SpecMessage that supersedes it, RESTRICTED to renames whose +# legacy payload already satisfies the new topic's consumers (so a transparent +# dual-emit is safe). Shape-changing renames are intentionally absent. +MIGRATION_MAP: Dict[str, SpecMessage] = { + "recognizer_loop:utterance": SpecMessage.UTTERANCE, + "speak": SpecMessage.SPEAK, + "recognizer_loop:audio_output_start": SpecMessage.AUDIO_OUTPUT_STARTED, + "recognizer_loop:audio_output_end": SpecMessage.AUDIO_OUTPUT_ENDED, + "mycroft.mic.listen": SpecMessage.MIC_LISTEN, + "recognizer_loop:record_begin": SpecMessage.LISTENER_RECORD_STARTED, + "recognizer_loop:record_end": SpecMessage.LISTENER_RECORD_ENDED, + "recognizer_loop:sleep": SpecMessage.LISTENER_SLEEP, + "mycroft.awoken": SpecMessage.LISTENER_AWOKEN, +} + +# reverse: spec topic (plain str) -> the legacy topic it replaces +SPEC_TO_LEGACY: Dict[str, str] = {v.value: k for k, v in MIGRATION_MAP.items()} + + +def migration_counterpart(topic: str) -> Optional[str]: + """Return the other-namespace topic for a migrating topic, else ``None``. + + A legacy topic returns its ``ovos.*`` replacement; a spec topic returns the + legacy name it replaces; an unmapped topic returns ``None``. Always a plain + ``str``. + """ + if topic in MIGRATION_MAP: + return MIGRATION_MAP[topic].value + return SPEC_TO_LEGACY.get(topic) diff --git a/test/test_messages.py b/test/test_messages.py new file mode 100644 index 0000000..fa6934d --- /dev/null +++ b/test/test_messages.py @@ -0,0 +1,58 @@ +"""Tests for the spec bus-message registry.""" +import unittest + +from ovos_spec_tools import ( + MIGRATION_MAP, + SPEC_TO_LEGACY, + SpecMessage, + migration_counterpart, +) + + +class TestSpecMessage(unittest.TestCase): + def test_members_are_ovos_namespaced_strings(self): + for m in SpecMessage: + self.assertTrue(m.value.startswith("ovos."), m.value) + + def test_usable_as_plain_string(self): + self.assertEqual(SpecMessage.SPEAK, "ovos.utterance.speak") + self.assertEqual(f"{SpecMessage.UTTERANCE}", "ovos.utterance.handle") + + def test_no_duplicate_values(self): + values = [m.value for m in SpecMessage] + self.assertEqual(len(values), len(set(values))) + + +class TestMigrationMap(unittest.TestCase): + def test_maps_legacy_to_specmessage(self): + self.assertEqual(MIGRATION_MAP["speak"], SpecMessage.SPEAK) + self.assertEqual(MIGRATION_MAP["recognizer_loop:utterance"], + SpecMessage.UTTERANCE) + + def test_reverse_is_consistent(self): + for legacy, spec in MIGRATION_MAP.items(): + self.assertEqual(SPEC_TO_LEGACY[str(spec)], legacy) + + def test_values_are_spec_members(self): + for spec in MIGRATION_MAP.values(): + self.assertIsInstance(spec, SpecMessage) + + +class TestMigrationCounterpart(unittest.TestCase): + def test_legacy_returns_spec(self): + self.assertEqual(migration_counterpart("speak"), "ovos.utterance.speak") + + def test_spec_returns_legacy(self): + self.assertEqual(migration_counterpart("ovos.utterance.speak"), "speak") + + def test_unmapped_returns_none(self): + self.assertIsNone(migration_counterpart("some.random.topic")) + + def test_roundtrip(self): + for legacy in MIGRATION_MAP: + spec = migration_counterpart(legacy) + self.assertEqual(migration_counterpart(spec), legacy) + + +if __name__ == "__main__": + unittest.main() From 73e9734b1a71e8a07fb77d3137e4ee658da99774 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:42:22 +0000 Subject: [PATCH 055/110] Increment Version to 0.9.0a1 --- ovos_spec_tools/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 495fc5e..d578501 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,8 +1,8 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 8 +VERSION_MINOR = 9 VERSION_BUILD = 0 -VERSION_ALPHA = 2 +VERSION_ALPHA = 1 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "") From 1b9eab65a816a2d8a957796184688efbd0eb7dc6 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:42:46 +0000 Subject: [PATCH 056/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ac6b14..82f2195 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.9.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.9.0a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.8.0a2...0.9.0a1) + +**Merged pull requests:** + +- feat: SpecMessage registry + legacy-\>spec MIGRATION\_MAP [\#26](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/26) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.8.0a2](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.8.0a2) (2026-06-13) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.8.0a1...0.8.0a2) From 27c6615b6e73d8b1e3481e1bf0e99693904b6237 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 20:14:56 +0100 Subject: [PATCH 057/110] feat: add NamespaceTranslator (shared bus-namespace migration logic) (#28) * feat: add NamespaceTranslator (shared migration logic for MessageBusClient + FakeBus) Pure, config-free logic (flags passed in) so the real bus and the FakeBus test double behave identically: counterpart_topics() for emit translation, is_migrated() and new_mirror_guard() for per-handler counterpart-dedup. Single source of truth. * fix: resolve mirror-guard clock per call so it stays monkeypatchable --- ovos_spec_tools/__init__.py | 2 ++ ovos_spec_tools/messages.py | 70 ++++++++++++++++++++++++++++++++++++- test/test_messages.py | 58 ++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index 34bde24..2cda9b2 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -44,6 +44,7 @@ from ovos_spec_tools.messages import ( MIGRATION_MAP, SPEC_TO_LEGACY, + NamespaceTranslator, SpecMessage, migration_counterpart, ) @@ -94,5 +95,6 @@ "MIGRATION_MAP", "SPEC_TO_LEGACY", "migration_counterpart", + "NamespaceTranslator", "__version__", ] diff --git a/ovos_spec_tools/messages.py b/ovos_spec_tools/messages.py index 6a63103..e189688 100644 --- a/ovos_spec_tools/messages.py +++ b/ovos_spec_tools/messages.py @@ -17,8 +17,10 @@ Topics with placeholders (``ovos.pipeline..intents.list``, ``:``) are not enum members. """ +import json as _json +import time from enum import Enum -from typing import Dict, Optional +from typing import Callable, Dict, List, Optional class SpecMessage(str, Enum): @@ -108,3 +110,69 @@ def migration_counterpart(topic: str) -> Optional[str]: if topic in MIGRATION_MAP: return MIGRATION_MAP[topic].value return SPEC_TO_LEGACY.get(topic) + + +class NamespaceTranslator: + """Shared legacy<->``ovos.*`` bus-namespace migration logic. + + Used by both ``ovos_bus_client.MessageBusClient`` and + ``ovos_utils.fakebus.FakeBus`` so the real bus and the test/satellite double + behave identically. Pure logic: the two flags are passed in (the caller reads + env/config), keeping ``ovos-spec-tools`` dependency-free. + + Args: + modernize: emitting a legacy topic also emits the ``ovos.*`` spec topic. + emit_legacy: emitting an ``ovos.*`` spec topic also emits the legacy topic. + window: seconds within which a counterpart re-delivery is a "mirror". + """ + + def __init__(self, modernize: bool = True, emit_legacy: bool = True, + window: float = 1.0): + self.modernize = modernize + self.emit_legacy = emit_legacy + self.window = window + + def counterpart_topics(self, msg_type: str) -> List[str]: + """Topic(s) a producer should ALSO emit for ``msg_type`` (0 or 1).""" + if self.modernize and msg_type in MIGRATION_MAP: + return [MIGRATION_MAP[msg_type].value] + if self.emit_legacy and msg_type in SPEC_TO_LEGACY: + return [SPEC_TO_LEGACY[msg_type]] + return [] + + def is_migrated(self, topic: str) -> bool: + """Whether ``topic`` participates in the migration (needs dedup).""" + return topic in MIGRATION_MAP or topic in SPEC_TO_LEGACY + + def new_mirror_guard(self, + clock: Optional[Callable[[], float]] = None + ) -> Callable[[object], bool]: + """Return a per-handler ``is_mirror(message) -> bool`` with private state. + + Returns True when ``message`` is the legacy/``ovos.*`` mirror of one just + handled — same payload+context re-delivered via the COUNTERPART topic + within the window — so the handler runs once. Two genuine events on the + SAME topic are never suppressed. ``clock`` defaults to + ``time.monotonic`` (resolved per call, so it stays monkeypatchable). + """ + seen: Dict[str, tuple] = {} + window = self.window + + def is_mirror(message) -> bool: + try: + mtype = message.msg_type + fingerprint = _json.dumps([message.data, message.context], + sort_keys=True, default=str) + except Exception: + return False + now = (clock or time.monotonic)() + for key in [k for k, (_, ts) in seen.items() if now - ts >= window]: + seen.pop(key, None) + prev = seen.get(fingerprint) + if prev is not None and prev[0] != mtype \ + and migration_counterpart(prev[0]) == mtype: + return True + seen[fingerprint] = (mtype, now) + return False + + return is_mirror diff --git a/test/test_messages.py b/test/test_messages.py index fa6934d..6fa58ba 100644 --- a/test/test_messages.py +++ b/test/test_messages.py @@ -4,11 +4,69 @@ from ovos_spec_tools import ( MIGRATION_MAP, SPEC_TO_LEGACY, + NamespaceTranslator, SpecMessage, migration_counterpart, ) +class _Msg: + def __init__(self, msg_type, data=None, context=None): + self.msg_type = msg_type + self.data = data or {} + self.context = context or {} + + +class TestNamespaceTranslator(unittest.TestCase): + def test_counterpart_topics_directions(self): + t = NamespaceTranslator(modernize=True, emit_legacy=True) + self.assertEqual(t.counterpart_topics("speak"), ["ovos.utterance.speak"]) + self.assertEqual(t.counterpart_topics("ovos.utterance.handle"), + ["recognizer_loop:utterance"]) + self.assertEqual(t.counterpart_topics("some.topic"), []) + + def test_flags_gate_each_direction(self): + only_mod = NamespaceTranslator(modernize=True, emit_legacy=False) + self.assertEqual(only_mod.counterpart_topics("speak"), ["ovos.utterance.speak"]) + self.assertEqual(only_mod.counterpart_topics("ovos.utterance.speak"), []) + only_leg = NamespaceTranslator(modernize=False, emit_legacy=True) + self.assertEqual(only_leg.counterpart_topics("speak"), []) + self.assertEqual(only_leg.counterpart_topics("ovos.utterance.speak"), ["speak"]) + + def test_is_migrated(self): + t = NamespaceTranslator() + self.assertTrue(t.is_migrated("speak")) + self.assertTrue(t.is_migrated("ovos.utterance.speak")) + self.assertFalse(t.is_migrated("random.topic")) + + def test_mirror_guard_drops_counterpart_pair(self): + guard = NamespaceTranslator().new_mirror_guard() + data = {"utterance": "hi"} + self.assertFalse(guard(_Msg("speak", data))) + self.assertTrue(guard(_Msg("ovos.utterance.speak", data))) # mirror + + def test_mirror_guard_keeps_same_topic_repeats(self): + guard = NamespaceTranslator().new_mirror_guard() + data = {"utterance": "ok"} + self.assertFalse(guard(_Msg("speak", data))) + self.assertFalse(guard(_Msg("speak", data))) # genuine repeat, same topic + + def test_mirror_guard_distinct_context_not_collapsed(self): + guard = NamespaceTranslator().new_mirror_guard() + data = {"utterance": "hi"} + self.assertFalse(guard(_Msg("speak", data, {"session": {"session_id": "A"}}))) + self.assertFalse(guard(_Msg("ovos.utterance.speak", data, + {"session": {"session_id": "B"}}))) + + def test_mirror_guard_window_expiry(self): + clk = {"t": 0.0} + guard = NamespaceTranslator(window=1.0).new_mirror_guard(clock=lambda: clk["t"]) + data = {"utterance": "hi"} + self.assertFalse(guard(_Msg("speak", data))) + clk["t"] = 2.0 + self.assertFalse(guard(_Msg("ovos.utterance.speak", data))) # window passed + + class TestSpecMessage(unittest.TestCase): def test_members_are_ovos_namespaced_strings(self): for m in SpecMessage: From 4f4589cf640ac4f0409fac99d327c1f64d41a6c7 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:15:06 +0000 Subject: [PATCH 058/110] Increment Version to 0.10.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index d578501..e92380c 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 9 +VERSION_MINOR = 10 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From c3e289d7adcc4b5c4e8a71d5d086df2666678518 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:15:25 +0000 Subject: [PATCH 059/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82f2195..6176c03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.10.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.10.0a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.9.0a1...0.10.0a1) + +**Merged pull requests:** + +- feat: add NamespaceTranslator \(shared bus-namespace migration logic\) [\#28](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/28) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.9.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.9.0a1) (2026-06-25) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.8.0a2...0.9.0a1) From a81f2f01ac565335d5cc89529b8c42ff4bed0fb8 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 14:21:44 +0100 Subject: [PATCH 060/110] feat: bridge PIPELINE-1 trio, STOP-1, intent-unmatched & INTENT-4 management topics (#30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: map the PIPELINE-1 handler trio + STOP-1 topics in MIGRATION_MAP Make ovos-spec-tools the single source of truth for ALL legacy<->ovos.* renames so the bus (NamespaceTranslator) bridges them transparently and producers never hand-roll a dual-emit. Adds the PIPELINE-1 §8 handler-lifecycle trio (mycroft.skill.handler.start/.complete/.error -> ovos.intent.handler.*) and STOP-1 skill.stop.pong/mycroft.stop. The trio payload is restructured to {skill_id,intent_name}; the bridged legacy message carries the spec payload (noted in the map docstring). Per-instance placeholder topics ({skill_id}.stop.ping etc.) stay producer/consumer subscribe-both since they can't be a static 1:1 map. * feat: map complete_intent_failure -> ovos.intent.unmatched (PIPELINE-1) 1:1 rename, payload-compatible. Documents why INTENT-4 registration topics are NOT mappable here (they consolidate/restructure legacy topics and need spec adoption in the producers/consumers, not a static bridge). * feat: map INTENT-4 intent-management topics (detach/enable/disable) detach_intent->ovos.intent.deregister, detach_skill->ovos.skill.deregister, mycroft.skill.enable_intent/disable_intent->ovos.intent.enable/.disable. These are 1:1 renames bridged transparently. INTENT-4 *registration* stays out (N register_vocab + register_intent consolidate into one register.keyword — needs spec adoption, not a static bridge); documented inline. --- ovos_spec_tools/messages.py | 40 ++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/ovos_spec_tools/messages.py b/ovos_spec_tools/messages.py index e189688..cd1cb7a 100644 --- a/ovos_spec_tools/messages.py +++ b/ovos_spec_tools/messages.py @@ -81,10 +81,21 @@ def __str__(self) -> str: LISTENER_AWOKEN = "ovos.listener.awoken" -# legacy topic -> SpecMessage that supersedes it, RESTRICTED to renames whose -# legacy payload already satisfies the new topic's consumers (so a transparent -# dual-emit is safe). Shape-changing renames are intentionally absent. +# legacy topic -> SpecMessage that supersedes it. This is the single source of +# truth for the legacy<->ovos.* topic renames; the bus (NamespaceTranslator) reads +# it to bridge BOTH namespaces transparently, so producers emit only the spec topic +# (from SpecMessage) and never hand-roll a dual-emit. Producers emit the spec +# payload; the bridged legacy message carries that same payload (consumers still on +# a legacy topic during the migration window receive the spec-shaped data — for the +# shape-changing renames below, e.g. the handler trio, that means the new +# skill_id/intent_name fields rather than the old `name`). +# +# Only 1:1 static renames live here. Per-instance topics with placeholders +# (``{skill_id}.stop.ping``, ``{skill_id}.stop`` -> the broadcast ``ovos.stop.ping`` +# / ``ovos.stop``) cannot be expressed as a static map and are handled by +# producers/consumers subscribing on both forms. MIGRATION_MAP: Dict[str, SpecMessage] = { + # --- AUDIO-IN-1 / PIPELINE-1 (payload-compatible) --- "recognizer_loop:utterance": SpecMessage.UTTERANCE, "speak": SpecMessage.SPEAK, "recognizer_loop:audio_output_start": SpecMessage.AUDIO_OUTPUT_STARTED, @@ -94,6 +105,29 @@ def __str__(self) -> str: "recognizer_loop:record_end": SpecMessage.LISTENER_RECORD_ENDED, "recognizer_loop:sleep": SpecMessage.LISTENER_SLEEP, "mycroft.awoken": SpecMessage.LISTENER_AWOKEN, + # --- PIPELINE-1 §8 handler-lifecycle trio (payload restructured to + # {skill_id, intent_name}; bridged transparently per the note above) --- + "mycroft.skill.handler.start": SpecMessage.INTENT_HANDLER_START, + "mycroft.skill.handler.complete": SpecMessage.INTENT_HANDLER_COMPLETE, + "mycroft.skill.handler.error": SpecMessage.INTENT_HANDLER_ERROR, + # --- STOP-1 §4.2/§5.3 (1:1 renames) --- + "skill.stop.pong": SpecMessage.STOP_PONG, + "mycroft.stop": SpecMessage.STOP, + # --- PIPELINE-1 intent outcome (1:1 rename) --- + "complete_intent_failure": SpecMessage.INTENT_UNMATCHED, + # --- INTENT-4 intent management (1:1 renames; payload restructured to + # {skill_id, intent_name}, bridged transparently) --- + "detach_intent": SpecMessage.INTENT_DEREGISTER, + "detach_skill": SpecMessage.SKILL_DEREGISTER, + "mycroft.skill.enable_intent": SpecMessage.INTENT_ENABLE, + "mycroft.skill.disable_intent": SpecMessage.INTENT_DISABLE, + # NOTE: INTENT-4 *registration* (ovos.intent.register.keyword/.template, + # ovos.entity.register) is NOT here — it is not a 1:1 rename. Adapt emits N + # `register_vocab` + one `register_intent` (vocab referenced by name); the spec + # consolidates all of that into ONE register.keyword message with inlined vocab + # descriptors. That N->1 + restructure requires INTENT-4 *adoption* in the + # producer (ovos_workshop/intents.py) and consumers (the pipeline plugins), not + # a static bus bridge. } # reverse: spec topic (plain str) -> the legacy topic it replaces From 679f015744bad223e701dceb2aa24349afb53f7e Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:21:57 +0000 Subject: [PATCH 061/110] Increment Version to 0.11.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index e92380c..a2e979e 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 10 +VERSION_MINOR = 11 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From daca8b9f20e40380bebb592c015e3ec27c2a527f Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:22:18 +0000 Subject: [PATCH 062/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6176c03..ff65511 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.11.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.11.0a1) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.10.0a1...0.11.0a1) + +**Merged pull requests:** + +- feat: bridge PIPELINE-1 trio, STOP-1, intent-unmatched & INTENT-4 management topics [\#30](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/30) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.10.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.10.0a1) (2026-06-25) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.9.0a1...0.10.0a1) From caedc634a4ad322c383523485f49c82ae0704a9b Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 17:05:46 +0100 Subject: [PATCH 063/110] feat: OVOS-SESSION-1 canonical Session reference implementation (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make `Session` a first-class ovos-spec-tools primitive — the canonical OVOS-SESSION-1 reference implementation — carrying the full §3 registered field set with spec-correct serialize/deserialize and helper methods, so downstream Session implementations subclass it instead of hand-rolling (mirroring how `Message` lives here and bus-client subclasses it). Fields (full §3 closed set): - §3.1 session_id (reserved "default"), §3.2 the six language signals, §3.3 site_id — owned by SESSION-1 (SESSION1_OWNED_FIELDS); - pipeline (PIPELINE-1 §5), intent_context (CONTEXT-1 §2), the three blacklists, the six *_transformers + their per-chain blacklists (TRANSFORM-1 §5/§5.2) — first-class, carried opaquely; - active_handlers (PIPELINE-1 §7.1), converse_handlers (CONVERSE-1 §2.1), response_mode (CONVERSE-1 §2.2) — with recency/dedup/cap/prune logic. API mirrors ovos-bus-client #234 exactly so the bus-client follow-up can subclass with zero churn: add/remove_active_handler, add/remove_converse_handler, prune_converse_handlers(ttl, now), set/clear_response_mode, the _coerce/_promote/_cap internals, the `active` property, and converse_handlers_cap (DEFAULT 64). Serialization honours SESSION-1 §2.1 omission-not-null (a field is present-with-value or absent, never JSON null) and §3.4 (empty list override ≡ omission). Unknown fields round-trip via `extras` (§2.4). Stays dependency-light: pure data + stdlib only. No ovos-bus-client, ovos-config, bus, or heavy ovos-utils imports; deployment defaults (pipeline, converse cap, lang) are injected, not read from a config singleton. SessionManager, dig_for_message, and the active_skills / utterance_states back-compat projections stay in the bus-client subclass. Tests: round-trip serialize/deserialize incl. omission-not-null, active_handlers dedup+recency, converse_handlers cap-64 eviction + TTL prune, response_mode single-holder + absence, full §3 field-set round trip (58 session tests, suite green). Co-authored-by: Claude Opus 4.8 --- ovos_spec_tools/__init__.py | 16 + ovos_spec_tools/session.py | 614 ++++++++++++++++++++++++++++++++++++ test/test_session.py | 446 ++++++++++++++++++++++++++ 3 files changed, 1076 insertions(+) create mode 100644 ovos_spec_tools/session.py create mode 100644 test/test_session.py diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index 2cda9b2..f206f83 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -24,6 +24,10 @@ grouping (§4.3) and utterance match / strip primitives; - :class:`~ovos_spec_tools.message.Message` — the OVOS-MSG-1 bus message envelope with the ``forward`` / ``reply`` / ``response`` derivations; +- :class:`~ovos_spec_tools.session.Session` — the OVOS-SESSION-1 session + carrier reference implementation, carrying the full §3 registered field + set with omission-not-null serialization and the + OVOS-PIPELINE-1 §7.1 / OVOS-CONVERSE-1 §2.1 / §2.2 handler-list helpers; - :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also exposed as the ``ovos-spec-lint`` command. """ @@ -49,6 +53,13 @@ migration_counterpart, ) from ovos_spec_tools.prompt import PromptRenderer, render_prompt +from ovos_spec_tools.session import ( + DEFAULT_CONVERSE_HANDLERS_CAP, + MalformedSession, + SESSION1_OWNED_FIELDS, + SESSION1_REGISTERED_FIELDS, + Session, +) from ovos_spec_tools.resources import ( LocaleResources, MalformedResource, @@ -67,6 +78,11 @@ "Message", "MalformedMessage", "DEFAULT_SESSION_ID", + "Session", + "MalformedSession", + "SESSION1_OWNED_FIELDS", + "SESSION1_REGISTERED_FIELDS", + "DEFAULT_CONVERSE_HANDLERS_CAP", "expand", "inline_keywords", "MalformedTemplate", diff --git a/ovos_spec_tools/session.py b/ovos_spec_tools/session.py new file mode 100644 index 0000000..fe38579 --- /dev/null +++ b/ovos_spec_tools/session.py @@ -0,0 +1,614 @@ +"""Reference implementation of OVOS-SESSION-1 — the session carrier. + +This module defines the **wire shape** of the JSON object that travels +inside ``Message.context.session``, and the rules consumers follow when +reading and propagating it. It is the canonical reference implementation +of the field set OVOS-SESSION-1 §3 claims, carrying every registered +field with spec-correct serialize / deserialize and the recency / cap / +prune helpers the field owners (OVOS-PIPELINE-1 §7.1, OVOS-CONVERSE-1 +§2.1 / §2.2) describe. + +Scope per OVOS-SESSION-1 §1: + +- the JSON shape (§2), including the *omissible-but-never-nullable* + rule (§2 / §2.1); +- the closed set of fields claimed in this version (§3), each carried + as a first-class attribute; +- the reserved ``"default"`` ``session_id`` value (§3.1); +- propagation across the OVOS-MSG-1 §5 derivations (§4) — modelled + here as round-trip equality; +- serialization per OVOS-MSG-1 §6 (§5). + +Design constraints — this is **pure data + stdlib only**. It does not +import ``ovos-bus-client``, ``ovos-config``, the bus, or any heavy +``ovos-utils`` machinery. Deployment defaults (the pipeline ordering, +the converse-handler cap, the configured language) are **injected** by +the caller, never read from a config singleton here. The lifecycle +surface that bus-client layers on top — ``SessionManager``, +``dig_for_message``, the ``active_skills`` / ``utterance_states`` +back-compat projections — lives in bus-client's subclass, not here. + +Out of scope (§1 non-goals): session lifecycle, a session store, +authentication / authorization, encryption, and the *semantics* of any +field whose owner is not OVOS-SESSION-1 — those are owned by the citing +specification. This module fixes only the wire contract, the §3.1 / +§3.2 / §3.3 fields whose meaning OVOS-SESSION-1 itself owns, and the +mechanical recency / cap / prune behaviour the owners describe for the +handler-list fields. +""" +from __future__ import annotations + +import json +import logging +import time +from copy import deepcopy +from typing import Any, Dict, List, Optional, Union + +from ovos_spec_tools.message import DEFAULT_SESSION_ID + +__all__ = [ + "Session", + "MalformedSession", + "DEFAULT_SESSION_ID", + "DEFAULT_CONVERSE_HANDLERS_CAP", + "SESSION1_OWNED_FIELDS", + "SESSION1_REGISTERED_FIELDS", +] + + +_log = logging.getLogger(__name__) + + +#: OVOS-CONVERSE-1 §2.1 default cap for the converse-handler recency +#: stack. A deployment MAY raise, lower, or set it to "unbounded" +#: (a value ``<= 0``). Injected via the ``converse_handlers_cap`` +#: constructor argument; this is only the fallback when none is given. +DEFAULT_CONVERSE_HANDLERS_CAP = 64 + + +#: The field names whose semantics OVOS-SESSION-1 itself owns +#: (§3.1 ``session_id`` + §3.2 language signals + §3.3 ``site_id``). +SESSION1_OWNED_FIELDS = frozenset({ + "session_id", "site_id", "lang", + "secondary_langs", "output_lang", + "stt_lang", "request_lang", "detected_lang", +}) + + +#: The six OVOS-TRANSFORM-1 §5 transformer-chain fields, each an ordered +#: array of plugin ids. +_TRANSFORMER_FIELDS = ( + "audio_transformers", "utterance_transformers", + "metadata_transformers", "intent_transformers", + "dialog_transformers", "tts_transformers", +) + +#: The list-valued denylist / override fields claimed by other specs +#: (OVOS-PIPELINE-1 §5, OVOS-TRANSFORM-1 §5.2). All are arrays of string +#: for which an empty array is wire-equivalent to omission (§3.4). +_LIST_OVERRIDE_FIELDS = ( + "pipeline", + "blacklisted_skills", "blacklisted_intents", "blacklisted_pipelines", + "blacklisted_audio_transformers", "blacklisted_utterance_transformers", + "blacklisted_metadata_transformers", "blacklisted_intent_transformers", + "blacklisted_dialog_transformers", "blacklisted_tts_transformers", +) + _TRANSFORMER_FIELDS + +#: The object-valued override field (OVOS-CONTEXT-1 §2 ``intent_context``). +_OBJECT_OVERRIDE_FIELDS = ("intent_context",) + +#: The handler-list / response-mode fields whose recency / cap / prune +#: behaviour this module reproduces (OVOS-PIPELINE-1 §7.1, +#: OVOS-CONVERSE-1 §2.1 / §2.2). +_HANDLER_FIELDS = ("active_handlers", "converse_handlers", "response_mode") + + +#: The full closed set of fields OVOS-SESSION-1 §3 recognizes in this +#: version. A consumer that recognizes any of these interprets it per +#: its owner specification; everything else is unknown-field passthrough +#: (§2.4) carried opaquely in :attr:`Session.extras`. +SESSION1_REGISTERED_FIELDS = frozenset(SESSION1_OWNED_FIELDS).union( + _LIST_OVERRIDE_FIELDS, _OBJECT_OVERRIDE_FIELDS, _HANDLER_FIELDS) + + +class MalformedSession(ValueError): + """A serialized session payload violates OVOS-SESSION-1 §2 / §5. + + Raised only for structural failures that the spec calls a hard + error: the carrier is not a JSON object, or the JSON is unparsable. + Per §2 an explicit ``null`` on a field is **not** a Message-level + rejection — it is logged and treated as omitted; see + :meth:`Session.from_dict`. + """ + + +def _is_bcp47(value: Any) -> bool: + """Cheap shape check: a BCP-47 tag is a non-empty string with no + whitespace. The full grammar is owned by language tools; this + module rejects only obvious type errors.""" + return isinstance(value, str) and bool(value) and not any( + c.isspace() for c in value) + + +class Session: + """OVOS-SESSION-1 carrier — the canonical reference implementation. + + Every field is **optional** on the wire (§2). The constructor and + :meth:`to_dict` honour the omissible-but-never-nullable rule: a + field whose value is ``None`` / empty is **absent** from the + serialized object, never emitted as ``null`` (§2.1, §3.4). Unknown + fields supplied via ``extras`` round-trip unchanged (§2.4). + + The Python-level default for ``session_id`` is ``None`` (omitted on + the wire). :meth:`resolved_session_id` returns the value a consumer + would fill in — ``"default"`` when omitted (§3.1, §2.1). + + :param session_id: §3.1 — opaque identity string. ``None`` ⇒ omitted + ⇒ resolves to ``"default"`` at consumption. + :param lang: §3.2.1 — user's preferred language (BCP-47). + :param secondary_langs: §3.2.2 — additional BCP-47 tags, ordered. + Empty list and ``None`` are equivalent on the wire (both omitted). + :param output_lang: §3.2.3 — preferred response language (BCP-47). + :param stt_lang: §3.2.4 — language STT actually transcribed in. + :param request_lang: §3.2.5 — language the emitter reported (hint). + :param detected_lang: §3.2.6 — language a detector classified. + :param site_id: §3.3 — opaque group / location identifier. + :param pipeline: OVOS-PIPELINE-1 §5 — ordered pipeline-plugin ids. + :param intent_context: OVOS-CONTEXT-1 §2 — the declarative intent + context object (opaque to this module). + :param blacklisted_skills: OVOS-PIPELINE-1 §5 — skill ids to ignore. + :param blacklisted_intents: OVOS-PIPELINE-1 §5 — intent ids to ignore. + :param blacklisted_pipelines: OVOS-PIPELINE-1 §5 — pipeline ids to skip. + :param audio_transformers ... tts_transformers: OVOS-TRANSFORM-1 §5 — + the six ordered transformer chains. + :param blacklisted_*_transformers: OVOS-TRANSFORM-1 §5.2 — per-chain + denylists. + :param active_handlers: OVOS-PIPELINE-1 §7.1 dispatch-recency record — + a head-first, deduplicated list of ``{skill_id, activated_at}``. + :param converse_handlers: OVOS-CONVERSE-1 §2.1 converse-eligibility + list — head-first, deduplicated, tail-dropped at the cap. + :param response_mode: OVOS-CONVERSE-1 §2.2 pending-response window — + a single ``{skill_id, expires_at}`` object, or ``None``. + :param converse_handlers_cap: OVOS-CONVERSE-1 §2.1 maximum length of + ``converse_handlers``; defaults to 64. A value ``<= 0`` means + "unbounded". + :param extras: passthrough mapping for fields claimed by future + specifications (anything outside :data:`SESSION1_REGISTERED_FIELDS`). + Treated opaquely per §2.4. + """ + + def __init__(self, + session_id: Optional[str] = None, + lang: Optional[str] = None, + secondary_langs: Optional[List[str]] = None, + output_lang: Optional[str] = None, + stt_lang: Optional[str] = None, + request_lang: Optional[str] = None, + detected_lang: Optional[str] = None, + site_id: Optional[str] = None, + pipeline: Optional[List[str]] = None, + intent_context: Optional[Dict[str, Any]] = None, + blacklisted_skills: Optional[List[str]] = None, + blacklisted_intents: Optional[List[str]] = None, + blacklisted_pipelines: Optional[List[str]] = None, + audio_transformers: Optional[List[str]] = None, + utterance_transformers: Optional[List[str]] = None, + metadata_transformers: Optional[List[str]] = None, + intent_transformers: Optional[List[str]] = None, + dialog_transformers: Optional[List[str]] = None, + tts_transformers: Optional[List[str]] = None, + blacklisted_audio_transformers: Optional[List[str]] = None, + blacklisted_utterance_transformers: Optional[List[str]] = None, + blacklisted_metadata_transformers: Optional[List[str]] = None, + blacklisted_intent_transformers: Optional[List[str]] = None, + blacklisted_dialog_transformers: Optional[List[str]] = None, + blacklisted_tts_transformers: Optional[List[str]] = None, + active_handlers: Optional[List[Dict[str, Any]]] = None, + converse_handlers: Optional[List[Dict[str, Any]]] = None, + response_mode: Optional[Dict[str, Any]] = None, + converse_handlers_cap: Optional[int] = None, + extras: Optional[Dict[str, Any]] = None): + if session_id is not None and (not isinstance(session_id, str) + or not session_id): + # §6 producer: non-empty string when set. + raise MalformedSession( + "session_id must be a non-empty string when set (§6)") + if site_id is not None and (not isinstance(site_id, str) + or not site_id): + raise MalformedSession( + "site_id must be a non-empty string when set (§3.3)") + for name, value in (("lang", lang), ("output_lang", output_lang), + ("stt_lang", stt_lang), + ("request_lang", request_lang), + ("detected_lang", detected_lang)): + if value is not None and not _is_bcp47(value): + raise MalformedSession( + f"{name} must be a non-empty BCP-47 string (§3.2)") + if secondary_langs is not None: + if not isinstance(secondary_langs, list): + raise MalformedSession( + "secondary_langs must be an array (§3.2.2)") + seen = set() + for tag in secondary_langs: + if not _is_bcp47(tag): + raise MalformedSession( + "secondary_langs entries must be BCP-47 strings " + "(§3.2.2)") + if tag in seen: + raise MalformedSession( + "secondary_langs MUST NOT contain duplicates " + "(§3.2.2)") + seen.add(tag) + # §3.2.2: MUST NOT contain `lang`. + if lang is not None and lang in seen: + raise MalformedSession( + "secondary_langs MUST NOT contain `lang` (§3.2.2)") + if extras is not None and not isinstance(extras, dict): + raise MalformedSession("extras must be a dict") + + # --- §3.1 / §3.2 / §3.3 owned scalars and lists --------------------- + self.session_id = session_id + self.lang = lang + self.secondary_langs = list(secondary_langs) if secondary_langs else None + self.output_lang = output_lang + self.stt_lang = stt_lang + self.request_lang = request_lang + self.detected_lang = detected_lang + self.site_id = site_id + + # --- other-spec list/object override fields (carried opaquely) ------ + self.pipeline = list(pipeline) if pipeline else None + self.intent_context = dict(intent_context) if intent_context else None + self.blacklisted_skills = self._as_str_list(blacklisted_skills) + self.blacklisted_intents = self._as_str_list(blacklisted_intents) + self.blacklisted_pipelines = self._as_str_list(blacklisted_pipelines) + self.audio_transformers = self._as_str_list(audio_transformers) + self.utterance_transformers = self._as_str_list(utterance_transformers) + self.metadata_transformers = self._as_str_list(metadata_transformers) + self.intent_transformers = self._as_str_list(intent_transformers) + self.dialog_transformers = self._as_str_list(dialog_transformers) + self.tts_transformers = self._as_str_list(tts_transformers) + self.blacklisted_audio_transformers = self._as_str_list( + blacklisted_audio_transformers) + self.blacklisted_utterance_transformers = self._as_str_list( + blacklisted_utterance_transformers) + self.blacklisted_metadata_transformers = self._as_str_list( + blacklisted_metadata_transformers) + self.blacklisted_intent_transformers = self._as_str_list( + blacklisted_intent_transformers) + self.blacklisted_dialog_transformers = self._as_str_list( + blacklisted_dialog_transformers) + self.blacklisted_tts_transformers = self._as_str_list( + blacklisted_tts_transformers) + + # --- PIPELINE-1 §7.1 / CONVERSE-1 §2.1 / §2.2 handler state --------- + if converse_handlers_cap is None: + converse_handlers_cap = DEFAULT_CONVERSE_HANDLERS_CAP + self.converse_handlers_cap = converse_handlers_cap + self.active_handlers: List[Dict[str, Any]] = self._coerce_handlers( + active_handlers) + self.converse_handlers: List[Dict[str, Any]] = self._coerce_handlers( + converse_handlers) + self._cap_handlers(self.converse_handlers) + self.response_mode: Optional[Dict[str, Any]] = self._coerce_response_mode( + response_mode) + + # --- §2.4 unknown-field passthrough -------------------------------- + self.extras: Dict[str, Any] = dict(extras) if extras else {} + + # --- helpers ------------------------------------------------------------ + + @staticmethod + def _as_str_list(value: Optional[List[str]]) -> Optional[List[str]]: + """Normalize a list-valued override into a copied list, or + ``None`` when empty / unset. An empty array is wire-equivalent + to omission (§3.4), so both collapse to ``None``.""" + if not value: + return None + if not isinstance(value, list): + raise MalformedSession("expected an array of string (§3)") + return list(value) + + # --- §3.1 reserved-default resolution ----------------------------------- + + @property + def is_default(self) -> bool: + """``True`` when this session resolves to the reserved + ``"default"`` identity (§3.1): an omitted, empty, or explicit + ``"default"`` ``session_id`` all map here.""" + return self.session_id in (None, DEFAULT_SESSION_ID) + + def resolved_session_id(self) -> str: + """The ``session_id`` a consumer would key per-session state on + — ``"default"`` when omitted (§3.1 + §2.1).""" + return self.session_id or DEFAULT_SESSION_ID + + # ------------------------------------------------------------------ + # OVOS-PIPELINE-1 §7.1 / OVOS-CONVERSE-1 §2.1 / §2.2 handler helpers + # ------------------------------------------------------------------ + @staticmethod + def _coerce_handlers( + handlers: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: + """Normalize a handler list into the spec ``{skill_id, + activated_at}`` object shape (PIPELINE-1 §7.1 / CONVERSE-1 §2.1). + + Accepts either the spec object shape (list of dicts) or the + legacy ``[skill_id, activated_at]`` pair shape (tolerated on + deserialization). Entries are deduplicated by ``skill_id`` + (head wins) and kept head-first by recency.""" + out: List[Dict[str, Any]] = [] + seen = set() + for entry in handlers or []: + if isinstance(entry, dict): + skill_id = entry.get("skill_id") + activated_at = entry.get("activated_at", time.time()) + elif isinstance(entry, (list, tuple)) and entry: + skill_id = entry[0] + activated_at = entry[1] if len(entry) > 1 else time.time() + else: + continue + if not skill_id or skill_id in seen: + continue + seen.add(skill_id) + out.append({"skill_id": skill_id, "activated_at": activated_at}) + return out + + @staticmethod + def _coerce_response_mode( + response_mode: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Normalize a ``response_mode`` value into the spec + ``{skill_id, expires_at}`` shape, or ``None`` when there is no + holder. Malformed values resolve to ``None`` (SESSION-1 §2.1, + CONVERSE-1 §2.2).""" + if not isinstance(response_mode, dict): + return None + skill_id = response_mode.get("skill_id") + if not skill_id: + return None + return {"skill_id": skill_id, + "expires_at": response_mode.get("expires_at", -1)} + + @staticmethod + def _promote_handler(handlers: List[Dict[str, Any]], skill_id: str, + activated_at: Optional[float] = None + ) -> List[Dict[str, Any]]: + """Dedup-and-promote ``skill_id`` to the head of ``handlers`` + (in place). + + Removes any existing entry with the same ``skill_id`` then + inserts a fresh ``{skill_id, activated_at}`` at index 0 — the + recency-stack rule shared by OVOS-PIPELINE-1 §7.1 and + OVOS-CONVERSE-1 §3.1.""" + if activated_at is None: + activated_at = time.time() + handlers[:] = [h for h in handlers if h.get("skill_id") != skill_id] + handlers.insert(0, {"skill_id": skill_id, "activated_at": activated_at}) + return handlers + + def _cap_handlers(self, + handlers: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Tail-drop ``handlers`` down to :attr:`converse_handlers_cap` + entries (in place). + + A cap ``<= 0`` means "unbounded" (OVOS-CONVERSE-1 §2.1). The + least-recent surviving owners (the tail) are dropped.""" + cap = self.converse_handlers_cap + if cap and cap > 0 and len(handlers) > cap: + del handlers[cap:] + return handlers + + @property + def active(self) -> bool: + """``True`` when any handler is active in ``active_handlers`` + (OVOS-PIPELINE-1 §7.1).""" + return len(self.active_handlers) > 0 + + # active_handlers (OVOS-PIPELINE-1 §7.1) + def add_active_handler(self, skill_id: str, + activated_at: Optional[float] = None): + """Push a handler onto ``active_handlers``, dedup-and-promoting + it to the head (OVOS-PIPELINE-1 §7.1). The list is head-first by + recency; any prior entry with the same ``skill_id`` is evicted.""" + self._promote_handler(self.active_handlers, skill_id, activated_at) + + def remove_active_handler(self, skill_id: str): + """Remove ``skill_id`` from ``active_handlers`` (e.g. a STOP-1 + drain).""" + self.active_handlers[:] = [h for h in self.active_handlers + if h.get("skill_id") != skill_id] + + # converse_handlers (OVOS-CONVERSE-1 §2.1 / §3.1) + def add_converse_handler(self, skill_id: str, + activated_at: Optional[float] = None): + """Stamp a handler onto ``converse_handlers``: dedup-promote to + head, then tail-drop at the §2.1 cap (OVOS-CONVERSE-1 §3.1).""" + self._promote_handler(self.converse_handlers, skill_id, activated_at) + self._cap_handlers(self.converse_handlers) + + def remove_converse_handler(self, skill_id: str): + """Remove ``skill_id`` from ``converse_handlers``.""" + self.converse_handlers[:] = [h for h in self.converse_handlers + if h.get("skill_id") != skill_id] + + def prune_converse_handlers(self, ttl: float, + now: Optional[float] = None): + """Drop ``converse_handlers`` entries older than ``ttl`` seconds + (OVOS-CONVERSE-1 §3.2). + + ``now - activated_at > ttl`` is dropped. A non-positive ``ttl`` + disables time-based pruning. The caller (the orchestrator) + invokes this at the pre-converse and pre-list-emission + boundaries.""" + if not ttl or ttl <= 0: + return + now = now if now is not None else time.time() + self.converse_handlers[:] = [ + h for h in self.converse_handlers + if now - h.get("activated_at", now) <= ttl + ] + + # response_mode (OVOS-CONVERSE-1 §2.2) — single-holder + def set_response_mode(self, skill_id: str, expires_at: float): + """Set the single-holder response window (OVOS-CONVERSE-1 §2.2). + + Overwrites any existing holder silently (single-holder + invariant).""" + self.response_mode = {"skill_id": skill_id, "expires_at": expires_at} + + def clear_response_mode(self, skill_id: Optional[str] = None): + """Clear the response window. + + When ``skill_id`` is given, clears it only if that skill + currently holds the window (a skill MUST NOT clear another's + hold); otherwise clears unconditionally.""" + if self.response_mode is None: + return + if skill_id is None or self.response_mode.get("skill_id") == skill_id: + self.response_mode = None + + # --- §2 / §5 wire shape ------------------------------------------------- + + def to_dict(self) -> Dict[str, Any]: + """Render the session as a JSON-friendly dict per §2 + §5. + + Fields whose Python value is ``None`` / empty are **omitted**, + never emitted as ``null`` (§2.1). An empty list-valued override + is wire-equivalent to omission and is dropped (§3.4). ``extras`` + are merged in last and round-trip unchanged (§2.4).""" + out: Dict[str, Any] = {} + + # §3.1 / §3.2 / §3.3 owned scalars + if self.session_id is not None: + out["session_id"] = self.session_id + if self.lang is not None: + out["lang"] = self.lang + if self.secondary_langs: + out["secondary_langs"] = list(self.secondary_langs) + if self.output_lang is not None: + out["output_lang"] = self.output_lang + if self.stt_lang is not None: + out["stt_lang"] = self.stt_lang + if self.request_lang is not None: + out["request_lang"] = self.request_lang + if self.detected_lang is not None: + out["detected_lang"] = self.detected_lang + if self.site_id is not None: + out["site_id"] = self.site_id + + # other-spec list/object override fields (omit-when-empty, §3.4) + for name in _LIST_OVERRIDE_FIELDS: + value = getattr(self, name) + if value: + out[name] = list(value) + for name in _OBJECT_OVERRIDE_FIELDS: + value = getattr(self, name) + if value: + out[name] = deepcopy(value) + + # PIPELINE-1 §7.1 / CONVERSE-1 §2.1 / §2.2 — omit-when-empty + if self.active_handlers: + out["active_handlers"] = deepcopy(self.active_handlers) + if self.converse_handlers: + out["converse_handlers"] = deepcopy(self.converse_handlers) + if self.response_mode: + out["response_mode"] = dict(self.response_mode) + + # §2.4 unknown-field tolerance — passthrough. + for k, v in self.extras.items(): + if k in out: + # An owned field set both ways is a programming error; + # owned wins (single source of truth). + continue + out[k] = deepcopy(v) + return out + + def serialize(self) -> str: + """Emit a JSON string per §5 (no NaN, UTF-8, single object).""" + return json.dumps(self.to_dict(), ensure_ascii=False, + allow_nan=False) + + # --- construction from the wire ----------------------------------------- + + @classmethod + def from_dict(cls, payload: Optional[Dict[str, Any]]) -> "Session": + """Construct a Session from a JSON-decoded dict per §2. + + Tolerant of the §2.1 deferral surface: an explicit ``null`` on + any registered field is logged and treated as omitted, not as a + rejection. Unknown fields are preserved verbatim in + :attr:`extras` (§2.4). Passing ``None`` (no ``session`` key in + ``context``) yields the same well-formed empty session as ``{}`` + (§2.1).""" + if payload is None: + return cls() + if not isinstance(payload, dict): + raise MalformedSession( + "session must be a JSON object (§2 / §5)") + + kwargs: Dict[str, Any] = {} + extras: Dict[str, Any] = {} + for key, value in payload.items(): + if key not in SESSION1_REGISTERED_FIELDS: + extras[key] = value + continue + if value is None: + # §2.1: explicit null is malformed — log, treat as omitted. + _log.warning( + "OVOS-SESSION-1 §2.1: explicit null on `%s` is " + "malformed; treating as omitted", key) + continue + kwargs[key] = value + if extras: + kwargs["extras"] = extras + return cls(**kwargs) + + @classmethod + def deserialize(cls, + payload: Union[str, bytes, bytearray, + Dict[str, Any], None]) -> "Session": + """Parse a serialized session per §5 and §2. + + Accepts bytes/str JSON, an already-parsed dict, or ``None``. + Raises :class:`MalformedSession` only for the structural + failures §5 calls hard errors (unparsable JSON, non-object + root). Field-level malformedness (an explicit ``null``) is + absorbed by :meth:`from_dict`.""" + if payload is None: + return cls() + if isinstance(payload, (bytes, bytearray)): + payload = payload.decode("utf-8") + if isinstance(payload, str): + try: + payload = json.loads(payload) + except json.JSONDecodeError as exc: + raise MalformedSession( + f"session payload is not valid JSON: {exc}") from exc + return cls.from_dict(payload) + + # --- §4 propagation ----------------------------------------------------- + + def propagate(self) -> "Session": + """Return a deep copy suitable for attaching to a derived + Message (§4). Every field — known and unknown — rides along + unchanged.""" + twin = Session.from_dict(self.to_dict()) + twin.converse_handlers_cap = self.converse_handlers_cap + return twin + + @classmethod + def materialize_default(cls) -> "Session": + """Materialize the default session per §4.1. + + Sets ``session_id`` to ``"default"`` and leaves every other + field omitted; the §4.1 rule forbids populating per-component + overrides on a materialized default.""" + return cls(session_id=DEFAULT_SESSION_ID) + + # --- value semantics ---------------------------------------------------- + + def __eq__(self, other: object) -> bool: + return (isinstance(other, Session) + and self.to_dict() == other.to_dict()) + + def __repr__(self) -> str: + return f"Session({self.to_dict()!r})" diff --git a/test/test_session.py b/test/test_session.py new file mode 100644 index 0000000..27d99a7 --- /dev/null +++ b/test/test_session.py @@ -0,0 +1,446 @@ +"""OVOS-SESSION-1 conformance tests for :class:`ovos_spec_tools.Session`. + +Every section heading below cites the spec section under test. The +handler-list sections additionally cite OVOS-PIPELINE-1 §7.1 and +OVOS-CONVERSE-1 §2.1 / §2.2 / §3, whose fields SESSION-1 §3 registers.""" +import unittest + +from ovos_spec_tools import ( + DEFAULT_CONVERSE_HANDLERS_CAP, DEFAULT_SESSION_ID, MalformedSession, + SESSION1_OWNED_FIELDS, SESSION1_REGISTERED_FIELDS, Session) + + +# --- §2 wire shape --------------------------------------------------------- + +class TestWireShape(unittest.TestCase): + def test_empty_session_is_wellformed(self): + s = Session() + self.assertEqual(s.to_dict(), {}) + self.assertEqual(s.serialize(), "{}") + + def test_only_session_id_is_wellformed(self): + s = Session(session_id="abc") + self.assertEqual(s.to_dict(), {"session_id": "abc"}) + + def test_omitted_field_never_serializes_as_null(self): + s = Session(session_id="abc", lang=None) + self.assertNotIn("lang", s.to_dict()) + self.assertNotIn("null", s.serialize()) + + def test_explicit_null_on_wire_is_logged_and_treated_as_omitted(self): + with self.assertLogs("ovos_spec_tools.session", level="WARNING"): + s = Session.from_dict({"session_id": "abc", "lang": None}) + self.assertIsNone(s.lang) + self.assertEqual(s.session_id, "abc") + + def test_explicit_null_on_registered_other_spec_field_treated_as_omitted(self): + # §2.1 — null on a field claimed by another spec is also omitted. + with self.assertLogs("ovos_spec_tools.session", level="WARNING"): + s = Session.from_dict({"session_id": "abc", + "active_handlers": None, + "response_mode": None}) + self.assertEqual(s.active_handlers, []) + self.assertIsNone(s.response_mode) + self.assertNotIn("active_handlers", s.to_dict()) + self.assertNotIn("response_mode", s.to_dict()) + + def test_unknown_field_passes_through(self): + # §2.4 unknown-field tolerance + §4 propagation + s = Session.from_dict({"session_id": "abc", + "novel_future_field": 42}) + self.assertEqual(s.extras["novel_future_field"], 42) + self.assertEqual(s.to_dict()["novel_future_field"], 42) + + def test_registered_other_spec_field_is_first_class(self): + # §3 — `pipeline` is registered (owner OVOS-PIPELINE-1), so it + # lands as a first-class attribute, not in `extras`. + s = Session.from_dict({"pipeline": ["padatious_high"]}) + self.assertEqual(s.pipeline, ["padatious_high"]) + self.assertNotIn("pipeline", s.extras) + self.assertEqual(s.to_dict()["pipeline"], ["padatious_high"]) + + def test_non_dict_payload_is_malformed(self): + with self.assertRaises(MalformedSession): + Session.from_dict(["not", "an", "object"]) # type: ignore[arg-type] + + def test_unparsable_json_is_malformed(self): + with self.assertRaises(MalformedSession): + Session.deserialize("{not valid") + + def test_none_payload_yields_empty_session(self): + # §2.1 — absent session ≡ empty object + self.assertEqual(Session.deserialize(None), Session()) + self.assertEqual(Session.from_dict(None), Session()) + + +# --- §3.1 session_id semantics -------------------------------------------- + +class TestSessionId(unittest.TestCase): + def test_omitted_resolves_to_default(self): + # §2.1 + §3.1 — three forms map to the same identity + self.assertEqual(Session().resolved_session_id(), DEFAULT_SESSION_ID) + self.assertEqual( + Session.from_dict({}).resolved_session_id(), DEFAULT_SESSION_ID) + self.assertEqual( + Session(session_id=DEFAULT_SESSION_ID).resolved_session_id(), + DEFAULT_SESSION_ID) + + def test_is_default_predicate(self): + self.assertTrue(Session().is_default) + self.assertTrue(Session(session_id=DEFAULT_SESSION_ID).is_default) + self.assertFalse(Session(session_id="remote-42").is_default) + + def test_empty_session_id_rejected(self): + with self.assertRaises(MalformedSession): + Session(session_id="") + + def test_non_string_session_id_rejected(self): + with self.assertRaises(MalformedSession): + Session(session_id=123) # type: ignore[arg-type] + + +# --- §3.2 language signals ------------------------------------------------- + +class TestLanguageSignals(unittest.TestCase): + def test_all_six_lang_fields_round_trip(self): + s = Session( + lang="en-US", secondary_langs=["es-ES", "fr-FR"], + output_lang="de-DE", stt_lang="en-GB", + request_lang="en-US", detected_lang="fr-FR") + self.assertEqual(Session.from_dict(s.to_dict()), s) + + def test_secondary_langs_must_not_contain_lang(self): + # §3.2.2 + with self.assertRaises(MalformedSession): + Session(lang="en-US", secondary_langs=["en-US", "fr-FR"]) + + def test_secondary_langs_no_duplicates(self): + with self.assertRaises(MalformedSession): + Session(secondary_langs=["fr-FR", "fr-FR"]) + + def test_secondary_langs_rejects_empty_string(self): + with self.assertRaises(MalformedSession): + Session(secondary_langs=[""]) + + def test_lang_must_be_string(self): + with self.assertRaises(MalformedSession): + Session(lang=42) # type: ignore[arg-type] + + +# --- §3.3 site_id ---------------------------------------------------------- + +class TestSiteId(unittest.TestCase): + def test_site_id_round_trip(self): + s = Session(site_id="kitchen") + self.assertEqual(s.to_dict(), {"site_id": "kitchen"}) + self.assertEqual(Session.from_dict(s.to_dict()), s) + + def test_empty_site_id_rejected(self): + with self.assertRaises(MalformedSession): + Session(site_id="") + + def test_unknown_site_id_carries_no_meaning(self): + # §3.3 — "unknown" is a normal opaque value, not reserved. + s = Session(site_id="unknown") + self.assertEqual(s.to_dict(), {"site_id": "unknown"}) + + +# --- §3 / §3.4 other-spec override fields ---------------------------------- + +class TestOverrideFields(unittest.TestCase): + def test_full_section_3_field_set_round_trips(self): + payload = { + "session_id": "abc", + "lang": "en-US", + "pipeline": ["stop_high", "converse", "adapt_high"], + "intent_context": {"frame_stack": [], "timeout": 120}, + "blacklisted_skills": ["skill-a"], + "blacklisted_intents": ["skill-b:foo"], + "blacklisted_pipelines": ["fallback_low"], + "audio_transformers": ["a"], + "utterance_transformers": ["b"], + "metadata_transformers": ["c"], + "intent_transformers": ["d"], + "dialog_transformers": ["e"], + "tts_transformers": ["f"], + "blacklisted_audio_transformers": ["ba"], + "blacklisted_utterance_transformers": ["bb"], + "blacklisted_metadata_transformers": ["bc"], + "blacklisted_intent_transformers": ["bd"], + "blacklisted_dialog_transformers": ["be"], + "blacklisted_tts_transformers": ["bf"], + } + s = Session.from_dict(payload) + self.assertEqual(s.to_dict(), payload) + self.assertEqual(Session.deserialize(s.serialize()), s) + + def test_empty_list_override_is_wire_equivalent_to_omission(self): + # §3.4 — an empty array on a list override is dropped. + s = Session(pipeline=[], blacklisted_skills=[], + audio_transformers=[]) + self.assertEqual(s.to_dict(), {}) + self.assertEqual(s.pipeline, None) + + def test_intent_context_object_round_trips(self): + s = Session(intent_context={"frame_stack": [["x", 1.0]]}) + self.assertEqual(s.to_dict()["intent_context"], + {"frame_stack": [["x", 1.0]]}) + + def test_context_key_is_not_registered_passes_through_as_extra(self): + # The spec field is `intent_context`, not `context`; a stray + # `context` key is unknown and rides in `extras` (§2.4). + s = Session.from_dict({"context": {"k": "v"}}) + self.assertEqual(s.extras["context"], {"k": "v"}) + self.assertEqual(s.to_dict()["context"], {"k": "v"}) + + +# --- OVOS-PIPELINE-1 §7.1 active_handlers ---------------------------------- + +class TestActiveHandlers(unittest.TestCase): + def test_add_active_handler_head_first(self): + s = Session() + s.add_active_handler("a", activated_at=1.0) + s.add_active_handler("b", activated_at=2.0) + self.assertEqual([h["skill_id"] for h in s.active_handlers], + ["b", "a"]) + + def test_add_active_handler_dedup_promotes_to_head(self): + # §7.1 — re-activation evicts the prior entry and re-inserts at head. + s = Session() + s.add_active_handler("a", activated_at=1.0) + s.add_active_handler("b", activated_at=2.0) + s.add_active_handler("a", activated_at=3.0) + self.assertEqual([h["skill_id"] for h in s.active_handlers], + ["a", "b"]) + self.assertEqual(s.active_handlers[0]["activated_at"], 3.0) + # no duplicate entry for "a" + self.assertEqual(len([h for h in s.active_handlers + if h["skill_id"] == "a"]), 1) + + def test_remove_active_handler(self): + s = Session() + s.add_active_handler("a") + s.add_active_handler("b") + s.remove_active_handler("a") + self.assertEqual([h["skill_id"] for h in s.active_handlers], ["b"]) + + def test_active_predicate(self): + s = Session() + self.assertFalse(s.active) + s.add_active_handler("a") + self.assertTrue(s.active) + + def test_active_handlers_round_trip(self): + s = Session() + s.add_active_handler("a", activated_at=1.0) + self.assertEqual(Session.from_dict(s.to_dict()), s) + + def test_legacy_pair_shape_coerced_on_deserialize(self): + # tolerant input — the legacy [skill_id, ts] pair shape coerces + # into the spec object shape. + s = Session.from_dict({"active_handlers": [["a", 1.0], ["b", 2.0]]}) + self.assertEqual(s.active_handlers, + [{"skill_id": "a", "activated_at": 1.0}, + {"skill_id": "b", "activated_at": 2.0}]) + + +# --- OVOS-CONVERSE-1 §2.1 converse_handlers -------------------------------- + +class TestConverseHandlers(unittest.TestCase): + def test_default_cap_is_64(self): + self.assertEqual(DEFAULT_CONVERSE_HANDLERS_CAP, 64) + self.assertEqual(Session().converse_handlers_cap, 64) + + def test_add_converse_handler_head_first_dedup(self): + s = Session() + s.add_converse_handler("a", activated_at=1.0) + s.add_converse_handler("b", activated_at=2.0) + s.add_converse_handler("a", activated_at=3.0) + self.assertEqual([h["skill_id"] for h in s.converse_handlers], + ["a", "b"]) + + def test_cap_evicts_tail(self): + # §2.1 — when the cap would be exceeded, drop the least-recent tail. + s = Session(converse_handlers_cap=3) + for i in range(5): + s.add_converse_handler(f"s{i}", activated_at=float(i)) + self.assertEqual([h["skill_id"] for h in s.converse_handlers], + ["s4", "s3", "s2"]) + self.assertEqual(len(s.converse_handlers), 3) + + def test_cap_applied_on_construction(self): + seed = [{"skill_id": f"s{i}", "activated_at": float(i)} + for i in range(10)] + s = Session(converse_handlers=seed, converse_handlers_cap=4) + self.assertEqual(len(s.converse_handlers), 4) + self.assertEqual(s.converse_handlers[0]["skill_id"], "s0") + + def test_cap_unbounded_when_non_positive(self): + s = Session(converse_handlers_cap=0) + for i in range(100): + s.add_converse_handler(f"s{i}", activated_at=float(i)) + self.assertEqual(len(s.converse_handlers), 100) + + def test_prune_drops_stale_entries(self): + # §3.2 — entries older than ttl are pruned at now - activated_at > ttl. + s = Session() + s.add_converse_handler("fresh", activated_at=100.0) + s.add_converse_handler("stale", activated_at=10.0) + s.prune_converse_handlers(ttl=50, now=100.0) + self.assertEqual([h["skill_id"] for h in s.converse_handlers], + ["fresh"]) + + def test_prune_noop_when_ttl_non_positive(self): + s = Session() + s.add_converse_handler("a", activated_at=1.0) + s.prune_converse_handlers(ttl=0, now=1e9) + self.assertEqual(len(s.converse_handlers), 1) + + def test_remove_converse_handler(self): + s = Session() + s.add_converse_handler("a") + s.add_converse_handler("b") + s.remove_converse_handler("a") + self.assertEqual([h["skill_id"] for h in s.converse_handlers], ["b"]) + + def test_converse_handlers_round_trip(self): + s = Session() + s.add_converse_handler("a", activated_at=1.0) + self.assertEqual(Session.from_dict(s.to_dict()), s) + + +# --- OVOS-CONVERSE-1 §2.2 response_mode ------------------------------------ + +class TestResponseMode(unittest.TestCase): + def test_absent_by_default(self): + s = Session() + self.assertIsNone(s.response_mode) + self.assertNotIn("response_mode", s.to_dict()) + + def test_set_response_mode_shape(self): + s = Session() + s.set_response_mode("skill-a", expires_at=123.0) + self.assertEqual(s.response_mode, + {"skill_id": "skill-a", "expires_at": 123.0}) + + def test_single_holder_overwrite(self): + # §2.2 — setting while another holds overwrites silently. + s = Session() + s.set_response_mode("skill-a", expires_at=1.0) + s.set_response_mode("skill-b", expires_at=2.0) + self.assertEqual(s.response_mode["skill_id"], "skill-b") + + def test_clear_unconditional(self): + s = Session() + s.set_response_mode("skill-a", expires_at=1.0) + s.clear_response_mode() + self.assertIsNone(s.response_mode) + + def test_clear_only_own_holder(self): + # a skill MUST NOT clear another's hold. + s = Session() + s.set_response_mode("skill-a", expires_at=1.0) + s.clear_response_mode("skill-b") + self.assertEqual(s.response_mode["skill_id"], "skill-a") + s.clear_response_mode("skill-a") + self.assertIsNone(s.response_mode) + + def test_malformed_response_mode_resolves_to_none(self): + # §2.1 — a holder with no skill_id is not a valid window. + self.assertIsNone(Session(response_mode={}).response_mode) + self.assertIsNone( + Session.from_dict({"response_mode": {"expires_at": 1}}).response_mode) + + def test_response_mode_round_trip(self): + s = Session() + s.set_response_mode("skill-a", expires_at=99.0) + self.assertEqual(Session.from_dict(s.to_dict()), s) + + +# --- §4 propagation -------------------------------------------------------- + +class TestPropagation(unittest.TestCase): + def test_propagate_is_deep_copy(self): + s = Session(session_id="abc", pipeline=["padatious_high"]) + copy = s.propagate() + self.assertEqual(copy, s) + copy.pipeline.append("adapt_high") + self.assertEqual(s.pipeline, ["padatious_high"]) + + def test_propagate_carries_handlers_and_cap(self): + s = Session(converse_handlers_cap=7) + s.add_active_handler("a", activated_at=1.0) + s.add_converse_handler("b", activated_at=2.0) + s.set_response_mode("c", expires_at=3.0) + copy = s.propagate() + self.assertEqual(copy, s) + self.assertEqual(copy.converse_handlers_cap, 7) + self.assertEqual(copy.active_handlers, s.active_handlers) + # deep copy — mutating the twin does not touch the source + copy.active_handlers[0]["activated_at"] = 999.0 + self.assertEqual(s.active_handlers[0]["activated_at"], 1.0) + + def test_unknown_field_survives_propagation(self): + s = Session.from_dict({"session_id": "abc", + "novel_future_field": [1, 2, 3]}) + self.assertEqual(s.propagate().to_dict()["novel_future_field"], + [1, 2, 3]) + + def test_materialize_default_sets_only_session_id(self): + # §4.1 + s = Session.materialize_default() + self.assertEqual(s.to_dict(), {"session_id": DEFAULT_SESSION_ID}) + + +# --- §5 serialization ------------------------------------------------------ + +class TestSerialization(unittest.TestCase): + def test_serialize_round_trip_via_json_str(self): + s = Session(session_id="abc", lang="en-US", + pipeline=["padatious_high"]) + wire = s.serialize() + self.assertEqual(Session.deserialize(wire), s) + + def test_serialize_rejects_nan(self): + # §5 + OVOS-MSG-1 §6 — numbers MUST be finite + s = Session(extras={"weird": float("nan")}) + with self.assertRaises(ValueError): + s.serialize() + + def test_session_id_not_distinguished_from_other_strings(self): + # §3.1 — "default" is a normal session value, structurally + self.assertEqual(Session(session_id=DEFAULT_SESSION_ID).to_dict(), + {"session_id": "default"}) + + def test_no_null_anywhere_on_a_fully_populated_session(self): + # §2.1 — nothing serializes as null even when many fields set. + s = Session(session_id="abc", lang="en-US", site_id="kitchen", + pipeline=["adapt_high"]) + s.add_active_handler("a", activated_at=1.0) + s.set_response_mode("a", expires_at=2.0) + self.assertNotIn("null", s.serialize()) + + +# --- §6 conformance, registry -------------------------------------------- + +class TestRegistry(unittest.TestCase): + def test_owned_fields_registry(self): + # The fields whose semantics SESSION-1 itself owns: §3.1 session_id, + # §3.2 the six language signals, §3.3 site_id. + self.assertEqual(SESSION1_OWNED_FIELDS, frozenset({ + "session_id", "site_id", "lang", "secondary_langs", "output_lang", + "stt_lang", "request_lang", "detected_lang", + })) + + def test_registered_fields_superset_of_owned(self): + self.assertTrue(SESSION1_OWNED_FIELDS.issubset( + SESSION1_REGISTERED_FIELDS)) + # the full §3 roster includes the other-spec registered fields + for f in ("pipeline", "intent_context", "active_handlers", + "converse_handlers", "response_mode", + "blacklisted_skills", "audio_transformers"): + self.assertIn(f, SESSION1_REGISTERED_FIELDS) + + +if __name__ == "__main__": + unittest.main() From 280fca6a58c25a7e091d50c82fdc324cb11df689 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 16:05:56 +0000 Subject: [PATCH 064/110] Increment Version to 0.12.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index a2e979e..83f0692 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 11 +VERSION_MINOR = 12 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From b0b2ab1303a315c6b6e3d3d156294010216238e6 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 16:06:15 +0000 Subject: [PATCH 065/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff65511..e779df1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.12.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.12.0a1) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.11.0a1...0.12.0a1) + +**Merged pull requests:** + +- feat: OVOS-SESSION-1 canonical Session reference implementation [\#16](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/16) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.11.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.11.0a1) (2026-06-26) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.10.0a1...0.11.0a1) From 403c48e6132d9b9f3861e3dc9de48ed3bbaeeaea Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 17:17:01 +0100 Subject: [PATCH 066/110] docs: spec traceability for the locale/template/lint domain (#33) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deepen docstrings, type hints and inline comments across the locale, template, dialog, prompt, language-matching and lint modules so every non-obvious decision cites the OVOS spec clause it implements. Docs, typing and comments only — no runtime behaviour change. - expansion.py (OVOS-INTENT-1): cite §4.1 step 1 / §3.7 on inline_keywords; flag its max_values cap and lenient unknown-ref handling as non-spec. - resources.py (OVOS-INTENT-2): Args/Returns + § citations on the common reader, locale-dir discovery, keyword_form and match helpers; add the Iterator return hint to iter_locale_dirs; note the §4.4 HTML-comment gap on read_prompt_file. - dialog.py / prompt.py (§4.2 / §4.4): cite the three substitution conditions and the fence heuristic; document the unimplemented §4.4 author-comment stripping as a known conformance gap, not a silent workaround. - language.py: module docstring now traces the BCP-47 rules to INTENT-2 §2 (case-insensitive) and §2.2 (fallback, threshold 10) and the SESSION-1 lang family; document the dialect-fallback ladder precisely. - lint.py: clause map in the module docstring; per-finding § citations in messages and comments; document the .intent union-slot relaxation that reconciles INTENT-1 §5.5 with INTENT-3 §5.3 (kept a warning by design). - docs: spec-coverage headers per page; worked §4.1 expansion example; dialect-fallback table; union-slot-set section; §4.4 prompt-comment gap. Co-authored-by: Claude Opus 4.8 --- docs/dialog.md | 24 ++++-- docs/language-matching.md | 28 +++++++ docs/linting.md | 32 ++++++- docs/locale-resources.md | 34 ++++++-- docs/templates.md | 41 ++++++++- ovos_spec_tools/expansion.py | 26 ++++-- ovos_spec_tools/language.py | 107 ++++++++++++++++++++++-- ovos_spec_tools/lint.py | 156 +++++++++++++++++++++++++++++------ ovos_spec_tools/prompt.py | 54 ++++++++++-- ovos_spec_tools/resources.py | 125 ++++++++++++++++++++++++---- 10 files changed, 548 insertions(+), 79 deletions(-) diff --git a/docs/dialog.md b/docs/dialog.md index b61af7a..ae3bae3 100644 --- a/docs/dialog.md +++ b/docs/dialog.md @@ -1,5 +1,13 @@ # 4. Dialog +> **Spec coverage.** This chapter is the reference for the **Dialog renderer** +> conformance role of OVOS-INTENT-1 §7 over the `.dialog` format of +> OVOS-INTENT-2 §4.2 (`dialog.py`), and for the `.prompt` format of +> OVOS-INTENT-2 §4.4 (`prompt.py`, at the end of the chapter). A dialog renderer +> MUST embed a conformant expander, verify slot-set consistency (§5.5), fill +> every `{name}` from caller-supplied values, and MUST NOT emit an unfilled slot +> (§5.1) — `UnfilledSlot` is that last MUST. + A `.dialog` file holds the phrases an assistant may speak for one response. **Rendering** a dialog means: pick one phrase, expand its `(a|b)` / `[x]` variety down to a single variant, and fill its `{name}` slots with values. This @@ -114,15 +122,21 @@ render_prompt("You are {role}. Answer: {query}", {"role": "concise"}) # 'You are concise. Answer: {query}' ``` -A `{name}` is replaced **only** when it is a well-formed name, the caller -supplied a value, and it is not inside a ``` ``` ``` fenced code block. -Everything else stays literal: +A `{name}` is replaced **only** when all three OVOS-INTENT-2 §4.4 conditions +hold: it is a well-formed name (§4.4 condition 1), the caller supplied a value +(condition 2), and it is not inside a ``` ``` ``` fenced code block +(condition 3). Everything else stays literal: -- an **unfilled** slot is left as `{name}` — slots are optional, the opposite - of `.dialog`, where every slot must be filled; +- an **unfilled** slot is left as `{name}` — §4.4 "slots are optional", the + deliberate opposite of `.dialog`, where every slot must be filled; - any other `{`/`}` — `{}`, `{ }`, JSON like `{"k": 1}` — is untouched; - a `{name}` inside a fenced code block is never substituted. +> **Known gap.** OVOS-INTENT-2 §4.4 also requires author-only HTML comments +> `` to be **stripped** before a prompt reaches a model (a MUST). +> `render_prompt()` does **not** yet do this — a comment passes through +> verbatim. Do not rely on comment removal until it is implemented. + `PromptRenderer` is the resource-backed, multilingual form — built from a `LocaleResources`, with the language given per call and optional default slots: diff --git a/docs/language-matching.md b/docs/language-matching.md index 8e93950..e62496d 100644 --- a/docs/language-matching.md +++ b/docs/language-matching.md @@ -1,5 +1,14 @@ # 5. Language matching +> **Spec coverage.** This chapter is the reference for the cross-spec BCP-47 +> language-matching rules (`language.py`). The case-insensitive tag comparison +> comes from **OVOS-INTENT-2 §2**; the nearest-language fallback and its +> "distance below 10 is a usable regional match" threshold come from +> **OVOS-INTENT-2 §2.2** (non-normative); the same rules decide whether a +> **SESSION-1** session language is served by an available resource, voice, or +> model. Everything here is a *reference policy*, not a conformance obligation — +> §2.2 is explicitly an implementation choice. + OVOS constantly needs to answer "the assistant was asked for language X — which of the languages I actually have is the best fit?" — for locale folders, TTS voices, STT models. The logic was reimplemented in several places @@ -90,6 +99,25 @@ This is exactly what `LocaleResources` uses for its smart language fallback ([chapter 3](locale-resources.md)) — and you can use it directly anywhere else the same question comes up. +### Dialect-fallback semantics, precisely + +A request resolves to a candidate **iff** their distance is below the threshold +(an exact match, distance `0`, always resolves). The distance ladder, nearest +to farthest: + +| Relationship | Distance | Resolves (default threshold 10)? | +|--------------|----------|----------------------------------| +| identical tag (case aside) | `0` | yes | +| bare tag vs its norm region (`pt` / `pt-PT`) | `0` | yes | +| two regions of one language (`en-US` / `en-GB`) | small (`<10`) | yes | +| a different primary language (`en` / `fr`) | large (`≥100` coarse) | no | + +So `en-AU` falls back to `en-US` over `de-DE`, `pt` prefers `pt-PT` over +`pt-BR`, and `zz-ZZ` resolves to nothing. The fallback is **dialect-only** by +design: it never crosses primary languages, because serving French wording to a +request for English is worse than failing (OVOS-INTENT-2 §2.2's caution that +"cross-region substitution can produce wording a user would not expect"). + ## Next [Linting](linting.md) — checking a whole locale folder at once. diff --git a/docs/linting.md b/docs/linting.md index 8e4d82c..1b8a6a5 100644 --- a/docs/linting.md +++ b/docs/linting.md @@ -1,5 +1,13 @@ # 6. Linting +> **Spec coverage.** This chapter is the reference for the locale linter +> (`lint.py`). Every finding enforces a specific clause — template syntax from +> **OVOS-INTENT-1 §3.6**, naming/layout/empty-file/uniqueness from +> **OVOS-INTENT-2 §2/§4.3/§5**, slot-set consistency from **OVOS-INTENT-1 §5.5** +> (relaxed to *union slot sets* for `.intent`, reconciling §5.5 with the +> multi-slot template-intent example of **OVOS-INTENT-3 §5.3**). Each message +> names the clause it failed, so a red lint points straight at the violated MUST. + `ovos-spec-lint` checks a skill's locale folder against both specs at once — the **syntax** of every template (OVOS-INTENT-1) and the **naming and layout** of every file (OVOS-INTENT-2) — and reports *every* problem rather than @@ -34,9 +42,10 @@ straight into a CI pipeline. With `--strict`, warnings fail the run too. - an empty file (no templates after comments and blank lines); - a file that is not valid UTF-8; - a named slot inside a slot-free role (`.entity` / `.voc` / `.blacklist`); -- templates within one `.intent` or `.dialog` declaring **different slot - sets** — every template of one definition must use the same `{slots}` - (OVOS-INTENT-1 §5.5); +- templates within one **`.dialog`** declaring **different slot sets** — every + phrase of one dialog must use the same `{slots}` so the caller fills the same + values whichever is chosen (OVOS-INTENT-1 §5.5). For **`.intent`** this is + only a *warning*, not an error: union slot sets are valid there (see below); - a base name outside the allowed charset (lowercase letters, digits, underscores); - an `.entity` whose base name — which names a slot — begins with a digit; @@ -56,6 +65,23 @@ straight into a CI pipeline. With `--strict`, warnings fail the run too. A `.prompt` is checked too, but not as a template — it is plain text, so only its naming and non-emptiness are checked, never template syntax. +### Union slot sets for `.intent` + +OVOS-INTENT-1 §5.5 says every template of one definition must declare the +identical slot set. The linter enforces that strictly for `.dialog` (an +**error**) but relaxes it for `.intent` to a **warning**, because +OVOS-INTENT-3 §5.3 presents a single template intent whose phrasings declare +*different* slots: + +``` +(play|put on) {query} +(play|put on) {query} (on|using) {engine} +``` + +Here `{engine}` is an optional capture — present in some phrasings, absent in +others. Treating the extra slots as optional makes this one valid intent, so +the linter warns ("verify this is intentional") rather than rejecting it. + ## Targeting an older spec version A skill may need to run on a device that has not been updated. `--spec-version` diff --git a/docs/locale-resources.md b/docs/locale-resources.md index 2c83ce7..100e2fa 100644 --- a/docs/locale-resources.md +++ b/docs/locale-resources.md @@ -1,5 +1,13 @@ # 3. Locale resources +> **Spec coverage.** This chapter is the reference for **OVOS-INTENT-2 — Locale +> Resource Formats** (`resources.py`). `LocaleResources` is the *conformant +> loader* of OVOS-INTENT-2 §5: it discovers languages (§5.1 / §2), locates a +> file under the override precedence (§5.2 / §2.1), applies the common reader +> (§5.3 / §3), applies the per-format rule (§5.4 / §4), and rejects an empty +> file (§5.5 / §5). Template expansion is delegated to the OVOS-INTENT-1 +> Expander ([chapter 2](templates.md)). + A skill ships its localized text as plain-text files under a `locale/` folder. This chapter covers what those files are and how `LocaleResources` loads them. It is the implementation of OVOS-INTENT-2. @@ -90,13 +98,20 @@ prompt renderer to fill ([chapter 4](dialog.md)). `` references inside an `.intent` resolve automatically against the `.voc` files of the same language — you do not pass vocabularies yourself. -A missing resource raises `FileNotFoundError`. A malformed one — an empty file, -a duplicate `(role, base name)` in one language tree, a slot in a slot-free -role — raises `MalformedResource`. +A missing resource raises `FileNotFoundError`. A malformed one raises +`MalformedResource`, each case tied to a spec MUST: + +- an **empty file** — OVOS-INTENT-2 §5 ("every file MUST contribute at least + one template"); +- a **duplicate `(role, base name)`** in one language tree — §2 ("MUST NOT share + a base name anywhere within one language directory tree"); +- a **named slot in a slot-free role** — §4.3 (`.entity`/`.voc`/`.blacklist` + are the slot-free format). ## Override precedence -A resource may come from three places, highest priority first: +A resource may come from three places (OVOS-INTENT-2 §2.1), highest priority +first; **first match wins**, and an override replaces the whole lower file: 1. **user** — per-user overrides, under a path the assistant decides; 2. **skill** — the files bundled with the skill; @@ -116,10 +131,13 @@ takes it as a parameter and imports no configuration of its own. ## Smart language fallback -When a skill has no directory for the requested language, `LocaleResources` -resolves to the **nearest available** language instead — a request for `en-AU` -loads `en-US`. This is re-evaluated on every call, so the same instance can -serve an exact language and a fallback one side by side. +OVOS-INTENT-2 §2.2 is explicit that fallback is **non-normative**: a loader +*SHOULD* prefer an exact match and *MAY* fall back to the nearest available +language. `LocaleResources` implements that suggested fallback. When a skill has +no directory for the requested language, it resolves to the **nearest +available** language instead — a request for `en-AU` loads `en-US`. This is +re-evaluated on every call, so the same instance can serve an exact language and +a fallback one side by side. ```python res.load_intent("play", "en-AU") # finds en-US/ if there is no en-AU/ diff --git a/docs/templates.md b/docs/templates.md index e7f041e..feac7fc 100644 --- a/docs/templates.md +++ b/docs/templates.md @@ -1,5 +1,10 @@ # 2. Sentence templates +> **Spec coverage.** This chapter is the reference for **OVOS-INTENT-1 — the +> Sentence Template Grammar** (`expansion.py`). `expand()` is the *Expander* +> conformance role of OVOS-INTENT-1 §7: it implements the token set of §3, the +> enumeration algorithm of §4.1, and the malformed-form rejection of §3.6. + A **sentence template** is a compact string that stands for a set of sentences. It is the grammar of OVOS-INTENT-1, and it is the foundation of this package: resource files are lists of templates, and the dialog renderer @@ -77,8 +82,8 @@ itself never does. ### Vocabulary references — `` -`` pulls in a named **vocabulary** — a reusable set of phrasings — and -expands it in place. Pass the vocabularies as a dict: +`` (OVOS-INTENT-1 §3.7) pulls in a named **vocabulary** — a reusable set +of phrasings — and expands it in place. Pass the vocabularies as a dict: ```python expand(" [there]", {"greeting": ["hello", "hi", "hey"]}) @@ -87,7 +92,37 @@ expand(" [there]", {"greeting": ["hello", "hi", "hey"]}) This is how you avoid repeating the same `(hello|hi|hey)` group across many templates: define it once as a vocabulary, reference it as ``. A -vocabulary may itself contain `` references; they resolve recursively. +vocabulary may itself contain `` references; they resolve recursively +(§4.1 step 1). A reference must resolve to a **slot-free** set — a vocabulary +may not introduce a `{slot}` — and a single-member vocabulary substitutes its +bare member rather than a one-branch group. + +## How expansion works — the §4.1 algorithm + +`expand()` follows OVOS-INTENT-1 §4.1 exactly, in order: + +1. **Resolve `` references** to alternative groups (recursively). +2. **Rewrite `[x]` as `(x|)`** — an optional is sugar for an empty branch. +3. **Cartesian product** of the innermost `(...)` groups, repeated until no + parenthesis remains. +4. **Normalize whitespace** — collapse runs of spaces, strip the ends (this is + what removes the double space an empty branch leaves behind). +5. **De-duplicate**, preserving first-seen order. + +Worked through on `(turn|switch) [the] (light|fan)` — three 2-branch groups +once `[the]` becomes `(the|)`, so `2×2×2 = 8` combinations collapse (after +whitespace normalization) to the 8-sentence sample set of §4.2: + +```python +expand("(turn|switch) [the] (light|fan)") +# ['turn the light', 'switch the light', 'turn light', 'switch light', +# 'turn the fan', 'switch the fan', 'turn fan', 'switch fan'] +``` + +The **sample set is a set** — §4 defines its membership, not an ordering — so +the eight sentences are exactly those of §4.2; the sequence above is just the +order `expand()` happens to emit them in. Throughout, `{name}` slots are +**opaque** — carried through and never enumerated (§4.1 final note). ## The input model diff --git a/ovos_spec_tools/expansion.py b/ovos_spec_tools/expansion.py index 806974c..ee9ff69 100644 --- a/ovos_spec_tools/expansion.py +++ b/ovos_spec_tools/expansion.py @@ -236,11 +236,21 @@ def inline_keywords( ) -> str: """Inline ```` references as ``(v1|v2|…)`` alternation groups. - Engines like Padatious don't look up ``.voc`` files at runtime — they - need keywords baked into the template body as standard ``(a|b|c)`` - alternations. This utility replaces every ```` reference - with its values in alternation syntax, handling nested references - recursively. + Partial application of OVOS-INTENT-1 §4.1 step 1 (resolve ```` + references) that stops **before** the rest of expansion: §3.7 / §4.1 define + a ```` reference as equivalent to the alternative group of the + vocabulary's members written in its place, and this returns exactly that + rewritten *template* rather than the fully enumerated sample set. It exists + because engines like Padatious do not look up ``.voc`` files at runtime — + they need keywords baked into the template body as standard ``(a|b|c)`` + alternations, then expand the result themselves. + + Unlike :func:`expand`, this is intentionally lenient and **not** a + conformant expander: it does no §3.6 validation, leaves an unknown keyword's + angle brackets stripped as literal text rather than raising + :class:`MalformedTemplate` for an undefined reference, and applies the + non-spec ``max_values`` cap below to bound the §4.3 sample-set blow-up. + Feed its output to :func:`expand` for the validated sample set. Parameters ---------- @@ -250,7 +260,11 @@ def inline_keywords( Flat ``{keyword: [values]}`` mapping. If ``None`` or empty the template is returned unchanged. max_values - Cap the number of values per keyword inlined. Default 10. + Cap the number of values per keyword inlined. Default 10. This is an + engineering limit, **not** a spec rule: OVOS-INTENT-1 §4.3 warns that a + sample set grows as the product of branch counts, and §4.3 permits an + expander to bound it; capping members here keeps a downstream engine's + expansion finite without that engine having to re-implement the limit. Returns ------- diff --git a/ovos_spec_tools/language.py b/ovos_spec_tools/language.py index c78701c..73358e8 100644 --- a/ovos_spec_tools/language.py +++ b/ovos_spec_tools/language.py @@ -1,18 +1,51 @@ """Language-tag utilities — standardization, distance, and closest match. +This module is the cross-spec implementation of OVOS's BCP-47 language-matching +and locale-resolution rules. It is referenced by — but not owned by — a single +spec; the rules it encodes are drawn from: + +- **OVOS-INTENT-2 §2** — locale directories are named with BCP-47 tags and tag + comparison is **case-insensitive** (``en-us`` and ``en-US`` denote the same + language). :func:`standardize_lang` is the canonicalization that makes that + comparison total, and :func:`lang_distance` returns ``0`` for tags that differ + only in case. +- **OVOS-INTENT-2 §2.2** — the *language fallback* suggestion: a loader MAY fall + back to the nearest available language, and the spec names ``langcodes``' + ``tag_distance()`` with a "distance below 10 is a usable regional match" + threshold. :data:`DEFAULT_MAX_LANGUAGE_DISTANCE` is exactly that ``10``; + :func:`closest_lang` is exactly that "nearest available" selection. §2.2 is + explicitly **non-normative** ("an implementation choice, not a requirement"), + so everything here is a reference policy, not a conformance obligation. +- the **SESSION-1 language family** — a session carries a ``lang`` tag; the same + matching rules decide whether a session's language is served by an available + resource/voice/model. + OVOS resolves "the closest available language for a request" in many places — locale resources, TTS voices, STT models — and the logic has been reimplemented repeatedly (``ovos_utils.lang.get_language_dir``, ``phoonnx.match_lang``, …) with subtle drift between the copies. This module is -intended to be the **single implementation**. +intended to be the **single implementation** behind all of them. It is built on one distance function, :func:`lang_distance`; :func:`closest_lang` is simply "the candidate with the smallest distance". All the policy — tag standardization, the norm-region preference, the behaviour when ``langcodes`` is absent — lives inside :func:`lang_distance`, not in branchy callers. -``langcodes`` is an optional dependency. Without it, :func:`lang_distance` -falls back to a coarse same-language / different-language measure. +**Dialect-fallback semantics**, precisely. A request resolves to a candidate iff +their :func:`lang_distance` is below the threshold (an exact match — distance +``0`` — always resolves). The distance ladder is, from nearest to farthest: +identical tag (``0``) < a bare tag vs its own norm region (``0``, see +:func:`_with_norm_region`) < two regions of one language (a small regional +distance) < the bare/generic form vs a region of the same language < a different +primary language (``>= 100`` with the coarse measure — never a usable match). +The asymmetry that matters: a bare ``pt`` is measured **from ``pt-PT``**, not +from the most-populous ``pt-BR``, so ``pt`` falls back to ``pt-PT`` before +``pt-BR`` — correcting ``langcodes``' population default (see :data:`_NORM_REGION`). + +``langcodes`` is an optional dependency (OVOS-INTENT-2 §2.2 names it). Without +it, :func:`lang_distance` falls back to a coarse same-language / +different-language measure that still honours the same ordering for the cases +locale resolution actually depends on. """ from __future__ import annotations @@ -45,6 +78,21 @@ def standardize_lang(tag: str) -> str: """Normalize a BCP-47 language tag for comparison. + Implements the canonicalization presupposed by OVOS-INTENT-2 §2's + "tag comparison is case-insensitive" rule: it folds underscores to hyphens, + lowercases the primary subtag, uppercases the region, and (via ``langcodes``) + resolves canonical script/region forms — so that two on-disk spellings of + one language (``en_us``, ``en-US``) reduce to a single comparable string. + + Args: + tag: a BCP-47-ish language tag, possibly underscore-separated or + mixed-case (as locale directory names and config values often are). + + Returns: + The standardized tag. With ``langcodes`` installed this is + ``langcodes.standardize_tag``'s output; without it, a primary-lowercase, + region-uppercase normalization. + Uses ``langcodes.standardize_tag`` when available — handling underscores, case, and script/region forms — and falls back to a simple normalization otherwise. @@ -108,11 +156,26 @@ def _coarse_distance(desired: str, supported: str) -> int: def lang_distance(desired: str, supported: str) -> int: """The distance between two BCP-47 language tags. + The numeric backbone of the OVOS-INTENT-2 §2.2 fallback: §2.2 names + ``langcodes``' ``tag_distance()`` and its "below 10 is a usable regional + match" reading, which this function adopts (with the norm-region correction + below) so every OVOS component ranks dialects identically. + ``0`` is identical; a larger number is further apart; a value of 10 or more is not a usable match. Both tags are standardized, and a bare tag is measured **from its norm region** — so ``lang_distance("pt", "pt-PT")`` is ``0`` while ``lang_distance("pt", "pt-BR")`` is a regional difference, - correcting ``langcodes``' population-based default. + correcting ``langcodes``' population-based default (see :data:`_NORM_REGION`). + + Args: + desired: the requested BCP-47 tag. + supported: the candidate BCP-47 tag to measure against. + + Returns: + A non-negative distance: ``0`` for identical (after standardization and + norm-region expansion), a small value for a regional difference, and a + large value (``>= 100`` under the coarse measure) for a different + primary language. Uses ``langcodes.tag_distance`` when available, and a coarse same-language measure otherwise. @@ -135,8 +198,19 @@ def lang_matches(a: str, b: str, Convenience wrapper around :func:`lang_distance` for the common ``if score < threshold`` check that cross-component boundaries (intent engines, TTS/STT plugin routing, locale lookup) reimplement by hand. The - default threshold matches :data:`DEFAULT_MAX_LANGUAGE_DISTANCE` — an exact - match always matches; pass ``max_distance=0`` to require an exact match. + default threshold matches :data:`DEFAULT_MAX_LANGUAGE_DISTANCE` — the + OVOS-INTENT-2 §2.2 "distance below 10 is a usable regional match" line. + + Args: + a: one BCP-47 tag. + b: the other BCP-47 tag. The relation is symmetric. + max_distance: the exclusive upper bound on an acceptable distance. An + exact match (distance ``0``) always matches regardless; pass + ``max_distance=0`` to require an exact match (OVOS-INTENT-2 §2.2's + "SHOULD prefer an exact match", with fallback disabled). + + Returns: + ``True`` iff the tags are identical or within ``max_distance``. """ distance = lang_distance(a, b) if distance == 0: @@ -149,13 +223,30 @@ def closest_lang(target: str, available: Sequence[str], ) -> Optional[str]: """Return the entry of ``available`` closest to ``target``. + This is the reference implementation of the OVOS-INTENT-2 §2.2 language + fallback ("a loader MAY fall back to the nearest available language"). §2.2 + is non-normative, so this selection is the package's recommended policy, not + a conformance requirement; a loader is free to disable it (``max_distance=0``, + yielding §2.2's "SHOULD prefer an exact match" with no fallback). + The candidate with the smallest :func:`lang_distance` wins. An exact match always resolves; any other match resolves only if its distance is **below** ``max_distance`` (so ``max_distance=0`` accepts exact matches only). ``None`` is returned when nothing qualifies. - The value returned is the original string from ``available``, so a caller - can map it back to a directory, a voice, a model, and so on. + Args: + target: the requested BCP-47 tag (e.g. a session/skill language). + available: the tags actually on hand — locale directory names, voice + ids, model languages. Iterated once; ties resolve to the first + smallest-distance candidate in iteration order. + max_distance: exclusive upper bound on an acceptable non-exact match + (default :data:`DEFAULT_MAX_LANGUAGE_DISTANCE` == §2.2's ``10``). + + Returns: + The original string from ``available`` that best matches ``target``, or + ``None`` when nothing qualifies. The value is returned **verbatim** (not + standardized) so a caller can map it straight back to a directory, a + voice, a model, and so on. """ best: Optional[str] = None best_distance: Optional[int] = None diff --git a/ovos_spec_tools/lint.py b/ovos_spec_tools/lint.py index 7f310f9..68d6b65 100644 --- a/ovos_spec_tools/lint.py +++ b/ovos_spec_tools/lint.py @@ -2,7 +2,33 @@ Validates the **syntax** (OVOS-INTENT-1) and the **naming and layout** (OVOS-INTENT-2) of every resource file under a locale directory, and reports -every problem found rather than stopping at the first. +every problem found rather than stopping at the first. Each finding is phrased +to point at the exact spec clause it enforces, so a failing lint tells the +author which MUST they violated. + +Clause map (which spec rule each rule enforces): + +- *empty file* → OVOS-INTENT-2 §5 step 5 — "Reject an empty file … every file + MUST contribute at least one template" (and for ``.prompt``, content). +- *duplicate (role, base name)* → OVOS-INTENT-2 §2 — "Two files with the same + extension MUST NOT share a base name anywhere within one language directory + tree". +- *base-name / ``.entity`` charset* → OVOS-INTENT-2 §2 + OVOS-INTENT-1 §3.4 — + base names are lowercase letters/digits/underscores; an ``.entity`` name also + obeys the slot-name rule (not beginning with a digit). +- *not-a-BCP-47 directory* → OVOS-INTENT-2 §2 — "Language directories are named + with BCP-47 language tags". +- *legacy extension / unknown role* → OVOS-INTENT-2 §1 + §5 — only the six + defined roles exist; a loader "MUST NOT introduce additional resource file + roles". Legacy Mycroft types are flagged, not parsed. +- *named slot in a slot-free role* → OVOS-INTENT-2 §4.3 — ``.entity`` / ``.voc`` + / ``.blacklist`` are slot-free. +- *template syntax* → OVOS-INTENT-1 §3.6 (delegated to + :func:`~ovos_spec_tools.expansion.expand`). +- *slot-set consistency* → OVOS-INTENT-1 §5.5 (see :func:`_lint_file` for the + ``.intent`` union-slot relaxation reconciling §5.5 with OVOS-INTENT-3 §5.3). +- *blacklist with no matching ``.intent``* → OVOS-INTENT-2 §4.3 — a + ``.blacklist`` "is paired by base name with exactly one ``.intent``". Exposed as the ``ovos-spec-lint`` command:: @@ -44,7 +70,13 @@ # File types OVOS-INTENT-2 deliberately does not define — flagged, not parsed. LEGACY_EXTENSIONS = (".rx", ".value", ".list", ".word", ".template", ".qml") -# The OVOS spec version that introduced each feature. +# The OVOS spec version that introduced each feature. This ladder is a *tooling* +# concept, not a clause in any single spec: it lets a skill target an older +# deployment by flagging roles/tokens that a runtime predating their +# introduction will silently ignore (a forward-compatibility lint, not a +# conformance check). Mapping: V1 = the formalized OVOS-INTENT-1/2 (adds the +# `.blacklist` role over the legacy V0 set), V2 = the `` inline vocabulary +# reference (OVOS-INTENT-1 §3.7), V3 = the `.prompt` role (OVOS-INTENT-2 §4.4). DEFAULT_SPEC_VERSION = 3 _BLACKLIST_SINCE = 1 # the `.blacklist` role _VOCABULARY_REFERENCE_SINCE = 2 # the `` inline vocabulary reference @@ -63,7 +95,19 @@ @dataclass class Finding: - """One problem found by the linter.""" + """One problem found by the linter. + + Severity is :data:`ERROR` for a spec **MUST** violation (a malformed + template, a duplicate ``(role, base name)``, an empty file, a slot in a + slot-free role) and :data:`WARNING` for a **SHOULD**/advisory issue (a + legacy file type, a non-BCP-47 directory name, an unpaired ``.blacklist``). + Only errors fail a non-``--strict`` run. + + Attributes: + severity: :data:`ERROR` or :data:`WARNING`. + path: the offending file or directory, as a string. + message: a human-readable description, citing the spec clause violated. + """ severity: str # ERROR or WARNING path: str @@ -76,12 +120,21 @@ def __str__(self) -> str: def lint_locale(path, spec_version: int = DEFAULT_SPEC_VERSION) -> List[Finding]: """Lint a locale directory, or a single language directory. - Returns every :class:`Finding`, in file order. A path whose name is a - BCP-47 tag is treated as a single language tree; otherwise it is treated as - a ``locale/`` directory and each language subdirectory is checked. - - ``spec_version`` is the target OVOS spec version: a resource using a - feature newer than it is flagged (see the module docstring). + The argument is dispatched by its name: a directory whose name matches the + BCP-47 shape (OVOS-INTENT-2 §2) is treated as a single ``/`` tree; + anything else is treated as the ``locale/`` root and each immediate + subdirectory is linted as a language tree. This lets the linter accept + either ``locale`` or ``locale/en-US`` as its target. + + Args: + path: a ``locale/`` directory or a single ``/`` directory. + spec_version: the target OVOS spec version; a resource using a feature + newer than it is flagged (see the module docstring's version ladder). + + Returns: + Every :class:`Finding`, in file order — never short-circuiting, so one + run surfaces all problems. A non-directory ``path`` yields a single + :data:`ERROR` finding. """ root = Path(path) if not root.is_dir(): @@ -93,13 +146,16 @@ def lint_locale(path, spec_version: int = DEFAULT_SPEC_VERSION) -> List[Finding] findings: List[Finding] = [] for child in sorted(root.iterdir()): if child.is_file() and child.suffix in ROLE_EXTENSIONS: + # §2: resources live under locale//, never loose at the root. findings.append(Finding( WARNING, str(child), - "resource file is not inside a language directory")) + "resource file is not inside a language directory " + "(OVOS-INTENT-2 §2)")) language_dirs = [c for c in sorted(root.iterdir()) if c.is_dir()] if not language_dirs: findings.append(Finding( - WARNING, str(root), "no language directories found")) + WARNING, str(root), + "no language directories found (OVOS-INTENT-2 §2)")) for language_dir in language_dirs: findings.extend(_lint_language_tree(language_dir, spec_version)) return findings @@ -109,10 +165,11 @@ def _lint_language_tree(language_dir: Path, spec_version: int) -> List[Finding]: """Lint one ``/`` directory and all its subdirectories.""" findings: List[Finding] = [] if not _LANG_TAG_RE.fullmatch(language_dir.name): + # §2: "Language directories are named with BCP-47 language tags." findings.append(Finding( WARNING, str(language_dir), f"directory name {language_dir.name!r} is not a BCP-47 " - f"language tag")) + f"language tag (OVOS-INTENT-2 §2)")) role_files: List[Path] = [] for path in sorted(language_dir.rglob("*")): @@ -121,15 +178,19 @@ def _lint_language_tree(language_dir: Path, spec_version: int) -> List[Finding]: if path.suffix in ROLE_EXTENSIONS: role_files.append(path) elif path.suffix in LEGACY_EXTENSIONS: + # §1 defines exactly six roles; §5 forbids a loader inventing more. + # Legacy Mycroft types are flagged (not parsed) so an author knows + # the file will be silently ignored by a conformant loader. findings.append(Finding( WARNING, str(path), f"{path.suffix} is a legacy file type, not an " - f"OVOS-INTENT-2 resource role")) + f"OVOS-INTENT-2 resource role (OVOS-INTENT-2 §1)")) if not role_files: findings.append(Finding( WARNING, str(language_dir), - "language directory contains no resource files")) + "language directory contains no resource files " + "(OVOS-INTENT-2 §2)")) # Duplicate (role, base name) within one language tree (OVOS-INTENT-2 §2). first_seen: Dict[tuple, Path] = {} @@ -139,7 +200,8 @@ def _lint_language_tree(language_dir: Path, spec_version: int) -> List[Finding]: findings.append(Finding( ERROR, str(path), f"duplicate {path.suffix} resource {path.stem!r} — also at " - f"{first_seen[key]}")) + f"{first_seen[key]} (OVOS-INTENT-2 §2: a (role, base name) " + f"must be unique per language tree)")) else: first_seen[key] = path @@ -154,10 +216,12 @@ def _lint_language_tree(language_dir: Path, spec_version: int) -> List[Finding]: f"the {path.suffix} role requires spec version {since}; a " f"version-{spec_version} runtime ignores it")) if path.suffix == ".blacklist" and path.stem not in intent_names: + # §4.3: a .blacklist "is paired by base name with exactly one + # .intent" whose match it suppresses; an unpaired one is inert. findings.append(Finding( WARNING, str(path), f"blacklist {path.stem!r} has no matching " - f"{path.stem}.intent to suppress")) + f"{path.stem}.intent to suppress (OVOS-INTENT-2 §4.3)")) # Vocabularies, for resolving references during expansion. vocabularies: Dict[str, List[str]] = {} @@ -176,7 +240,23 @@ def _lint_language_tree(language_dir: Path, spec_version: int) -> List[Finding]: def _lint_file(path: Path, vocabularies: Dict[str, Sequence[str]], spec_version: int) -> List[Finding]: - """Lint one resource file: naming, then the syntax of every template.""" + """Lint one resource file: naming, then the syntax of every template. + + Order mirrors the loader steps of OVOS-INTENT-2 §5: naming (§2), then the + common reader (§3), then the per-format rule (§4) — for templated roles via + a conformant expander (OVOS-INTENT-1), for ``.prompt`` as a whole-file + non-empty check (§4.4). + + Args: + path: the resource file to lint. + vocabularies: every ``.voc`` in this language tree, so a ```` + reference (OVOS-INTENT-1 §3.7) resolves during expansion instead of + being mis-reported as undefined. + spec_version: the target spec version, for the feature gates. + + Returns: + Every :class:`Finding` for this file. + """ findings: List[Finding] = [] extension = path.suffix base_name = path.stem @@ -186,15 +266,21 @@ def _lint_file(path: Path, findings.append(Finding( ERROR, str(path), f"base name {base_name!r} must be lowercase ASCII letters, " - f"digits and underscores only")) + f"digits and underscores only (OVOS-INTENT-2 §2)")) if extension == ".entity" and not _SLOT_NAME_RE.fullmatch(base_name): + # §2: where a base name names a slot (an .entity names its {slot}), it + # additionally obeys the slot-name rule of OVOS-INTENT-1 §3.4. findings.append(Finding( ERROR, str(path), f".entity base name {base_name!r} names a slot and must not " - f"begin with a digit")) + f"begin with a digit (OVOS-INTENT-2 §2 / OVOS-INTENT-1 §3.4)")) if path.name != path.name.lower(): + # §2: base names and extensions are lowercase. Reported as a warning + # (not an error) because the casefold is recoverable on case-insensitive + # filesystems; on case-sensitive ones it will fail lookup. findings.append(Finding( - WARNING, str(path), "file name should be lowercase")) + WARNING, str(path), + "file name should be lowercase (OVOS-INTENT-2 §2)")) # --- `.prompt` — a whole-file document, not a template list (§4.4) ------ if extension == PROMPT_ROLE: @@ -242,15 +328,29 @@ def _lint_file(path: Path, ERROR, str(path), f"{exc} [in: {template!r}]")) continue if slot_free and any("{" in sample for sample in samples): + # §4.3: .entity/.voc/.blacklist are the slot-free format — no {name}. findings.append(Finding( ERROR, str(path), f"{extension} is slot-free but a template contains a named " - f"slot [in: {template!r}]")) + f"slot (OVOS-INTENT-2 §4.3) [in: {template!r}]")) if slot_bearing: slot_sets.append(frozenset(_SLOT_RE.findall(template))) # --- slot consistency (OVOS-INTENT-1 §5.5) ------------------------------ - # .dialog requires identical slot sets; .intent allows union slot sets. + # §5.5 (and OVOS-INTENT-3 §5.1) require every template of one definition to + # declare the identical slot set, and a tool "MUST reject" one that does + # not. For a `.dialog` that is enforced as an ERROR: the caller fills the + # same slots whichever phrase is chosen (§4.2), so a divergent slot set is a + # genuine fault. + # + # For an `.intent` the rule is *relaxed to union slot sets* (WARNING, not + # ERROR) to reconcile §5.5 with the worked example in OVOS-INTENT-3 §5.3, + # which presents `(play|put on) {query}` and + # `(play|put on) {query} (on|using) {engine}` as ONE valid template intent + # despite their differing slot sets ({query} vs {query},{engine}). Treating + # the extra slots as optional captures is the documented intent there, so + # the linter warns rather than rejects. Changing this from a warning to an + # error would regress that valid pattern — hence it stays a WARNING. if len(set(slot_sets)) > 1: if extension == ".dialog": findings.append(Finding( @@ -267,7 +367,17 @@ def _lint_file(path: Path, def main(argv: Optional[Sequence[str]] = None) -> int: - """Entry point for the ``ovos-spec-lint`` command.""" + """Entry point for the ``ovos-spec-lint`` command. + + Lints the target, prints every :class:`Finding`, then a count summary. + + Args: + argv: command-line arguments (defaults to ``sys.argv[1:]``). + + Returns: + ``0`` on success; ``1`` if any error was found, or — with ``--strict`` + — if any warning was found (so a CI step can gate on locale health). + """ parser = argparse.ArgumentParser( prog="ovos-spec-lint", description="Validate OVOS locale resource files against " diff --git a/ovos_spec_tools/prompt.py b/ovos_spec_tools/prompt.py index 2a16727..140ff6f 100644 --- a/ovos_spec_tools/prompt.py +++ b/ovos_spec_tools/prompt.py @@ -8,14 +8,31 @@ code and JSON, and rendering it must never corrupt text the author did not write as a slot. -A ``{name}`` is replaced by a caller-supplied value only when all three hold: +A ``{name}`` is replaced by a caller-supplied value only when all three hold +(OVOS-INTENT-2 §4.4, the three numbered substitution conditions): 1. it is a well-formed slot name — lowercase ASCII letters, digits and underscores, not beginning with a digit (so ``{}``, ``{ }`` and JSON such - as ``{"key": 1}`` are left untouched); + as ``{"key": 1}`` are left untouched). The charset is INTENT-2 §4.4's + "lowercase ASCII letters, digits, and underscores … MUST NOT begin with a + digit", identical to the slot-name rule of OVOS-INTENT-1 §3.4; 2. the caller supplied a value for that name — an **unfilled** slot is left as - literal text, not an error (the opposite of ``.dialog``, §4.2); -3. it does not lie inside a ```` ``` ```` fenced code block. + literal text, not an error (the deliberate opposite of ``.dialog``, where + §4.2/OVOS-INTENT-1 §5.1 require **every** slot be filled before TTS); +3. it does not lie inside a ```` ``` ```` fenced code block — §4.4 condition 3. + Fence detection here is the "simpler heuristic (counting triple backticks)" + §4.4 explicitly permits: a line whose first non-whitespace content is three + or more backticks toggles the fence, and an unterminated fence extends to + end-of-file. §4.4 marks nested/indented fences as implementation-defined. + +.. note:: + **Known conformance gap — author-only comments (OVOS-INTENT-2 §4.4).** §4.4 + requires that an HTML-style comment ```` be **stripped** before + the prompt reaches a language model, and that an unterminated ```` is passed + through verbatim. Until that is implemented, authors must not rely on + comments being removed. This is documented, not silently worked around. Two interfaces are provided: @@ -35,7 +52,21 @@ def _is_fence(line: str) -> bool: - """Whether a line opens or closes a ```` ``` ```` fenced code block.""" + """Whether a line opens or closes a ```` ``` ```` fenced code block. + + Implements the OVOS-INTENT-2 §4.4 fence test in its permitted simplified + form: a line whose first non-whitespace content is three backticks toggles + the fenced-block state. ``lstrip(" \\t")`` mirrors §4.4's "first + non-whitespace content"; only the opening delimiter is checked (info + strings and longer fences are treated the same), which §4.4 allows as a + "simpler heuristic" so long as well-formed prompts render identically. + + Args: + line: one line of the prompt, with or without its trailing newline. + + Returns: + ``True`` if this line is a fence delimiter, else ``False``. + """ return line.lstrip(" \t").startswith("```") @@ -48,12 +79,19 @@ def render_prompt(text: str, docstring — conservatively, and leaving an unfilled slot as literal text. Args: - text: the whole-file content of a ``.prompt``. + text: the whole-file content of a ``.prompt`` (§4.4 "the whole file, + verbatim, is one prompt"). slots: caller-supplied values, keyed by slot name. Values are - converted to text. Names not present here are left as ``{name}``. + converted to text via ``str()``. A name absent here, or any + malformed ``{…}`` (per §4.4 condition 1), is left as literal + ``{name}`` text — §4.4's "slots are optional". Returns: - The prompt with its supplied slots substituted, otherwise verbatim. + The prompt with its supplied slots substituted, otherwise verbatim — + byte-for-byte unchanged outside the substituted ``{name}`` points + (``splitlines(keepends=True)`` preserves the original line endings, and + text inside a fenced block is reproduced untouched). An empty ``text`` + returns ``""``. """ values = slots or {} diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index e1575b4..19efa80 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -70,10 +70,23 @@ class MalformedResource(ValueError): def read_resource_file(path: Path) -> List[str]: """Apply the OVOS-INTENT-2 §3 common reader to one file. - The file is read as UTF-8, a leading byte-order mark is discarded, and both - ``LF`` and ``CRLF`` line endings are accepted. Each line is stripped; blank - lines and ``#``-comment lines are dropped. The surviving lines — each one - template — are returned in order. + The file is read as UTF-8 (§3: "the file is UTF-8 … a reader that encounters + [a BOM] MUST discard it"), a leading byte-order mark is discarded, and both + ``LF`` and ``CRLF`` line endings are accepted (§3: "a reader MUST accept + both"). Each line is stripped; blank lines and ``#``-comment lines are + dropped (§3: "a blank line is skipped … a line whose first character is + ``#`` is a comment"). There are no inline comments — a ``#`` mid-line is + literal — so only a line *beginning* with ``#`` is dropped. The surviving + lines — each one template (OVOS-INTENT-1) — are returned in order. + + Args: + path: the resource file to read (a line-oriented role, not ``.prompt``; + see :func:`read_prompt_file` for the whole-file role). + + Returns: + The template lines, in file order, with blanks and comments removed. An + all-blank/all-comment file yields ``[]`` — the caller treats that as the + §5 "empty file" fault. """ text = path.read_text(encoding="utf-8-sig") # utf-8-sig discards a BOM templates: List[str] = [] @@ -88,9 +101,13 @@ def read_resource_file(path: Path) -> List[str]: def iter_locale_dirs(root: Path, native_langs: Optional[Sequence[str]] = None, max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE - ): + ) -> Iterator[Tuple[str, Path]]: """Iterate ``/locale//`` subdirs as ``(lang, path)`` pairs. + Implements the "discover languages" loader step (OVOS-INTENT-2 §5 step 1, + §2 layout): each immediate subdirectory of ``/locale/`` is one + language tree, named with a BCP-47 tag. + Each immediate subdirectory of ``/locale/`` is treated as a locale tree; its name is normalized with :func:`standardize_lang` and yielded as the first item of the pair. The second item is the directory ``Path``. @@ -105,7 +122,16 @@ def iter_locale_dirs(root: Path, this and the disagreement goes away — locales are always discovered as full-tag directories, ``closest_lang`` reconciles at query time. - ``root`` without a ``locale/`` child yields nothing. + Args: + root: the skill root containing a ``locale/`` directory. + native_langs: if given, only locales whose closest native is within + ``max_distance`` are yielded (§2.2 nearness, applied as a filter). + max_distance: the §2.2 distance threshold for that filter. + + Yields: + ``(standardized_lang, dir_path)`` for each accepted locale directory, + in sorted directory order. A non-tag or unparseable subdir name is + skipped. ``root`` without a ``locale/`` child yields nothing. """ root = Path(root) locales_root = root / "locale" @@ -181,9 +207,23 @@ def keyword_form(template_line: str, A ``.voc`` line like ``(hi|hello|hey)`` becomes one keyword whose entity value is ``"hi"`` and whose aliases are ``["hello", "hey"]`` — any consumer that distinguishes a canonical form from synonyms can use this - grouping directly (OVOS-INTENT-2 §4.3). - - An empty or whitespace-only line yields ``("", [])``. + grouping directly (OVOS-INTENT-2 §4.3 slot-free roles). + + The canonical/alias split is a tooling convention layered *on top of* the + spec's unordered sample set (OVOS-INTENT-1 §4): the spec defines no + "canonical" member, so "first after case-fold + sort" is chosen here purely + to be deterministic. + + Args: + template_line: one slot-free template line (a ``.voc``/``.entity`` line). + vocabularies: vocabularies for any ```` reference in the line + (OVOS-INTENT-1 §3.7). + + Returns: + ``(entity, aliases)`` — the canonical form and its synonyms. An empty or + whitespace-only line, or a line that fails to expand, yields + ``("", [])`` (a malformed line is dropped, not raised, so one bad line + does not poison a batch keyword load). """ if not template_line.strip(): return "", [] @@ -215,6 +255,20 @@ def normalize_for_match(text: str, *, Set either flag to ``False`` for languages where the distinction is semantic (e.g. French ``ou``/``où``). + + This is a *match-time* normalization the resource consumers apply, distinct + from the upstream ASR normalization OVOS-INTENT-1 §2 presumes; it exists + because real utterances and authored ``.voc`` lines drift in accent and + punctuation. ``{``/``}`` are preserved so a slot marker survives a + pre-render pass. + + Args: + text: the string to normalize. + strip_diacritics: fold combining marks via NFD decomposition. + strip_punct: drop ASCII punctuation except ``{`` and ``}``. + + Returns: + The normalized, lowercased, whitespace-trimmed string. """ import unicodedata text = text.strip().lower() @@ -246,7 +300,22 @@ def utterance_contains(utterance: str, samples: Sequence[str], ``strip_diacritics`` and ``strip_punct`` flags are independent — forward each one as required by the target language. - An empty utterance or empty sample set returns ``False``. + The default whole-word (non-``exact``) mode implements the + OVOS-INTENT-2 §4.3 occurrence rule used for ``.voc``/``.blacklist`` testing: + a phrase "occurs" when its words appear "as a contiguous sequence of whole + words … not a raw substring" — which is why ``art`` does not match within + ``start``. + + Args: + utterance: the (ASR-normalized) text to test. + samples: the phrase set to look for, e.g. an expanded ``.voc``. + exact: require equality after normalization rather than substring. + strip_diacritics: forwarded to :func:`normalize_for_match`. + strip_punct: forwarded to :func:`normalize_for_match`. + + Returns: + ``True`` iff any sample matches. An empty utterance or empty sample set + returns ``False``. """ if not utterance or not samples: return False @@ -273,7 +342,18 @@ def strip_samples(utterance: str, samples: Sequence[str]) -> str: Samples are stripped longest first so that a composite match is consumed before any of its shorter constituents (``"give up"`` before ``"up"``). The match is whole-word and case-insensitive; the utterance - is otherwise returned with original casing and punctuation. + is otherwise returned with original casing and punctuation. The whole-word + anchoring is the same OVOS-INTENT-2 §4.3 "contiguous whole words" rule as + :func:`utterance_contains`. + + Args: + utterance: the text to strip from. + samples: phrases to remove (e.g. an expanded ``.voc`` of filler words). + + Returns: + ``utterance`` with every whole-word sample occurrence removed; double + spaces left behind are **not** collapsed (the caller normalizes if it + needs to). """ import re for s in sorted({s for s in samples if s and s.strip()}, @@ -290,11 +370,26 @@ def read_prompt_file(path: Path) -> str: """Read a ``.prompt`` whole and verbatim (OVOS-INTENT-2 §3, §4.4). A ``.prompt`` is **not** line-oriented: it is read whole, with no line - stripping and no blank- or ``#``-comment-line filtering, because every - character is part of the prompt. The file is UTF-8 and a leading - byte-order mark is discarded. + stripping and no blank- or ``#``-comment-line filtering, because §4.4 states + "every character is part of the prompt" (``#`` lines are ordinary prompt + text, unlike in the line-oriented roles of §3). The file is UTF-8 and a + leading byte-order mark is discarded (§3's "a reader … MUST discard" a BOM). + + Args: + path: the resolved ``.prompt`` file. + + Returns: + The file's whole content as a single string, byte-for-byte except the + stripped BOM and Python's universal-newline decoding. + + .. note:: + **Known conformance gap (OVOS-INTENT-2 §4.4).** This reader does **not** + strip the author-only ```` HTML comments §4.4 requires be + removed before the prompt reaches a language model, nor does it report an + unterminated ```` be **stripped** before - the prompt reaches a language model, and that an unterminated ```` is passed - through verbatim. Until that is implemented, authors must not rely on - comments being removed. This is documented, not silently worked around. +literal. The one special construct is the **double-brace** ``{{name}}`` +substitution point, and it is applied **conservatively** — a prompt is +free-form text that routinely embeds code and JSON, and rendering it must +never corrupt text the author did not write as a slot. + +A ``.prompt`` uses the double-brace form **only** (OVOS-INTENT-2 §4.4): a +single ``{name}`` is **literal text** and is never substituted, as are any +lone ``{`` or ``}``. This is the rationale for the double-brace requirement — +prompts are natural-language LLM text that routinely contains literal single +braces (JSON such as ``{"key": 1}``, code), so a single brace must never +trigger substitution. The single-brace ``{name}`` form (OVOS-INTENT-1 §3.4) +remains the slot token of the *template* roles (``.intent``/``.dialog``); it +has no special meaning in a ``.prompt``. + +A ``{{name}}`` is replaced by a caller-supplied value only when all three hold: + +1. it is a well-formed double-brace token ``{{name}}`` whose name is lowercase + ASCII letters, digits and underscores, not beginning with a digit (so + ``{{}}``, ``{{ }}``, a single ``{name}``, and JSON such as ``{"key": 1}`` + are left untouched). The name charset is identical to the slot-name rule of + OVOS-INTENT-1 §3.4; +2. the caller supplied a value for that name — an **unfilled** ``{{name}}`` is + left as literal text, not an error (the deliberate opposite of ``.dialog``, + where §4.2/OVOS-INTENT-1 §5.1 require **every** slot be filled before TTS); +3. it does not lie inside a ```` ``` ```` fenced code block. Fence detection + here is a simple heuristic (counting triple backticks): a line whose first + non-whitespace content is three or more backticks toggles the fence, and an + unterminated fence extends to end-of-file. Nested/indented fences are + implementation-defined. Two interfaces are provided: @@ -47,8 +46,12 @@ __all__ = ["render_prompt", "PromptRenderer"] -# A {name} substitution point — the OVOS-INTENT-2 §4.4 slot-name charset. -_SLOT_RE = re.compile(r"\{([a-z][a-z0-9_]*)\}") +# A {{name}} substitution point — the OVOS-INTENT-2 §4.4 double-brace form, +# the ONLY substitution token in a .prompt. The name uses the §3.4 charset. +# A single-brace {name} is deliberately NOT matched: in a prompt it is literal +# text. The interior of the braces forbids ``{``/``}`` so the token is flat and +# a single ``{name}`` nested inside ``{{ }}`` cannot accidentally extend it. +_SLOT_RE = re.compile(r"\{\{([a-z][a-z0-9_]*)\}\}") def _is_fence(line: str) -> bool: @@ -75,20 +78,23 @@ def render_prompt(text: str, """Render a ``.prompt`` string (stateless). The whole ``text`` is the prompt; every character is significant. - ``{name}`` substitution points are filled per the rules in the module + ``{{name}}`` substitution points are filled per the rules in the module docstring — conservatively, and leaving an unfilled slot as literal text. + A single-brace ``{name}`` and any literal brace are passed through + unchanged. Args: text: the whole-file content of a ``.prompt`` (§4.4 "the whole file, verbatim, is one prompt"). slots: caller-supplied values, keyed by slot name. Values are - converted to text via ``str()``. A name absent here, or any - malformed ``{…}`` (per §4.4 condition 1), is left as literal - ``{name}`` text — §4.4's "slots are optional". + converted to text via ``str()``. A name absent here, any + malformed ``{{…}}`` (per §4.4 condition 1), and every single-brace + ``{name}`` are left as literal text — §4.4's "slots are optional" + and the double-brace-only rule. Returns: The prompt with its supplied slots substituted, otherwise verbatim — - byte-for-byte unchanged outside the substituted ``{name}`` points + byte-for-byte unchanged outside the substituted ``{{name}}`` points (``splitlines(keepends=True)`` preserves the original line endings, and text inside a fenced block is reproduced untouched). An empty ``text`` returns ``""``. @@ -99,7 +105,7 @@ def replace(match: "re.Match") -> str: name = match.group(1) if name in values: return str(values[name]) - return match.group(0) # an unfilled slot stays literal (§4.4) + return match.group(0) # an unfilled {{name}} stays literal (§4.4) rendered: List[str] = [] in_fence = False diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index 19efa80..8d79d66 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -375,19 +375,20 @@ def read_prompt_file(path: Path) -> str: text, unlike in the line-oriented roles of §3). The file is UTF-8 and a leading byte-order mark is discarded (§3's "a reader … MUST discard" a BOM). + Args: + path: the resolved ``.prompt`` file. + + A ``.prompt`` has **no** special syntax beyond ``{{name}}`` substitution + (OVOS-INTENT-2 §4.4): a single ``{name}``, any lone brace, and an + ```` HTML comment are all **literal pass-through text** — they + reach the model unchanged. The whole file is returned verbatim. + Args: path: the resolved ``.prompt`` file. Returns: The file's whole content as a single string, byte-for-byte except the stripped BOM and Python's universal-newline decoding. - - .. note:: - **Known conformance gap (OVOS-INTENT-2 §4.4).** This reader does **not** - strip the author-only ```` HTML comments §4.4 requires be - removed before the prompt reaches a language model, nor does it report an - unterminated ```` is literal text.""" + text = " Body for {{name}}." + assert render_prompt(text, {"name": "Sam"}) == " Body for Sam." + + def test_numeric_slot_value_is_stringified(): - assert render_prompt("count {n}", {"n": 0}) == "count 0" + assert render_prompt("count {{n}}", {"n": 0}) == "count 0" def test_filled_value_is_not_re_scanned_for_slots(): - """A value containing `{...}` is inserted literally, not substituted again.""" - assert render_prompt("say {x}", {"x": "{y}"}) == "say {y}" + """A value containing `{{...}}` is inserted literally, not substituted again.""" + assert render_prompt("say {{x}}", {"x": "{{y}}"}) == "say {{y}}" def test_text_with_no_trailing_newline_is_preserved(): - assert render_prompt("no newline {x}", {"x": "here"}) == "no newline here" + assert render_prompt("no newline {{x}}", {"x": "here"}) == "no newline here" # --- PromptRenderer — stateful, multilingual --------------------------------- def test_renderer_renders_from_resources(tmp_path): - res = _resources(tmp_path, {"en-US/sys.prompt": "You are {role}."}) + res = _resources(tmp_path, {"en-US/sys.prompt": "You are {{role}}."}) renderer = PromptRenderer(res, "sys") assert renderer.render("en-US", {"role": "a helper"}) == "You are a helper." @@ -109,14 +136,14 @@ def test_renderer_serves_multiple_languages(tmp_path): def test_renderer_default_slots_are_reused(tmp_path): - res = _resources(tmp_path, {"en-US/g.prompt": "You are {assistant}."}) + res = _resources(tmp_path, {"en-US/g.prompt": "You are {{assistant}}."}) renderer = PromptRenderer(res, "g", slots={"assistant": "OVOS"}) assert renderer.render("en-US") == "You are OVOS." assert renderer.render("en-US") == "You are OVOS." def test_per_call_slot_overrides_a_default(tmp_path): - res = _resources(tmp_path, {"en-US/g.prompt": "You are {assistant}."}) + res = _resources(tmp_path, {"en-US/g.prompt": "You are {{assistant}}."}) renderer = PromptRenderer(res, "g", slots={"assistant": "OVOS"}) assert renderer.render("en-US", {"assistant": "Mycroft"}) == "You are Mycroft." @@ -131,7 +158,14 @@ def test_renderer_missing_prompt_raises(tmp_path): # --- load_prompt() ----------------------------------------------------------- def test_load_prompt_returns_the_whole_file(tmp_path): - text = "# Heading\n\nBody {slot} text.\n" + text = "# Heading\n\nBody {{slot}} text.\n" + res = _resources(tmp_path, {"en-US/p.prompt": text}) + assert res.load_prompt("p", "en-US") == text + + +def test_load_prompt_passes_single_brace_and_comments_verbatim(tmp_path): + """read_prompt_file/load_prompt does no stripping — all text is literal.""" + text = ' {"json": 1} single {slot} double {{slot}}\n' res = _resources(tmp_path, {"en-US/p.prompt": text}) assert res.load_prompt("p", "en-US") == text From 3238c5e69d8e313c6647419fb2fe04a57724362b Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:32:55 +0000 Subject: [PATCH 076/110] Increment Version to 0.13.0a1 --- ovos_spec_tools/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 6d9c9e2..4e69203 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 12 -VERSION_BUILD = 1 +VERSION_MINOR = 13 +VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From be82af889636ecd9f308d9f85ad9ec3800417aa5 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:33:13 +0000 Subject: [PATCH 077/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 771e840..537cd1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.13.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.13.0a1) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.12.1a1...0.13.0a1) + +**Merged pull requests:** + +- feat: dual-brace template refs \({x} and {{x}}\); .prompt double-brace-only [\#38](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/38) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.12.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.12.1a1) (2026-06-26) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.12.0a3...0.12.1a1) From 881d04b0b9ff65d91409f5d4e21a24c1cfb8b546 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:38:14 +0100 Subject: [PATCH 078/110] feat: add persona_id to canonical Session (OVOS-PERSONA-1 registered field) (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add persona_id to canonical Session (OVOS-PERSONA-1 registered field) Add persona_id as a first-class field on the canonical ovos_spec_tools.Session (OVOS-SESSION-1 reference implementation), mirroring the existing other-spec field pattern. persona_id is registered by OVOS-PERSONA-1 (recognized per the OVOS-SESSION-1 §2.2 field registry), so it joins SESSION1_REGISTERED_FIELDS via a new _SCALAR_OVERRIDE_FIELDS tuple — the registered (not owned) bucket, same as active_handlers/converse_handlers. Carried opaquely: present only when set, omitted (never JSON null) when None per §2.1. Co-Authored-By: Claude Opus 4.8 * Update session.py * refactor: rename _SCALAR_OVERRIDE_FIELDS -> _STRING_OVERRIDE_FIELDS (persona_id is a string; match _LIST_/_OBJECT_ naming) --------- Co-authored-by: Claude Opus 4.8 --- ovos_spec_tools/session.py | 29 ++++++++++++++++++++++++++++- test/test_session.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/ovos_spec_tools/session.py b/ovos_spec_tools/session.py index 71275a7..cd45aa6 100644 --- a/ovos_spec_tools/session.py +++ b/ovos_spec_tools/session.py @@ -104,13 +104,21 @@ #: OVOS-CONVERSE-1 §2.1 / §2.2). _HANDLER_FIELDS = ("active_handlers", "converse_handlers", "response_mode") +#: String fields claimed by other specs. ``persona_id`` is registered by +#: OVOS-PERSONA-1 and recognized here per the OVOS-SESSION-1 §2.2 field +#: registry; OVOS-SESSION-1 carries it opaquely (its semantics are owned +#: by OVOS-PERSONA-1). An empty / unset value is wire-equivalent to +#: omission (§2.1). +_STRING_OVERRIDE_FIELDS = ("persona_id",) + #: The full closed set of fields OVOS-SESSION-1 §3 recognizes in this #: version. A consumer that recognizes any of these interprets it per #: its owner specification; everything else is unknown-field passthrough #: (§2.4) carried opaquely in :attr:`Session.extras`. SESSION1_REGISTERED_FIELDS = frozenset(SESSION1_OWNED_FIELDS).union( - _LIST_OVERRIDE_FIELDS, _OBJECT_OVERRIDE_FIELDS, _HANDLER_FIELDS) + _LIST_OVERRIDE_FIELDS, _OBJECT_OVERRIDE_FIELDS, _HANDLER_FIELDS, + _STRING_OVERRIDE_FIELDS) class MalformedSession(ValueError): @@ -175,6 +183,10 @@ class Session: time via the ``cap`` argument to :meth:`add_converse_handler`. :param response_mode: OVOS-CONVERSE-1 §2.2 pending-response window — a single ``{skill_id, expires_at}`` object, or ``None``. + :param persona_id: OVOS-PERSONA-1 — opaque identifier of the persona + bound to this session. Registered as a session field by + OVOS-PERSONA-1 (recognized here per the OVOS-SESSION-1 §2.2 field + registry); ``None`` ⇒ omitted on the wire (§2.1). :param extras: passthrough mapping for fields claimed by future specifications (anything outside :data:`SESSION1_REGISTERED_FIELDS`). Treated opaquely per §2.4. @@ -209,6 +221,7 @@ def __init__(self, active_handlers: Optional[List[Dict[str, Any]]] = None, converse_handlers: Optional[List[Dict[str, Any]]] = None, response_mode: Optional[Dict[str, Any]] = None, + persona_id: Optional[str] = None, extras: Optional[Dict[str, Any]] = None): if session_id is not None and (not isinstance(session_id, str) or not session_id): @@ -219,6 +232,12 @@ def __init__(self, or not site_id): raise MalformedSession( "site_id must be a non-empty string when set (§3.3)") + if persona_id is not None and (not isinstance(persona_id, str) + or not persona_id): + # OVOS-PERSONA-1 registered field: non-empty string when set. + raise MalformedSession( + "persona_id must be a non-empty string when set " + "(OVOS-PERSONA-1)") for name, value in (("lang", lang), ("output_lang", output_lang), ("stt_lang", stt_lang), ("request_lang", request_lang), @@ -261,6 +280,8 @@ def __init__(self, # --- other-spec list/object override fields (carried opaquely) ------ self.pipeline = list(pipeline) if pipeline else None self.intent_context = dict(intent_context) if intent_context else None + # OVOS-PERSONA-1 registered scalar (carried opaquely) + self.persona_id = persona_id self.blacklisted_skills = self._as_str_list(blacklisted_skills) self.blacklisted_intents = self._as_str_list(blacklisted_intents) self.blacklisted_pipelines = self._as_str_list(blacklisted_pipelines) @@ -515,6 +536,12 @@ def to_dict(self) -> Dict[str, Any]: if value: out[name] = deepcopy(value) + # OVOS-PERSONA-1 registered scalar(s) (omit-when-empty, §2.1) + for name in _STRING_OVERRIDE_FIELDS: + value = getattr(self, name) + if value is not None: + out[name] = value + # PIPELINE-1 §7.1 / CONVERSE-1 §2.1 / §2.2 — omit-when-empty if self.active_handlers: out["active_handlers"] = deepcopy(self.active_handlers) diff --git a/test/test_session.py b/test/test_session.py index e922a15..b0231d5 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -146,6 +146,43 @@ def test_unknown_site_id_carries_no_meaning(self): self.assertEqual(s.to_dict(), {"site_id": "unknown"}) +# --- OVOS-PERSONA-1 registered persona_id ---------------------------------- + +class TestPersonaId(unittest.TestCase): + def test_persona_id_round_trip(self): + s = Session(persona_id="default") + self.assertEqual(s.to_dict(), {"persona_id": "default"}) + self.assertEqual(Session.from_dict(s.to_dict()), s) + + def test_persona_id_omitted_not_null_when_none(self): + # §2.1 omission-not-null: absent persona_id never serializes. + s = Session() + self.assertIsNone(s.persona_id) + self.assertNotIn("persona_id", s.to_dict()) + self.assertNotIn("persona_id", s.serialize()) + + def test_persona_id_is_registered_not_owned(self): + # OVOS-PERSONA-1 registers it; OVOS-SESSION-1 does not own it. + self.assertIn("persona_id", SESSION1_REGISTERED_FIELDS) + self.assertNotIn("persona_id", SESSION1_OWNED_FIELDS) + + def test_persona_id_is_first_class_not_extra(self): + # Registered ⇒ lands as a first-class attribute, not in `extras`. + s = Session.from_dict({"persona_id": "assistant"}) + self.assertEqual(s.persona_id, "assistant") + self.assertNotIn("persona_id", s.extras) + + def test_empty_persona_id_rejected(self): + with self.assertRaises(MalformedSession): + Session(persona_id="") + + def test_explicit_null_persona_id_treated_as_omitted(self): + # §2.1: explicit null on a registered field ⇒ omitted, not error. + s = Session.from_dict({"persona_id": None}) + self.assertIsNone(s.persona_id) + self.assertNotIn("persona_id", s.to_dict()) + + # --- §3 / §3.4 other-spec override fields ---------------------------------- class TestOverrideFields(unittest.TestCase): From 31399d768c4892ef35f3575746a7f0c107d6b6ef Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:38:24 +0000 Subject: [PATCH 079/110] Increment Version to 0.14.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 4e69203..26fffad 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 13 +VERSION_MINOR = 14 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From d1135d335aa6923995d0a32f32fe62a01a6ce0c6 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:38:40 +0000 Subject: [PATCH 080/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 537cd1a..b2a6a25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.14.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.14.0a1) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.13.0a1...0.14.0a1) + +**Merged pull requests:** + +- feat: add persona\_id to canonical Session \(OVOS-PERSONA-1 registered field\) [\#37](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/37) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.13.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.13.0a1) (2026-06-26) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.12.1a1...0.13.0a1) From 9a344534ebcd7324716e10fdcbad678bae18632b Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 03:18:10 +0100 Subject: [PATCH 081/110] fix: locale/template/lint/language spec-conformance (audit remediation) (#41) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: locale/template/lint/language spec-conformance (audit remediation) Remediate the spec-conformance flags from the adversarial audit, each tied to the cited INTENT-1/2/3 clause: - expansion.py inline_keywords: drop the silent max_values=10 truncation (data loss). Default now inlines ALL values; an explicit max_values bound REFUSES (raises MalformedTemplate) when exceeded per INTENT-1 §4.3 (refuse-and-document, never silently drop). - expansion.py inline_keywords: replace the `for _ in range(8)` depth cap with proper recursion + cycle detection (INTENT-1 §4.1) — unbounded resolution that raises on a reference cycle, no magic depth limit. - dialog.py: implement the INTENT-1 §7 / §5.5 MUST — verify_slot_consistency checks all phrases of a dialog declare the same slot set; both render paths call it before rendering and raise on divergence. - lint.py: a divergent .intent slot set is now an ERROR (was WARNING), matching .dialog. INTENT-1 §5.5, INTENT-2 §4.1/§4.2, INTENT-3 §5.1 all mandate rejection; removed the misread INTENT-3 §5.3 (a worked example that itself violates §5.5) justification. - language.py: relabel the pt-PT norm-region and tl/tgl Tagalog overrides as NON-NORMATIVE implementation policy (they deliberately diverge from langcodes, which §2.2 endorses); annotate _coarse_distance 3/5/100 as arbitrary ordering-only values with no spec basis (§2.2 silent on the no-langcodes case). - resources.py keyword_form: comment clarifying its deliberate leniency is confined to the best-effort keyword extractors; the conformant loaders (_load_expanded etc.) call expand() directly and raise. Docs (linting.md, dialog.md traceability, spec-traceability.md) updated to match. Tests updated for the new ERROR severity, no-truncation, cycle rejection, and §5.5 dialog checks. Co-Authored-By: Claude Opus 4.8 * fix: .intent MAY declare different slot sets (union); slot-consistency ERROR is .dialog-only Corrects the audit-remediation overreach: INTENT-2 §4.1 / INTENT-3 §5.1 (per #56/#67) allow .intent templates to declare different slot sets (union); only .dialog (§4.2) requires identical slots. * feat: validate required_slots are declared by a template (OVOS-INTENT-3 §5.3) A required slot MUST be declared by at least one template in the intent; declaring a required slot no template mentions is malformed and a tool MUST reject the definition at registration time (OVOS-INTENT-3 §5.3). required_slots is an intent-definition field above the raw .intent file, so the locale linter (which sees only files) cannot enforce it. Expose the check as public functions for the registration/loading path that has both in hand: - declared_slots(templates): the union of named slots across templates, folding {{name}} to {name} (§3.4). - validate_required_slots(required_slots, templates): raises MalformedTemplate if any required slot is declared by no template. - lint_required_slots(path, required_slots, templates): a Finding-returning wrapper for linting callers. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- docs/linting.md | 36 +++++---- docs/spec-traceability.md | 1 + ovos_spec_tools/__init__.py | 19 ++++- ovos_spec_tools/dialog.py | 63 +++++++++++++-- ovos_spec_tools/expansion.py | 93 +++++++++++++++------ ovos_spec_tools/language.py | 50 ++++++++---- ovos_spec_tools/lint.py | 152 ++++++++++++++++++++++++++++------- ovos_spec_tools/resources.py | 12 +++ test/test_dialog.py | 46 ++++++++++- test/test_expansion.py | 36 +++++++-- test/test_lint.py | 98 +++++++++++++++++++--- 11 files changed, 499 insertions(+), 107 deletions(-) diff --git a/docs/linting.md b/docs/linting.md index 1b8a6a5..7e4b8e8 100644 --- a/docs/linting.md +++ b/docs/linting.md @@ -4,9 +4,9 @@ > (`lint.py`). Every finding enforces a specific clause — template syntax from > **OVOS-INTENT-1 §3.6**, naming/layout/empty-file/uniqueness from > **OVOS-INTENT-2 §2/§4.3/§5**, slot-set consistency from **OVOS-INTENT-1 §5.5** -> (relaxed to *union slot sets* for `.intent`, reconciling §5.5 with the -> multi-slot template-intent example of **OVOS-INTENT-3 §5.3**). Each message -> names the clause it failed, so a red lint points straight at the violated MUST. +> (restated for `.intent` by **OVOS-INTENT-2 §4.1** / **OVOS-INTENT-3 §5.1**, +> and for `.dialog` by **OVOS-INTENT-2 §4.2**). Each message names the clause it +> failed, so a red lint points straight at the violated MUST. `ovos-spec-lint` checks a skill's locale folder against both specs at once — the **syntax** of every template (OVOS-INTENT-1) and the **naming and layout** @@ -42,10 +42,11 @@ straight into a CI pipeline. With `--strict`, warnings fail the run too. - an empty file (no templates after comments and blank lines); - a file that is not valid UTF-8; - a named slot inside a slot-free role (`.entity` / `.voc` / `.blacklist`); -- templates within one **`.dialog`** declaring **different slot sets** — every - phrase of one dialog must use the same `{slots}` so the caller fills the same - values whichever is chosen (OVOS-INTENT-1 §5.5). For **`.intent`** this is - only a *warning*, not an error: union slot sets are valid there (see below); +- templates within one **`.intent`** or **`.dialog`** declaring **different + slot sets** — every line of one definition MUST declare the same `{slots}` so + the engine captures, or the caller fills, the same slots whichever line + matched or was chosen (OVOS-INTENT-1 §5.5; OVOS-INTENT-2 §4.1/§4.2; + OVOS-INTENT-3 §5.1); - a base name outside the allowed charset (lowercase letters, digits, underscores); - an `.entity` whose base name — which names a slot — begins with a digit; @@ -65,22 +66,25 @@ straight into a CI pipeline. With `--strict`, warnings fail the run too. A `.prompt` is checked too, but not as a template — it is plain text, so only its naming and non-emptiness are checked, never template syntax. -### Union slot sets for `.intent` +### Slot-set consistency -OVOS-INTENT-1 §5.5 says every template of one definition must declare the -identical slot set. The linter enforces that strictly for `.dialog` (an -**error**) but relaxes it for `.intent` to a **warning**, because -OVOS-INTENT-3 §5.3 presents a single template intent whose phrasings declare -*different* slots: +OVOS-INTENT-1 §5.5 says every template of one definition MUST declare the +identical slot set, and a tool "MUST reject" one that does not. This is restated +normatively for each slot-bearing role — `.intent` by OVOS-INTENT-2 §4.1 and +OVOS-INTENT-3 §5.1 ("Every template in one intent MUST declare the same set of +named slots"), `.dialog` by OVOS-INTENT-2 §4.2 — so the linter treats a +divergent slot set as an **error** for both. A phrasing that genuinely needs a +different slot, such as: ``` (play|put on) {query} (play|put on) {query} (on|using) {engine} ``` -Here `{engine}` is an optional capture — present in some phrasings, absent in -others. Treating the extra slots as optional makes this one valid intent, so -the linter warns ("verify this is intentional") rather than rejecting it. +is **two** intents, not one: §5.5 says "place them in separate files and handle +them individually". (The two-line block above appears as a *worked example* in +OVOS-INTENT-3 §5.3, but that example is illustrative and itself violates the +§5.5 MUST it does not restate; the normative rule governs.) ## Targeting an older spec version diff --git a/docs/spec-traceability.md b/docs/spec-traceability.md index 4587004..84bea9b 100644 --- a/docs/spec-traceability.md +++ b/docs/spec-traceability.md @@ -133,6 +133,7 @@ symbols it exports as imported by the package `__init__`.) | `render` | OVOS-INTENT-2 | §4.2 (select + fill a `.dialog` phrase) | | `DialogRenderer` | OVOS-INTENT-2 | §4.2 (stateful, multilingual renderer) | | `UnfilledSlot` | OVOS-INTENT-2 | §4.2 (a chosen phrase has an unfilled slot) | +| `verify_slot_consistency` | OVOS-INTENT-1 | §7 + §5.5 (Dialog renderer MUST verify all phrases declare the same slot set) | ### `prompt.py` — OVOS-INTENT-2 §4.4 (`.prompt`) diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index f206f83..295fdcc 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -31,7 +31,12 @@ - :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also exposed as the ``ovos-spec-lint`` command. """ -from ovos_spec_tools.dialog import DialogRenderer, UnfilledSlot, render +from ovos_spec_tools.dialog import ( + DialogRenderer, + UnfilledSlot, + render, + verify_slot_consistency, +) from ovos_spec_tools.expansion import MalformedTemplate, expand, inline_keywords from ovos_spec_tools.message import ( DEFAULT_SESSION_ID, @@ -44,7 +49,13 @@ lang_matches, standardize_lang, ) -from ovos_spec_tools.lint import Finding, lint_locale +from ovos_spec_tools.lint import ( + Finding, + declared_slots, + lint_locale, + lint_required_slots, + validate_required_slots, +) from ovos_spec_tools.messages import ( MIGRATION_MAP, SPEC_TO_LEGACY, @@ -98,6 +109,7 @@ "utterance_contains", "render", "DialogRenderer", + "verify_slot_consistency", "UnfilledSlot", "render_prompt", "PromptRenderer", @@ -106,6 +118,9 @@ "lang_matches", "closest_lang", "lint_locale", + "declared_slots", + "validate_required_slots", + "lint_required_slots", "Finding", "SpecMessage", "MIGRATION_MAP", diff --git a/ovos_spec_tools/dialog.py b/ovos_spec_tools/dialog.py index bd67a24..e2809ca 100644 --- a/ovos_spec_tools/dialog.py +++ b/ovos_spec_tools/dialog.py @@ -1,9 +1,12 @@ """Reference dialog renderer for OVOS-INTENT-2 §4.2. This is the reference implementation of the **Dialog renderer** conformance -role of OVOS-INTENT-1 §7. Rendering a dialog means: select one phrase from a -loaded ``.dialog``, expand its ``(a|b)`` / ``[x]`` variety to a single variant, -and fill every ``{name}`` slot with a value. +role of OVOS-INTENT-1 §7. Rendering a dialog means: verify all phrases declare +the same slot set (§5.5), select one phrase from a loaded ``.dialog``, expand +its ``(a|b)`` / ``[x]`` variety to a single variant, and fill every ``{name}`` +slot with a value. The §7 role MUST "verify that all phrases in a dialog +definition declare the same slot set (§5.5)"; :func:`verify_slot_consistency` +performs that check and both render paths call it before rendering. Two interfaces are provided: @@ -25,13 +28,49 @@ import re from typing import Dict, Optional, Protocol, Sequence -from ovos_spec_tools.expansion import expand +from ovos_spec_tools.expansion import MalformedTemplate, expand -__all__ = ["render", "DialogRenderer", "Chooser", "UnfilledSlot"] +__all__ = [ + "render", + "DialogRenderer", + "Chooser", + "UnfilledSlot", + "verify_slot_consistency", +] _SLOT_TOKEN_RE = re.compile(r"\{([a-z][a-z0-9_]*)\}") +def verify_slot_consistency(phrases: Sequence[str]) -> None: + """Verify every phrase of a dialog declares the identical slot set (§5.5). + + OVOS-INTENT-1 §7 makes this a **MUST** for the *Dialog renderer* role: it + "verify[s] that all phrases in a dialog definition declare the same slot + set (§5.5)". A definition MUST NOT mix templates that declare different + slots, so the caller supplies the same values whichever phrase is chosen + (§5.1, OVOS-INTENT-2 §4.2). + + A phrase *declares* a slot name if that name appears anywhere in it; + optionality does not change this (§5.5), so ``say [{x}]`` and ``say {x}`` + declare the same slot set ``{x}`` and may coexist. + + Args: + phrases: the phrase strings of one ``.dialog`` definition. + + Raises: + MalformedTemplate: two phrases declare different slot sets (§5.5). + """ + slot_sets = {frozenset(_SLOT_TOKEN_RE.findall(phrase)) for phrase in phrases} + if len(slot_sets) > 1: + rendered = sorted( + "{" + ", ".join(sorted(s)) + "}" for s in slot_sets) + raise MalformedTemplate( + "dialog phrases declare different slot sets " + f"({', '.join(rendered)}): every phrase in one dialog definition " + "MUST declare the same {slots} (OVOS-INTENT-1 §5.5); a phrasing " + "that needs different slots belongs in a separate dialog") + + class Chooser(Protocol): """A source of random selection: anything with a ``choice`` method. @@ -75,10 +114,15 @@ def render(phrases: Sequence[str], Raises: UnfilledSlot: a slot in the chosen phrase has no value. ValueError: ``phrases`` is empty. - MalformedTemplate: the chosen phrase is not a valid template. + MalformedTemplate: the chosen phrase is not a valid template, or the + phrases declare divergent slot sets (OVOS-INTENT-1 §5.5). """ if not phrases: raise ValueError("no dialog phrases to render") + # §7 Dialog-renderer MUST: all phrases in a dialog declare the same slot + # set (§5.5). Enforce before choosing, so a divergent definition is + # rejected regardless of which phrase the chooser would pick. + verify_slot_consistency(phrases) chooser = rng if rng is not None else _random phrase = chooser.choice(list(phrases)) return _render_phrase(phrase, slots or {}, vocabularies, chooser) @@ -141,9 +185,14 @@ def render(self, lang: str, Raises: UnfilledSlot: a slot in the chosen phrase has no value. FileNotFoundError: the dialog does not exist for ``lang``. - MalformedTemplate: the chosen phrase is not a valid template. + MalformedTemplate: the chosen phrase is not a valid template, or + the dialog's phrases declare divergent slot sets + (OVOS-INTENT-1 §5.5). """ phrases = self.resources.load_dialog(self.name, lang) + # §7 Dialog-renderer MUST verify all phrases declare the same slot set + # (§5.5) before rendering. + verify_slot_consistency(phrases) effective = dict(self.default_slots) if slots: effective.update(slots) diff --git a/ovos_spec_tools/expansion.py b/ovos_spec_tools/expansion.py index 344477c..3027963 100644 --- a/ovos_spec_tools/expansion.py +++ b/ovos_spec_tools/expansion.py @@ -263,7 +263,7 @@ def inline_keywords( template: str, vocabularies: Dict[str, Sequence[str]] | None = None, *, - max_values: int = 10, + max_values: Optional[int] = None, ) -> str: """Inline ```` references as ``(v1|v2|…)`` alternation groups. @@ -277,11 +277,15 @@ def inline_keywords( alternations, then expand the result themselves. Unlike :func:`expand`, this is intentionally lenient and **not** a - conformant expander: it does no §3.6 validation, leaves an unknown keyword's - angle brackets stripped as literal text rather than raising - :class:`MalformedTemplate` for an undefined reference, and applies the - non-spec ``max_values`` cap below to bound the §4.3 sample-set blow-up. - Feed its output to :func:`expand` for the validated sample set. + conformant expander: it does no §3.6 validation and leaves an unknown + keyword's angle brackets stripped as literal text rather than raising + :class:`MalformedTemplate` for an undefined reference. Feed its output to + :func:`expand` for the validated sample set. + + By default **every** value of a keyword is inlined (no silent truncation). + Resolution recurses through nested references — ```` inside ```` — and + a reference cycle raises :class:`MalformedTemplate` per OVOS-INTENT-1 §4.1, + rather than being cut off at an arbitrary depth. Parameters ---------- @@ -291,11 +295,12 @@ def inline_keywords( Flat ``{keyword: [values]}`` mapping. If ``None`` or empty the template is returned unchanged. max_values - Cap the number of values per keyword inlined. Default 10. This is an - engineering limit, **not** a spec rule: OVOS-INTENT-1 §4.3 warns that a - sample set grows as the product of branch counts, and §4.3 permits an - expander to bound it; capping members here keeps a downstream engine's - expansion finite without that engine having to re-implement the limit. + Optional bound on the number of values inlined per keyword. ``None`` + (the default) inlines **all** values. OVOS-INTENT-1 §4.3 permits an + expander to *refuse* (not silently drop) a template whose expansion + exceeds a documented limit; accordingly, when a keyword has more than + ``max_values`` members this **raises** :class:`MalformedTemplate`. It + never silently truncates the value list. Returns ------- @@ -304,6 +309,13 @@ def inline_keywords( groups. Keywords not found in ``vocabularies`` have their angle brackets stripped and become literal text. + Raises + ------ + MalformedTemplate + On a reference cycle (OVOS-INTENT-1 §4.1), or when a keyword's value + count exceeds an explicit ``max_values`` bound (OVOS-INTENT-1 §4.3, + refuse-and-document). + Example ------- >>> inline_keywords(" [the] {name}", @@ -312,17 +324,50 @@ def inline_keywords( """ if not vocabularies: return template + return _inline_keywords(template, vocabularies, max_values, ()) - def _sub(m: re.Match[str]) -> str: - vals = vocabularies.get(m.group(1)) - if vals: - return "(" + "|".join(vals[:max_values]) + ")" - return m.group(0) # keep brackets for unresolvable keywords - - # Iterate until stable — handles nested refs like inside - for _ in range(8): - new = _VOC_TOKEN_RE.sub(_sub, template) - if new == template: - break - template = new - return template + +def _inline_keywords(template: str, + vocabularies: Dict[str, Sequence[str]], + max_values: Optional[int], + stack: Tuple[str, ...]) -> str: + """Resolve ```` references recursively, leftmost first, with cycle + detection (OVOS-INTENT-1 §4.1). ``stack`` is the chain of references being + resolved; a name already on it is a cycle and is rejected.""" + while True: + match = _VOC_TOKEN_RE.search(template) + if match is None: + return template + name = match.group(1) + + if name in stack: + raise MalformedTemplate( + f"cyclic vocabulary reference <{name}> " + f"(chain: {' -> '.join(stack + (name,))})") + + vals = vocabularies.get(name) + if not vals: + # Lenient: an unknown keyword is left as literal text (brackets + # stripped) rather than raising. Skip past it so the leftmost-first + # scan continues and an undefined reference cannot loop forever. + template = (template[:match.start()] + name + + template[match.end():]) + continue + + if max_values is not None and len(vals) > max_values: + # §4.3 permits a documented limit enforced by REFUSING — never by + # silently dropping values (which would be data loss). + raise MalformedTemplate( + f"vocabulary <{name}> has {len(vals)} values, exceeding the " + f"max_values bound of {max_values} (OVOS-INTENT-1 §4.3: a limit " + f"is enforced by refusing, not by truncating)") + + # Resolve any nested references inside each value before substituting, + # carrying this name on the stack for cycle detection. + resolved_vals = [ + _inline_keywords(v, vocabularies, max_values, stack + (name,)) + for v in vals + ] + substitution = "(" + "|".join(resolved_vals) + ")" + template = (template[:match.start()] + substitution + + template[match.end():]) diff --git a/ovos_spec_tools/language.py b/ovos_spec_tools/language.py index 73358e8..38f89bb 100644 --- a/ovos_spec_tools/language.py +++ b/ovos_spec_tools/language.py @@ -38,14 +38,20 @@ :func:`_with_norm_region`) < two regions of one language (a small regional distance) < the bare/generic form vs a region of the same language < a different primary language (``>= 100`` with the coarse measure — never a usable match). -The asymmetry that matters: a bare ``pt`` is measured **from ``pt-PT``**, not -from the most-populous ``pt-BR``, so ``pt`` falls back to ``pt-PT`` before -``pt-BR`` — correcting ``langcodes``' population default (see :data:`_NORM_REGION`). + +The norm-region preference is **non-normative implementation policy**, not a +spec rule: a bare ``pt`` is measured **from ``pt-PT``**, not from the +most-populous ``pt-BR``, so ``pt`` falls back to ``pt-PT`` before ``pt-BR``. +This deliberately *diverges* from ``langcodes``' population default — which +§2.2 endorses — and is a choice this package makes, not a requirement of any +spec (see :data:`_NORM_REGION`). ``langcodes`` is an optional dependency (OVOS-INTENT-2 §2.2 names it). Without it, :func:`lang_distance` falls back to a coarse same-language / different-language measure that still honours the same ordering for the cases -locale resolution actually depends on. +locale resolution actually depends on. That fallback, and its distance values, +are likewise this package's own implementation policy — §2.2 is silent on the +no-``langcodes`` case. """ from __future__ import annotations @@ -63,13 +69,14 @@ # see the langcodes distance-values documentation). DEFAULT_MAX_LANGUAGE_DISTANCE = 10 -# The norm region for a bare language tag. `langcodes` resolves a bare tag to -# its *most-populous* region — for "pt" that is "pt-BR" — but the unmarked -# form of a language should resolve to its reference variety. Portuguese is -# "from Portugal" by name, and every Lusophone country except Brazil follows -# the pt-PT norm, so a bare "pt" is measured from "pt-PT". Add a language here -# only when its bare tag has a clear reference region distinct from the -# populous one. +# NON-NORMATIVE IMPLEMENTATION POLICY (no spec backing). `langcodes` resolves a +# bare tag to its *most-populous* region — for "pt" that is "pt-BR". This map +# deliberately overrides that for languages whose unmarked form OVOS prefers to +# resolve to a reference variety instead: a bare "pt" is measured from "pt-PT". +# OVOS-INTENT-2 §2.2 endorses `langcodes`; this preference diverges from it on +# purpose and is this package's policy, not a requirement of any spec. Add a +# language here only when its bare tag has a clear reference region distinct +# from the populous one. _NORM_REGION = { "pt": "PT", } @@ -98,8 +105,10 @@ def standardize_lang(tag: str) -> str: otherwise. """ if tag.lower() in ("tl", "tgl"): - # langcodes folds Tagalog into the `fil` macrolanguage; OVOS keeps - # `tl` distinct, so this one tag is normalized by hand. + # NON-NORMATIVE IMPLEMENTATION POLICY (no spec backing). langcodes folds + # Tagalog into the `fil` macrolanguage; OVOS keeps `tl` distinct, so + # this one tag is normalized by hand. This is a deliberate divergence + # from langcodes (which §2.2 endorses), not a spec requirement. return "tl" try: from langcodes import standardize_tag @@ -116,6 +125,8 @@ def _with_norm_region(tag: str) -> str: """Give a bare language tag its norm region (``pt`` -> ``pt-PT``). A regioned tag, or a language with no norm region, is returned unchanged. + The norm-region map is non-normative implementation policy (see + :data:`_NORM_REGION`); it diverges from ``langcodes`` on purpose. """ if "-" in tag: return tag @@ -145,6 +156,13 @@ def _coarse_distance(desired: str, supported: str) -> int: The two tags are already standardized and norm-expanded. A shared primary subtag is near, a differing one is far; the generic (region-less) form of a language counts as nearer than a sibling region of it. + + The values 3 / 5 / 100 are **arbitrary, ordering-only** numbers with no + spec basis: OVOS-INTENT-2 §2.2 names ``langcodes``' ``tag_distance`` and is + silent on the no-``langcodes`` case, so this fallback exists purely to + preserve the same nearest-to-farthest *ordering* that locale resolution + depends on. Only the relative order (and being below/above the threshold of + 10) is meaningful; the magnitudes are not. """ if desired.split("-")[0].lower() != supported.split("-")[0].lower(): return 100 # a different language — beyond any usable threshold @@ -164,8 +182,10 @@ def lang_distance(desired: str, supported: str) -> int: ``0`` is identical; a larger number is further apart; a value of 10 or more is not a usable match. Both tags are standardized, and a bare tag is measured **from its norm region** — so ``lang_distance("pt", "pt-PT")`` is - ``0`` while ``lang_distance("pt", "pt-BR")`` is a regional difference, - correcting ``langcodes``' population-based default (see :data:`_NORM_REGION`). + ``0`` while ``lang_distance("pt", "pt-BR")`` is a regional difference. The + norm-region step is non-normative implementation policy that diverges from + ``langcodes``' population-based default on purpose (see :data:`_NORM_REGION`), + not a §2.2 requirement. Args: desired: the requested BCP-47 tag. diff --git a/ovos_spec_tools/lint.py b/ovos_spec_tools/lint.py index cb44c93..9313d95 100644 --- a/ovos_spec_tools/lint.py +++ b/ovos_spec_tools/lint.py @@ -25,10 +25,23 @@ / ``.blacklist`` are slot-free. - *template syntax* → OVOS-INTENT-1 §3.6 (delegated to :func:`~ovos_spec_tools.expansion.expand`). -- *slot-set consistency* → OVOS-INTENT-1 §5.5 (see :func:`_lint_file` for the - ``.intent`` union-slot relaxation reconciling §5.5 with OVOS-INTENT-3 §5.3). +- *slot-set consistency* → OVOS-INTENT-2 §4.2 (``.dialog`` only): a ``.dialog`` + whose phrases declare different slot sets is an ERROR (the caller fills the + slots of the rendered phrase, so all phrases must expose the same slots). + ``.intent`` templates MAY declare different slot sets — their union is the + intent's slot set (OVOS-INTENT-2 §4.1, OVOS-INTENT-3 §5.1) — and are NOT + flagged. - *blacklist with no matching ``.intent``* → OVOS-INTENT-2 §4.3 — a ``.blacklist`` "is paired by base name with exactly one ``.intent``". +- *required slot declared by no template* → OVOS-INTENT-3 §5.3 — "A required + slot MUST be declared by at least one template in the intent … a tool MUST + reject the definition at registration time." This is an intent-**definition** + rule, not a single-file rule: ``required_slots`` lives above the raw + ``.intent`` file. The locale linter (which sees only files) cannot enforce + it; the check is exposed as :func:`validate_required_slots` (raises) and + :func:`lint_required_slots` (returns a :class:`Finding`) for the + registration/loading path that has both the ``required_slots`` list and the + templates in hand. Exposed as the ``ovos-spec-lint`` command:: @@ -67,7 +80,14 @@ read_resource_file, ) -__all__ = ["Finding", "lint_locale", "main"] +__all__ = [ + "Finding", + "lint_locale", + "declared_slots", + "validate_required_slots", + "lint_required_slots", + "main", +] # The six OVOS-INTENT-2 resource roles. ROLE_EXTENSIONS = SLOT_BEARING_ROLES + SLOT_FREE_ROLES + (PROMPT_ROLE,) @@ -121,6 +141,91 @@ def __str__(self) -> str: return f"{self.path}: {self.severity}: {self.message}" +def declared_slots(templates: Sequence[str]) -> frozenset: + """The union of the named slots declared across ``templates``. + + An intent's slot set is the **union** of the slots declared by its + templates (OVOS-INTENT-2 §4.1, OVOS-INTENT-3 §5.1): the engine extracts + only the slots of whichever template matched, so a slot the intent can + ever fill is one declared by *some* template. Both equivalent spellings + ``{name}`` and ``{{name}}`` (OVOS-INTENT-1 §3.4) count, so each template is + folded to the single-brace canonical form before its slots are read. + + Args: + templates: the template lines of one ``.intent`` definition. + + Returns: + The frozen union of every named slot any template declares. + """ + slots: set = set() + for template in templates: + slots.update(_SLOT_RE.findall(fold_double_braces(template))) + return frozenset(slots) + + +def validate_required_slots( + required_slots: Sequence[str], + templates: Sequence[str]) -> None: + """Verify every required slot is declared by at least one template (§5.3). + + OVOS-INTENT-3 §5.3 makes this a **MUST** at registration time: "A required + slot MUST be declared by at least one template in the intent. Declaring a + required slot that no template mentions is malformed: the intent can never + match, and a tool MUST reject the definition at registration time." + + ``required_slots`` is an intent-**definition** field — it lives above the + raw ``.intent`` file, alongside the templates. This validator is the place + that sees both together; call it when registering or loading an intent + definition so a malformed one (a required slot no template can fill) is + rejected before it can silently never fire. + + Args: + required_slots: the slot names the intent declares as required. + templates: the intent's template lines (its ``.intent`` samples). + + Raises: + MalformedTemplate: a required slot is declared by no template, so the + intent can never match (OVOS-INTENT-3 §5.3). + """ + available = declared_slots(templates) + missing = [name for name in required_slots if name not in available] + if missing: + have = "{" + ", ".join(sorted(available)) + "}" if available else "{}" + raise MalformedTemplate( + f"required slot(s) {{{', '.join(missing)}}} declared by no " + f"template — the intent's templates declare only {have}, so the " + "intent can never match. A required slot MUST be declared by at " + "least one template (OVOS-INTENT-3 §5.3)") + + +def lint_required_slots( + path: str, + required_slots: Sequence[str], + templates: Sequence[str]) -> List[Finding]: + """Lint an intent's ``required_slots`` against its templates (§5.3). + + A :class:`Finding`-returning wrapper over :func:`validate_required_slots`, + for callers that lint intent definitions (where ``required_slots`` metadata + is available alongside the templates) and want findings rather than an + exception. Each undeclared required slot is an :data:`ERROR` (a §5.3 MUST). + + Args: + path: the offending intent's identifier (file path or name), reported + on the finding. + required_slots: the slot names the intent declares as required. + templates: the intent's template lines. + + Returns: + One :data:`ERROR` finding if any required slot is undeclared, else an + empty list. + """ + try: + validate_required_slots(required_slots, templates) + except MalformedTemplate as exc: + return [Finding(ERROR, path, str(exc))] + return [] + + def lint_locale(path, spec_version: int = DEFAULT_SPEC_VERSION) -> List[Finding]: """Lint a locale directory, or a single language directory. @@ -344,32 +449,23 @@ def _lint_file(path: Path, slot_sets.append( frozenset(_SLOT_RE.findall(fold_double_braces(template)))) - # --- slot consistency (OVOS-INTENT-1 §5.5) ------------------------------ - # §5.5 (and OVOS-INTENT-3 §5.1) require every template of one definition to - # declare the identical slot set, and a tool "MUST reject" one that does - # not. For a `.dialog` that is enforced as an ERROR: the caller fills the - # same slots whichever phrase is chosen (§4.2), so a divergent slot set is a - # genuine fault. + # --- slot consistency: `.dialog` ONLY ----------------------------------- + # `.dialog` phrases MUST all declare the same slot set: the caller fills the + # slots of whichever phrase is rendered, so every phrase must expose the + # identical slots (OVOS-INTENT-2 §4.2; OVOS-INTENT-1 §5.5). A `.dialog` + # whose phrases diverge is malformed — ERROR. # - # For an `.intent` the rule is *relaxed to union slot sets* (WARNING, not - # ERROR) to reconcile §5.5 with the worked example in OVOS-INTENT-3 §5.3, - # which presents `(play|put on) {query}` and - # `(play|put on) {query} (on|using) {engine}` as ONE valid template intent - # despite their differing slot sets ({query} vs {query},{engine}). Treating - # the extra slots as optional captures is the documented intent there, so - # the linter warns rather than rejects. Changing this from a warning to an - # error would regress that valid pattern — hence it stays a WARNING. - if len(set(slot_sets)) > 1: - if extension == ".dialog": - findings.append(Finding( - ERROR, str(path), - f"templates declare different slot sets — every template in one " - f"{extension} must use the same {{slots}} (OVOS-INTENT-1 §5.5)")) - else: - findings.append(Finding( - WARNING, str(path), - f"templates declare different slot sets — this is valid for " - f".intent but unusual; verify this is intentional (OVOS-INTENT-1 §5.5)")) + # `.intent` is the deliberate OPPOSITE: its templates MAY declare different + # slot sets, and the intent's slot set is their union — the engine extracts + # only the slots of the template that matched (OVOS-INTENT-2 §4.1, + # OVOS-INTENT-3 §5.1). A tool MUST NOT reject a `.intent` for divergent + # slots, so divergence is NOT flagged for the `.intent` role. + if extension == ".dialog" and len(set(slot_sets)) > 1: + findings.append(Finding( + ERROR, str(path), + "a .dialog's phrases declare different slot sets — every phrase in " + "one .dialog MUST declare the same {slots} (OVOS-INTENT-2 §4.2; " + "OVOS-INTENT-1 §5.5)")) return findings diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py index 8d79d66..d3d845a 100644 --- a/ovos_spec_tools/resources.py +++ b/ovos_spec_tools/resources.py @@ -224,6 +224,16 @@ def keyword_form(template_line: str, whitespace-only line, or a line that fails to expand, yields ``("", [])`` (a malformed line is dropped, not raised, so one bad line does not poison a batch keyword load). + + This leniency is **deliberate and confined to this low-level helper**: it + serves only the best-effort keyword extractors (:meth:`vocabulary_keywords`, + :meth:`entity_keywords`), which group ``.voc``/``.entity`` lines into + canonical/alias pairs and skip a line that will not expand. It is **not** a + conformant load path — the conformant loaders (:meth:`_load_expanded` and + everything built on it: :meth:`load_intent`, :meth:`load_vocabulary`, …) + call :func:`expand` directly and **raise** on a malformed template or a + slot in a slot-free role per OVOS-INTENT-2 §5 / OVOS-INTENT-1 §3.6. No + conformant load relies on this function's silent drop. """ if not template_line.strip(): return "", [] @@ -231,6 +241,8 @@ def keyword_form(template_line: str, samples = expand(template_line, vocabularies) except Exception: # A malformed line yields no keyword rather than poisoning the batch. + # Lenient by design — see the note in the docstring; conformant loaders + # call expand() directly and raise instead. return "", [] options = sorted({s.lower() for s in samples}) if not options: diff --git a/test/test_dialog.py b/test/test_dialog.py index e53192e..72e55bf 100644 --- a/test/test_dialog.py +++ b/test/test_dialog.py @@ -1,7 +1,14 @@ """Conformance tests for the OVOS-INTENT-2 §4.2 reference dialog renderer.""" import pytest -from ovos_spec_tools import DialogRenderer, LocaleResources, UnfilledSlot, render +from ovos_spec_tools import ( + DialogRenderer, + LocaleResources, + MalformedTemplate, + UnfilledSlot, + render, + verify_slot_consistency, +) class _FixedRng: @@ -199,3 +206,40 @@ def test_renderer_missing_dialog_raises(tmp_path): renderer = DialogRenderer(res, "absent") with pytest.raises(FileNotFoundError): renderer.render("en-US") + + +# --- §5.5 slot-set consistency (the §7 Dialog-renderer MUST) ----------------- + +def test_verify_slot_consistency_accepts_identical_slot_sets(): + # Optionality does not change a declared slot set (§5.5): `say [{x}]` and + # `say {x}` both declare {x}. + verify_slot_consistency(["say [{x}]", "say {x}"]) # no raise + + +def test_verify_slot_consistency_rejects_divergent_slot_sets(): + with pytest.raises(MalformedTemplate): + verify_slot_consistency(["hello {name}", "hi {name} and {title}"]) + + +def test_render_rejects_divergent_slot_sets(): + # §7: the renderer MUST verify all phrases declare the same slot set (§5.5), + # regardless of which phrase the chooser would pick. + phrases = ["hello {name}", "hi {name} and {title}"] + with pytest.raises(MalformedTemplate): + render(phrases, slots={"name": "Sam", "title": "Dr"}) + + +def test_renderer_rejects_divergent_slot_sets(tmp_path): + res = _resources(tmp_path, { + "en-US/g.dialog": "hello {name}\nhi {name} and {title}\n"}) + renderer = DialogRenderer(res, "g") + with pytest.raises(MalformedTemplate): + renderer.render("en-US", {"name": "Sam", "title": "Dr"}) + + +def test_render_consistent_dialog_still_works(tmp_path): + res = _resources(tmp_path, { + "en-US/g.dialog": "hello {name}\nhi there {name}\n"}) + renderer = DialogRenderer(res, "g") + assert renderer.render("en-US", {"name": "Sam"}) in ( + "hello Sam", "hi there Sam") diff --git a/test/test_expansion.py b/test/test_expansion.py index a164dcf..bc403a1 100644 --- a/test/test_expansion.py +++ b/test/test_expansion.py @@ -311,16 +311,42 @@ def test_inline_keywords_none_vocab(): def test_inline_keywords_strips_unresolved(): + # An unknown keyword is left as literal text with its angle brackets + # stripped (the documented lenient behaviour), not raised on. vocab = {"known": ["yes"]} result = inline_keywords(" ", vocab) - assert result == " (yes)" # brackets kept for unresolvable + assert result == "unknown (yes)" -def test_inline_keywords_max_values(): +def test_inline_keywords_inlines_all_values_by_default(): + # No silent truncation: every value is inlined (OVOS-INTENT-1 §4.3 — a + # limit is enforced by refusing, not by dropping). vocab = {"x": [str(i) for i in range(20)]} - result = inline_keywords("", vocab, max_values=5) - assert "10" not in result - assert "0" in result + result = inline_keywords("", vocab) + for i in range(20): + assert f"|{i}|" in f"|{result[1:-1]}|" + assert result.count("|") == 19 + + +def test_inline_keywords_max_values_refuses(): + # §4.3: an explicit bound, when exceeded, REFUSES (raises) — it never + # silently truncates the value list. + vocab = {"x": [str(i) for i in range(20)]} + with pytest.raises(MalformedTemplate): + inline_keywords("", vocab, max_values=5) + + +def test_inline_keywords_max_values_within_bound(): + vocab = {"x": ["a", "b", "c"]} + assert inline_keywords("", vocab, max_values=5) == "(a|b|c)" + + +def test_inline_keywords_cycle_raises(): + # A reference cycle is rejected (OVOS-INTENT-1 §4.1), not cut off at an + # arbitrary recursion depth. + vocab = {"a": [""], "b": [""]} + with pytest.raises(MalformedTemplate): + inline_keywords("", vocab) def test_inline_keywords_no_refs(): diff --git a/test/test_lint.py b/test/test_lint.py index c7db33c..cae02ae 100644 --- a/test/test_lint.py +++ b/test/test_lint.py @@ -1,7 +1,16 @@ """Tests for the locale resource linter (`ovos-spec-lint`).""" import pytest -from ovos_spec_tools.lint import ERROR, WARNING, lint_locale, main +from ovos_spec_tools.expansion import MalformedTemplate +from ovos_spec_tools.lint import ( + ERROR, + WARNING, + declared_slots, + lint_locale, + lint_required_slots, + main, + validate_required_slots, +) def _write(path, text): @@ -145,23 +154,22 @@ def test_lint_accepts_a_single_language_directory(tmp_path): assert lint_locale(locale / "en-US") == [] -# --- slot consistency (OVOS-INTENT-1 §5.5) ---------------------------------- +# --- slot consistency: .dialog ONLY (OVOS-INTENT-2 §4.2) --------------------- -# .intent allows union slot sets — templates MAY declare different slots. +# .intent templates MAY declare different slot sets — the engine extracts only +# the matched template's slots and the intent's slot set is their union +# (OVOS-INTENT-2 §4.1, OVOS-INTENT-3 §5.1). A tool MUST NOT reject .intent for +# divergent slots, so divergence is NOT flagged for the .intent role. -def test_inconsistent_slots_in_one_intent_warns(tmp_path): +def test_divergent_slots_in_one_intent_is_allowed(tmp_path): locale = tmp_path / "locale" _write(locale / "en-US" / "p.intent", "play {query}\nstop {engine}\n") - warnings = _warnings(lint_locale(locale)) - assert any("slot sets" in f.message for f in warnings) assert not any("slot sets" in f.message for f in _errors(lint_locale(locale))) -def test_mixing_slotted_and_slotless_lines_in_intent_warns(tmp_path): +def test_mixing_slotted_and_slotless_lines_in_intent_is_allowed(tmp_path): locale = tmp_path / "locale" _write(locale / "en-US" / "p.intent", "play {query}\njust stop\n") - warnings = _warnings(lint_locale(locale)) - assert any("slot sets" in f.message for f in warnings) assert not any("slot sets" in f.message for f in _errors(lint_locale(locale))) @@ -297,3 +305,75 @@ def test_default_spec_version_does_not_flag_prompt(tmp_path): locale = tmp_path / "locale" _write(locale / "en-US" / "system.prompt", "You are helpful.\n") assert not any("spec version" in f.message for f in lint_locale(locale)) + + +# --- required_slots validation (OVOS-INTENT-3 §5.3) ---------------------- + +def test_declared_slots_is_the_union_across_templates(): + templates = [ + "(play|put on) {query}", + "(play|put on) {query} (on|using) {engine}", + "i want to listen to {query}", + ] + assert declared_slots(templates) == frozenset({"query", "engine"}) + + +def test_declared_slots_folds_double_brace_spelling(): + # {{name}} and {name} are the same slot (OVOS-INTENT-1 §3.4). + assert declared_slots(["say {{name}}", "say {name}!"]) == frozenset({"name"}) + + +def test_required_slot_declared_by_a_template_is_accepted(): + templates = [ + "(play|put on) {query}", + "(play|put on) {query} (on|using) {engine}", + ] + # both required slots are declared by at least one template — no raise. + validate_required_slots(["query", "engine"], templates) + + +def test_required_slot_declared_by_no_template_is_rejected(): + templates = ["(play|put on) {query}"] + with pytest.raises(MalformedTemplate) as exc: + validate_required_slots(["query", "engine"], templates) + assert "engine" in str(exc.value) + assert "§5.3" in str(exc.value) + + +def test_required_slot_in_only_one_of_several_templates_is_accepted(): + # the engine extracts only the matched template's slots, so a required slot + # declared by a *single* template still satisfies §5.3 (it can fire). + templates = [ + "i want to listen to {query}", + "(play|put on) {query} (on|using) {engine}", + ] + validate_required_slots(["engine"], templates) + + +def test_no_required_slots_is_always_accepted(): + validate_required_slots([], ["(play|put on) {query}"]) + + +def test_required_slot_against_slotless_templates_is_rejected(): + with pytest.raises(MalformedTemplate): + validate_required_slots(["query"], ["just hello", "say hi"]) + + +def test_required_slot_declared_only_in_double_brace_is_accepted(): + # {{engine}} declares the slot just as {engine} would (§3.4 fold). + validate_required_slots(["engine"], ["play {query} on {{engine}}"]) + + +def test_lint_required_slots_returns_an_error_finding(): + findings = lint_required_slots( + "play.intent", ["query", "engine"], ["(play|put on) {query}"]) + assert len(findings) == 1 + assert findings[0].severity == ERROR + assert findings[0].path == "play.intent" + assert "engine" in findings[0].message + + +def test_lint_required_slots_clean_returns_no_findings(): + findings = lint_required_slots( + "play.intent", ["query"], ["(play|put on) {query}"]) + assert findings == [] From b68b2b884be55dfaca196a3d99e1b15f9b8704f0 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:18:22 +0000 Subject: [PATCH 082/110] Increment Version to 0.14.1a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 26fffad..8db7b16 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 14 -VERSION_BUILD = 0 +VERSION_BUILD = 1 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 1b5e00a6710771736a8b2fbc89d5372dbb25f170 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:18:45 +0000 Subject: [PATCH 083/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2a6a25..09a1776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.14.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.14.1a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.14.0a1...0.14.1a1) + +**Merged pull requests:** + +- fix: locale/template/lint/language spec-conformance \(audit remediation\) [\#41](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/41) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.14.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.14.0a1) (2026-06-26) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.13.0a1...0.14.0a1) From ee01f98ed7b43dec4c4800ba0388050fcb2af0a8 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 03:19:22 +0100 Subject: [PATCH 084/110] feat: add fallback_handlers to canonical Session (OVOS-FALLBACK-1 registered field) (#45) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SESSION-1 §3 registers fallback_handlers (array of string, owner OVOS-FALLBACK-1 §4). It was falling through to opaque extras instead of being a first-class registered field. Add it to _LIST_OVERRIDE_FIELDS (same array-of-string, other-spec bucket as the blacklists / transformer chains): constructor param, _as_str_list attribute, generic to_dict / from_dict round-trip with empty-list-as- omission (§3.4 / §2.1), and SESSION1_REGISTERED_FIELDS membership via the _LIST_OVERRIDE_FIELDS union. Like converse_handlers, OVOS-FALLBACK-1 is a forward reference (unmerged); the field is carried on the strength of the SESSION-1 §3 registration. Refs SESSION-1 §3, OVOS-FALLBACK-1 §4. Co-authored-by: Claude Opus 4.8 --- ovos_spec_tools/session.py | 18 ++++++++++++-- test/test_session.py | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/ovos_spec_tools/session.py b/ovos_spec_tools/session.py index cd45aa6..da7789d 100644 --- a/ovos_spec_tools/session.py +++ b/ovos_spec_tools/session.py @@ -86,14 +86,17 @@ ) #: The list-valued denylist / override fields claimed by other specs -#: (OVOS-PIPELINE-1 §5, OVOS-TRANSFORM-1 §5.2). All are arrays of string -#: for which an empty array is wire-equivalent to omission (§3.4). +#: (OVOS-PIPELINE-1 §5, OVOS-TRANSFORM-1 §5.2, OVOS-FALLBACK-1 §4). All are +#: arrays of string for which an empty array is wire-equivalent to omission +#: (§3.4). ``fallback_handlers`` is registered here per the SESSION-1 §3 +#: field table (owner OVOS-FALLBACK-1 §4); see the constructor docstring. _LIST_OVERRIDE_FIELDS = ( "pipeline", "blacklisted_skills", "blacklisted_intents", "blacklisted_pipelines", "blacklisted_audio_transformers", "blacklisted_utterance_transformers", "blacklisted_metadata_transformers", "blacklisted_intent_transformers", "blacklisted_dialog_transformers", "blacklisted_tts_transformers", + "fallback_handlers", ) + _TRANSFORMER_FIELDS #: The object-valued override field (OVOS-CONTEXT-1 §2 ``intent_context``). @@ -173,6 +176,14 @@ class Session: the six ordered transformer chains. :param blacklisted_*_transformers: OVOS-TRANSFORM-1 §5.2 — per-chain denylists. + :param fallback_handlers: OVOS-FALLBACK-1 §4 — an ordered array of + skill-id strings registered as a session field by the SESSION-1 §3 + field table (owner OVOS-FALLBACK-1 §4). Carried opaquely here; its + semantics are owned by OVOS-FALLBACK-1. Empty list and ``None`` are + wire-equivalent (both omitted, §3.4 / §2.1). Like + ``converse_handlers``, OVOS-FALLBACK-1 is a **forward reference** + (it cites this field but is not yet merged); the field is carried + regardless, on the strength of the SESSION-1 §3 registration. :param active_handlers: OVOS-PIPELINE-1 §7.1 dispatch-recency record — a head-first, deduplicated list of ``{skill_id, activated_at}``. :param converse_handlers: OVOS-CONVERSE-1 §2.1 converse-eligibility @@ -218,6 +229,7 @@ def __init__(self, blacklisted_intent_transformers: Optional[List[str]] = None, blacklisted_dialog_transformers: Optional[List[str]] = None, blacklisted_tts_transformers: Optional[List[str]] = None, + fallback_handlers: Optional[List[str]] = None, active_handlers: Optional[List[Dict[str, Any]]] = None, converse_handlers: Optional[List[Dict[str, Any]]] = None, response_mode: Optional[Dict[str, Any]] = None, @@ -303,6 +315,8 @@ def __init__(self, blacklisted_dialog_transformers) self.blacklisted_tts_transformers = self._as_str_list( blacklisted_tts_transformers) + # OVOS-FALLBACK-1 §4 registered array-of-string (carried opaquely) + self.fallback_handlers = self._as_str_list(fallback_handlers) # --- PIPELINE-1 §7.1 / CONVERSE-1 §2.1 / §2.2 handler state --------- # The §2.1 converse-handler cap is NOT session state: a constructed diff --git a/test/test_session.py b/test/test_session.py index b0231d5..58c132c 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -183,6 +183,55 @@ def test_explicit_null_persona_id_treated_as_omitted(self): self.assertNotIn("persona_id", s.to_dict()) +class TestFallbackHandlers(unittest.TestCase): + """OVOS-SESSION-1 §3 registers ``fallback_handlers`` (array of string, + owner OVOS-FALLBACK-1 §4). It is an array-of-string override field — + same bucket / wire rules as the blacklists and transformer chains.""" + + def test_fallback_handlers_round_trip(self): + s = Session(fallback_handlers=["skill-a", "skill-b"]) + self.assertEqual(s.to_dict(), + {"fallback_handlers": ["skill-a", "skill-b"]}) + self.assertEqual(Session.from_dict(s.to_dict()), s) + self.assertEqual(Session.deserialize(s.serialize()), s) + + def test_fallback_handlers_omitted_not_null_when_none(self): + # §2.1 omission-not-null: absent fallback_handlers never serializes. + s = Session() + self.assertIsNone(s.fallback_handlers) + self.assertNotIn("fallback_handlers", s.to_dict()) + self.assertNotIn("fallback_handlers", s.serialize()) + + def test_empty_fallback_handlers_is_wire_equivalent_to_omission(self): + # §3.4 — an empty array on a list override is dropped to omission. + s = Session(fallback_handlers=[]) + self.assertIsNone(s.fallback_handlers) + self.assertEqual(s.to_dict(), {}) + + def test_fallback_handlers_is_registered_not_owned(self): + # OVOS-FALLBACK-1 registers it; OVOS-SESSION-1 does not own it. + self.assertIn("fallback_handlers", SESSION1_REGISTERED_FIELDS) + self.assertNotIn("fallback_handlers", SESSION1_OWNED_FIELDS) + + def test_fallback_handlers_is_first_class_not_extra(self): + # Registered ⇒ lands as a first-class attribute, not in `extras`. + s = Session.from_dict({"fallback_handlers": ["skill-x"]}) + self.assertEqual(s.fallback_handlers, ["skill-x"]) + self.assertNotIn("fallback_handlers", s.extras) + + def test_explicit_null_fallback_handlers_treated_as_omitted(self): + # §2.1: explicit null on a registered field ⇒ omitted, not error. + s = Session.from_dict({"fallback_handlers": None}) + self.assertIsNone(s.fallback_handlers) + self.assertNotIn("fallback_handlers", s.to_dict()) + + def test_non_list_fallback_handlers_rejected(self): + # Wrong wire type for an array-of-string override is malformed, + # same as the blacklists / transformer chains (§3). + with self.assertRaises(MalformedSession): + Session(fallback_handlers="not-a-list") + + # --- §3 / §3.4 other-spec override fields ---------------------------------- class TestOverrideFields(unittest.TestCase): From 227fa1fac588f0529792a0f33e323e86fb64d7da Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:19:30 +0000 Subject: [PATCH 085/110] Increment Version to 0.15.0a1 --- ovos_spec_tools/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 8db7b16..7f7a874 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 14 -VERSION_BUILD = 1 +VERSION_MINOR = 15 +VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 2a1382719db5a306f565f970eb7d88409f194890 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:19:54 +0000 Subject: [PATCH 086/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09a1776..13e5f3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.15.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.15.0a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.14.1a1...0.15.0a1) + +**Merged pull requests:** + +- feat: add fallback\_handlers to canonical Session \(OVOS-FALLBACK-1 registered field\) [\#45](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/45) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.14.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.14.1a1) (2026-06-27) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.14.0a1...0.14.1a1) From 4c544fdd6d4b856550de6752b04fbe23c2ae6d57 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 03:22:47 +0100 Subject: [PATCH 087/110] feat: plugin-agnostic IntentBuilder/Intent + voc_match (INTENT-4 keyword model, adapt-free) (#46) Co-authored-by: Claude Opus 4.8 --- ovos_spec_tools/__init__.py | 15 ++ ovos_spec_tools/intent.py | 411 ++++++++++++++++++++++++++++++++++++ test/test_intent.py | 181 ++++++++++++++++ 3 files changed, 607 insertions(+) create mode 100644 ovos_spec_tools/intent.py create mode 100644 test/test_intent.py diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index 295fdcc..0af74ba 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -28,6 +28,11 @@ carrier reference implementation, carrying the full §3 registered field set with omission-not-null serialization and the OVOS-PIPELINE-1 §7.1 / OVOS-CONVERSE-1 §2.1 / §2.2 handler-list helpers; +- :class:`~ovos_spec_tools.intent.IntentBuilder` / + :class:`~ovos_spec_tools.intent.Intent` — the adapt-free, plugin-agnostic + keyword intent-definition primitives mapping to the OVOS-INTENT-4 §5 keyword + registration model, with :func:`~ovos_spec_tools.intent.open_intent_envelope` + and the :func:`~ovos_spec_tools.intent.voc_match` ``.voc`` matching helper; - :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also exposed as the ``ovos-spec-lint`` command. """ @@ -49,6 +54,12 @@ lang_matches, standardize_lang, ) +from ovos_spec_tools.intent import ( + Intent, + IntentBuilder, + open_intent_envelope, + voc_match, +) from ovos_spec_tools.lint import ( Finding, declared_slots, @@ -94,6 +105,10 @@ "SESSION1_OWNED_FIELDS", "SESSION1_REGISTERED_FIELDS", "DEFAULT_CONVERSE_HANDLERS_CAP", + "Intent", + "IntentBuilder", + "open_intent_envelope", + "voc_match", "expand", "inline_keywords", "MalformedTemplate", diff --git a/ovos_spec_tools/intent.py b/ovos_spec_tools/intent.py new file mode 100644 index 0000000..fdff44c --- /dev/null +++ b/ovos_spec_tools/intent.py @@ -0,0 +1,411 @@ +"""Plugin-agnostic intent-definition primitives for the OVOS-INTENT-4 keyword model. + +This module is a clean, dependency-light reimplementation of the +``IntentBuilder`` / ``Intent`` pair historically provided by ``ovos-workshop`` +(which vendored Mycroft's ``adapt`` data classes). It carries **no** ``adapt`` +dependency: it is pure data describing the **structure** of a keyword intent — +which vocabularies are *required*, *optional*, *one_of*, or *excluded* — exactly +as OVOS-INTENT-4 §5 defines the ``ovos.intent.register.keyword`` payload. + +The split of responsibility, per OVOS-INTENT-4 §5.1, is: + +- the **builder** captures the intent *structure* (vocabulary **names** under + each role) — that is all a skill expresses when it writes + ``IntentBuilder("Foo").require("Set").one_of("Up", "Down").build()``; +- a **producer** (the skill loader) later inlines each vocabulary's expanded + ``samples`` to form the wire :meth:`Intent.to_keyword_payload` descriptors — + file paths never cross the bus (§5.1). + +Because samples are inlined downstream, the descriptors this module emits carry +``name`` only by default. A producer that already has the expanded samples may +supply them through the ``samples`` argument of :meth:`Intent.to_keyword_payload`. + +The public API is **source-compatible** with the ``ovos-workshop`` classes it +replaces, so skills and intent engines can re-point their imports here without +code changes: + +- ``IntentBuilder(name).require(t, attribute_name=None, optional=False)``, + ``.optionally(t, attribute_name=None)``, ``.one_of(*args)``, + ``.exclude(t)``, ``.build()``, ``.name``; +- ``Intent(name, requires, at_least_one, optional, excludes)`` exposing + ``.name``, ``.requires``, ``.at_least_one``, ``.optional``, ``.excludes``; +- :func:`open_intent_envelope` reconstructing an :class:`Intent` from a Message. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence, Tuple + +__all__ = [ + "Intent", + "IntentBuilder", + "open_intent_envelope", + "voc_match", +] + +# A keyword role entry as the legacy adapt/workshop classes stored it: +# ``(entity_type, attribute_name)`` for required/optional, a bare entity type +# for excludes, and a tuple of entity types for each one_of group. +RoleEntry = Tuple[str, str] + + +class Intent: + """A built keyword-intent **definition** — name plus the four role lists. + + This mirrors the attribute surface of the legacy ``ovos-workshop`` / + ``adapt`` ``Intent`` so existing readers keep working: + + - ``name`` (str) — the intent name (skills munge it to + ``:`` before registration); + - ``requires`` — ``list[(entity_type, attribute_name)]``; every entry MUST + occur (OVOS-INTENT-4 §5.2 ``required``); + - ``at_least_one`` — ``list[tuple[entity_type, ...]]``; each inner tuple is + a group, one member of which MUST occur (§5.2 ``one_of``); + - ``optional`` — ``list[(entity_type, attribute_name)]``; captured if + present (§5.2 ``optional``); + - ``excludes`` — ``list[entity_type]``; any occurrence suppresses the + match (§5.2 ``excluded``). + + Unlike the legacy class this carries **no matching logic** — matching is a + pipeline-plugin concern (OVOS-PIPELINE-1). It is pure data: the structural + half of a ``ovos.intent.register.keyword`` registration, ready for a + producer to inline vocabulary samples into (§5.1). + """ + + def __init__(self, name: str = "", + requires: Optional[Sequence] = None, + at_least_one: Optional[Sequence] = None, + optional: Optional[Sequence] = None, + excludes: Optional[Sequence] = None): + """ + Args: + name: the intent name. + requires: required entries, each ``(entity_type, attribute_name)`` + or a bare ``entity_type`` (normalized to a pair). + at_least_one: ``one_of`` groups, each a sequence of entity types. + optional: optional entries, same shape as ``requires``. + excludes: excluded entity types, each a bare ``entity_type`` (a + ``(type, attr)`` pair is accepted and reduced to its type). + """ + self.name = name + self.requires: List[RoleEntry] = [self._as_pair(r) + for r in (requires or [])] + self.at_least_one: List[Tuple[str, ...]] = [tuple(self._as_name(e) + for e in group) + for group in + (at_least_one or [])] + self.optional: List[RoleEntry] = [self._as_pair(o) + for o in (optional or [])] + self.excludes: List[str] = [self._as_name(e) for e in (excludes or [])] + + @staticmethod + def _as_pair(entry) -> RoleEntry: + """Normalize an entry to ``(entity_type, attribute_name)``. + + Accepts a bare string (attribute defaults to the type) or any 2-tuple. + """ + if isinstance(entry, str): + return (entry, entry) + entity_type, attribute_name = entry[0], entry[1] + return (entity_type, attribute_name or entity_type) + + @staticmethod + def _as_name(entry) -> str: + """Reduce an entry to its bare entity-type name.""" + if isinstance(entry, str): + return entry + return entry[0] + + # -- OVOS-INTENT-4 §5 emission ------------------------------------------- + + @staticmethod + def _descriptor(name: str, + samples: Optional[Dict[str, List[str]]]) -> Dict[str, Any]: + """Build one §5.1 vocabulary descriptor for ``name``. + + ``samples`` is an optional ``vocab_name -> samples`` map a producer may + supply when it has already expanded the vocabularies. When absent — the + usual builder-side case — only ``name`` is emitted and the producer is + expected to inline ``samples`` before the payload crosses the bus + (§5.1). + """ + descriptor: Dict[str, Any] = {"name": name} + if samples and name in samples: + descriptor["samples"] = list(samples[name]) + return descriptor + + def to_keyword_payload(self, skill_id: Optional[str] = None, + lang: Optional[str] = None, + samples: Optional[Dict[str, List[str]]] = None + ) -> Dict[str, Any]: + """Emit the OVOS-INTENT-4 §5.2 keyword-registration structure. + + Returns a dict with the four shape-stable role keys ``required``, + ``optional``, ``one_of``, ``excluded`` (§5.2 mandates all four are + present, even when empty), each a list of vocabulary descriptors (§5.1). + ``one_of`` is a list of groups, each a list of descriptors. + + Identity fields (``skill_id``, ``intent_name``, ``lang``; §3.2) are + included when provided — ``intent_name`` is always set from ``name``. + Vocabulary ``samples`` are inlined per descriptor only when the + ``samples`` map carries them; otherwise the producer inlines them + before emission (§5.1). + + Args: + skill_id: optional ``skill_id`` identity field (§3.2). + lang: optional ``lang`` identity field (§3.2). + samples: optional ``vocab_name -> expanded samples`` map; when a + name is present its descriptor gains a ``samples`` entry. + + Returns: + the §5.2 keyword payload structure. + """ + payload: Dict[str, Any] = {} + if skill_id is not None: + payload["skill_id"] = skill_id + payload["intent_name"] = self.name + if lang is not None: + payload["lang"] = lang + payload["required"] = [self._descriptor(t, samples) + for t, _ in self.requires] + payload["optional"] = [self._descriptor(t, samples) + for t, _ in self.optional] + payload["one_of"] = [[self._descriptor(t, samples) for t in group] + for group in self.at_least_one] + payload["excluded"] = [self._descriptor(t, samples) + for t in self.excludes] + return payload + + def __repr__(self) -> str: + return (f"Intent(name={self.name!r}, requires={self.requires!r}, " + f"at_least_one={self.at_least_one!r}, " + f"optional={self.optional!r}, excludes={self.excludes!r})") + + def __eq__(self, other) -> bool: + if not isinstance(other, Intent): + return NotImplemented + return (self.name == other.name and + self.requires == other.requires and + self.at_least_one == other.at_least_one and + self.optional == other.optional and + self.excludes == other.excludes) + + +class IntentBuilder: + """Fluent builder for a keyword :class:`Intent` — adapt-free. + + Source-compatible with the ``ovos-workshop`` / ``adapt`` ``IntentBuilder``: + it accumulates vocabulary **names** under the four OVOS-INTENT-4 §5 roles + and :meth:`build` freezes them into an :class:`Intent`. It captures only the + intent *structure* (§5.1) — no samples, no matching. + + Example: + >>> intent = (IntentBuilder("SetBrightness") + ... .require("Set") + ... .require("Brightness") + ... .one_of("Up", "Down") + ... .optionally("Politely") + ... .exclude("Question") + ... .build()) + >>> intent.to_keyword_payload()["required"] + [{'name': 'Set'}, {'name': 'Brightness'}] + """ + + def __init__(self, intent_name: str): + """ + Args: + intent_name: the name of the intent being built. + """ + self.name = intent_name + self.requires: List[RoleEntry] = [] + self.at_least_one: List[Tuple[str, ...]] = [] + self.optional: List[RoleEntry] = [] + self.excludes: List[str] = [] + + def require(self, entity_type: str, attribute_name: Optional[str] = None, + optional: bool = False) -> "IntentBuilder": + """Require (or, with ``optional=True``, optionally capture) a vocabulary. + + Args: + entity_type: the vocabulary name. + attribute_name: name of the captured attribute on the match result; + defaults to ``entity_type``. + optional: when True, behaves like :meth:`optionally` — kept for + source-compatibility with the legacy signature. + + Returns: + self, for chaining. + """ + if not attribute_name: + attribute_name = entity_type + if optional: + self.optional.append((entity_type, attribute_name)) + else: + self.requires.append((entity_type, attribute_name)) + return self + + def optionally(self, entity_type: str, + attribute_name: Optional[str] = None) -> "IntentBuilder": + """Optionally capture a vocabulary (OVOS-INTENT-4 §5.2 ``optional``). + + Args: + entity_type: the vocabulary name. + attribute_name: captured-attribute name; defaults to ``entity_type``. + + Returns: + self, for chaining. + """ + if not attribute_name: + attribute_name = entity_type + self.optional.append((entity_type, attribute_name)) + return self + + def one_of(self, *args: str) -> "IntentBuilder": + """Require at least one of the given vocabularies (§5.2 ``one_of``). + + Each call adds one **group**; at least one member of each group must + occur. Separate calls express ``one_of(A, B)`` *and* ``one_of(C, D)``. + + Args: + *args: vocabulary names forming one group. + + Returns: + self, for chaining. + """ + self.at_least_one.append(tuple(args)) + return self + + def exclude(self, entity_type: str) -> "IntentBuilder": + """Forbid a vocabulary (§5.2 ``excluded``): its presence suppresses the + match. + + Args: + entity_type: the vocabulary name to exclude. + + Returns: + self, for chaining. + """ + self.excludes.append(entity_type) + return self + + def build(self) -> Intent: + """Freeze the accumulated roles into an :class:`Intent`.""" + return Intent(self.name, self.requires, self.at_least_one, + self.optional, self.excludes) + + +def open_intent_envelope(message) -> Intent: + """Reconstruct an :class:`Intent` from a Message payload. + + Mirrors the legacy ``ovos-workshop`` helper: it reads an intent definition + out of ``message.data``. Both the legacy serialization keys + (``name`` / ``requires`` / ``at_least_one`` / ``optional`` / ``excludes``, + as produced by ``Intent.__dict__``) and the OVOS-INTENT-4 §5.2 wire keys + (``intent_name`` / ``required`` / ``one_of`` / ``optional`` / ``excluded``, + whose role entries are vocabulary descriptors) are accepted, so the helper + round-trips a definition serialized by either generation. + + For §5.2 descriptors only the ``name`` is read into the role list — the + inlined ``samples`` are not part of the structural :class:`Intent`. + + Args: + message: a Message-like object exposing a ``data`` mapping. + + Returns: + the reconstructed :class:`Intent`. + """ + data = getattr(message, "data", None) + if data is None: + data = message # tolerate being handed a raw dict + + name = data.get("name") or data.get("intent_name") or "" + + def _names(role) -> List[str]: + # Accept §5.2 descriptors ({"name": ...}), bare strings, or legacy + # (type, attr) pairs — reduce each to its vocabulary name. + out = [] + for entry in role or []: + if isinstance(entry, dict): + out.append(entry.get("name")) + elif isinstance(entry, str): + out.append(entry) + else: # (entity_type, attribute_name) + out.append(entry[0]) + return out + + requires = data.get("requires") + if requires is None: + requires = _names(data.get("required")) + + optional = data.get("optional") + # legacy `optional` is already pairs; §5.2 `optional` is descriptors + if optional and isinstance(optional[0], dict): + optional = _names(optional) + + excludes = data.get("excludes") + if excludes is None: + excludes = _names(data.get("excluded")) + + at_least_one = data.get("at_least_one") + if at_least_one is None: + # §5.2 `one_of`: list of groups, each a list of descriptors. + at_least_one = [_names(group) for group in data.get("one_of") or []] + + return Intent(name, requires, at_least_one, optional, excludes) + + +def voc_match(utterance: str, voc_name: str, lang: str, + locale, *, + exact: bool = False, + strip_diacritics: bool = True, + strip_punct: bool = True) -> bool: + """Load a named ``.voc`` and test whether ``utterance`` matches it. + + This is the plugin-agnostic replacement for + ``OVOSAbstractApplication.voc_match`` / the skill ``voc_match`` that + common-query, OCP, and other pipelines historically reached into + ``ovos-workshop`` for. It loads the ``.voc`` for ``lang`` and + matches with whole-word OVOS-INTENT-2 §4.3 semantics — identical to the + skill helper (so pipelines behave unchanged): a sample ``yes`` matches + ``"yes, please"`` but not ``"yesterday"``. + + Args: + utterance: the (ASR-normalized) text to test. + voc_name: the ``.voc`` base name (no extension). + lang: BCP-47 language tag of the resource to load. + locale: either a :class:`~ovos_spec_tools.resources.LocaleResources` + instance, or a ``locale/`` directory path (``str`` / ``Path``), or + a sequence of such paths searched in override-precedence order + (user, skill, core — see + :class:`~ovos_spec_tools.resources.LocaleResources`). + exact: require equality after normalization rather than whole-word + substring containment. + strip_diacritics: forwarded to the matcher. + strip_punct: forwarded to the matcher. + + Returns: + ``True`` iff any ``.voc`` sample matches; ``False`` when the resource + does not exist for the language. + """ + from ovos_spec_tools.resources import LocaleResources + + if isinstance(locale, LocaleResources): + resources = locale + elif isinstance(locale, (str, bytes)) or hasattr(locale, "__fspath__"): + resources = LocaleResources(str(locale)) + else: # a sequence of locale dirs, highest precedence first + dirs = [str(p) for p in locale] + if not dirs: + return False + # Map the precedence-ordered dirs onto the user/skill/core slots, which + # LocaleResources searches in that same order. A single dir is the + # skill locale; the spare slots stay empty. + if len(dirs) == 1: + resources = LocaleResources(skill_locale=dirs[0]) + else: + resources = LocaleResources( + user_locale=dirs[0], + skill_locale=dirs[1], + core_locale=dirs[2] if len(dirs) > 2 else None) + return resources.voc_match( + utterance, voc_name, lang, exact=exact, + strip_diacritics=strip_diacritics, strip_punct=strip_punct) diff --git a/test/test_intent.py b/test/test_intent.py new file mode 100644 index 0000000..042456d --- /dev/null +++ b/test/test_intent.py @@ -0,0 +1,181 @@ +"""Conformance tests for the OVOS-INTENT-4 keyword intent primitives.""" +import pytest + +from ovos_spec_tools import ( + Intent, + IntentBuilder, + LocaleResources, + open_intent_envelope, + voc_match, +) +from ovos_spec_tools.message import Message + + +def _write(path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +# --- IntentBuilder API surface (workshop/adapt-compatible) ------------------- + +def test_builder_accumulates_roles(): + b = (IntentBuilder("SetBrightness") + .require("Set") + .require("Brightness") + .one_of("Up", "Down") + .optionally("Politely") + .exclude("Question")) + assert b.name == "SetBrightness" + assert b.requires == [("Set", "Set"), ("Brightness", "Brightness")] + assert b.at_least_one == [("Up", "Down")] + assert b.optional == [("Politely", "Politely")] + assert b.excludes == ["Question"] + + +def test_builder_require_attribute_name(): + b = IntentBuilder("X").require("Color", "favourite_colour") + assert b.requires == [("Color", "favourite_colour")] + + +def test_builder_require_optional_flag(): + # legacy signature: require(..., optional=True) routes to optional + b = IntentBuilder("X").require("Y", optional=True) + assert b.requires == [] + assert b.optional == [("Y", "Y")] + + +def test_builder_chaining_returns_self(): + b = IntentBuilder("X") + assert b.require("A") is b + assert b.optionally("B") is b + assert b.one_of("C", "D") is b + assert b.exclude("E") is b + + +def test_build_returns_intent_with_attributes(): + intent = IntentBuilder("Name").require("XKeyword").optionally("Y").build() + assert isinstance(intent, Intent) + assert intent.name == "Name" + assert intent.requires == [("XKeyword", "XKeyword")] + assert intent.optional == [("Y", "Y")] + assert intent.at_least_one == [] + assert intent.excludes == [] + + +# --- OVOS-INTENT-4 §5.2 keyword payload mapping ------------------------------ + +def test_to_keyword_payload_structure(): + intent = (IntentBuilder("SetBrightness") + .require("Set") + .require("Brightness") + .one_of("Up", "Down") + .optionally("Politely") + .exclude("Question") + .build()) + payload = intent.to_keyword_payload(skill_id="lighting.skill", + lang="en-US") + assert payload["skill_id"] == "lighting.skill" + assert payload["intent_name"] == "SetBrightness" + assert payload["lang"] == "en-US" + assert payload["required"] == [{"name": "Set"}, {"name": "Brightness"}] + assert payload["optional"] == [{"name": "Politely"}] + assert payload["one_of"] == [[{"name": "Up"}, {"name": "Down"}]] + assert payload["excluded"] == [{"name": "Question"}] + + +def test_to_keyword_payload_all_four_keys_present_when_empty(): + # §5.2: a producer MUST include all four role keys, even when empty. + intent = IntentBuilder("Bare").require("Only").build() + payload = intent.to_keyword_payload() + for key in ("required", "optional", "one_of", "excluded"): + assert key in payload + assert payload["optional"] == [] + assert payload["one_of"] == [] + assert payload["excluded"] == [] + assert payload["intent_name"] == "Bare" + # identity fields omitted when not provided + assert "skill_id" not in payload + assert "lang" not in payload + + +def test_to_keyword_payload_inlines_samples_when_supplied(): + intent = IntentBuilder("X").require("Set").one_of("Up", "Down").build() + samples = {"Set": ["set", "change"], "Up": ["up", "higher"]} + payload = intent.to_keyword_payload(samples=samples) + assert payload["required"] == [{"name": "Set", "samples": ["set", "change"]}] + assert payload["one_of"] == [[ + {"name": "Up", "samples": ["up", "higher"]}, + {"name": "Down"}, # no samples supplied -> name-only + ]] + + +# --- open_intent_envelope round-trip ----------------------------------------- + +def test_open_intent_envelope_legacy_keys(): + intent = (IntentBuilder("Foo").require("A").optionally("B") + .one_of("C", "D").exclude("E").build()) + # legacy serialization == Intent.__dict__ shape + msg = Message("register_intent", { + "name": intent.name, + "requires": intent.requires, + "at_least_one": [list(g) for g in intent.at_least_one], + "optional": intent.optional, + "excludes": intent.excludes, + }) + rebuilt = open_intent_envelope(msg) + assert rebuilt == intent + + +def test_open_intent_envelope_intent4_descriptors(): + intent = (IntentBuilder("Foo").require("A").optionally("B") + .one_of("C", "D").exclude("E").build()) + payload = intent.to_keyword_payload(skill_id="s", lang="en-US") + msg = Message("ovos.intent.register.keyword", payload) + rebuilt = open_intent_envelope(msg) + assert rebuilt == intent + + +def test_open_intent_envelope_accepts_raw_dict(): + rebuilt = open_intent_envelope({"intent_name": "Z", + "required": [{"name": "A"}], + "optional": [], "one_of": [], + "excluded": []}) + assert rebuilt.name == "Z" + assert rebuilt.requires == [("A", "A")] + + +# --- voc_match convenience --------------------------------------------------- + +@pytest.fixture +def locale(tmp_path): + _write(tmp_path / "locale" / "en-US" / "yes.voc", + "yes\nyeah\nyep\nof course\n") + return tmp_path / "locale" + + +def test_voc_match_whole_word_hit(locale): + res = LocaleResources(str(locale)) + assert voc_match("yes, please", "yes", "en-US", res) is True + + +def test_voc_match_whole_word_no_substring(locale): + res = LocaleResources(str(locale)) + # 'yes' must NOT match inside 'yesterday' (whole-word §4.3 semantics) + assert voc_match("yesterday is gone", "yes", "en-US", res) is False + + +def test_voc_match_multiword_entry(locale): + res = LocaleResources(str(locale)) + assert voc_match("well of course i will", "yes", "en-US", res) is True + + +def test_voc_match_accepts_path(locale): + assert voc_match("yeah right", "yes", "en-US", str(locale)) is True + + +def test_voc_match_accepts_sequence_of_dirs(locale): + assert voc_match("yep indeed", "yes", "en-US", [str(locale)]) is True + + +def test_voc_match_missing_voc_returns_false(locale): + assert voc_match("anything", "nonexistent", "en-US", str(locale)) is False From eab420f855dec08f9aa5bb4536b987d266f62889 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:22:59 +0000 Subject: [PATCH 088/110] Increment Version to 0.16.0a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 7f7a874..636a774 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,6 +1,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 15 +VERSION_MINOR = 16 VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 2f0d76dccef5f7d487042a62130bb61da07d16b3 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:23:23 +0000 Subject: [PATCH 089/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13e5f3e..b042ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.16.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.16.0a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.15.0a1...0.16.0a1) + +**Merged pull requests:** + +- feat: plugin-agnostic IntentBuilder/Intent + voc\_match \(INTENT-4 keyword model, adapt-free\) [\#46](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/46) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.15.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.15.0a1) (2026-06-27) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.14.1a1...0.15.0a1) From d07375fcc04b7334c1d68be061ed96d4516f5720 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 03:41:10 +0100 Subject: [PATCH 090/110] fix: message-domain conformance + NamespaceTranslator per-topic payload translation (#42) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: message-domain conformance + NamespaceTranslator per-topic payload translation Part 1 — citation/docstring conformance fixes: - messages.py: AUDIO_OUTPUT_STARTED/ENDED + MIC_LISTEN were mis-cited to AUDIO-IN-1; they are owned by OVOS-AUDIO-1 (audio-out.md, open PR #38, unmerged) §5.1/§5.2/§4.4 -> re-cited and marked genuinely PROVISIONAL. - messages.py: LISTENER_RECORD_STARTED/ENDED, LISTENER_SLEEP, LISTENER_AWOKEN are AUDIO-IN-1 §6.1-§6.4 (MERGED) -> mandated, stale 'provisional' removed. - messages.py: relabel NamespaceTranslator/new_mirror_guard dual-emit + mirror-window dedup as non-normative IMPLEMENTATION POLICY (MSG-1 §5.4 disavows host-side correlation). - message.py: serialize() now refuses an empty 'type' (MSG-1 §2.1/§7 gate). - message.py: deserialize() rejects present-but-non-object data/context ([], 0, false, ...) instead of silently coercing to {} (MSG-1 §6/§7). Part 2 — per-topic payload translation (maintainer-directed feature): - Add MIGRATION_PAYLOAD_TRANSFORMS (legacy topic -> (legacy_to_spec, spec_to_legacy)) with transforms for the 5 shape-changing renames (handler trio, detach_intent, enable/disable). Payload-compatible renames default to identity. - Add NamespaceTranslator.translate_payload(from_topic, to_topic, data) that selects direction and applies the transform (identity if none). Bus wiring (ovos-bus-client / ovos-utils FakeBus) is a follow-up. - Update MIGRATION_MAP / module docstrings: drop the false 'legacy consumers keep working without code changes' blanket claim; document lossy cases. Tests: +25 (serialize empty-type, deserialize wrong-type, transform round-trips, identity, direction selection). 410 pass. Co-Authored-By: Claude Opus 4.8 * fix: AUDIO-1 merged — drop provisional labels + add §7 bus-surface topics AUDIO-1 (audio-out.md) is merged, so its mandated topics are no longer provisional. Strip the unmerged/PR#38/PROVISIONAL qualifiers from the audio output signals (ovos.audio.output.{started,ended} §5.1/§5.2, ovos.mic.listen §4.4) across the module docstring, the SpecMessage block, and the MIGRATION_MAP comments. Add the 6 AUDIO-1 §7 bus-surface topics the enum was missing (enum-only, not legacy renames): SPEAK_B64 (§3.4), AUDIO_SPEECH (§4.3), AUDIO_QUEUE (§4.1), AUDIO_PLAY_SOUND (§4.2), AUDIO_STOP (§6), AUDIO_IS_SPEAKING (§5.3). Co-Authored-By: Claude Opus 4.8 * docs: make message-domain docstrings timeless and standalone Drop development-process framing (merged/now-mandated/no-longer/historically) from the message.py and messages.py docstrings; state the current spec-defined behaviour directly. Runtime-state phrasing (a 'previously disabled' intent, a 'previously-seen' mirror Message) is unchanged. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- ovos_spec_tools/__init__.py | 2 + ovos_spec_tools/message.py | 61 +++-- ovos_spec_tools/messages.py | 430 ++++++++++++++++++++++++++++++------ test/test_message.py | 39 ++++ test/test_messages.py | 179 +++++++++++++++ 5 files changed, 631 insertions(+), 80 deletions(-) diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index 0af74ba..7159fb6 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -69,6 +69,7 @@ ) from ovos_spec_tools.messages import ( MIGRATION_MAP, + MIGRATION_PAYLOAD_TRANSFORMS, SPEC_TO_LEGACY, NamespaceTranslator, SpecMessage, @@ -139,6 +140,7 @@ "Finding", "SpecMessage", "MIGRATION_MAP", + "MIGRATION_PAYLOAD_TRANSFORMS", "SPEC_TO_LEGACY", "migration_counterpart", "NamespaceTranslator", diff --git a/ovos_spec_tools/message.py b/ovos_spec_tools/message.py index 78ca7c4..6366174 100644 --- a/ovos_spec_tools/message.py +++ b/ovos_spec_tools/message.py @@ -70,12 +70,11 @@ class MalformedMessage(ValueError, AssertionError): keys (§2), missing ``type`` (§2), wrong value types, or unparsable JSON (§6). - Inherits from both :class:`ValueError` (the modern, - correctly-typed exception class) **and** :class:`AssertionError` - (the type historically raised by ``ovos_bus_client.Message``'s - bare ``assert`` constructor checks), so legacy ``except - AssertionError`` handlers in downstream code continue to catch - the same conditions. + Inherits from both :class:`ValueError` (the correctly-typed + exception class) **and** :class:`AssertionError` (the type raised by + ``ovos_bus_client.Message``'s bare ``assert`` constructor checks), so + ``except AssertionError`` handlers in downstream code also catch the + same conditions. """ @@ -127,8 +126,8 @@ def __init__(self, msg_type: str, # pattern (``Message("").forward(real_type, data)``) is widely # used to build a routing scaffold before the real topic is # known, so the constructor accepts an empty string here and - # :meth:`serialize` is the gate that flags non-conformant wire - # output (see §7 producer rules). + # :meth:`serialize` is the gate that *rejects* non-conformant wire + # output (an empty ``type`` raises there, per §7 producer rules). if not isinstance(msg_type, str): raise MalformedMessage("msg_type must be a string (§2.1)") if data is not None and not isinstance(data, dict): @@ -229,14 +228,33 @@ def serialize(self) -> str: leave out — framing, encryption, alternative encoders — without touching this envelope contract. + This method is the §7 producer conformance gate: §2.1 requires + ``type`` to be a **non-empty** string and §7 makes a non-empty + ``type`` a producer ``MUST``. The constructor tolerates an empty + ``msg_type`` to allow the construct-then-``forward`` scaffold + pattern, but emitting such a Message would put a malformed + envelope on the wire, so ``serialize`` **refuses** it. + Returns: A single UTF-8 JSON object string conforming to §6. Raises: + MalformedMessage: when ``msg_type`` is empty — §2.1/§7 require a + producer to emit a non-empty ``type``; an empty-``type`` + scaffold Message must be ``forward``/``reply``-derived into + a real topic before it can be serialized. ValueError: from :func:`json.dumps` when ``data`` / ``context`` contain a non-finite number (``allow_nan=False``), enforcing the §6 "Numbers MUST be finite" rule at emit time. """ + # §2.1 / §7 producer MUST: the emitted ``type`` must be a non-empty + # string. The constructor accepts "" for the construct-then-forward + # scaffold; this is the gate that refuses to put it on the wire. + if not self.msg_type: + raise MalformedMessage( + "cannot serialize a Message with an empty 'type' — §2.1/§7 " + "require a producer to emit a non-empty topic; derive a real " + "topic via forward()/reply() before serializing") return json.dumps( {"type": self.msg_type, "data": self._to_jsonable(self.data), @@ -261,7 +279,9 @@ def deserialize(cls, Raises :class:`MalformedMessage` per the §7 ``MUST reject`` conformance rules: unparsable JSON, non-object root, unknown - top-level keys, missing ``type``, or wrong value types. + top-level keys, missing ``type``, or wrong value types — including + a present-but-non-object ``data`` / ``context`` (``[]``, ``0``, + ``false``, …), which §6 forbids silently coercing to ``{}``. """ if isinstance(payload, (bytes, bytearray)): payload = payload.decode("utf-8") @@ -285,9 +305,22 @@ def deserialize(cls, "forbids any key other than 'type', 'data', 'context'") if "type" not in obj: raise MalformedMessage("missing required key 'type' (§2)") - # data and context default to {} per §2.2/§2.3 ("MAY be empty"). - return cls(obj["type"], obj.get("data") or {}, - obj.get("context") or {}) + # §2.2/§2.3: when present, ``data`` and ``context`` MUST be JSON + # objects. An ABSENT key defaults to ``{}`` ("MAY be empty"), but a + # *present* wrong-typed value ([], 0, false, "", null) is malformed + # and §6 forbids silently coercing it — reject per the §7 consumer + # "wrong types" MUST instead of papering over it with ``or {}``. + for key in ("data", "context"): + if key in obj and obj[key] is not None \ + and not isinstance(obj[key], dict): + raise MalformedMessage( + f"'{key}' must be a JSON object when present, got " + f"{type(obj[key]).__name__} (§2.{2 if key == 'data' else 3}/§7)") + # An absent (or explicit null) data/context defaults to {} per + # §2.2/§2.3 ("an absent data/context is equivalent to {}"). + return cls(obj["type"], + obj["data"] if obj.get("data") is not None else {}, + obj["context"] if obj.get("context") is not None else {}) # --- §5 derivations ----------------------------------------------------- @@ -337,8 +370,8 @@ def reply(self, msg_type: str, unchanged (§5.2 step 3). Any keys supplied via ``context`` are overlaid on the copied context - **before** the swap, matching the historical - ``ovos_bus_client.Message.reply`` behaviour: passing + **before** the swap, matching ``ovos_bus_client.Message.reply`` + behaviour: passing ``context={"source": "C", "destination": "D"}`` yields ``source=D, destination=C`` because the §5.2 swap is the final step. Non-routing keys overlaid this way (a custom ``session`` shape, a diff --git a/ovos_spec_tools/messages.py b/ovos_spec_tools/messages.py index 912048a..af50578 100644 --- a/ovos_spec_tools/messages.py +++ b/ovos_spec_tools/messages.py @@ -3,7 +3,7 @@ Specs implemented ----------------- This module is the vocabulary and migration map for the OVOS bus-namespace -move from the historical Mycroft topics to the ``ovos.*`` namespace. Each +move from the Mycroft topics to the ``ovos.*`` namespace. Each :class:`SpecMessage` member names a topic an OVOS specification defines, and is tied to its owning spec section in the per-member comments below: @@ -15,11 +15,17 @@ introspection topics (§§5–8, §10); - **OVOS-STOP-1** — §4.2 ping/pong and §5 global-stop broadcast (``ovos.stop.*``); -- **OVOS-AUDIO-IN-1** — listener / mic / audio-output signals - (``ovos.mic.listen``, ``ovos.listener.*``, ``ovos.audio.output.*``). - These topic *names* are settled (memory: listening signals → AUDIO-IN-1 - ``ovos.listener.*``); the owning spec prose is still landing, so they are - flagged **provisional** at their definitions. +- **OVOS-AUDIO-IN-1** — listener lifecycle signals (``ovos.listener.*``). + AUDIO-IN-1 §6.1–§6.4 mandates ``ovos.listener.record.started`` / + ``.record.ended`` / ``ovos.listener.sleep`` / ``ovos.listener.awoken``; +- **OVOS-AUDIO-1** — the audio **output** service bus surface (§7), + mandated. Two rendering modes (``ovos.utterance.speak.b64`` + §3.4 → ``ovos.audio.speech`` §4.3 for remote clients); the playback model + (``ovos.audio.queue`` §4.1 scheduled sounds, ``ovos.audio.play_sound`` §4.2 + instant sounds); the output-lifecycle signals (``ovos.audio.output.started`` + / ``.output.ended`` §5.1/§5.2); the speaking-status query + (``ovos.audio.is_speaking`` §5.3); stop integration (``ovos.audio.stop`` + §6); and the listen-flag mic re-open (``ovos.mic.listen`` §4.4). Why an enum ----------- @@ -29,23 +35,45 @@ visibly legacy or implementation-specific. The enum is the *vocabulary* half of the migration; :data:`MIGRATION_MAP` is the *rename* half. -The transparent bridge (``MIGRATION_MAP`` / :class:`NamespaceTranslator`) ------------------------------------------------------------------------- +The legacy↔``ovos.*`` bridge (``MIGRATION_MAP`` / :class:`NamespaceTranslator`) +------------------------------------------------------------------------------- ``MIGRATION_MAP`` maps each legacy topic to the ``SpecMessage`` that replaces -it. The bridge is **payload-compatible and transparent**: a producer emits -only the spec topic, and the bus dual-emit (``ovos-bus-client``'s -``MessageBusClient`` and ``ovos_utils.fakebus.FakeBus``, both driven by -:class:`NamespaceTranslator`) *also* delivers the **same payload** under the -counterpart topic, so consumers still subscribed to the legacy name during -the migration window keep working without code changes. The receive side -deduplicates the mirror (see :meth:`NamespaceTranslator.new_mirror_guard`) so -a handler subscribed to both names runs once. - -Because the bridge mirrors the payload **verbatim**, only renames that need -no payload transformation can live in the map. The two deliberate exclusions -are documented at :data:`MIGRATION_MAP`: the OVOS-INTENT-4 registration -*consolidation* (an N→1 restructure the bus cannot synthesize) and the -per-skill placeholder ``ovos.stop.*`` ping topics (not static strings). +it. The bus dual-emit (``ovos-bus-client``'s ``MessageBusClient`` and +``ovos_utils.fakebus.FakeBus``, both driven by :class:`NamespaceTranslator`) +mirrors an emission onto its counterpart topic so a consumer still subscribed +to the legacy name during the migration window keeps receiving the event. The +receive side deduplicates the mirror (see +:meth:`NamespaceTranslator.new_mirror_guard`) so a handler subscribed to both +names runs once. + +Payload translation, not verbatim mirroring. Some renames are *payload +compatible* (the legacy and spec topics carry the same ``data`` shape) and the +mirror forwards the payload unchanged. Others are *shape-changing* renames +(the handler trio, the INTENT-4 management topics): for those, forwarding the +producer's payload verbatim would hand a legacy-only consumer spec-shaped +``data`` it cannot read. :data:`MIGRATION_PAYLOAD_TRANSFORMS` therefore pairs +each shape-changing topic with two pure functions and +:meth:`NamespaceTranslator.translate_payload` applies the right one per +direction, so the mirrored payload is in the **recipient's** shape. Several +transforms are **best-effort / lossy** (a field that does not exist in the +other shape is synthesized or dropped); each lossy case is documented at +:data:`MIGRATION_PAYLOAD_TRANSFORMS`. The "legacy consumers keep working +without code changes" guarantee is therefore scoped: it holds outright for +payload-compatible renames, and holds *best-effort* (with documented loss) for +the shape-changing ones. + +Two renames stay out of the map entirely because no per-topic payload +transform can bridge them: the OVOS-INTENT-4 *registration* consolidation (an +N→1 restructure the stateless bus cannot synthesize) and the per-skill +placeholder ``ovos.stop.*`` ping topics (not static strings). Both are +documented at :data:`MIGRATION_MAP`. + +Note on dual-emit and MSG-1 §5.4. The dual-emit/mirror-window behaviour is a +non-normative **implementation policy** of this migration tooling, not a +spec-mandated mechanism. No OVOS specification mandates dual-emit, and +OVOS-MSG-1 §5.4 explicitly disavows any host-side correlation/bookkeeping of +the kind a mirror window resembles; the window here is a pragmatic dedup +heuristic for the migration period, scoped to :class:`NamespaceTranslator`. Out of enum / map scope (OVOS-MSG-1 §2.1.1 runtime-assembled topics) -------------------------------------------------------------------- @@ -58,7 +86,7 @@ import json as _json import time from enum import Enum -from typing import Callable, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple class SpecMessage(str, Enum): @@ -72,9 +100,8 @@ class SpecMessage(str, Enum): bus.on(SpecMessage.SPEAK, handler) bus.emit(Message(SpecMessage.UTTERANCE, {...})) - The enum grows as specs are catalogued/merged; absence of a topic here - does not imply it is not spec-defined. Topics from specs still landing are - flagged *provisional* in the comments. + Absence of a topic here does not imply it is not spec-defined; topics from + specs not yet catalogued are flagged *provisional* in the comments. """ def __str__(self) -> str: @@ -138,45 +165,86 @@ def __str__(self) -> str: #: §5.3 — universal stop broadcast; ``ovos.stop.*`` namespace is reserved by STOP-1. STOP = "ovos.stop" - # --- OVOS-AUDIO-IN-1 listener / mic / audio-output signals (provisional) --- - # Topic names settled; owning AUDIO-IN-1 prose still landing (see module docstring). + # --- OVOS-AUDIO-1 audio-OUTPUT service bus surface (§7, mandated) --- + # Defined by OVOS-AUDIO-1 (audio-out.md), NOT by AUDIO-IN-1. + #: §3.4 — remote-client rendering mode; same TTS pipeline as ``SPEAK`` but + #: emits ``ovos.audio.speech`` (b64) instead of enqueueing local playback. + SPEAK_B64 = "ovos.utterance.speak.b64" + #: §4.3 — synthesised audio (base64) emitted for a ``SPEAK_B64`` Message; a + #: bridge relays it to the remote client; audio-output → broadcast. + AUDIO_SPEECH = "ovos.audio.speech" + #: §4.1 — queue a sound for sequential (FIFO) scheduled playback; any → audio. + AUDIO_QUEUE = "ovos.audio.queue" + #: §4.2 — fire-and-forget instant sound, played immediately over the queue; + #: any → audio. + AUDIO_PLAY_SOUND = "ovos.audio.play_sound" + #: §6 — stop audio output: clear the scheduled queue and terminate playback; + #: any → audio. + AUDIO_STOP = "ovos.audio.stop" + #: §5.3 — query whether the audio output service is currently speaking; the + #: service answers ``{"speaking": bool}`` (§7 names no distinct response + #: topic, so the reply is ``reply``-derived, MSG-1 §5.2); any → audio. + AUDIO_IS_SPEAKING = "ovos.audio.is_speaking" + #: §5.1 — playback session started (idle→active); audio-output → broadcast. AUDIO_OUTPUT_STARTED = "ovos.audio.output.started" + #: §5.2 — playback session ended (queue empty, last item done); audio-output + #: → broadcast. AUDIO_OUTPUT_ENDED = "ovos.audio.output.ended" + #: §4.4 — re-open the mic after a ``listen: true`` item; audio-output → + #: broadcast. MIC_LISTEN = "ovos.mic.listen" + + # --- OVOS-AUDIO-IN-1 listener lifecycle signals (§6, mandated) --- + #: §6.1 — voice-command capture began; audio-input → broadcast. LISTENER_RECORD_STARTED = "ovos.listener.record.started" + #: §6.2 — voice-command capture ended; audio-input → broadcast. LISTENER_RECORD_ENDED = "ovos.listener.record.ended" + #: §6.3 — controller → audio-input: enter sleep mode and suspend capture. LISTENER_SLEEP = "ovos.listener.sleep" + #: §6.4 — left sleep mode (sleep→awake transition); audio-input → broadcast. LISTENER_AWOKEN = "ovos.listener.awoken" #: Legacy (Mycroft-era) topic -> the :class:`SpecMessage` that supersedes it. #: This is the single source of truth for the legacy↔``ovos.*`` renames that -#: the bus bridges transparently. ``NamespaceTranslator`` reads it (and its -#: reverse, :data:`SPEC_TO_LEGACY`) so a producer emits only the spec topic and -#: the bus dual-emits the legacy counterpart carrying the **same payload** — -#: consumers still on a legacy topic during the migration window keep receiving -#: data. Every entry here is a rename the bus can bridge **without transforming -#: the payload**; the two deliberate exclusions are documented at the bottom. +#: the bus bridges. ``NamespaceTranslator`` reads it (and its reverse, +#: :data:`SPEC_TO_LEGACY`) to pick the counterpart topic; it then translates the +#: payload into the recipient's shape via :data:`MIGRATION_PAYLOAD_TRANSFORMS` +#: (identity for payload-compatible renames), so a consumer still on the legacy +#: topic during the migration window receives ``data`` it can read. +#: +#: Two kinds of entry live here: +#: +#: * **Payload-compatible renames** — legacy and spec topics carry the same +#: ``data`` shape; the mirror forwards the payload unchanged (identity +#: transform). Most entries are of this kind. +#: * **Shape-changing renames** — the handler trio +#: (``mycroft.skill.handler.*``) and the INTENT-4 management topics +#: (``detach_intent``, ``enable_intent``/``disable_intent``) change the +#: payload shape across the rename. For these the bridge does NOT forward the +#: payload verbatim: :data:`MIGRATION_PAYLOAD_TRANSFORMS` carries a +#: best-effort, sometimes lossy transform pair (documented there) that +#: reshapes the payload per direction. #: -#: For the renames marked "payload restructured" below (the handler trio and the -#: INTENT-4 management topics), the *bridge itself* still does not transform -#: anything: once the **producer** has adopted the spec payload shape -#: ({skill_id, intent_name}), the mirror carries that already-modern payload on -#: the legacy topic too. The restructure is a producer-side adoption, the bridge -#: is the topic rename — they compose, but the map only owns the rename. +#: The two deliberate exclusions (registration consolidation, per-skill stop +#: ping placeholders) are documented at the bottom — no per-topic payload +#: transform can bridge them. MIGRATION_MAP: Dict[str, SpecMessage] = { - # --- AUDIO-IN-1 / PIPELINE-1 §9 (payload-compatible 1:1 renames) --- + # --- PIPELINE-1 §9 utterance layer (payload-compatible 1:1 renames) --- "recognizer_loop:utterance": SpecMessage.UTTERANCE, # PIPELINE-1 §9.1 "speak": SpecMessage.SPEAK, # PIPELINE-1 §9.6 - "recognizer_loop:audio_output_start": SpecMessage.AUDIO_OUTPUT_STARTED, # AUDIO-IN-1 (provisional) - "recognizer_loop:audio_output_end": SpecMessage.AUDIO_OUTPUT_ENDED, # AUDIO-IN-1 (provisional) - "mycroft.mic.listen": SpecMessage.MIC_LISTEN, # AUDIO-IN-1 (provisional) - "recognizer_loop:record_begin": SpecMessage.LISTENER_RECORD_STARTED, # AUDIO-IN-1 (provisional) - "recognizer_loop:record_end": SpecMessage.LISTENER_RECORD_ENDED, # AUDIO-IN-1 (provisional) - "recognizer_loop:sleep": SpecMessage.LISTENER_SLEEP, # AUDIO-IN-1 (provisional) - "mycroft.awoken": SpecMessage.LISTENER_AWOKEN, # AUDIO-IN-1 (provisional) - # --- PIPELINE-1 §8 handler-lifecycle trio (rename; payload adopted by the - # producer to {skill_id, intent_name} — bridge mirrors verbatim) --- + # --- AUDIO-1 §5.1/§5.2/§4.4 audio-output signals (payload-compatible + # 1:1 renames) --- + "recognizer_loop:audio_output_start": SpecMessage.AUDIO_OUTPUT_STARTED, # AUDIO-1 §5.1 + "recognizer_loop:audio_output_end": SpecMessage.AUDIO_OUTPUT_ENDED, # AUDIO-1 §5.2 + "mycroft.mic.listen": SpecMessage.MIC_LISTEN, # AUDIO-1 §4.4 + # --- AUDIO-IN-1 §6 listener lifecycle (payload-compatible renames) --- + "recognizer_loop:record_begin": SpecMessage.LISTENER_RECORD_STARTED, # AUDIO-IN-1 §6.1 + "recognizer_loop:record_end": SpecMessage.LISTENER_RECORD_ENDED, # AUDIO-IN-1 §6.2 + "recognizer_loop:sleep": SpecMessage.LISTENER_SLEEP, # AUDIO-IN-1 §6.3 + "mycroft.awoken": SpecMessage.LISTENER_AWOKEN, # AUDIO-IN-1 §6.4 + # --- PIPELINE-1 §8 handler-lifecycle trio (SHAPE-CHANGING rename; payload + # reshaped per direction by MIGRATION_PAYLOAD_TRANSFORMS, best-effort) --- "mycroft.skill.handler.start": SpecMessage.INTENT_HANDLER_START, # §8.1 "mycroft.skill.handler.complete": SpecMessage.INTENT_HANDLER_COMPLETE, # §8.1 "mycroft.skill.handler.error": SpecMessage.INTENT_HANDLER_ERROR, # §8.1 @@ -185,12 +253,14 @@ def __str__(self) -> str: "mycroft.stop": SpecMessage.STOP, # STOP-1 §5.3 universal stop broadcast # --- PIPELINE-1 §9.3 intent outcome (1:1 rename) --- "complete_intent_failure": SpecMessage.INTENT_UNMATCHED, - # --- INTENT-4 §8 intent management (renames; producer adopts the - # {skill_id, intent_name} payload — bridge mirrors verbatim) --- - "detach_intent": SpecMessage.INTENT_DEREGISTER, # §8.2 - "detach_skill": SpecMessage.SKILL_DEREGISTER, # §8.4 - "mycroft.skill.enable_intent": SpecMessage.INTENT_ENABLE, # §8.5 - "mycroft.skill.disable_intent": SpecMessage.INTENT_DISABLE, # §8.5 + # --- INTENT-4 §8 intent management --- + # detach_intent / enable_intent / disable_intent are SHAPE-CHANGING renames + # (legacy munged/partial payload vs spec {skill_id, intent_name, lang}) — + # reshaped per direction by MIGRATION_PAYLOAD_TRANSFORMS (best-effort). + "detach_intent": SpecMessage.INTENT_DEREGISTER, # §8.2 (shape-changing) + "detach_skill": SpecMessage.SKILL_DEREGISTER, # §8.4 (payload-compatible: {skill_id}) + "mycroft.skill.enable_intent": SpecMessage.INTENT_ENABLE, # §8.5 (shape-changing) + "mycroft.skill.disable_intent": SpecMessage.INTENT_DISABLE, # §8.5 (shape-changing) # # ---- DELIBERATE EXCLUSION 1: INTENT-4 §5–§7 *registration* ---- # ovos.intent.register.keyword / .register.template / ovos.entity.register @@ -218,6 +288,162 @@ def __str__(self) -> str: SPEC_TO_LEGACY: Dict[str, str] = {v.value: k for k, v in MIGRATION_MAP.items()} +# --------------------------------------------------------------------------- +# Per-topic payload translation +# --------------------------------------------------------------------------- +# A migrating topic that is a *payload-compatible* rename carries the same +# ``data`` shape on both names, so the mirror forwards the payload unchanged. +# A *shape-changing* rename does not: the legacy and spec topics describe the +# same event with different fields, so forwarding the producer's payload +# verbatim would hand the recipient ``data`` in the wrong shape. For those, the +# bridge reshapes the payload with a pair of pure functions. + +#: One ``(legacy_to_spec, spec_to_legacy)`` transform pair. Each function takes +#: a payload ``dict`` and returns a NEW payload ``dict`` in the other shape; it +#: must never mutate its input. The pair is keyed by the **legacy** topic in +#: :data:`MIGRATION_PAYLOAD_TRANSFORMS`. +PayloadTransform = Callable[[Dict[str, Any]], Dict[str, Any]] + + +def _identity(data: Dict[str, Any]) -> Dict[str, Any]: + """Payload-compatible default: copy the payload through unchanged.""" + return dict(data) + + +# --- handler trio: mycroft.skill.handler.{start,complete,error} ↔ +# ovos.intent.handler.{start,complete,error} (PIPELINE-1 §8) --- +# +# Legacy shape (Mycroft-era producers): +# start/complete: {"handler": ""} (+ "duration" on complete) +# error: {"handler": "", "traceback": ""} +# (skill_id rode in Message.context, not data — see ovos_workshop/skills/ovos.py) +# Spec shape (PIPELINE-1 §8): {"skill_id", "intent_name"} (+ "exception" on error) +# +# LOSSY mapping (documented): +# - legacy ``handler`` (a handler **function** name) is mapped to/from spec +# ``intent_name`` BEST-EFFORT only: they are related but not identical +# (a handler fn name is not guaranteed to equal the intent name). +# - legacy→spec cannot recover ``skill_id`` from ``data`` (it lived in +# ``context``); it is omitted. The bus SHOULD lift it from ``context`` when +# wiring — out of scope for this pure-data transform. +# - ``complete``'s legacy ``duration`` has no spec field → DROPPED on +# legacy→spec; spec→legacy cannot synthesize it → OMITTED. +# - ``error``: legacy ``traceback`` ↔ spec ``exception`` are mapped to each +# other (both human-readable failure text). + +def _handler_legacy_to_spec(data: Dict[str, Any]) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if "skill_id" in data: # rarely present in data, but honour it if so + out["skill_id"] = data["skill_id"] + if "handler" in data: + out["intent_name"] = data["handler"] # best-effort + # error: legacy ``traceback`` -> spec ``exception`` + if "traceback" in data: + out["exception"] = data["traceback"] + elif "exception" in data: + out["exception"] = data["exception"] + # ``duration`` (complete) has no spec field -> dropped. + return out + + +def _handler_spec_to_legacy(data: Dict[str, Any]) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if "intent_name" in data: + out["handler"] = data["intent_name"] # best-effort + if "skill_id" in data: + out["skill_id"] = data["skill_id"] # harmless extra on legacy topic + # error: spec ``exception`` -> legacy ``traceback`` + if "exception" in data: + out["traceback"] = data["exception"] + # spec has no ``duration`` to restore -> omitted. + return out + + +# --- detach_intent ↔ ovos.intent.deregister (INTENT-4 §8.2) --- +# Legacy: {"intent_name": ":"} (the munged form — see +# ovos_workshop/intents.py::detach_intent). +# Spec: {"skill_id", "intent_name", "lang"?} (§8.2; lang optional). +# CLEANLY BIDIRECTIONAL on skill_id/intent_name: +# spec->legacy joins ":"; legacy->spec splits on the +# FIRST ":" (MSG-1 §2.1.1: skill_id must not contain ":"). +# LOSSY: legacy carries no ``lang`` -> legacy->spec omits it (spec §8.2 allows +# an omitted lang = "all languages"); spec->legacy drops ``lang``. + +def _detach_legacy_to_spec(data: Dict[str, Any]) -> Dict[str, Any]: + munged = data.get("intent_name", "") + if ":" in munged: + skill_id, intent_name = munged.split(":", 1) + return {"skill_id": skill_id, "intent_name": intent_name} + # No separator -> cannot split; best-effort pass the whole as intent_name. + return {"intent_name": munged} if munged else {} + + +def _detach_spec_to_legacy(data: Dict[str, Any]) -> Dict[str, Any]: + skill_id = data.get("skill_id", "") + intent_name = data.get("intent_name", "") + if skill_id: + return {"intent_name": f"{skill_id}:{intent_name}"} + # No skill_id to join -> emit the bare intent_name (best-effort). + return {"intent_name": intent_name} + + +# --- mycroft.skill.{enable,disable}_intent ↔ ovos.intent.{enable,disable} +# (INTENT-4 §8.5) --- +# Legacy: {"intent_name": ""} (no skill_id, no lang) +# Spec: {"skill_id", "intent_name", "lang"?} (§8.5) +# LOSSY: legacy->spec CANNOT recover ``skill_id`` (absent in the legacy +# payload) -> it is OMITTED, and ``lang`` likewise (spec treats an omitted +# lang as "all languages", §8.5). This is the documented limitation: a +# legacy-sourced enable/disable bridged to the spec topic targets the intent +# name without a skill scope; a consumer that needs skill scoping must obtain +# it elsewhere (e.g. Message.context["skill_id"]). spec->legacy drops +# skill_id/lang, keeping only intent_name. + +def _toggle_legacy_to_spec(data: Dict[str, Any]) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if "intent_name" in data: + out["intent_name"] = data["intent_name"] + # skill_id / lang not recoverable from the legacy payload -> omitted. + return out + + +def _toggle_spec_to_legacy(data: Dict[str, Any]) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if "intent_name" in data: + out["intent_name"] = data["intent_name"] + # spec skill_id / lang have no legacy field -> dropped. + return out + + +#: **legacy topic** -> ``(legacy_to_spec, spec_to_legacy)`` payload transforms. +#: +#: Only the SHAPE-CHANGING renames need an entry; every other migrating topic +#: is payload-compatible and uses the IDENTITY transform implicitly (see +#: :meth:`NamespaceTranslator.translate_payload`). Keyed by the legacy topic so +#: both :data:`MIGRATION_MAP` (legacy->spec) and :data:`SPEC_TO_LEGACY` +#: (spec->legacy) resolve to the same pair. +#: +#: Lossy cases (each documented at the transform above): +#: +#: * **handler trio** — ``handler`` ↔ ``intent_name`` is best-effort (a handler +#: function name is not guaranteed to equal the intent name); ``skill_id`` is +#: not recoverable from legacy ``data``; ``duration`` (complete) is dropped; +#: ``traceback`` ↔ ``exception`` are mapped. +#: * **detach_intent** — cleanly bidirectional on ``skill_id``/``intent_name``; +#: ``lang`` is not present in the legacy payload. +#: * **enable/disable** — legacy carries neither ``skill_id`` nor ``lang``, so +#: legacy->spec omits both (a legacy-sourced toggle has no skill scope in +#: ``data``). +MIGRATION_PAYLOAD_TRANSFORMS: Dict[str, Tuple[PayloadTransform, PayloadTransform]] = { + "mycroft.skill.handler.start": (_handler_legacy_to_spec, _handler_spec_to_legacy), + "mycroft.skill.handler.complete": (_handler_legacy_to_spec, _handler_spec_to_legacy), + "mycroft.skill.handler.error": (_handler_legacy_to_spec, _handler_spec_to_legacy), + "detach_intent": (_detach_legacy_to_spec, _detach_spec_to_legacy), + "mycroft.skill.enable_intent": (_toggle_legacy_to_spec, _toggle_spec_to_legacy), + "mycroft.skill.disable_intent": (_toggle_legacy_to_spec, _toggle_spec_to_legacy), +} + + def migration_counterpart(topic: str) -> Optional[str]: """Return the other-namespace counterpart of a migrating topic, else ``None``. @@ -243,20 +469,31 @@ def migration_counterpart(topic: str) -> Optional[str]: class NamespaceTranslator: """Shared legacy↔``ovos.*`` bus-namespace migration logic (OVOS bus bridge). - This is the **reference implementation of the transparent dual-emit bridge** - that ``ovos-bus-client``'s ``MessageBusClient`` and ``ovos_utils``' - ``FakeBus`` both delegate to, so the real websocket bus and the - test/satellite double behave identically. It carries no I/O and no config: - the two direction flags are passed in (the caller reads env/config), - keeping ``ovos-spec-tools`` dependency-free. + This is the shared **migration tooling** that ``ovos-bus-client``'s + ``MessageBusClient`` and ``ovos_utils``' ``FakeBus`` both delegate to, so + the real websocket bus and the test/satellite double behave identically. It + carries no I/O and no config: the two direction flags are passed in (the + caller reads env/config), keeping ``ovos-spec-tools`` dependency-free. + + .. note:: + + **Non-normative implementation policy.** The dual-emit + mirror-window + dedup behaviour described below is a pragmatic migration *policy* of + this tooling, **not** a spec-mandated mechanism. No OVOS specification + mandates dual-emit, and OVOS-MSG-1 §5.4 explicitly disavows any + host-side correlation/bookkeeping of the kind a mirror window resembles. + The window is a best-effort dedup heuristic scoped to the migration + period and to this class; it is not part of any conformance surface. The bridge has two halves, matching the two halves of a dual-emit bus: - **send side** — :meth:`counterpart_topics` tells the bus which extra topic to mirror an emission onto, so a producer that emits only the spec - topic still reaches legacy subscribers (and vice-versa). The payload is - copied verbatim (only renames that need no payload transformation are in - :data:`MIGRATION_MAP`). + topic still reaches legacy subscribers (and vice-versa); the bus calls + :meth:`translate_payload` to reshape the mirrored payload into the + recipient's shape (identity for payload-compatible renames, a best-effort + transform for shape-changing ones — see + :data:`MIGRATION_PAYLOAD_TRANSFORMS`). - **receive side** — :meth:`new_mirror_guard` lets a handler subscribed to *both* names run exactly once, by recognising the mirror re-delivery and dropping it. :meth:`is_migrated` is the cheap pre-check ("does this topic @@ -283,7 +520,9 @@ def counterpart_topics(self, msg_type: str) -> List[str]: """Extra topic(s) the bus should ALSO emit ``msg_type`` on (0 or 1). The result is the **send-side** half of the dual-emit: the bus emits the - original Message, then re-emits the same payload on each returned topic. + original Message, then re-emits — on each returned topic — the payload + produced by :meth:`translate_payload` (reshaped into the counterpart + topic's shape, identity for payload-compatible renames). Args: msg_type: the topic the producer asked to emit. @@ -301,6 +540,59 @@ def counterpart_topics(self, msg_type: str) -> List[str]: return [SPEC_TO_LEGACY[msg_type]] return [] + def translate_payload(self, from_topic: str, to_topic: str, + data: Dict[str, Any]) -> Dict[str, Any]: + """Reshape ``data`` from ``from_topic``'s shape into ``to_topic``'s shape. + + This is the **payload half** of the bridge, the companion to + :meth:`counterpart_topics`: when the bus mirrors an emission onto the + counterpart topic, it calls this to translate the payload so a consumer + on the counterpart topic receives ``data`` in *its* shape rather than + the producer's. + + Direction is inferred from ``from_topic``: + + - ``from_topic`` is a **legacy** topic (in :data:`MIGRATION_MAP`) → + apply the ``legacy_to_spec`` transform; + - ``from_topic`` is a **spec** topic (in :data:`SPEC_TO_LEGACY`) → + apply the ``spec_to_legacy`` transform. + + If the migrating topic has no entry in + :data:`MIGRATION_PAYLOAD_TRANSFORMS` (a payload-compatible rename) the + transform is the **identity** — a shallow copy of ``data`` unchanged. + The same identity copy is returned when ``from_topic``/``to_topic`` are + not a migrating pair at all, so the method is always safe to call. + + The transforms are pure: ``data`` is never mutated, and a NEW dict is + always returned. Some transforms are **best-effort / lossy** — see + :data:`MIGRATION_PAYLOAD_TRANSFORMS` for the per-topic loss notes. + + Args: + from_topic: the topic the Message was emitted on (legacy or spec). + to_topic: the counterpart topic the mirror is being emitted on. Used + only to confirm direction; the transform is keyed off the legacy + topic of the pair. + data: the source Message's payload. + + Returns: + A new payload ``dict`` in ``to_topic``'s shape. + """ + data = data or {} + # Resolve the legacy topic of this migrating pair (the transform key), + # and which direction we are translating. + if from_topic in MIGRATION_MAP: + legacy_topic, direction = from_topic, 0 # legacy -> spec + elif from_topic in SPEC_TO_LEGACY: + legacy_topic, direction = SPEC_TO_LEGACY[from_topic], 1 # spec -> legacy + else: + # Not a migrating topic — nothing to translate, return a copy. + return dict(data) + transform = MIGRATION_PAYLOAD_TRANSFORMS.get(legacy_topic) + if transform is None: + # Payload-compatible rename: identity. + return dict(data) + return transform[direction](data) + def is_migrated(self, topic: str) -> bool: """Whether ``topic`` participates in the migration (so needs dedup). @@ -319,6 +611,12 @@ def new_mirror_guard(self, ) -> Callable[[object], bool]: """Build a stateful ``is_mirror(message) -> bool`` for receive-side dedup. + **Non-normative implementation policy** (migration tooling): the + mirror-window dedup below is a pragmatic heuristic for the migration + period, not a spec-mandated behaviour. OVOS-MSG-1 §5.4 disavows + host-side correlation; this window is deliberately narrow and scoped to + :class:`NamespaceTranslator` so it cannot be mistaken for one. + Each call returns a **fresh closure with its own private ``seen`` state**, so one guard is created per handler (or per subscription) and guards do not cross-talk. The returned predicate answers: *is this Message the diff --git a/test/test_message.py b/test/test_message.py index 27b5f8e..ec2a401 100644 --- a/test/test_message.py +++ b/test/test_message.py @@ -137,6 +137,45 @@ def test_serialize_data_can_be_empty(self): assert parsed["data"] == {} assert parsed["context"] == {} + def test_serialize_rejects_empty_type(self): + """§2.1/§7 producer MUST: ``serialize`` is the conformance gate and + MUST refuse to emit an empty ``type``.""" + with pytest.raises(MalformedMessage): + Message("").serialize() + + def test_serialize_after_forward_from_empty_scaffold_ok(self): + """The construct-then-forward scaffold becomes serializable once it + has a real topic.""" + m = Message("").forward("ovos.real", {"k": "v"}) + parsed = json.loads(m.serialize()) + assert parsed["type"] == "ovos.real" + + def test_deserialize_rejects_wrong_type_data(self): + """§2.2/§6: a present-but-non-object ``data`` MUST be rejected, not + silently coerced to ``{}``.""" + for bad in ([], 0, False, "x"): + with pytest.raises(MalformedMessage): + Message.deserialize({"type": "ovos.test", "data": bad}) + + def test_deserialize_rejects_wrong_type_context(self): + """§2.3/§6: a present-but-non-object ``context`` MUST be rejected.""" + for bad in ([], 0, False, "x"): + with pytest.raises(MalformedMessage): + Message.deserialize({"type": "ovos.test", "context": bad}) + + def test_deserialize_absent_or_null_data_defaults_to_empty(self): + """§2.2/§2.3: an absent (or explicit null) ``data``/``context`` is + equivalent to ``{}`` — that is still allowed.""" + m = Message.deserialize({"type": "ovos.test"}) + assert m.data == {} and m.context == {} + m2 = Message.deserialize({"type": "ovos.test", "data": None, + "context": None}) + assert m2.data == {} and m2.context == {} + + def test_deserialize_preserves_empty_dict_data(self): + m = Message.deserialize({"type": "ovos.test", "data": {}}) + assert m.data == {} + # --- §5.1 forward ----------------------------------------------------------- diff --git a/test/test_messages.py b/test/test_messages.py index 6fa58ba..88847b1 100644 --- a/test/test_messages.py +++ b/test/test_messages.py @@ -3,6 +3,7 @@ from ovos_spec_tools import ( MIGRATION_MAP, + MIGRATION_PAYLOAD_TRANSFORMS, SPEC_TO_LEGACY, NamespaceTranslator, SpecMessage, @@ -80,6 +81,25 @@ def test_no_duplicate_values(self): values = [m.value for m in SpecMessage] self.assertEqual(len(values), len(set(values))) + def test_audio1_bus_surface_members(self): + # OVOS-AUDIO-1 §7 bus surface — enum-only topics (not legacy renames). + self.assertEqual(SpecMessage.SPEAK_B64, "ovos.utterance.speak.b64") # §3.4 + self.assertEqual(SpecMessage.AUDIO_SPEECH, "ovos.audio.speech") # §4.3 + self.assertEqual(SpecMessage.AUDIO_QUEUE, "ovos.audio.queue") # §4.1 + self.assertEqual(SpecMessage.AUDIO_PLAY_SOUND, "ovos.audio.play_sound") # §4.2 + self.assertEqual(SpecMessage.AUDIO_STOP, "ovos.audio.stop") # §6 + self.assertEqual(SpecMessage.AUDIO_IS_SPEAKING, "ovos.audio.is_speaking") # §5.3 + + def test_audio1_members_are_enum_only_not_migration(self): + # These 6 are not legacy renames -> must not appear in the migration map + # or its payload transforms. + for m in (SpecMessage.SPEAK_B64, SpecMessage.AUDIO_SPEECH, + SpecMessage.AUDIO_QUEUE, SpecMessage.AUDIO_PLAY_SOUND, + SpecMessage.AUDIO_STOP, SpecMessage.AUDIO_IS_SPEAKING): + self.assertNotIn(m.value, SPEC_TO_LEGACY) + self.assertIsNone(migration_counterpart(m.value)) + self.assertNotIn(m.value, MIGRATION_PAYLOAD_TRANSFORMS) + class TestMigrationMap(unittest.TestCase): def test_maps_legacy_to_specmessage(self): @@ -112,5 +132,164 @@ def test_roundtrip(self): self.assertEqual(migration_counterpart(spec), legacy) +class TestPayloadTransforms(unittest.TestCase): + """Per-topic payload translation (MIGRATION_PAYLOAD_TRANSFORMS).""" + + def setUp(self): + self.t = NamespaceTranslator() + + # --- structure invariants --- + def test_transform_keys_are_legacy_topics(self): + for legacy in MIGRATION_PAYLOAD_TRANSFORMS: + self.assertIn(legacy, MIGRATION_MAP, + f"{legacy} is not a known legacy topic") + + def test_only_shape_changing_topics_have_transforms(self): + # The 6 shape-changing entries: handler trio + detach + enable/disable. + self.assertEqual(set(MIGRATION_PAYLOAD_TRANSFORMS), { + "mycroft.skill.handler.start", + "mycroft.skill.handler.complete", + "mycroft.skill.handler.error", + "detach_intent", + "mycroft.skill.enable_intent", + "mycroft.skill.disable_intent", + }) + + def test_transforms_do_not_mutate_input(self): + for legacy, (l2s, s2l) in MIGRATION_PAYLOAD_TRANSFORMS.items(): + src = {"intent_name": "a:b", "handler": "h", "skill_id": "s", + "duration": 1, "traceback": "tb", "exception": "ex", + "lang": "en-US"} + before = dict(src) + l2s(src) + s2l(src) + self.assertEqual(src, before, f"{legacy} mutated its input") + + # --- detach_intent: cleanly bidirectional --- + def test_detach_spec_to_legacy_joins(self): + out = self.t.translate_payload( + "ovos.intent.deregister", "detach_intent", + {"skill_id": "music.skill", "intent_name": "play", "lang": "en-US"}) + self.assertEqual(out, {"intent_name": "music.skill:play"}) + + def test_detach_legacy_to_spec_splits(self): + out = self.t.translate_payload( + "detach_intent", "ovos.intent.deregister", + {"intent_name": "music.skill:play"}) + self.assertEqual(out, {"skill_id": "music.skill", + "intent_name": "play"}) + + def test_detach_roundtrip_spec_legacy_spec(self): + spec = {"skill_id": "music.skill", "intent_name": "play"} + legacy = self.t.translate_payload( + "ovos.intent.deregister", "detach_intent", spec) + back = self.t.translate_payload( + "detach_intent", "ovos.intent.deregister", legacy) + self.assertEqual(back, spec) + + def test_detach_split_on_first_colon_only(self): + out = self.t.translate_payload( + "detach_intent", "ovos.intent.deregister", + {"intent_name": "skill:a:b"}) + self.assertEqual(out, {"skill_id": "skill", "intent_name": "a:b"}) + + # --- handler trio: best-effort / lossy --- + def test_handler_complete_legacy_to_spec_drops_duration(self): + out = self.t.translate_payload( + "mycroft.skill.handler.complete", "ovos.intent.handler.complete", + {"handler": "play_music", "duration": 0.4}) + # handler->intent_name best-effort; duration has no spec field. + self.assertEqual(out, {"intent_name": "play_music"}) + + def test_handler_error_maps_traceback_to_exception(self): + out = self.t.translate_payload( + "mycroft.skill.handler.error", "ovos.intent.handler.error", + {"handler": "play_music", "traceback": "RuntimeError: boom"}) + self.assertEqual(out, {"intent_name": "play_music", + "exception": "RuntimeError: boom"}) + + def test_handler_spec_to_legacy_maps_exception_to_traceback(self): + out = self.t.translate_payload( + "ovos.intent.handler.error", "mycroft.skill.handler.error", + {"skill_id": "music.skill", "intent_name": "play_music", + "exception": "RuntimeError: boom"}) + self.assertEqual(out, {"handler": "play_music", + "skill_id": "music.skill", + "traceback": "RuntimeError: boom"}) + + def test_handler_best_effort_roundtrip(self): + # handler<->intent_name and traceback<->exception round-trip; + # skill_id is not recoverable from a legacy-only payload. + legacy = {"handler": "play_music", "traceback": "Err"} + spec = self.t.translate_payload( + "mycroft.skill.handler.error", "ovos.intent.handler.error", legacy) + back = self.t.translate_payload( + "ovos.intent.handler.error", "mycroft.skill.handler.error", spec) + self.assertEqual(back, {"handler": "play_music", "traceback": "Err"}) + + # --- enable/disable: documented loss (no skill_id on legacy side) --- + def test_toggle_spec_to_legacy_drops_skill_and_lang(self): + out = self.t.translate_payload( + "ovos.intent.enable", "mycroft.skill.enable_intent", + {"skill_id": "music.skill", "intent_name": "play", "lang": "en-US"}) + self.assertEqual(out, {"intent_name": "play"}) + + def test_toggle_legacy_to_spec_cannot_recover_skill_id(self): + out = self.t.translate_payload( + "mycroft.skill.disable_intent", "ovos.intent.disable", + {"intent_name": "play"}) + # documented limitation: no skill_id / lang available. + self.assertEqual(out, {"intent_name": "play"}) + self.assertNotIn("skill_id", out) + + +class TestTranslatePayload(unittest.TestCase): + """translate_payload direction selection + identity behaviour.""" + + def setUp(self): + self.t = NamespaceTranslator() + + def test_identity_for_payload_compatible_rename(self): + data = {"utterances": ["hello"], "lang": "en-US"} + # speak <-> ovos.utterance.speak has no transform entry -> identity. + self.assertEqual( + self.t.translate_payload("speak", "ovos.utterance.speak", data), + data) + self.assertEqual( + self.t.translate_payload("ovos.utterance.speak", "speak", data), + data) + + def test_identity_returns_new_dict(self): + data = {"a": 1} + out = self.t.translate_payload("speak", "ovos.utterance.speak", data) + self.assertIsNot(out, data) + + def test_non_migrating_topic_returns_copy(self): + data = {"x": 1} + out = self.t.translate_payload("random.a", "random.b", data) + self.assertEqual(out, data) + self.assertIsNot(out, data) + + def test_direction_legacy_source_uses_legacy_to_spec(self): + # from_topic legacy -> applies legacy_to_spec (split form). + out = self.t.translate_payload( + "detach_intent", "ovos.intent.deregister", + {"intent_name": "s:n"}) + self.assertEqual(out, {"skill_id": "s", "intent_name": "n"}) + + def test_direction_spec_source_uses_spec_to_legacy(self): + # from_topic spec -> applies spec_to_legacy (join form). + out = self.t.translate_payload( + "ovos.intent.deregister", "detach_intent", + {"skill_id": "s", "intent_name": "n"}) + self.assertEqual(out, {"intent_name": "s:n"}) + + def test_handles_empty_data(self): + self.assertEqual( + self.t.translate_payload("detach_intent", + "ovos.intent.deregister", {}), + {}) + + if __name__ == "__main__": unittest.main() From 03554da68ae7cdec19ac8d834cb61ac5616c70ce Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:41:20 +0000 Subject: [PATCH 091/110] Increment Version to 0.16.1a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 636a774..10211f5 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 16 -VERSION_BUILD = 0 +VERSION_BUILD = 1 VERSION_ALPHA = 1 # END_VERSION_BLOCK From c0567eea07283cd87f8f6a1d20de790afdbe2e38 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:41:35 +0000 Subject: [PATCH 092/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b042ccf..554df6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.16.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.16.1a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.0a1...0.16.1a1) + +**Merged pull requests:** + +- fix: message-domain conformance + NamespaceTranslator per-topic payload translation [\#42](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/42) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.16.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.16.0a1) (2026-06-27) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.15.0a1...0.16.0a1) From 08625778d4712b0a7513f5d54a7b7617ab0bf5e9 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 03:51:05 +0100 Subject: [PATCH 093/110] docs: make intent + bus-namespace docs timeless and standalone (#50) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: make intent + bus-namespace docs timeless and standalone Restate the intent primitives and the namespace-migration docs in terms of current behaviour rather than development history: drop 'historically provided by ovos-workshop', 'reimplementation/replacement', 'OVOS is moving ... off the historical names', and 'still landing'. Describes what the code and the migration map ARE, readable without knowing the project's history. Co-Authored-By: Claude Opus 4.8 * docs: state AUDIO-IN-1/AUDIO-1 bus topics as defined, not provisional The listener-lifecycle (AUDIO-IN-1 §6) and mic/audio-output (AUDIO-1 §4.4/§5) specs are merged; split the conflated table row by owning spec and drop the 'provisional / prose not yet finalized' caveat. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- docs/bus-namespaces.md | 17 +++++++++-------- docs/message.md | 4 ++-- ovos_spec_tools/intent.py | 18 +++++++++--------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/docs/bus-namespaces.md b/docs/bus-namespaces.md index ce17af8..acbd88b 100644 --- a/docs/bus-namespaces.md +++ b/docs/bus-namespaces.md @@ -1,12 +1,12 @@ # Bus namespaces — the legacy ↔ `ovos.*` migration -OVOS is moving its bus topics off the historical Mycroft-era names +OVOS bus topics span two namespaces: the Mycroft-era names (`speak`, `recognizer_loop:utterance`, `mycroft.skill.handler.start`, …) -and onto the `ovos.*` namespace the specifications define +and the `ovos.*` namespace the specifications define (`ovos.utterance.speak`, `ovos.utterance.handle`, `ovos.intent.handler.start`, …). `ovos-spec-tools` owns the **vocabulary** -of that move and the **transparent bridge** that lets it happen without a -flag day. Two things implement it, both in `ovos_spec_tools/messages.py`: +of the mapping between them and the **transparent bridge** that lets a +deployment cross from one to the other without a flag day. Two things implement it, both in `ovos_spec_tools/messages.py`: - `SpecMessage` — the enum of spec-defined `ovos.*` topics; - `MIGRATION_MAP` + `NamespaceTranslator` — the rename map and the @@ -41,11 +41,12 @@ spec section is cited in the source and in the table below. | Registration / management | OVOS-INTENT-4 §§5–8 | `INTENT_REGISTER_KEYWORD` (§5), `INTENT_REGISTER_TEMPLATE` (§6), `ENTITY_REGISTER` (§7), `INTENT_DEREGISTER` (§8.2), `ENTITY_DEREGISTER` (§8.3), `SKILL_DEREGISTER` (§8.4), `INTENT_ENABLE`/`INTENT_DISABLE` (§8.5) | | Introspection | OVOS-INTENT-4 §10 | `INTENT_LIST`, `INTENT_LIST_RESPONSE`, `INTENT_DESCRIBE`, `INTENT_DESCRIBE_RESPONSE` | | Stop cascade | OVOS-STOP-1 §4–§5 | `STOP_PING` (§4.2), `STOP_PONG` (§4.2), `STOP` (§5.3) | -| Listener / mic / output | OVOS-AUDIO-IN-1 (provisional) | `MIC_LISTEN`, `LISTENER_RECORD_STARTED`, `LISTENER_RECORD_ENDED`, `LISTENER_SLEEP`, `LISTENER_AWOKEN`, `AUDIO_OUTPUT_STARTED`, `AUDIO_OUTPUT_ENDED` | +| Listener lifecycle | OVOS-AUDIO-IN-1 §6 | `LISTENER_RECORD_STARTED`, `LISTENER_RECORD_ENDED`, `LISTENER_SLEEP`, `LISTENER_AWOKEN` | +| Mic / audio output | OVOS-AUDIO-1 §4.4, §5 | `MIC_LISTEN` (§4.4), `AUDIO_OUTPUT_STARTED` (§5.1), `AUDIO_OUTPUT_ENDED` (§5.2) | -The AUDIO-IN-1 group is **provisional**: the topic *names* are settled -(listening signals belong on `ovos.listener.*`), but the spec prose that -formally defines them is still landing. +The listener lifecycle signals live on the `ovos.listener.*` namespace +(OVOS-AUDIO-IN-1 §6); the mic re-open flag and audio-output session signals +belong to the audio-output service (OVOS-AUDIO-1 §4.4, §5). ## `MIGRATION_MAP` — the rename map diff --git a/docs/message.md b/docs/message.md index 2838fdc..67b33a8 100644 --- a/docs/message.md +++ b/docs/message.md @@ -90,8 +90,8 @@ addressed back to the original producer: an array — exact choice is implementation-defined). Other `context` keys, including `session`, pass through unchanged. The -optional `context` argument is overlaid before the swap, matching the -historical `ovos-bus-client.Message.reply` behaviour: +optional `context` argument is overlaid before the swap, matching +`ovos-bus-client.Message.reply` behaviour: ```python ack = m.reply("speak", {"utterance": "got it"}) diff --git a/ovos_spec_tools/intent.py b/ovos_spec_tools/intent.py index fdff44c..eb37a8e 100644 --- a/ovos_spec_tools/intent.py +++ b/ovos_spec_tools/intent.py @@ -1,9 +1,9 @@ """Plugin-agnostic intent-definition primitives for the OVOS-INTENT-4 keyword model. -This module is a clean, dependency-light reimplementation of the -``IntentBuilder`` / ``Intent`` pair historically provided by ``ovos-workshop`` -(which vendored Mycroft's ``adapt`` data classes). It carries **no** ``adapt`` -dependency: it is pure data describing the **structure** of a keyword intent — +This module is a clean, dependency-light ``IntentBuilder`` / ``Intent`` pair — +the plugin-agnostic form of the intent-definition classes ``ovos-workshop`` +exposes to skills. It carries **no** ``adapt`` dependency: it is pure data +describing the **structure** of a keyword intent — which vocabularies are *required*, *optional*, *one_of*, or *excluded* — exactly as OVOS-INTENT-4 §5 defines the ``ovos.intent.register.keyword`` payload. @@ -360,12 +360,12 @@ def voc_match(utterance: str, voc_name: str, lang: str, strip_punct: bool = True) -> bool: """Load a named ``.voc`` and test whether ``utterance`` matches it. - This is the plugin-agnostic replacement for - ``OVOSAbstractApplication.voc_match`` / the skill ``voc_match`` that - common-query, OCP, and other pipelines historically reached into - ``ovos-workshop`` for. It loads the ``.voc`` for ``lang`` and + This is the plugin-agnostic equivalent of + ``OVOSAbstractApplication.voc_match`` / the skill ``voc_match`` — the + helper common-query, OCP, and other pipelines use without depending on + ``ovos-workshop``. It loads the ``.voc`` for ``lang`` and matches with whole-word OVOS-INTENT-2 §4.3 semantics — identical to the - skill helper (so pipelines behave unchanged): a sample ``yes`` matches + skill helper (so pipelines behave the same): a sample ``yes`` matches ``"yes, please"`` but not ``"yesterday"``. Args: From eb1f5baf092fadf6be23ec980aa9d0ee606b7b15 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:51:14 +0000 Subject: [PATCH 094/110] Increment Version to 0.16.1a2 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 10211f5..436e3ec 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -2,7 +2,7 @@ VERSION_MAJOR = 0 VERSION_MINOR = 16 VERSION_BUILD = 1 -VERSION_ALPHA = 1 +VERSION_ALPHA = 2 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "") From a902c0a5c7648706e72c04f60de6ac4295508808 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 02:51:30 +0000 Subject: [PATCH 095/110] Update Changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 554df6f..1cbab0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.16.1a2](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.16.1a2) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a1...0.16.1a2) + +**Merged pull requests:** + +- docs: make intent + bus-namespace docs timeless and standalone [\#50](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/50) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.16.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.16.1a1) (2026-06-27) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.0a1...0.16.1a1) From 0e5c4aa08161a68a3c5da8b111cb1c135a6c6972 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:58:04 +0100 Subject: [PATCH 096/110] =?UTF-8?q?feat:=20bridge=20AUDIO-1=20=C2=A77=20ou?= =?UTF-8?q?tput=20topics=20in=20MIGRATION=5FMAP=20(#55)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The six OVOS-AUDIO-1 §7 output topics are present in the SpecMessage enum but were enum-only — they had no MIGRATION_MAP entry, so the bus could not bridge the legacy Mycroft-era handler names during the migration window. Add the legacy->spec renames (legacy names verified against ovos-audio register_handlers); all are payload-compatible 1:1 renames, so they bridge with the identity payload transform (no MIGRATION_PAYLOAD_TRANSFORMS entry): - speak:b64_audio -> ovos.utterance.speak.b64 (§3.4) - speak:b64_audio.response -> ovos.audio.speech (§4.3) - mycroft.audio.queue -> ovos.audio.queue (§4.1) - mycroft.audio.play_sound -> ovos.audio.play_sound (§4.2) - mycroft.audio.speak.status -> ovos.audio.is_speaking (§5.3) - mycroft.audio.speech.stop -> ovos.audio.stop (§6) Tests: enum membership + legacy->spec mapping + round-trip; replace the now-stale assertion that the AUDIO-1 members carry no migration counterpart. Co-authored-by: Claude Opus 4.8 --- ovos_spec_tools/messages.py | 9 +++++++++ test/test_messages.py | 37 +++++++++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/ovos_spec_tools/messages.py b/ovos_spec_tools/messages.py index af50578..4857c41 100644 --- a/ovos_spec_tools/messages.py +++ b/ovos_spec_tools/messages.py @@ -238,6 +238,15 @@ def __str__(self) -> str: "recognizer_loop:audio_output_start": SpecMessage.AUDIO_OUTPUT_STARTED, # AUDIO-1 §5.1 "recognizer_loop:audio_output_end": SpecMessage.AUDIO_OUTPUT_ENDED, # AUDIO-1 §5.2 "mycroft.mic.listen": SpecMessage.MIC_LISTEN, # AUDIO-1 §4.4 + # --- AUDIO-1 §3.4/§4.1/§4.2/§5.3/§6 audio-output bus surface + # (payload-compatible 1:1 renames; legacy handler names verified against + # ovos-audio register_handlers) --- + "speak:b64_audio": SpecMessage.SPEAK_B64, # AUDIO-1 §3.4 {utterance, listen} + "speak:b64_audio.response": SpecMessage.AUDIO_SPEECH, # AUDIO-1 §4.3 {audio, listen, ...} + "mycroft.audio.queue": SpecMessage.AUDIO_QUEUE, # AUDIO-1 §4.1 {uri} + "mycroft.audio.play_sound": SpecMessage.AUDIO_PLAY_SOUND, # AUDIO-1 §4.2 {uri} + "mycroft.audio.speak.status": SpecMessage.AUDIO_IS_SPEAKING, # AUDIO-1 §5.3 (query, empty) + "mycroft.audio.speech.stop": SpecMessage.AUDIO_STOP, # AUDIO-1 §6 (empty) # --- AUDIO-IN-1 §6 listener lifecycle (payload-compatible renames) --- "recognizer_loop:record_begin": SpecMessage.LISTENER_RECORD_STARTED, # AUDIO-IN-1 §6.1 "recognizer_loop:record_end": SpecMessage.LISTENER_RECORD_ENDED, # AUDIO-IN-1 §6.2 diff --git a/test/test_messages.py b/test/test_messages.py index 88847b1..ea1f4b5 100644 --- a/test/test_messages.py +++ b/test/test_messages.py @@ -82,7 +82,7 @@ def test_no_duplicate_values(self): self.assertEqual(len(values), len(set(values))) def test_audio1_bus_surface_members(self): - # OVOS-AUDIO-1 §7 bus surface — enum-only topics (not legacy renames). + # OVOS-AUDIO-1 §7 bus surface members. self.assertEqual(SpecMessage.SPEAK_B64, "ovos.utterance.speak.b64") # §3.4 self.assertEqual(SpecMessage.AUDIO_SPEECH, "ovos.audio.speech") # §4.3 self.assertEqual(SpecMessage.AUDIO_QUEUE, "ovos.audio.queue") # §4.1 @@ -90,15 +90,32 @@ def test_audio1_bus_surface_members(self): self.assertEqual(SpecMessage.AUDIO_STOP, "ovos.audio.stop") # §6 self.assertEqual(SpecMessage.AUDIO_IS_SPEAKING, "ovos.audio.is_speaking") # §5.3 - def test_audio1_members_are_enum_only_not_migration(self): - # These 6 are not legacy renames -> must not appear in the migration map - # or its payload transforms. - for m in (SpecMessage.SPEAK_B64, SpecMessage.AUDIO_SPEECH, - SpecMessage.AUDIO_QUEUE, SpecMessage.AUDIO_PLAY_SOUND, - SpecMessage.AUDIO_STOP, SpecMessage.AUDIO_IS_SPEAKING): - self.assertNotIn(m.value, SPEC_TO_LEGACY) - self.assertIsNone(migration_counterpart(m.value)) - self.assertNotIn(m.value, MIGRATION_PAYLOAD_TRANSFORMS) + def test_audio1_output_topics_are_payload_compatible_renames(self): + # The AUDIO-1 §7 output topics are payload-compatible 1:1 renames of the + # Mycroft-era handler names (verified against ovos-audio + # register_handlers), so they migrate via MIGRATION_MAP with no payload + # transform (identity). + expected = { + "speak:b64_audio": SpecMessage.SPEAK_B64, + "speak:b64_audio.response": SpecMessage.AUDIO_SPEECH, + "mycroft.audio.queue": SpecMessage.AUDIO_QUEUE, + "mycroft.audio.play_sound": SpecMessage.AUDIO_PLAY_SOUND, + "mycroft.audio.speak.status": SpecMessage.AUDIO_IS_SPEAKING, + "mycroft.audio.speech.stop": SpecMessage.AUDIO_STOP, + } + for legacy, spec in expected.items(): + self.assertEqual(MIGRATION_MAP[legacy], spec) + # payload-compatible: identity transform (no per-topic entry) + self.assertNotIn(legacy, MIGRATION_PAYLOAD_TRANSFORMS) + + def test_audio1_output_topics_round_trip(self): + # legacy -> spec -> legacy and spec -> legacy round-trip. + for legacy in ("speak:b64_audio", "speak:b64_audio.response", + "mycroft.audio.queue", "mycroft.audio.play_sound", + "mycroft.audio.speak.status", "mycroft.audio.speech.stop"): + spec = migration_counterpart(legacy) + self.assertIsNotNone(spec) + self.assertEqual(migration_counterpart(spec), legacy) class TestMigrationMap(unittest.TestCase): From dc0bd6bc59787ddddfde626a361c641d843c5d6e Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:58:08 +0100 Subject: [PATCH 097/110] =?UTF-8?q?fix:=20enforce=20OVOS-MSG-1=20=C2=A72.1?= =?UTF-8?q?=20type=20syntax=20on=20serialize=20(#53)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit serialize() already gates the §2.1 non-emptiness rule but not the §2.1 character/whitespace rule: a topic may contain only ASCII letters, digits, '.', ':', '_', '-' and no whitespace. Message("a b").serialize() emitted an embedded space instead of rejecting it. Add the §2.1 charset/whitespace validation at the serialize() wire gate (after the non-empty check) so a malformed topic never reaches the wire, matching the §7 producer MUST. The constructor still tolerates an empty scaffold type for the construct-then-forward pattern. Co-authored-by: Claude Opus 4.8 --- ovos_spec_tools/message.py | 25 ++++++++++++++++++++++++- test/test_message.py | 22 ++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/ovos_spec_tools/message.py b/ovos_spec_tools/message.py index 6366174..096ba64 100644 --- a/ovos_spec_tools/message.py +++ b/ovos_spec_tools/message.py @@ -42,11 +42,21 @@ from __future__ import annotations import json +import re from copy import deepcopy from typing import Any, Dict, List, Optional, Union __all__ = ["Message", "MalformedMessage", "DEFAULT_SESSION_ID"] +#: The OVOS-MSG-1 §2.1 ``type`` syntax: a non-empty run of ASCII letters, +#: digits, and the four punctuation characters the spec permits — ``.`` ``:`` +#: ``_`` ``-``. §2.1 forbids whitespace and any other character outright +#: ("ASCII letters, digits, ``.``, ``:``, ``_``, ``-``; no whitespace"), so +#: anything outside this set — an embedded space, a slash, a non-ASCII letter — +#: is a malformed topic the producer MUST NOT emit. Lowercase is only +#: RECOMMENDED (§2.1), so uppercase is accepted. +_TYPE_SYNTAX_RE = re.compile(r"[A-Za-z0-9.:_-]+") + #: A routing key value as it may appear on the wire (OVOS-MSG-1 §3): a single #: opaque identifier string, or — for ``destination`` only — an array of them #: (§3.3). ``None`` models the absent/broadcast case (§3.3). The envelope @@ -242,7 +252,10 @@ def serialize(self) -> str: MalformedMessage: when ``msg_type`` is empty — §2.1/§7 require a producer to emit a non-empty ``type``; an empty-``type`` scaffold Message must be ``forward``/``reply``-derived into - a real topic before it can be serialized. + a real topic before it can be serialized. Also when + ``msg_type`` violates the §2.1 syntax (a character outside + ASCII letters/digits/``.``/``:``/``_``/``-``, e.g. an embedded + space): such a topic must not reach the wire. ValueError: from :func:`json.dumps` when ``data`` / ``context`` contain a non-finite number (``allow_nan=False``), enforcing the §6 "Numbers MUST be finite" rule at emit time. @@ -255,6 +268,16 @@ def serialize(self) -> str: "cannot serialize a Message with an empty 'type' — §2.1/§7 " "require a producer to emit a non-empty topic; derive a real " "topic via forward()/reply() before serializing") + # §2.1 ``type`` syntax: ASCII letters, digits, ``.`` ``:`` ``_`` ``-``; + # no whitespace. The non-emptiness check above is the §7 producer MUST + # for an empty topic; this is the §2.1 producer MUST for the charset. + # An embedded space (``Message("a b")``) or any other character must + # not reach the wire, so ``serialize`` refuses it here. + if not _TYPE_SYNTAX_RE.fullmatch(self.msg_type): + raise MalformedMessage( + f"'type' {self.msg_type!r} violates OVOS-MSG-1 §2.1 syntax — a " + "topic may contain only ASCII letters, digits, '.', ':', '_', " + "'-' and no whitespace") return json.dumps( {"type": self.msg_type, "data": self._to_jsonable(self.data), diff --git a/test/test_message.py b/test/test_message.py index ec2a401..3eb7ef7 100644 --- a/test/test_message.py +++ b/test/test_message.py @@ -150,6 +150,28 @@ def test_serialize_after_forward_from_empty_scaffold_ok(self): parsed = json.loads(m.serialize()) assert parsed["type"] == "ovos.real" + def test_serialize_accepts_valid_type_syntax(self): + """§2.1: ASCII letters, digits, ``.`` ``:`` ``_`` ``-`` are all valid + topic characters; a dot/colon-segmented topic serializes unchanged.""" + m = Message("assistant.intent.register.keyword") + assert json.loads(m.serialize())["type"] == \ + "assistant.intent.register.keyword" + # the full permitted charset, including the colon separator and a digit + assert json.loads(Message("Foo_bar-1:Baz").serialize())["type"] == \ + "Foo_bar-1:Baz" + + def test_serialize_rejects_type_with_whitespace(self): + """§2.1 producer MUST: a topic carries no whitespace, so an embedded + space MUST be rejected at the wire gate rather than emitted.""" + with pytest.raises(MalformedMessage): + Message("a b").serialize() + + def test_serialize_rejects_type_with_illegal_char(self): + """§2.1: characters outside [A-Za-z0-9.:_-] (here a slash) are not + permitted in a topic.""" + with pytest.raises(MalformedMessage): + Message("ovos/test").serialize() + def test_deserialize_rejects_wrong_type_data(self): """§2.2/§6: a present-but-non-object ``data`` MUST be rejected, not silently coerced to ``{}``.""" From 99d7e8dc9b45e2082b0f6e90d647e320986f1c0a Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:58:12 +0000 Subject: [PATCH 098/110] Increment Version to 0.17.0a1 --- ovos_spec_tools/version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 436e3ec..e70c8ba 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,8 +1,8 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 16 -VERSION_BUILD = 1 -VERSION_ALPHA = 2 +VERSION_MINOR = 17 +VERSION_BUILD = 0 +VERSION_ALPHA = 1 # END_VERSION_BLOCK __version__ = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}" + (f"a{VERSION_ALPHA}" if VERSION_ALPHA else "") From 3efa43d4499e2b8f32b8773e2941bcb19f846a11 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:58:19 +0000 Subject: [PATCH 099/110] Increment Version to 0.17.1a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index e70c8ba..b3b8911 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 17 -VERSION_BUILD = 0 +VERSION_BUILD = 1 VERSION_ALPHA = 1 # END_VERSION_BLOCK From a45276921f3c920f2869ccfda23b80bbaffa23a9 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:58:32 +0000 Subject: [PATCH 100/110] Update Changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cbab0d..3c9a905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.17.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.0a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a2...0.17.0a1) + +**Merged pull requests:** + +- feat: bridge AUDIO-1 §7 output topics in MIGRATION\_MAP [\#55](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/55) ([JarbasAl](https://github.com/JarbasAl)) +- fix: enforce OVOS-MSG-1 §2.1 type syntax on serialize [\#53](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/53) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.16.1a2](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.16.1a2) (2026-06-27) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a1...0.16.1a2) From 18bed52059ac27360c6f9ac5b7528aa005bcd28a Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:58:39 +0000 Subject: [PATCH 101/110] Update Changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c9a905..ae9d7bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog -## [0.17.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.0a1) (2026-06-27) +## [0.17.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.1a1) (2026-06-27) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a2...0.17.0a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a2...0.17.1a1) **Merged pull requests:** From 5283a338144addc079dc117ce801f0109a0ef227 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:03:17 +0100 Subject: [PATCH 102/110] =?UTF-8?q?fix:=20reject=20malformed=20keyword=20i?= =?UTF-8?q?ntents=20at=20build/emit=20(INTENT-3=20=C2=A74.2)=20(#54)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The INTENT-3 §4.2 well-formedness MUSTs were enforced only in lint_locale, not in the IntentBuilder/Intent data model, so an invalid intent could be built and registered on the bus. Add Intent.validate() enforcing both §4.2 MUSTs and call it from the construction path (IntentBuilder.build) and the emission path (Intent.to_keyword_payload): - (a) a keyword intent MUST declare at least one required or one-of constraint — an optional/excluded-only intent has nothing that must be present and is malformed; - (b) a vocabulary MUST appear under at most one role — the same vocab under two roles (e.g. required and excluded) is contradictory and malformed. Raises the new MalformedIntent. Raw Intent(...) construction stays permissive (the scaffold / wire round-trip path via open_intent_envelope), so validation fires only at build/emit and a subclass overriding build() is unaffected. INTENT-1 §5.5 (DialogRenderer rejecting non-identical slot sets) is already enforced on dev via verify_slot_consistency in both render paths. Co-authored-by: Claude Opus 4.8 --- ovos_spec_tools/__init__.py | 1 + ovos_spec_tools/intent.py | 88 ++++++++++++++++++++++++++++++++++++- test/test_intent.py | 55 +++++++++++++++++++++++ 3 files changed, 142 insertions(+), 2 deletions(-) diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py index 7159fb6..1ea2022 100644 --- a/ovos_spec_tools/__init__.py +++ b/ovos_spec_tools/__init__.py @@ -57,6 +57,7 @@ from ovos_spec_tools.intent import ( Intent, IntentBuilder, + MalformedIntent, open_intent_envelope, voc_match, ) diff --git a/ovos_spec_tools/intent.py b/ovos_spec_tools/intent.py index eb37a8e..e98d9ab 100644 --- a/ovos_spec_tools/intent.py +++ b/ovos_spec_tools/intent.py @@ -38,6 +38,7 @@ __all__ = [ "Intent", "IntentBuilder", + "MalformedIntent", "open_intent_envelope", "voc_match", ] @@ -48,6 +49,26 @@ RoleEntry = Tuple[str, str] +class MalformedIntent(ValueError): + """A keyword intent that violates an OVOS-INTENT-3 §4.2 structural MUST. + + Raised when a built / emitted intent breaks one of the two §4.2 + well-formedness MUSTs: + + - it declares **no** ``required`` and **no** ``one-of`` constraint, so + nothing must be present for it to match ("an intent with only optional + and excluded constraints has nothing that must be present and is + malformed"); + - it lists the **same vocabulary under two different roles** ("a vocabulary + MUST appear under at most one role within a single intent. Listing the + same vocabulary under two roles … is contradictory and malformed"). + + These are the data-model counterparts of the checks the locale linter + already performs; raising at build/emit time means an invalid keyword + intent is rejected before it can be registered on the bus. + """ + + class Intent: """A built keyword-intent **definition** — name plus the four role lists. @@ -115,6 +136,56 @@ def _as_name(entry) -> str: return entry return entry[0] + # -- OVOS-INTENT-3 §4.2 well-formedness ---------------------------------- + + def validate(self) -> "Intent": + """Reject an intent that violates an OVOS-INTENT-3 §4.2 structural MUST. + + Enforces the two §4.2 well-formedness rules the spec calls ``MUST``: + + - **(a)** a keyword intent **MUST** declare at least one ``required`` + or ``one-of`` constraint — "an intent with only optional and + excluded constraints has nothing that must be present and is + malformed"; + - **(b)** a vocabulary **MUST** appear under at most one role — listing + the same vocabulary under two roles (e.g. both required and excluded) + "is contradictory and malformed". + + Returns ``self`` so it can be used inline (``Intent(...).validate()``). + + Raises: + MalformedIntent: if either §4.2 rule is violated. + """ + # (a) at least one required or one-of must be present. + if not self.requires and not self.at_least_one: + raise MalformedIntent( + f"keyword intent {self.name!r} declares no required and no " + "one-of constraint — it has nothing that must be present " + "(OVOS-INTENT-3 §4.2)") + + # (b) a vocabulary appears under at most one role. Collect every + # (vocabulary -> roles it appears in) and reject any vocabulary that + # spans more than one role. one-of vocabularies count once per name + # regardless of how many groups list them. + roles: Dict[str, set] = {} + for name, _ in self.requires: + roles.setdefault(name, set()).add("required") + for name, _ in self.optional: + roles.setdefault(name, set()).add("optional") + for group in self.at_least_one: + for name in group: + roles.setdefault(name, set()).add("one_of") + for name in self.excludes: + roles.setdefault(name, set()).add("excluded") + clashes = {name: sorted(r) for name, r in roles.items() if len(r) > 1} + if clashes: + detail = "; ".join(f"{name!r} under {roles}" + for name, roles in sorted(clashes.items())) + raise MalformedIntent( + f"keyword intent {self.name!r} lists a vocabulary under more " + f"than one role — {detail} (OVOS-INTENT-3 §4.2)") + return self + # -- OVOS-INTENT-4 §5 emission ------------------------------------------- @staticmethod @@ -158,7 +229,14 @@ def to_keyword_payload(self, skill_id: Optional[str] = None, Returns: the §5.2 keyword payload structure. + + Raises: + MalformedIntent: if the intent violates an OVOS-INTENT-3 §4.2 + well-formedness MUST — a register payload MUST NOT be emitted + for an intent with no required/one-of constraint or with a + vocabulary listed under two roles. """ + self.validate() payload: Dict[str, Any] = {} if skill_id is not None: payload["skill_id"] = skill_id @@ -288,9 +366,15 @@ def exclude(self, entity_type: str) -> "IntentBuilder": return self def build(self) -> Intent: - """Freeze the accumulated roles into an :class:`Intent`.""" + """Freeze the accumulated roles into a validated :class:`Intent`. + + The result is checked against the OVOS-INTENT-3 §4.2 well-formedness + MUSTs (:meth:`Intent.validate`): a built intent must declare at least + one required or one-of constraint, and must not list a vocabulary under + two roles. A malformed builder state raises :class:`MalformedIntent`. + """ return Intent(self.name, self.requires, self.at_least_one, - self.optional, self.excludes) + self.optional, self.excludes).validate() def open_intent_envelope(message) -> Intent: diff --git a/test/test_intent.py b/test/test_intent.py index 042456d..1eae7a7 100644 --- a/test/test_intent.py +++ b/test/test_intent.py @@ -5,6 +5,7 @@ Intent, IntentBuilder, LocaleResources, + MalformedIntent, open_intent_envelope, voc_match, ) @@ -179,3 +180,57 @@ def test_voc_match_accepts_sequence_of_dirs(locale): def test_voc_match_missing_voc_returns_false(locale): assert voc_match("anything", "nonexistent", "en-US", str(locale)) is False + + +# --- OVOS-INTENT-3 §4.2 well-formedness MUSTs -------------------------------- + +def test_build_rejects_intent_with_no_required_and_no_one_of(): + """§4.2: a keyword intent MUST declare at least one required or one-of + constraint — only optional + excluded "has nothing that must be present + and is malformed".""" + builder = IntentBuilder("Bad").optionally("Politely").exclude("Question") + with pytest.raises(MalformedIntent): + builder.build() + + +def test_build_accepts_intent_with_only_one_of(): + """A single one-of group satisfies §4.2 (at least one of required/one-of).""" + intent = IntentBuilder("OK").one_of("Up", "Down").build() + assert intent.at_least_one == [("Up", "Down")] + + +def test_build_rejects_same_vocab_under_two_roles(): + """§4.2: a vocabulary MUST appear under at most one role; required + excluded + of the same vocab is contradictory and malformed.""" + builder = IntentBuilder("Bad").require("Light").exclude("Light") + with pytest.raises(MalformedIntent): + builder.build() + + +def test_build_rejects_vocab_required_and_one_of(): + builder = IntentBuilder("Bad").require("Set").one_of("Set", "Down") + with pytest.raises(MalformedIntent): + builder.build() + + +def test_validate_returns_self_for_well_formed_intent(): + intent = Intent("Good", requires=[("Set", "Set")]) + assert intent.validate() is intent + + +def test_to_keyword_payload_rejects_malformed_intent(): + """The §5.2 register payload MUST NOT be emitted for a §4.2-malformed + intent — emission validates.""" + intent = Intent("Bad", optional=[("Politely", "Politely")], + excludes=["Question"]) + with pytest.raises(MalformedIntent): + intent.to_keyword_payload() + + +def test_raw_intent_construction_does_not_validate(): + """Raw ``Intent(...)`` construction stays permissive (the scaffold / wire + round-trip path); validation happens at build()/emit. The empty-default + Intent and open_intent_envelope reconstruction must not raise.""" + Intent() # scaffold default, no raise + rebuilt = open_intent_envelope({"intent_name": "Z"}) + assert rebuilt.name == "Z" From 2beef0fc67e09ea8c80dcf76dc14686fa2a358fc Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:03:31 +0000 Subject: [PATCH 103/110] Increment Version to 0.17.2a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index b3b8911..080cd98 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 17 -VERSION_BUILD = 1 +VERSION_BUILD = 2 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 2d254d47be7a1b5aa18a2b95ecd644d040db781e Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:03:58 +0000 Subject: [PATCH 104/110] Update Changelog --- CHANGELOG.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae9d7bb..10e9076 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,20 @@ # Changelog +## [0.17.2a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.2a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.1a1...0.17.2a1) + +**Merged pull requests:** + +- fix: reject malformed keyword intents at build/emit \(INTENT-3 §4.2\) [\#54](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/54) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.17.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.1a1) (2026-06-27) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a2...0.17.1a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.0a1...0.17.1a1) + +## [0.17.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.0a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a2...0.17.0a1) **Merged pull requests:** From 750b739ec77cfd821164f4ed6efeea2a578f9fd4 Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 21:18:24 +0100 Subject: [PATCH 105/110] =?UTF-8?q?fix:=20warn=20(not=20raise)=20on=20malf?= =?UTF-8?q?ormed=20intent=20at=20build/emit=20(OVOS-INTENT-3=20=C2=A74.2)?= =?UTF-8?q?=20(#59)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IntentBuilder.build() and Intent.to_keyword_payload() validated the intent and RAISED MalformedIntent. Because ovos-workshop re-exports this IntentBuilder, build()-raises broke skills that build a constraint-less intent — a backward-compat regression on consumers' dev (e.g. workshop's test_build_preserves_name). Build/emit now LOG a warning instead; the raising enforcement stays available via the explicit Intent.validate() method and the locale linter, so §4.2 is still enforced where rejection is wanted, without breaking construction. Co-authored-by: Claude Opus 4.8 --- ovos_spec_tools/intent.py | 43 +++++++++++++++++++++++------------ test/test_intent.py | 47 ++++++++++++++++++++++++++------------- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/ovos_spec_tools/intent.py b/ovos_spec_tools/intent.py index e98d9ab..9b65815 100644 --- a/ovos_spec_tools/intent.py +++ b/ovos_spec_tools/intent.py @@ -33,8 +33,11 @@ """ from __future__ import annotations +import logging from typing import Any, Dict, List, Optional, Sequence, Tuple +_log = logging.getLogger(__name__) + __all__ = [ "Intent", "IntentBuilder", @@ -230,13 +233,16 @@ def to_keyword_payload(self, skill_id: Optional[str] = None, Returns: the §5.2 keyword payload structure. - Raises: - MalformedIntent: if the intent violates an OVOS-INTENT-3 §4.2 - well-formedness MUST — a register payload MUST NOT be emitted - for an intent with no required/one-of constraint or with a - vocabulary listed under two roles. + A malformed intent (OVOS-INTENT-3 §4.2 — no required/one-of + constraint, or a vocabulary listed under two roles) is **logged as a + warning** rather than raised, so emitting stays backward-compatible; + call :meth:`validate` explicitly to reject before emit. """ - self.validate() + try: + self.validate() + except MalformedIntent as err: + _log.warning("emitting register payload for malformed intent %r " + "(OVOS-INTENT-3 §4.2): %s", self.name, err) payload: Dict[str, Any] = {} if skill_id is not None: payload["skill_id"] = skill_id @@ -366,15 +372,24 @@ def exclude(self, entity_type: str) -> "IntentBuilder": return self def build(self) -> Intent: - """Freeze the accumulated roles into a validated :class:`Intent`. - - The result is checked against the OVOS-INTENT-3 §4.2 well-formedness - MUSTs (:meth:`Intent.validate`): a built intent must declare at least - one required or one-of constraint, and must not list a vocabulary under - two roles. A malformed builder state raises :class:`MalformedIntent`. + """Freeze the accumulated roles into an :class:`Intent`. + + A built intent should satisfy the OVOS-INTENT-3 §4.2 well-formedness + MUSTs (:meth:`Intent.validate`): declare at least one required or + one-of constraint, and not list a vocabulary under two roles. A + malformed builder state is **logged as a warning** rather than raised, + so ``build()`` stays backward-compatible — call :meth:`Intent.validate` + explicitly (or rely on the locale linter) to enforce §4.2 where you + want to reject. """ - return Intent(self.name, self.requires, self.at_least_one, - self.optional, self.excludes).validate() + intent = Intent(self.name, self.requires, self.at_least_one, + self.optional, self.excludes) + try: + intent.validate() + except MalformedIntent as err: + _log.warning("built intent %r is malformed per OVOS-INTENT-3 " + "§4.2: %s", self.name, err) + return intent def open_intent_envelope(message) -> Intent: diff --git a/test/test_intent.py b/test/test_intent.py index 1eae7a7..6ae3d7f 100644 --- a/test/test_intent.py +++ b/test/test_intent.py @@ -182,15 +182,27 @@ def test_voc_match_missing_voc_returns_false(locale): assert voc_match("anything", "nonexistent", "en-US", str(locale)) is False -# --- OVOS-INTENT-3 §4.2 well-formedness MUSTs -------------------------------- +# --- OVOS-INTENT-3 §4.2 well-formedness (validate raises; build/emit warn) --- -def test_build_rejects_intent_with_no_required_and_no_one_of(): +def test_validate_rejects_intent_with_no_required_and_no_one_of(): """§4.2: a keyword intent MUST declare at least one required or one-of constraint — only optional + excluded "has nothing that must be present - and is malformed".""" - builder = IntentBuilder("Bad").optionally("Politely").exclude("Question") + and is malformed". Explicit validate() enforces this.""" + intent = Intent("Bad", optional=[("Politely", "Politely")], + excludes=["Question"]) with pytest.raises(MalformedIntent): - builder.build() + intent.validate() + + +def test_build_warns_but_does_not_raise_on_malformed_intent(caplog): + """build() stays backward-compatible: a §4.2-malformed builder logs a + warning rather than raising. Enforcement is via explicit validate / lint.""" + builder = IntentBuilder("Bad").optionally("Politely").exclude("Question") + with caplog.at_level("WARNING"): + intent = builder.build() # must NOT raise + assert any("malformed" in r.message.lower() for r in caplog.records) + with pytest.raises(MalformedIntent): # explicit validation still rejects + intent.validate() def test_build_accepts_intent_with_only_one_of(): @@ -199,18 +211,19 @@ def test_build_accepts_intent_with_only_one_of(): assert intent.at_least_one == [("Up", "Down")] -def test_build_rejects_same_vocab_under_two_roles(): +def test_validate_rejects_same_vocab_under_two_roles(): """§4.2: a vocabulary MUST appear under at most one role; required + excluded of the same vocab is contradictory and malformed.""" - builder = IntentBuilder("Bad").require("Light").exclude("Light") + intent = Intent("Bad", requires=[("Light", "Light")], excludes=["Light"]) with pytest.raises(MalformedIntent): - builder.build() + intent.validate() -def test_build_rejects_vocab_required_and_one_of(): - builder = IntentBuilder("Bad").require("Set").one_of("Set", "Down") +def test_validate_rejects_vocab_required_and_one_of(): + intent = Intent("Bad", requires=[("Set", "Set")], + at_least_one=[("Set", "Down")]) with pytest.raises(MalformedIntent): - builder.build() + intent.validate() def test_validate_returns_self_for_well_formed_intent(): @@ -218,13 +231,15 @@ def test_validate_returns_self_for_well_formed_intent(): assert intent.validate() is intent -def test_to_keyword_payload_rejects_malformed_intent(): - """The §5.2 register payload MUST NOT be emitted for a §4.2-malformed - intent — emission validates.""" +def test_to_keyword_payload_warns_but_emits_malformed_intent(caplog): + """Emitting a §4.2-malformed register payload warns rather than raising, + keeping the producer backward-compatible.""" intent = Intent("Bad", optional=[("Politely", "Politely")], excludes=["Question"]) - with pytest.raises(MalformedIntent): - intent.to_keyword_payload() + with caplog.at_level("WARNING"): + payload = intent.to_keyword_payload() # must NOT raise + assert payload["intent_name"] == "Bad" + assert any("malformed" in r.message.lower() for r in caplog.records) def test_raw_intent_construction_does_not_validate(): From 7bb70e19a9bade93c17bf8d15906df56a80e2bc3 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:18:38 +0000 Subject: [PATCH 106/110] Increment Version to 0.17.3a1 --- ovos_spec_tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 080cd98..4f27e99 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 17 -VERSION_BUILD = 2 +VERSION_BUILD = 3 VERSION_ALPHA = 1 # END_VERSION_BLOCK From 4a1d796cf3353e45c7546af4e328f4a03d3715f1 Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:19:02 +0000 Subject: [PATCH 107/110] Update Changelog --- CHANGELOG.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10e9076..37f00c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,20 +1,28 @@ # Changelog +## [0.17.3a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.3a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.2a1...0.17.3a1) + +**Merged pull requests:** + +- fix: warn \(not raise\) on malformed intent at build/emit \(INTENT-3 §4.2\) [\#59](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/59) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.17.2a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.2a1) (2026-06-27) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.1a1...0.17.2a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.0a1...0.17.2a1) **Merged pull requests:** - fix: reject malformed keyword intents at build/emit \(INTENT-3 §4.2\) [\#54](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/54) ([JarbasAl](https://github.com/JarbasAl)) -## [0.17.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.1a1) (2026-06-27) +## [0.17.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.0a1) (2026-06-27) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.0a1...0.17.1a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.1a1...0.17.0a1) -## [0.17.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.0a1) (2026-06-27) +## [0.17.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.1a1) (2026-06-27) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a2...0.17.0a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a2...0.17.1a1) **Merged pull requests:** From 72a5005ebd55797866d4550567f51d58867a14bf Mon Sep 17 00:00:00 2001 From: JarbasAI <33701864+JarbasAl@users.noreply.github.com> Date: Sun, 28 Jun 2026 00:42:34 +0100 Subject: [PATCH 108/110] feat: make Session and Message hashable (#61) Both classes defined __eq__ but no __hash__, so Python set __hash__ to None and they were unhashable -- forcing downstream lru_cache callers (padacioso/linha-fina/nebulento) to pass frozenset workarounds instead of the objects themselves. Add __hash__ to both, derived from the SAME fields each __eq__ compares so equal objects always hash equal (the hash/eq contract): - Session.__hash__ from to_dict() - Message.__hash__ from (msg_type, data, context) A shared _freeze() helper (in message.py, imported by session.py) recursively converts nested dict/list/set payloads into a deterministic hashable form. It preserves value-equality: {"x": 1} and {"x": 1.0} freeze equal (unlike a json.dumps digest, which would diverge "1" vs "1.0" and violate the contract). The hash is a point-in-time snapshot of these mutable objects -- safe for lru_cache keys and short-lived dict/set membership, documented in the method docstrings. Not a spec change: no message/session semantics or spec docs touched. Co-authored-by: Claude Opus 4.8 --- ovos_spec_tools/message.py | 47 +++++++++++++++++++++++++++++++ ovos_spec_tools/session.py | 12 +++++++- test/test_message.py | 57 ++++++++++++++++++++++++++++++++++++++ test/test_session.py | 48 ++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 1 deletion(-) diff --git a/ovos_spec_tools/message.py b/ovos_spec_tools/message.py index 096ba64..1cca277 100644 --- a/ovos_spec_tools/message.py +++ b/ovos_spec_tools/message.py @@ -72,6 +72,41 @@ DEFAULT_SESSION_ID = "default" +def _freeze(value: Any) -> Any: + """Recursively convert ``value`` into a deterministic, hashable form. + + Used to derive ``__hash__`` for :class:`Message` and + :class:`~ovos_spec_tools.session.Session` from the same nested + dict/list payloads their ``__eq__`` compares. The mapping preserves + Python's value-equality invariant: anything that compares equal + freezes to an equal (and therefore equally-hashing) form. In + particular ``{"x": 1}`` and ``{"x": 1.0}`` freeze equal because the + underlying ``int``/``float`` are themselves equal and hash equal — + unlike a ``json.dumps`` digest, which would diverge on ``"1"`` vs + ``"1.0"``. + + - ``dict`` → ``frozenset`` of ``(key, _freeze(val))`` pairs + (order-independent, mirroring dict equality); + - ``list`` / ``tuple`` → ``tuple`` of frozen items (order-preserving); + - ``set`` / ``frozenset`` → ``frozenset`` of frozen items; + - already-hashable scalars (``str``, ``int``, ``float``, ``bool``, + ``None``, …) → returned unchanged; + - any remaining exotic unhashable → its ``repr`` (deterministic + last resort). + """ + if isinstance(value, dict): + return frozenset((k, _freeze(v)) for k, v in value.items()) + if isinstance(value, (list, tuple)): + return tuple(_freeze(v) for v in value) + if isinstance(value, (set, frozenset)): + return frozenset(_freeze(v) for v in value) + try: + hash(value) + except TypeError: + return repr(value) + return value + + class MalformedMessage(ValueError, AssertionError): """A serialized payload that does not conform to OVOS-MSG-1 §2 / §6. @@ -159,6 +194,18 @@ def __eq__(self, other: object) -> bool: and other.data == self.data and other.context == self.context) + def __hash__(self) -> int: + # Hash over the same §2 fields ``__eq__`` compares (type + data + + # context), frozen into a deterministic hashable form so equal + # Messages always hash equal (the hash/eq contract). NOTE: ``data`` + # and ``context`` are mutable dicts stored by reference, so this is + # a *point-in-time snapshot* — safe for ``functools.lru_cache`` keys + # and short-lived set/dict membership, but do not mutate a Message + # while it is live as a dict key. + return hash((self.msg_type, + _freeze(self.data), + _freeze(self.context))) + def __repr__(self) -> str: # Eval-friendly debug form; not the wire format (use serialize()). return (f"{self.__class__.__name__}(" diff --git a/ovos_spec_tools/session.py b/ovos_spec_tools/session.py index da7789d..3782408 100644 --- a/ovos_spec_tools/session.py +++ b/ovos_spec_tools/session.py @@ -44,7 +44,7 @@ from copy import deepcopy from typing import Any, Dict, List, Optional, Union -from ovos_spec_tools.message import DEFAULT_SESSION_ID +from ovos_spec_tools.message import DEFAULT_SESSION_ID, _freeze __all__ = [ "Session", @@ -659,5 +659,15 @@ def __eq__(self, other: object) -> bool: return (isinstance(other, Session) and self.to_dict() == other.to_dict()) + def __hash__(self) -> int: + # Hash over the same canonical ``to_dict()`` view ``__eq__`` + # compares, frozen into a deterministic hashable form so equal + # Sessions always hash equal (the hash/eq contract). NOTE: a + # Session is mutable, so this is a *point-in-time snapshot* — safe + # for ``functools.lru_cache`` keys and short-lived set/dict + # membership, but do not mutate a Session while it is live as a + # dict key. + return hash(_freeze(self.to_dict())) + def __repr__(self) -> str: return f"Session({self.to_dict()!r})" diff --git a/test/test_message.py b/test/test_message.py index 3eb7ef7..4e7d253 100644 --- a/test/test_message.py +++ b/test/test_message.py @@ -389,3 +389,60 @@ def serialize(self): m = Message("ovos.test", {}, {"session": _Carrier()}) assert m.as_dict["context"]["session"] == {"session_id": "s-7"} + + +# --- hashability (library-usefulness; not a spec rule) ---------------------- + +class TestHashable: + def test_equal_messages_hash_equal(self): + a = Message("ovos.test", {"x": 1}, {"source": "skills"}) + b = Message("ovos.test", {"x": 1}, {"source": "skills"}) + assert a == b + assert hash(a) == hash(b) + + def test_usable_as_dict_key_and_set_member(self): + a = Message("ovos.test", {"x": 1}) + b = Message("ovos.test", {"x": 1}) + c = Message("ovos.other", {"x": 1}) + d = {a: "value"} + assert d[b] == "value" # equal key hits the same bucket + assert len({a, b, c}) == 2 # a and b collapse, c distinct + + def test_usable_in_lru_cache(self): + import functools + + calls = [] + + @functools.lru_cache(maxsize=None) + def handler(msg): + calls.append(msg.msg_type) + return msg.msg_type.upper() + + m1 = Message("ovos.test", {"x": 1}, {"source": "a"}) + m2 = Message("ovos.test", {"x": 1}, {"source": "a"}) # equal to m1 + assert handler(m1) == "OVOS.TEST" + assert handler(m2) == "OVOS.TEST" + assert calls == ["ovos.test"] # cached: only one underlying call + + def test_nested_dict_and_list_data_hashes(self): + m = Message("ovos.test", + {"items": [1, 2, {"k": "v"}], "meta": {"a": [3, 4]}}, + {"session": {"session_id": "s", "langs": ["en", "pt"]}}) + # must not raise, and must be stable across repeated calls + assert hash(m) == hash(m) + + def test_int_and_float_data_hash_equal(self): + # value-equality invariant: {"x": 1} == {"x": 1.0} so they MUST + # hash equal (a json.dumps digest would break this). + a = Message("ovos.test", {"x": 1}) + b = Message("ovos.test", {"x": 1.0}) + assert a == b + assert hash(a) == hash(b) + + def test_mutation_then_rehash_changes_hash(self): + # documented behavior: the hash is a point-in-time snapshot of the + # mutable payload; mutating data after hashing changes the hash. + m = Message("ovos.test", {"x": 1}) + h1 = hash(m) + m.data["x"] = 2 + assert hash(m) != h1 diff --git a/test/test_session.py b/test/test_session.py index 58c132c..170602d 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -560,5 +560,53 @@ def test_registered_fields_superset_of_owned(self): self.assertIn(f, SESSION1_REGISTERED_FIELDS) +# --- hashability (library-usefulness; not a spec rule) --------------------- + +class TestHashable(unittest.TestCase): + def test_equal_sessions_hash_equal(self): + a = Session(session_id="abc", lang="en-US") + b = Session(session_id="abc", lang="en-US") + self.assertEqual(a, b) + self.assertEqual(hash(a), hash(b)) + + def test_usable_as_dict_key_and_set_member(self): + a = Session(session_id="abc") + b = Session(session_id="abc") + c = Session(session_id="xyz") + d = {a: "value"} + self.assertEqual(d[b], "value") + self.assertEqual(len({a, b, c}), 2) + + def test_usable_in_lru_cache(self): + import functools + + calls = [] + + @functools.lru_cache(maxsize=None) + def resolve(sess): + calls.append(sess.session_id) + return sess.session_id + + s1 = Session(session_id="abc", lang="en-US") + s2 = Session(session_id="abc", lang="en-US") # equal to s1 + self.assertEqual(resolve(s1), "abc") + self.assertEqual(resolve(s2), "abc") + self.assertEqual(calls, ["abc"]) # cached: only one underlying call + + def test_nested_override_fields_hash(self): + s = Session(session_id="abc", + secondary_langs=["pt-PT", "es-ES"], + pipeline=["padatious_high"]) + self.assertEqual(hash(s), hash(s)) # stable, must not raise + + def test_mutation_then_rehash_changes_hash(self): + # documented behavior: the hash snapshots the mutable Session at the + # moment of hashing; mutating a field afterwards changes the hash. + s = Session(session_id="abc") + h1 = hash(s) + s.lang = "en-US" + self.assertNotEqual(hash(s), h1) + + if __name__ == "__main__": unittest.main() From 5d6b1e43fcd362c6962ad17041141065e02eaf2c Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 23:42:44 +0000 Subject: [PATCH 109/110] Increment Version to 0.18.0a1 --- ovos_spec_tools/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ovos_spec_tools/version.py b/ovos_spec_tools/version.py index 4f27e99..32aee2b 100644 --- a/ovos_spec_tools/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 -VERSION_MINOR = 17 -VERSION_BUILD = 3 +VERSION_MINOR = 18 +VERSION_BUILD = 0 VERSION_ALPHA = 1 # END_VERSION_BLOCK From c5556aeada55df0301d05a01bd9b1b7f77fcecdc Mon Sep 17 00:00:00 2001 From: JarbasAl <33701864+JarbasAl@users.noreply.github.com> Date: Sat, 27 Jun 2026 23:42:59 +0000 Subject: [PATCH 110/110] Update Changelog --- CHANGELOG.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37f00c0..662f855 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.18.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.18.0a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.3a1...0.18.0a1) + +**Merged pull requests:** + +- feat: make Session and Message hashable [\#61](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/61) ([JarbasAl](https://github.com/JarbasAl)) + ## [0.17.3a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.3a1) (2026-06-27) [Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.2a1...0.17.3a1) @@ -10,19 +18,19 @@ ## [0.17.2a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.2a1) (2026-06-27) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.0a1...0.17.2a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.1a1...0.17.2a1) **Merged pull requests:** - fix: reject malformed keyword intents at build/emit \(INTENT-3 §4.2\) [\#54](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/54) ([JarbasAl](https://github.com/JarbasAl)) -## [0.17.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.0a1) (2026-06-27) +## [0.17.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.1a1) (2026-06-27) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.1a1...0.17.0a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.0a1...0.17.1a1) -## [0.17.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.1a1) (2026-06-27) +## [0.17.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.0a1) (2026-06-27) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a2...0.17.1a1) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a2...0.17.0a1) **Merged pull requests:**