diff --git a/.github/workflows/build-tests.yml b/.github/workflows/build-tests.yml new file mode 100644 index 00000000..dff64be8 --- /dev/null +++ b/.github/workflows/build-tests.yml @@ -0,0 +1,14 @@ +name: Build Tests + +on: + pull_request: + branches: [dev, master, main] + workflow_dispatch: + +jobs: + build: + uses: OpenVoiceOS/gh-automations/.github/workflows/build-tests.yml@dev + with: + python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + install_extras: 'test' + test_path: 'test' diff --git a/.github/workflows/conventional-label.yml b/.github/workflows/conventional-label.yml new file mode 100644 index 00000000..9894c1ba --- /dev/null +++ b/.github/workflows/conventional-label.yml @@ -0,0 +1,10 @@ +# auto add labels to PRs +on: + pull_request_target: + types: [ opened, edited ] +name: conventional-release-labels +jobs: + label: + runs-on: ubuntu-latest + steps: + - uses: bcoe/conventional-release-labels@v1 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000..33fe452b --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,16 @@ +name: Code Coverage + +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + coverage: + uses: OpenVoiceOS/gh-automations/.github/workflows/coverage.yml@dev + with: + python_version: '3.11' + coverage_source: 'ovos_spec_tools' + test_path: 'test/' + install_extras: '.[test]' + min_coverage: 0 diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml new file mode 100644 index 00000000..faf7754a --- /dev/null +++ b/.github/workflows/license_check.yml @@ -0,0 +1,17 @@ +name: License Check + +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + license_check: + uses: OpenVoiceOS/gh-automations/.github/workflows/license-check.yml@dev + with: + # pilosus/action-pip-license-checker queries PyPI metadata for each + # installed package. The PyPI release of ovos-spec-tools predates the + # SPDX License classifier in pyproject.toml, so the package audits + # itself as "Error" (unknown). Run in warn-only mode until the next + # alpha republishes — transitive dependencies still get reported. + warn_only: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..0cb95641 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,13 @@ +name: Lint + +on: + pull_request: + branches: [dev, master, main] + workflow_dispatch: + +jobs: + lint: + uses: OpenVoiceOS/gh-automations/.github/workflows/lint.yml@dev + with: + ruff: true + pre_commit: false # set true if .pre-commit-config.yaml exists diff --git a/.github/workflows/pip_audit.yml b/.github/workflows/pip_audit.yml new file mode 100644 index 00000000..131320dc --- /dev/null +++ b/.github/workflows/pip_audit.yml @@ -0,0 +1,10 @@ +name: PIP Audit + +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + pip_audit: + uses: OpenVoiceOS/gh-automations/.github/workflows/pip-audit.yml@dev diff --git a/.github/workflows/publish_stable.yml b/.github/workflows/publish_stable.yml new file mode 100644 index 00000000..82c2a468 --- /dev/null +++ b/.github/workflows/publish_stable.yml @@ -0,0 +1,23 @@ +name: Publish Stable Release + +on: + workflow_dispatch: + push: + branches: [master, main] + +permissions: + contents: write # required for version bump commit and release tag + +jobs: + publish_stable: + if: github.actor != 'github-actions[bot]' + uses: OpenVoiceOS/gh-automations/.github/workflows/publish-stable.yml@dev + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + MATRIX_TOKEN: ${{ secrets.MATRIX_TOKEN }} + with: + version_file: 'ovos_spec_tools/version.py' + publish_pypi: true + publish_release: true + sync_dev: true + notify_matrix: true diff --git a/.github/workflows/release-preview.yml b/.github/workflows/release-preview.yml new file mode 100644 index 00000000..6057e42d --- /dev/null +++ b/.github/workflows/release-preview.yml @@ -0,0 +1,13 @@ +name: Release Preview + +on: + pull_request: + branches: [dev] + workflow_dispatch: + +jobs: + release_preview: + uses: OpenVoiceOS/gh-automations/.github/workflows/release-preview.yml@dev + with: + package_name: 'ovos_spec_tools' + version_file: 'ovos_spec_tools/version.py' diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml new file mode 100644 index 00000000..06776ae0 --- /dev/null +++ b/.github/workflows/release_workflow.yml @@ -0,0 +1,28 @@ +name: Release Alpha and Propose Stable + +on: + workflow_dispatch: + pull_request: + types: [closed] + branches: [dev] + +permissions: + contents: write + pull-requests: write + +jobs: + publish_alpha: + if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' + uses: OpenVoiceOS/gh-automations/.github/workflows/publish-alpha.yml@dev + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + MATRIX_TOKEN: ${{ secrets.MATRIX_TOKEN }} + with: + branch: 'dev' + version_file: 'ovos_spec_tools/version.py' + update_changelog: true + publish_prerelease: true + propose_release: true + changelog_max_issues: 50 + publish_pypi: true + notify_matrix: true diff --git a/.github/workflows/repo-health.yml b/.github/workflows/repo-health.yml new file mode 100644 index 00000000..187d3978 --- /dev/null +++ b/.github/workflows/repo-health.yml @@ -0,0 +1,12 @@ +name: Repo Health + +on: + pull_request: + branches: [dev, master, main] + workflow_dispatch: + +jobs: + repo_health: + uses: OpenVoiceOS/gh-automations/.github/workflows/repo-health.yml@dev + with: + version_file: 'ovos_spec_tools/version.py' diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..6d707478 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,279 @@ +# Changelog + +## [1.2.2a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/1.2.2a1) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/1.2.1a1...1.2.2a1) + +**Merged pull requests:** + +- fix: default session folds like any session \(drop owner-only reservation\) [\#71](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/71) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.2.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/1.2.1a1) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/1.2.0a1...1.2.1a1) + +**Merged pull requests:** + +- fix: default session single source of truth \(sessions dict\) [\#69](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/69) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.2.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/1.2.0a1) (2026-06-29) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/1.1.0a1...1.2.0a1) + +**Merged pull requests:** + +- feat: SessionManager singleton registry + forward/reply session stamping [\#67](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/67) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.1.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/1.1.0a1) (2026-06-28) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/1.0.0a1...1.1.0a1) + +**Merged pull requests:** + +- feat: complete SpecMessage with all spec-defined bus topics [\#65](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/65) ([JarbasAl](https://github.com/JarbasAl)) + +## [1.0.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/1.0.0a1) (2026-06-28) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.18.0a1...1.0.0a1) + +**Breaking changes:** + +- fix!: drop handler-trio from the namespace migration bridge \(orchestrator-owned\) [\#63](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/63) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.18.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.18.0a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.3a1...0.18.0a1) + +**Merged pull requests:** + +- feat: make Session and Message hashable [\#61](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/61) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.17.3a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.3a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.2a1...0.17.3a1) + +**Merged pull requests:** + +- fix: warn \(not raise\) on malformed intent at build/emit \(INTENT-3 §4.2\) [\#59](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/59) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.17.2a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.2a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.0a1...0.17.2a1) + +**Merged pull requests:** + +- fix: reject malformed keyword intents at build/emit \(INTENT-3 §4.2\) [\#54](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/54) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.17.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.0a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.17.1a1...0.17.0a1) + +## [0.17.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.17.1a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a2...0.17.1a1) + +**Merged pull requests:** + +- feat: bridge AUDIO-1 §7 output topics in MIGRATION\_MAP [\#55](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/55) ([JarbasAl](https://github.com/JarbasAl)) +- fix: enforce OVOS-MSG-1 §2.1 type syntax on serialize [\#53](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/53) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.16.1a2](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.16.1a2) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.1a1...0.16.1a2) + +**Merged pull requests:** + +- docs: make intent + bus-namespace docs timeless and standalone [\#50](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/50) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.16.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.16.1a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.16.0a1...0.16.1a1) + +**Merged pull requests:** + +- fix: message-domain conformance + NamespaceTranslator per-topic payload translation [\#42](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/42) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.16.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.16.0a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.15.0a1...0.16.0a1) + +**Merged pull requests:** + +- feat: plugin-agnostic IntentBuilder/Intent + voc\_match \(INTENT-4 keyword model, adapt-free\) [\#46](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/46) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.15.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.15.0a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.14.1a1...0.15.0a1) + +**Merged pull requests:** + +- feat: add fallback\_handlers to canonical Session \(OVOS-FALLBACK-1 registered field\) [\#45](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/45) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.14.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.14.1a1) (2026-06-27) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.14.0a1...0.14.1a1) + +**Merged pull requests:** + +- fix: locale/template/lint/language spec-conformance \(audit remediation\) [\#41](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/41) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.14.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.14.0a1) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.13.0a1...0.14.0a1) + +**Merged pull requests:** + +- feat: add persona\_id to canonical Session \(OVOS-PERSONA-1 registered field\) [\#37](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/37) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.13.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.13.0a1) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.12.1a1...0.13.0a1) + +**Merged pull requests:** + +- feat: dual-brace template refs \({x} and {{x}}\); .prompt double-brace-only [\#38](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/38) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.12.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.12.1a1) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.12.0a3...0.12.1a1) + +**Merged pull requests:** + +- fix: converse-handler cap is a per-insertion deployment value, not Session state [\#39](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/39) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.12.0a3](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.12.0a3) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.12.0a2...0.12.0a3) + +## [0.12.0a2](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.12.0a2) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.12.0a1...0.12.0a2) + +**Merged pull requests:** + +- docs: spec-traceability for the bus/message domain \(MSG-1\) [\#34](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/34) ([JarbasAl](https://github.com/JarbasAl)) +- docs: spec traceability for the locale/template/lint domain [\#33](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/33) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.12.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.12.0a1) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.11.0a1...0.12.0a1) + +**Merged pull requests:** + +- feat: OVOS-SESSION-1 canonical Session reference implementation [\#16](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/16) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.11.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.11.0a1) (2026-06-26) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.10.0a1...0.11.0a1) + +**Merged pull requests:** + +- feat: bridge PIPELINE-1 trio, STOP-1, intent-unmatched & INTENT-4 management topics [\#30](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/30) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.10.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.10.0a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.9.0a1...0.10.0a1) + +**Merged pull requests:** + +- feat: add NamespaceTranslator \(shared bus-namespace migration logic\) [\#28](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/28) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.9.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.9.0a1) (2026-06-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.8.0a2...0.9.0a1) + +**Merged pull requests:** + +- feat: SpecMessage registry + legacy-\>spec MIGRATION\_MAP [\#26](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/26) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.8.0a2](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.8.0a2) (2026-06-13) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.8.0a1...0.8.0a2) + +**Merged pull requests:** + +- docs: standardize NGI0 Commons Fund attribution [\#24](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/24) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.8.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.8.0a1) (2026-06-02) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.7.0a1...0.8.0a1) + +**Merged pull requests:** + +- feat: inline\_keywords — resolve \ refs as \(a|b|c\) for engines without .voc support [\#22](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/22) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.7.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.7.0a1) (2026-06-02) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.6.0a1...0.7.0a1) + +**Merged pull requests:** + +- feat: union slot sets for .intent \(OVOS-INTENT-1 §5.5\) [\#19](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/19) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.6.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.6.0a1) (2026-05-25) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.5.1a1...0.6.0a1) + +**Merged pull requests:** + +- feat: find\_lang\_dir — standalone language-aware directory resolver [\#17](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/17) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.5.1a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.5.1a1) (2026-05-24) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.5.0a1...0.5.1a1) + +**Merged pull requests:** + +- fix\(Message\): three follow-ups missed by the 0.5.0a1 cut [\#14](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/14) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.5.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.5.0a1) (2026-05-24) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.4.0a1...0.5.0a1) + +**Merged pull requests:** + +- feat: OVOS-MSG-1 Message envelope [\#12](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/12) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.4.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.4.0a1) (2026-05-23) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.3.0a1...0.4.0a1) + +**Merged pull requests:** + +- feat: LocaleResources.voc\_list / .voc\_match / .remove\_voc [\#10](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/10) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.3.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.3.0a1) (2026-05-23) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.2.0a1...0.3.0a1) + +**Merged pull requests:** + +- feat: keyword\_form / vocabulary\_keywords + utterance\_contains / strip\_samples [\#8](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/8) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.2.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.2.0a1) (2026-05-23) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.1.0a1...0.2.0a1) + +**Merged pull requests:** + +- feat: LocaleResources.find — public resource lookup by \(name, ext, lang\) [\#6](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/6) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.1.0a1](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.1.0a1) (2026-05-23) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/0.0.1a2...0.1.0a1) + +**Merged pull requests:** + +- feat: lang\_matches and iter\_locale\_dirs — pin the cross-component lang policy [\#4](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/4) ([JarbasAl](https://github.com/JarbasAl)) + +## [0.0.1a2](https://github.com/OpenVoiceOS/ovos-spec-tools/tree/0.0.1a2) (2026-05-22) + +[Full Changelog](https://github.com/OpenVoiceOS/ovos-spec-tools/compare/13d694e2960b7ede0533f4029442301c21c86a8c...0.0.1a2) + +**Merged pull requests:** + +- Configure Renovate [\#1](https://github.com/OpenVoiceOS/ovos-spec-tools/pull/1) ([renovate[bot]](https://github.com/apps/renovate)) + + + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* diff --git a/README.md b/README.md index 16ebf41e..d231647c 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,97 @@ -# ovos-intent-primitives +# ovos-spec-tools -Reference implementation of the OVOS intent specifications — the low-level, -dependency-light primitives that the [formal -specifications](https://github.com/OpenVoiceOS/formal-specifications) describe. +Reference implementation of the OVOS [formal +specifications](https://github.com/OpenVoiceOS/architecture) — the +low-level, dependency-light primitives those specifications describe. -OVOS components reimplement template expansion and resource loading in several -places, and the copies drift. This package is the single conformant -implementation those components — and any third-party tool — can depend on. +OVOS components reimplement template expansion, resource loading, and language +matching in several places, and the copies drift. This package is the single +conformant implementation those components — and any third-party tool — can +depend on. ## Status -| Primitive | Spec | State | -|-----------|------|-------| -| Sentence template expander | OVOS-INTENT-1 v2 | available | -| Locale resource loader | OVOS-INTENT-2 | planned | -| Dialog renderer | OVOS-INTENT-2 §4.2 | planned | +| Tool | Spec | Code | +|------|------|-------| +| Sentence template expander | OVOS-INTENT-1 v2 | `expansion.py` | +| Locale resource loader | OVOS-INTENT-2 | `resources.py` | +| Dialog renderer | OVOS-INTENT-2 §4.2 | `dialog.py` | +| Prompt renderer | OVOS-INTENT-2 §4.4 | `prompt.py` | +| Language-tag matching | OVOS-INTENT-2 §2.2 | `language.py` | +| Bus message envelope | OVOS-MSG-1 | `message.py` | +| `ovos-spec-lint` locale linter | OVOS-INTENT-1 / -2 | `lint.py` | -## The expander +## Install + +```bash +pip install ovos-spec-tools # core — no dependencies +pip install ovos-spec-tools[langcodes] # adds the smart language fallback +``` + +Requires Python 3.10+. -`expand()` turns a sentence template into its **sample set** — the finite set -of sentences it denotes (OVOS-INTENT-1 §4). It resolves `(a|b)` alternatives, -`[x]` optionals, and `` inline vocabulary references; named `{name}` -slots are opaque and carried through unchanged. +## Quick taste ```python -from ovos_intent_primitives import expand +from ovos_spec_tools import (expand, LocaleResources, render, render_prompt, + closest_lang, Message) -expand("(turn|switch) [the] (light|fan)") -# ['turn the light', 'turn the fan', 'turn light', 'turn fan', -# 'switch the light', 'switch the fan', 'switch light', 'switch fan'] +expand("(turn|switch) [the] light") # all 4 sentences it denotes +res = LocaleResources("my-skill/locale") +res.load_intent("play", "en-US") # a skill's intent samples +render(res.load_dialog("weather", "en-US"), # a spoken response + slots={"temperature": 21}) +render_prompt(res.load_prompt("system", "en-US"), # a language-model prompt + slots={"query": "what time is it"}) +closest_lang("en-AU", ["pt-BR", "en-US"]) # 'en-US' -expand(" {name}", {"greeting": ["hello", "hi"]}) -# ['hello {name}', 'hi {name}'] +# OVOS-MSG-1 envelope: a bus message and its 'send back to the asker' reply +m = Message("ovos.intent.list", {}, {"source": "skill.id"}) +res = m.response({"intents": ["..."]}) # 'ovos.intent.list.response' +res.serialize() # single UTF-8 JSON object ``` -A template that violates OVOS-INTENT-1 §3.6 — unbalanced metacharacters, a -single-branch group, an empty sample, a slot-only template, adjacent slots, a -repeated slot name, or an undefined or cyclic vocabulary reference — raises -`MalformedTemplate`. +```bash +ovos-spec-lint my-skill/locale # validate a locale folder +``` -Input is assumed already ASR-normalized (OVOS-INTENT-1 §2): lowercase, -single-spaced, alphanumeric. This package expands; it does not normalize. +## Documentation -## Install +A zero-to-hero guide lives in [`docs/`](docs/README.md): -```bash -pip install ovos-intent-primitives -``` +1. [Getting started](docs/getting-started.md) — install, and a first taste of + every tool. +2. [Sentence templates](docs/templates.md) — the grammar: alternatives, + optionals, slots, vocabulary references, malformed forms. +3. [Locale resources](docs/locale-resources.md) — the `locale/` folder, the + six file roles, loading across languages, override precedence. +4. [Dialog](docs/dialog.md) — choosing and filling a spoken response, and + rendering language-model prompts. +5. [Language matching](docs/language-matching.md) — tag standardization, + distance, and closest-match resolution. +6. [Bus messages](docs/message.md) — the on-the-wire envelope, the + `forward` / `reply` / `response` derivations, and the session carrier. +7. [Linting](docs/linting.md) — validating a locale folder, on the CLI or in CI. +7. [API reference](docs/api-reference.md) — every public name, in brief. + +Runnable example scripts are in [`examples/`](examples/README.md). + +--- + +## Credits + +Developed by [TigreGótico](https://tigregotico.pt) for +[OpenVoiceOS](https://openvoiceos.org). + +[![NGI0 Commons Fund](./ngi.png)](https://nlnet.nl/project/OpenVoiceOS) + +This project was funded through the [NGI0 Commons Fund](https://nlnet.nl/commonsfund), +a fund established by [NLnet](https://nlnet.nl) with financial support from the +European Commission's [Next Generation Internet](https://ngi.eu) programme, under +the aegis of [DG Communications Networks, Content and Technology](https://commission.europa.eu/about-european-commission/departments-and-executive-agencies/communications-networks-content-and-technology_en) +under grant agreement No [101135429](https://cordis.europa.eu/project/id/101135429). + +--- ## License diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..d8cb6ccb --- /dev/null +++ b/docs/README.md @@ -0,0 +1,62 @@ +# ovos-spec-tools — documentation + +`ovos-spec-tools` is the reference implementation of the +[OVOS formal specifications](https://github.com/OpenVoiceOS/architecture): +the small, dependency-light primitives those specs describe, in one place, so +OVOS components and third-party tools stop reimplementing them and drifting +apart. + +It gives you five things: + +- an **expander** — turns a sentence template into the set of sentences it + stands for (OVOS-INTENT-1); +- a **resource loader** — reads a skill's `locale/` folder (OVOS-INTENT-2); +- a **dialog renderer** — picks and fills a spoken response (OVOS-INTENT-2 §4.2); +- **language matching** — normalizes tags and finds the closest one; +- a **bus message envelope** — the `type` / `data` / `context` JSON contract + and its `forward` / `reply` / `response` derivations (OVOS-MSG-1); + +plus **`ovos-spec-lint`**, a command-line linter for locale folders. + +## The guide + +Read it in order — each chapter builds on the one before, and **templates** +are the foundation everything else rests on. + +1. [Getting started](getting-started.md) — install it, and a first taste of + every tool. +2. [Sentence templates](templates.md) — the grammar: alternatives, optionals, + slots, vocabulary references, and what counts as malformed. +3. [Locale resources](locale-resources.md) — the `locale/` folder, the five + file roles, and loading them across languages. +4. [Dialog](dialog.md) — choosing and filling a spoken response, with the + stateless function and the stateful renderer. +5. [Language matching](language-matching.md) — tag standardization, distance, + and closest-match resolution. +6. [Bus messages](message.md) — the on-the-wire envelope, the three + derivations (`forward` / `reply` / `response`), and the session carrier. +7. [Bus namespaces](bus-namespaces.md) — the spec topic vocabulary + (`SpecMessage`), the legacy↔`ovos.*` `MIGRATION_MAP`, and the transparent + dual-emit bridge with its migration window. +8. [Linting](linting.md) — validating a locale folder, from the command line + or in CI. +9. [API reference](api-reference.md) — every public name, in brief. + +### Proving the scope + +- **[Spec traceability](spec-traceability.md)** — every public symbol in + `ovos-spec-tools` mapped to its authoritative OVOS spec section. This is + the index that backs the scope note below: the package is *exclusively* a + reference implementation of the specs, with no general-purpose code. + +## A note on scope + +This package **expands, loads, renders, matches language, lints, and provides +the bus message envelope**. It does not *recognize* intents — matching an +utterance to an intent is the job of an intent engine, and is deliberately +out of scope (see OVOS-INTENT-1 §4). It does not *transport* messages either +— wire framing, encryption, websocket clients, multi-tenant routing, and +session lifecycle all belong to the layers that consume the envelope +(`ovos-bus-client` for the websocket transport; HiveMind for layer-2 +routing). What you get here is the data those engines consume, the message +shape they exchange, and the tooling around them. diff --git a/docs/api-reference.md b/docs/api-reference.md new file mode 100644 index 00000000..699d0bb1 --- /dev/null +++ b/docs/api-reference.md @@ -0,0 +1,188 @@ +# 7. API reference + +Every public name, in brief. All are importable from the top-level package: + +```python +from ovos_spec_tools import expand, LocaleResources, render, closest_lang # etc. +``` + +For the why and the worked examples, see the chapter each section links to. + +## Expansion — [chapter 2](templates.md) + +### `expand(template, vocabularies=None) -> list[str]` + +Expand a sentence template to its sample set. `vocabularies` is a +`name -> list[str]` dict, needed only if the template uses `` references. +Raises `MalformedTemplate`. + +### `MalformedTemplate` + +`ValueError` subclass — a template violates OVOS-INTENT-1 §3.6. + +## Locale resources — [chapter 3](locale-resources.md) + +### `LocaleResources(skill_locale, core_locale=None, user_locale=None, lang_resolver=None, max_language_distance=10)` + +Loads a skill's locale resource files. The three `*_locale` arguments are paths +to `locale/` directories, in ascending override precedence. `lang_resolver` is +a `(target, available, max_distance) -> str | None` callable (default +`closest_lang`); `max_language_distance` caps the smart fallback (`0` disables +it). + +Methods — each takes a resource base name and a BCP-47 `lang`: + +| Method | Returns | +|--------|---------| +| `load_intent(name, lang)` | expanded sample set, slots intact | +| `load_entity(name, lang)` | expanded value set | +| `load_vocabulary(name, lang)` | expanded phrase set | +| `load_blacklist(name, lang)` | expanded phrase set | +| `load_dialog(name, lang)` | raw phrase strings (not expanded) | +| `load_prompt(name, lang)` | the whole `.prompt` file, as one string | +| `vocabularies(lang)` | `name -> templates` for every `.voc` | +| `entities(lang)` | `name -> values` for every `.entity` | + +A missing resource raises `FileNotFoundError`; a malformed one raises +`MalformedResource`. + +### `read_resource_file(path) -> list[str]` + +Apply the OVOS-INTENT-2 §3 common reader to one file: UTF-8, BOM discarded, +`LF`/`CRLF` accepted, lines stripped, blank and `#`-comment lines dropped. + +### `read_prompt_file(path) -> str` + +Read a `.prompt` whole and verbatim — UTF-8, BOM discarded, no line filtering. + +### `MalformedResource` + +`ValueError` subclass — a resource file or layout violates OVOS-INTENT-2 +(empty file, duplicate `(role, base name)`, a slot in a slot-free role). + +## Dialog — [chapter 4](dialog.md) + +### `render(phrases, slots=None, vocabularies=None, rng=None) -> str` + +Render one phrase from an explicit list. `slots` fills `{name}` slots; +`vocabularies` resolves `` references; `rng` is any object with a +`choice` method (for reproducible output). Raises `UnfilledSlot`, or +`ValueError` if `phrases` is empty. + +### `DialogRenderer(resources, name, rng=None, slots=None)` + +A stateful, multilingual renderer for the dialog `name`, backed by a +`LocaleResources`. `slots` are default slot values reused on every call. + +- **`render(lang, slots=None) -> str`** — render one phrase in `lang`. Avoids + repeating the previous phrase (per language). Slot precedence: per-call, + then default, then a random `.entity` value, then `UnfilledSlot`. + +### `UnfilledSlot` + +`ValueError` subclass — a chosen phrase has a slot with no value. + +## Prompts — [chapter 4](dialog.md) + +### `render_prompt(text, slots=None) -> str` + +Render a `.prompt` string. The whole `text` is the prompt; a `{name}` is +substituted only when it is a well-formed name, `slots` supplies a value, and +it is not inside a ``` ``` ``` fenced code block. An unfilled slot, and any +other `{`/`}`, is left literal. Never raises for an unfilled slot. + +### `PromptRenderer(resources, name, slots=None)` + +A stateful, multilingual renderer for the prompt `name`, backed by a +`LocaleResources`. `slots` are default values reused on every call. + +- **`render(lang, slots=None) -> str`** — render the prompt in `lang`; a + per-call value overrides a default. Raises `FileNotFoundError` if the prompt + is missing for `lang`, `MalformedResource` if the file is empty. + +## Language matching — [chapter 5](language-matching.md) + +### `standardize_lang(tag) -> str` + +Normalize a BCP-47 tag (underscores, case, canonical forms). + +### `lang_distance(desired, supported) -> int` + +Distance between two tags: `0` is identical, `>= 10` is not a usable match. A +bare tag is measured from its norm region. + +### `closest_lang(target, available, max_distance=10) -> str | None` + +The entry of `available` with the smallest `lang_distance`, if it is below +`max_distance` (or exact). Returns the original string, or `None`. + +## Bus messages — [chapter 6](message.md) + +### `Message(msg_type, data=None, context=None)` + +The OVOS-MSG-1 envelope: exactly three top-level fields — `msg_type` +(the wire field `type`), `data`, `context`. The constructor rejects +malformed input as `MalformedMessage`. `data` and `context` default to +empty dicts and are stored by reference. + +#### Derivations + +- `m.forward(msg_type, data=None) -> Message` — same context, new + type/data; the forwarder does not become the new `source`. +- `m.reply(msg_type, data=None, context=None) -> Message` — copies + context, overlays the optional `context`, swaps `source` and + `destination`. Other context keys (including `session`) pass through + unchanged. +- `m.response(data=None, context=None) -> Message` — sugar for + `reply(msg_type + ".response", ...)`. + +All three preserve the runtime class — subclasses get back instances +of their own subclass. + +#### Serialization + +- `m.serialize() -> str` — single UTF-8 JSON object per OVOS-MSG-1 §6; + recursively converts nested objects exposing a `.serialize()` method + (e.g. `Session`). +- `m.as_dict -> dict` (property) — JSON-decoded envelope, equivalent + to `json.loads(m.serialize())`. Nested `.serialize()`-walking carriers + are converted the same way they would be on the wire. +- `Message.deserialize(payload) -> Message` — parse a JSON string, + bytes, or already-parsed dict. Rejects unknown top-level keys, missing + `type`, wrong value types, NaN/Infinity as `MalformedMessage`. + +### `MalformedMessage` + +`ValueError` subclass raised by the constructor and `deserialize` when a +payload violates OVOS-MSG-1 §2 / §6 / §7. + +### `DEFAULT_SESSION_ID` + +The reserved `"default"` value (OVOS-MSG-1 §4.1) — *the Message +originates from the device itself*. An absent `session` on a Message is +equivalent for every policy decision. + +## Linting — [chapter 7](linting.md) + +### `lint_locale(path, spec_version=2) -> list[Finding]` + +Validate every resource file under a locale (or single-language) directory. +`spec_version` (0, 1, or 2) flags features newer than that target — see +[chapter 7](linting.md). + +### `Finding` + +A dataclass with `severity` (`"error"` / `"warning"`), `path`, and `message`. +`str(finding)` formats it as one line. + +### `ovos-spec-lint` (command) + +CLI wrapper over `lint_locale`. `ovos-spec-lint [--strict] +[--spec-version {0,1,2}]`; exit code is non-zero on errors (with `--strict`, +on warnings too). + +## Package + +### `__version__` + +The installed package version string. diff --git a/docs/bus-namespaces.md b/docs/bus-namespaces.md new file mode 100644 index 00000000..acbd88b0 --- /dev/null +++ b/docs/bus-namespaces.md @@ -0,0 +1,209 @@ +# Bus namespaces — the legacy ↔ `ovos.*` migration + +OVOS bus topics span two namespaces: the Mycroft-era names +(`speak`, `recognizer_loop:utterance`, `mycroft.skill.handler.start`, …) +and the `ovos.*` namespace the specifications define +(`ovos.utterance.speak`, `ovos.utterance.handle`, +`ovos.intent.handler.start`, …). `ovos-spec-tools` owns the **vocabulary** +of the mapping between them and the **transparent bridge** that lets a +deployment cross from one to the other without a flag day. Two things implement it, both in `ovos_spec_tools/messages.py`: + +- `SpecMessage` — the enum of spec-defined `ovos.*` topics; +- `MIGRATION_MAP` + `NamespaceTranslator` — the rename map and the + dual-emit/dedup bridge the bus runs on top of it. + +Everything here references the spec that *owns* each topic, so a reader can +always answer "which document made this a topic?" — see the +[spec traceability map](spec-traceability.md) for the full index. + +## `SpecMessage` — the spec topic vocabulary + +`SpecMessage` is a `str` enum (OVOS-MSG-1 §2.1 topics are strings, so a +member *is* a usable topic): + +```python +from ovos_spec_tools import SpecMessage, Message + +bus.on(SpecMessage.SPEAK, handler) # 'ovos.utterance.speak' +bus.emit(Message(SpecMessage.UTTERANCE, {...})) # 'ovos.utterance.handle' +str(SpecMessage.STOP) # 'ovos.stop' (StrEnum-like) +``` + +Referencing `SpecMessage.SPEAK` instead of the bare string makes code +self-documenting: a `SpecMessage` member is *provably* spec-defined; a bare +string is visibly legacy or implementation-specific. Each member's owning +spec section is cited in the source and in the table below. + +| Group | Owning spec | Topics | +|---|---|---| +| Utterance lifecycle | OVOS-PIPELINE-1 §9 | `UTTERANCE` (§9.1), `SPEAK` (§9.6), `UTTERANCE_HANDLED` (§9.5), `UTTERANCE_CANCELLED` (§6.4), `INTENT_MATCHED` (§9.2), `INTENT_UNMATCHED` (§9.3) | +| Handler-lifecycle trio | OVOS-PIPELINE-1 §8 | `INTENT_HANDLER_START`, `INTENT_HANDLER_COMPLETE`, `INTENT_HANDLER_ERROR` | +| Registration / management | OVOS-INTENT-4 §§5–8 | `INTENT_REGISTER_KEYWORD` (§5), `INTENT_REGISTER_TEMPLATE` (§6), `ENTITY_REGISTER` (§7), `INTENT_DEREGISTER` (§8.2), `ENTITY_DEREGISTER` (§8.3), `SKILL_DEREGISTER` (§8.4), `INTENT_ENABLE`/`INTENT_DISABLE` (§8.5) | +| Introspection | OVOS-INTENT-4 §10 | `INTENT_LIST`, `INTENT_LIST_RESPONSE`, `INTENT_DESCRIBE`, `INTENT_DESCRIBE_RESPONSE` | +| Stop cascade | OVOS-STOP-1 §4–§5 | `STOP_PING` (§4.2), `STOP_PONG` (§4.2), `STOP` (§5.3) | +| Listener lifecycle | OVOS-AUDIO-IN-1 §6 | `LISTENER_RECORD_STARTED`, `LISTENER_RECORD_ENDED`, `LISTENER_SLEEP`, `LISTENER_AWOKEN` | +| Mic / audio output | OVOS-AUDIO-1 §4.4, §5 | `MIC_LISTEN` (§4.4), `AUDIO_OUTPUT_STARTED` (§5.1), `AUDIO_OUTPUT_ENDED` (§5.2) | + +The listener lifecycle signals live on the `ovos.listener.*` namespace +(OVOS-AUDIO-IN-1 §6); the mic re-open flag and audio-output session signals +belong to the audio-output service (OVOS-AUDIO-1 §4.4, §5). + +## `MIGRATION_MAP` — the rename map + +`MIGRATION_MAP` maps each **legacy** topic to the `SpecMessage` that +supersedes it. `SPEC_TO_LEGACY` is its reverse, and `migration_counterpart` +resolves either direction: + +```python +from ovos_spec_tools import migration_counterpart + +migration_counterpart("speak") # 'ovos.utterance.speak' +migration_counterpart("ovos.utterance.speak") # 'speak' +migration_counterpart("some.app.topic") # None (does not migrate) +``` + +Each entry encodes one spec rename. A sample: + +| Legacy topic | `SpecMessage` | Spec § | +|---|---|---| +| `recognizer_loop:utterance` | `UTTERANCE` | PIPELINE-1 §9.1 | +| `speak` | `SPEAK` | PIPELINE-1 §9.6 | +| `complete_intent_failure` | `INTENT_UNMATCHED` | PIPELINE-1 §9.3 | +| `mycroft.skill.handler.start` | `INTENT_HANDLER_START` | PIPELINE-1 §8.1 | +| `skill.stop.pong` | `STOP_PONG` | STOP-1 §4.2 | +| `mycroft.stop` | `STOP` | STOP-1 §5.3 | +| `detach_intent` | `INTENT_DEREGISTER` | INTENT-4 §8.2 | +| `mycroft.skill.enable_intent` | `INTENT_ENABLE` | INTENT-4 §8.5 | + +### Two deliberate exclusions + +The bridge mirrors the **payload verbatim** (see below), so only renames that +need no payload transformation can live in the map. Two spec renames are +intentionally *not* mapped: + +1. **INTENT-4 §5–§7 registration.** The rename is not 1:1. Legacy Adapt emits + *N* `register_vocab` messages plus one `register_intent` (the intent + references its vocab by name); INTENT-4 §5.2 consolidates all of that into + **one** `ovos.intent.register.keyword` Message with the vocab descriptors + **inlined** (§5.1). A transparent bus bridge cannot synthesize that N→1 + join — it would have to buffer and merge messages it has no schema for — so + registration is adopted in the **producer** (`ovos_workshop/intents.py`) + and **consumers** (the pipeline plugins), not bridged here. + +2. **STOP-1 per-skill ping placeholders.** The legacy stoppability handshake + used per-skill topics shaped `{skill_id}.stop.ping` / `{skill_id}.stop` + (OVOS-MSG-1 §2.1.1 *runtime-assembled* topics). STOP-1 §4.2/§5.3 replace + them with the single broadcast `ovos.stop.ping` / `ovos.stop`. A + `{skill_id}.*` placeholder is not a static string, so it cannot be a dict + key; that migration is handled by producers/consumers subscribing on both + forms. + +> A subtlety: some mapped entries are commented "payload restructured to +> `{skill_id, intent_name}`" (the handler trio, the INTENT-4 management +> topics). The bridge *itself* still transforms nothing — once the **producer** +> adopts the modern payload shape, the mirror carries that already-modern +> payload on the legacy topic too. The restructure is producer-side adoption; +> the map owns only the topic rename. The two are composed, never conflated. + +## The transparent bridge — `NamespaceTranslator` + +`NamespaceTranslator` is the reference implementation of the **dual-emit +bus bridge**. `ovos-bus-client`'s `MessageBusClient` and `ovos_utils`' +`FakeBus` both delegate to it, so the real websocket bus and the +test/satellite double behave identically. It is pure logic — no I/O, no +config — so `ovos-spec-tools` stays dependency-free; the caller reads +env/config and passes the two direction flags in. + +```python +from ovos_spec_tools import NamespaceTranslator +xlat = NamespaceTranslator(modernize=True, emit_legacy=True, window=1.0) +``` + +| Flag | Effect | +|---|---| +| `modernize` | emitting a **legacy** topic also emits its `ovos.*` counterpart — carries the migration forward | +| `emit_legacy` | emitting an `ovos.*` topic also emits the legacy counterpart — keeps un-migrated consumers working | +| `window` | the **mirror window** (seconds) for receive-side dedup (below) | + +### Send side — `counterpart_topics` + +When a producer emits one topic, the bus asks the translator which extra +topic to mirror onto, then re-emits the **same payload** there: + +```python +xlat.counterpart_topics("speak") # ['ovos.utterance.speak'] +xlat.counterpart_topics("ovos.utterance.speak") # ['speak'] +xlat.counterpart_topics("some.app.topic") # [] (never mirrored) +``` + +At most one counterpart per topic. The flags gate each direction +independently (a `modernize`-only bus mirrors legacy→spec but not the +reverse). + +### Receive side — `new_mirror_guard` and the mirror window + +A handler subscribed to **both** namespaces would otherwise fire twice on a +dual-emit. `new_mirror_guard()` returns a stateful predicate +`is_mirror(message) -> bool` — **one per handler**, each with private +`seen` state — that the bus uses to run the handler exactly once: + +```python +guard = xlat.new_mirror_guard() +guard(Message("speak", {"utterance": "hi"})) # False — first arrival, run it +guard(Message("ovos.utterance.speak", {"utterance": "hi"})) # True — mirror, drop it +``` + +Mirror-window semantics, precisely: + +- A Message is a **mirror** iff a recently-seen Message had the same + payload+context fingerprint, a **different** `msg_type`, **and** that + earlier type's `migration_counterpart` equals this one — i.e. the same + event re-delivered on the counterpart topic. The payload fingerprint is + order-independent (OVOS-MSG-1 §6: key order is not significant). +- Two genuine events on the **same** topic are never suppressed — identical + repeats on one topic are real repeats, not a mirror. +- Entries older than `window` are evicted before each check, so a + counterpart that arrives *after* the window counts as a new event. The + bridge only ever collapses a near-simultaneous dual-emit **pair**, never + two deliberate emissions spaced apart. +- The guard is defensive: anything not Message-shaped (no `msg_type` / + `data` / `context`, or an unfingerprintable payload) returns `False` — + it never drops what it cannot prove is a mirror. + +## Worked example — a skill says "hello" during the migration + +A skill emits the modern spec topic only: + +```python +bus.emit(Message(SpecMessage.SPEAK, {"utterance": "hello"})) # 'ovos.utterance.speak' +``` + +On a bus configured `emit_legacy=True`: + +1. **Send.** The bus emits `ovos.utterance.speak`, then calls + `counterpart_topics("ovos.utterance.speak")` → `['speak']` and re-emits + the **same** `{"utterance": "hello"}` payload on the legacy `speak` + topic. Old TTS components still listening on `speak` receive it. +2. **Receive.** A modernized audio component subscribed to *both* + `ovos.utterance.speak` and `speak` runs its `is_mirror` guard: + - `ovos.utterance.speak` arrives first → `False` → handler runs once; + - `speak` arrives within the window, same payload, counterpart topic → + `True` → dropped. + +The skill wrote one modern emission; legacy and modern consumers both got +exactly one delivery. When the migration completes and `emit_legacy` is +turned off, the legacy mirror simply stops — no skill code changes. + +## What this is *not* + +The bridge is a **topic rename**, not a payload translator and not a +correlation mechanism. It never invents fields, never joins messages (hence +exclusion 1), and never touches `context` routing or `session` — those follow +the OVOS-MSG-1 §5 derivation rules implemented by [`Message`](message.md). + +## See also + +- [Bus messages](message.md) — the OVOS-MSG-1 envelope and the `forward` / + `reply` / `response` derivations these topics ride on. +- [Spec traceability](spec-traceability.md) — every public symbol mapped to + its authoritative spec section. diff --git a/docs/dialog.md b/docs/dialog.md new file mode 100644 index 00000000..ae3bae39 --- /dev/null +++ b/docs/dialog.md @@ -0,0 +1,157 @@ +# 4. Dialog + +> **Spec coverage.** This chapter is the reference for the **Dialog renderer** +> conformance role of OVOS-INTENT-1 §7 over the `.dialog` format of +> OVOS-INTENT-2 §4.2 (`dialog.py`), and for the `.prompt` format of +> OVOS-INTENT-2 §4.4 (`prompt.py`, at the end of the chapter). A dialog renderer +> MUST embed a conformant expander, verify slot-set consistency (§5.5), fill +> every `{name}` from caller-supplied values, and MUST NOT emit an unfilled slot +> (§5.1) — `UnfilledSlot` is that last MUST. + +A `.dialog` file holds the phrases an assistant may speak for one response. +**Rendering** a dialog means: pick one phrase, expand its `(a|b)` / `[x]` +variety down to a single variant, and fill its `{name}` slots with values. This +chapter covers the two ways to do it. It implements OVOS-INTENT-2 §4.2. + +A `.dialog` phrase is spoken *output*, not ASR input, so — unlike the +input-direction roles — it may contain mixed case and punctuation: + +``` +# weather.dialog +It is {temperature} degrees. +Right now it's {temperature} degrees out. +(Currently|At the moment) {temperature} degrees. +``` + +## `render()` — the stateless function + +`render()` takes an explicit list of phrases and returns one rendered sentence: + +```python +from ovos_spec_tools import render + +phrases = res.load_dialog("weather", "en-US") +render(phrases, slots={"temperature": 21}) +# 'It is 21 degrees.' +``` + +Slots are filled from the `slots` dict. The phrase and the `(a|b)` variant are +chosen at random; pass `rng=` (anything with a `choice` method, e.g. a seeded +`random.Random`) for reproducible output. `vocabularies=` supplies any +`` references. + +Expansion runs **before** filling, with slots kept opaque — so a slot value can +never be mis-parsed as grammar. A value of `"(a|b)"` is filled in literally, +not expanded. + +If the chosen phrase has a slot with no value, `render()` raises +`UnfilledSlot` — a half-filled phrase must never reach text-to-speech. + +## `DialogRenderer` — the stateful, multilingual renderer + +`render()` is fine for one-off use. For a skill that speaks a response +repeatedly, `DialogRenderer` does better. It is built from a `LocaleResources` +and a dialog name: + +```python +from ovos_spec_tools import DialogRenderer + +renderer = DialogRenderer(res, "weather") +renderer.render("en-US", {"temperature": 21}) +renderer.render("pt-PT", {"temperature": 22}) # same renderer, another language +``` + +It adds three things over the bare function. + +### It is multilingual + +The language is a parameter of `render()`, not of the constructor — one +renderer serves every language the dialog ships in. + +### It avoids repeating itself + +`DialogRenderer` remembers the phrase it chose last time and avoids picking it +again — tracked **per language** — so a frequently-spoken response does not +sound mechanical. With two phrases, consecutive renders strictly alternate. + +### It carries default slots and an `.entity` fallback + +A slot is resolved in a clear order of precedence: + +1. a value passed to this `render()` call; +2. a **default** value, set once on the constructor and reused every call; +3. a random value from the slot's **`.entity`** value set; +4. otherwise — `UnfilledSlot`. + +```python +renderer = DialogRenderer(res, "greeting", slots={"assistant": "OVOS"}) +renderer.render("en-US") +``` + +Here `{assistant}` is filled from the default every time. A `{weekday}` slot +that the caller does not supply is filled from `weekday.entity` if the skill +ships one — useful for a slot whose value is a free pick from a known set +rather than a computed value. + +## Choosing between them + +| | `render()` | `DialogRenderer` | +|--|------------|------------------| +| Input | an explicit phrase list | a `LocaleResources` + dialog name | +| Language | decided by the caller | per `render()` call | +| Repetition avoidance | no | yes, per language | +| Default slots / `.entity` fallback | no | yes | + +Use `render()` when you already hold the phrases and want one sentence. Use +`DialogRenderer` for a skill response spoken more than once. + +## Rendering `.prompt` files + +A `.prompt` is the localized prompt a skill feeds to a language model. Unlike a +`.dialog` it is **not** a template — it is plain text — and the whole file, +verbatim, is one prompt. The only special construct is `{name}` substitution, +and it is **conservative**: a prompt is free-form text full of code and JSON, +so rendering must never corrupt a brace the author did not write as a slot. + +`render_prompt()` is the stateless function: + +```python +from ovos_spec_tools import render_prompt + +render_prompt("You are {role}. Answer: {query}", {"role": "concise"}) +# 'You are concise. Answer: {query}' +``` + +A `{name}` is replaced **only** when all three OVOS-INTENT-2 §4.4 conditions +hold: it is a well-formed name (§4.4 condition 1), the caller supplied a value +(condition 2), and it is not inside a ``` ``` ``` fenced code block +(condition 3). Everything else stays literal: + +- an **unfilled** slot is left as `{name}` — §4.4 "slots are optional", the + deliberate opposite of `.dialog`, where every slot must be filled; +- any other `{`/`}` — `{}`, `{ }`, JSON like `{"k": 1}` — is untouched; +- a `{name}` inside a fenced code block is never substituted. + +> **Known gap.** OVOS-INTENT-2 §4.4 also requires author-only HTML comments +> `` to be **stripped** before a prompt reaches a model (a MUST). +> `render_prompt()` does **not** yet do this — a comment passes through +> verbatim. Do not rely on comment removal until it is implemented. + +`PromptRenderer` is the resource-backed, multilingual form — built from a +`LocaleResources`, with the language given per call and optional default slots: + +```python +from ovos_spec_tools import PromptRenderer + +renderer = PromptRenderer(res, "system", slots={"assistant": "OVOS"}) +renderer.render("en-US", {"query": "what time is it"}) +renderer.render("pt-PT", {"query": "que horas são"}) # same prompt, another language +``` + +It has no phrase selection, no repetition avoidance, and no `.entity` fallback +— a prompt is one whole-file body and its slots are optional. + +## Next + +[Language matching](language-matching.md) — the tag logic behind the loader's +smart fallback. diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 00000000..864fbf4b --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,112 @@ +# 1. Getting started + +## What this package is + +OVOS skills ship plain-text resource files — sentence templates, vocabularies, +spoken-response phrases — grouped by language under a `locale/` folder. Several +OVOS components each grew their own code to parse and expand those files, and +the copies drifted: the same template could expand differently depending on +which copy ran. + +`ovos-spec-tools` is the **one conformant implementation** of that machinery, +matching the formal specifications. Depend on it instead of rolling your own. + +## Install + +```bash +pip install ovos-spec-tools # core — no dependencies +pip install ovos-spec-tools[langcodes] # adds the smart language fallback +``` + +The core package has **no dependencies**. The optional `langcodes` extra +improves only one thing — how a missing language falls back to a near one +(see [Language matching](language-matching.md)); everything else works without +it. + +Requires Python 3.8 or newer. + +## A first taste + +Every tool is one import away. The four snippets below are the whole package +in miniature; the later chapters go deep on each. + +### Expand a template + +A *template* describes many sentences at once. `expand()` enumerates them: + +```python +from ovos_spec_tools import expand + +expand("(turn|switch) [the] (light|fan)") +# ['turn the light', 'switch the light', 'turn light', 'switch light', +# 'turn the fan', 'switch the fan', 'turn fan', 'switch fan'] +``` + +→ [Sentence templates](templates.md) + +### Load a skill's locale folder + +`LocaleResources` reads the resource files under `locale//`: + +```python +from ovos_spec_tools import LocaleResources + +res = LocaleResources("my-skill/locale") +res.load_intent("play", "en-US") # the expanded training samples +res.load_dialog("weather", "en-US") # the spoken-response phrases +``` + +→ [Locale resources](locale-resources.md) + +### Render a spoken response + +`render()` picks one phrase from a `.dialog` and fills its slots: + +```python +from ovos_spec_tools import render + +render(res.load_dialog("weather", "en-US"), slots={"temperature": 21}) +# 'It is 21 degrees.' +``` + +→ [Dialog](dialog.md) + +### Match a language tag + +`closest_lang()` finds the nearest available language: + +```python +from ovos_spec_tools import closest_lang + +closest_lang("en-AU", ["pt-BR", "en-US", "de-DE"]) # 'en-US' +``` + +→ [Language matching](language-matching.md) + +### Construct a bus message + +`Message` is the JSON envelope OVOS components exchange: + +```python +from ovos_spec_tools import Message + +m = Message("ovos.intent.list", {}, {"source": "skill.id"}) +res = m.response({"intents": ["..."]}) # 'ovos.intent.list.response' +``` + +→ [Bus messages](message.md) + +### Lint a locale folder + +From the command line: + +```bash +ovos-spec-lint my-skill/locale +``` + +→ [Linting](linting.md) + +## Where to next + +Continue with [Sentence templates](templates.md) — the grammar is the +foundation the resource loader and the dialog renderer both build on. diff --git a/docs/language-matching.md b/docs/language-matching.md new file mode 100644 index 00000000..e62496d8 --- /dev/null +++ b/docs/language-matching.md @@ -0,0 +1,123 @@ +# 5. Language matching + +> **Spec coverage.** This chapter is the reference for the cross-spec BCP-47 +> language-matching rules (`language.py`). The case-insensitive tag comparison +> comes from **OVOS-INTENT-2 §2**; the nearest-language fallback and its +> "distance below 10 is a usable regional match" threshold come from +> **OVOS-INTENT-2 §2.2** (non-normative); the same rules decide whether a +> **SESSION-1** session language is served by an available resource, voice, or +> model. Everything here is a *reference policy*, not a conformance obligation — +> §2.2 is explicitly an implementation choice. + +OVOS constantly needs to answer "the assistant was asked for language X — which +of the languages I actually have is the best fit?" — for locale folders, TTS +voices, STT models. The logic was reimplemented in several places +(`ovos_utils.lang`, `phoonnx`, …) and drifted. This module is the one +implementation. + +Three functions: + +```python +from ovos_spec_tools import standardize_lang, lang_distance, closest_lang +``` + +## `standardize_lang(tag)` + +Normalizes a BCP-47 tag for comparison — underscores to hyphens, consistent +case, canonical script/region forms: + +```python +standardize_lang("en_us") # 'en-US' +standardize_lang("PT") # 'pt' +``` + +It uses `langcodes` when installed and a simple normalization otherwise. One +deliberate exception: `tl` and `tgl` (Tagalog) are kept as `tl` — `langcodes` +would fold them into the `fil` macrolanguage, which OVOS keeps distinct. + +## `lang_distance(a, b)` + +The heart of the module. It returns an integer: **`0` is identical**, larger is +further apart, and `10` or more is not a usable match. + +```python +lang_distance("en-US", "en_us") # 0 — the same tag +lang_distance("en-US", "en-GB") # small — a regional difference +lang_distance("en-US", "fr-FR") # large — a different language +``` + +All the policy lives here, so callers never need special cases. + +### A bare tag is measured from its norm region + +`langcodes` resolves a bare tag to its *most-populous* region — it considers +`pt` closest to `pt-BR`. But the unmarked form of a language should mean its +**reference variety**: Portuguese is "from Portugal" by name, and every +Lusophone country except Brazil follows the pt-PT norm. So `lang_distance` +measures a bare tag from its norm region: + +```python +lang_distance("pt", "pt-PT") # 0 — the norm region +lang_distance("pt", "pt-BR") # > 0 — a regional variant +``` + +The norm regions are a small explicit table (currently just `pt → PT`); a +language is added only when its bare tag has a clear reference region distinct +from the populous one. + +### Without `langcodes` + +`langcodes` is optional. Without it, `lang_distance` falls back to a coarse +measure: a shared primary subtag is near, a different language is far, and the +generic region-less form counts as nearer than a sibling region. It is enough +to keep a request for `en-AU` resolving against `en`, `en-GB`, `en-US` — just +without `langcodes`' finer regional ranking. Install the `langcodes` extra for +that: + +```bash +pip install ovos-spec-tools[langcodes] +``` + +## `closest_lang(target, available, max_distance=10)` + +Given a wanted tag and the tags you have, returns the closest one — or `None`. +It is simply *the candidate with the smallest `lang_distance`*: + +```python +closest_lang("en-AU", ["pt-BR", "en-US", "de-DE"]) # 'en-US' +closest_lang("pt", ["pt-BR", "pt-PT"]) # 'pt-PT' (norm region) +closest_lang("zz-ZZ", ["en-US", "pt-BR"]) # None +``` + +An exact match always resolves. Any other match resolves only if its distance +is **below** `max_distance` (default 10, the OVOS-INTENT-2 §2.2 threshold); +`max_distance=0` accepts exact matches only. The return value is the original +string from `available`, so you can map it straight back to a directory, a +voice file, a model name. + +This is exactly what `LocaleResources` uses for its smart language fallback +([chapter 3](locale-resources.md)) — and you can use it directly anywhere else +the same question comes up. + +### Dialect-fallback semantics, precisely + +A request resolves to a candidate **iff** their distance is below the threshold +(an exact match, distance `0`, always resolves). The distance ladder, nearest +to farthest: + +| Relationship | Distance | Resolves (default threshold 10)? | +|--------------|----------|----------------------------------| +| identical tag (case aside) | `0` | yes | +| bare tag vs its norm region (`pt` / `pt-PT`) | `0` | yes | +| two regions of one language (`en-US` / `en-GB`) | small (`<10`) | yes | +| a different primary language (`en` / `fr`) | large (`≥100` coarse) | no | + +So `en-AU` falls back to `en-US` over `de-DE`, `pt` prefers `pt-PT` over +`pt-BR`, and `zz-ZZ` resolves to nothing. The fallback is **dialect-only** by +design: it never crosses primary languages, because serving French wording to a +request for English is worse than failing (OVOS-INTENT-2 §2.2's caution that +"cross-region substitution can produce wording a user would not expect"). + +## Next + +[Linting](linting.md) — checking a whole locale folder at once. diff --git a/docs/linting.md b/docs/linting.md new file mode 100644 index 00000000..7e4b8e8b --- /dev/null +++ b/docs/linting.md @@ -0,0 +1,143 @@ +# 6. Linting + +> **Spec coverage.** This chapter is the reference for the locale linter +> (`lint.py`). Every finding enforces a specific clause — template syntax from +> **OVOS-INTENT-1 §3.6**, naming/layout/empty-file/uniqueness from +> **OVOS-INTENT-2 §2/§4.3/§5**, slot-set consistency from **OVOS-INTENT-1 §5.5** +> (restated for `.intent` by **OVOS-INTENT-2 §4.1** / **OVOS-INTENT-3 §5.1**, +> and for `.dialog` by **OVOS-INTENT-2 §4.2**). Each message names the clause it +> failed, so a red lint points straight at the violated MUST. + +`ovos-spec-lint` checks a skill's locale folder against both specs at once — +the **syntax** of every template (OVOS-INTENT-1) and the **naming and layout** +of every file (OVOS-INTENT-2) — and reports *every* problem rather than +stopping at the first. + +## The command + +```bash +ovos-spec-lint path/to/locale +``` + +The argument may be a whole `locale/` directory (every language subdirectory is +checked) or a single `/` directory. Output is one line per finding: + +``` +locale/en-US/play.intent: error: single-branch group (button): ... +locale/en-US/old.rx: warning: .rx is a legacy file type, not an OVOS-INTENT-2 role +locale/english: warning: directory name 'english' is not a BCP-47 language tag + +2 error(s), 1 warning(s) +``` + +The exit code is **non-zero when there are errors** — so the command drops +straight into a CI pipeline. With `--strict`, warnings fail the run too. + +## What it checks + +**Errors** — the file is wrong: + +- a template that does not parse — any malformed form of + [chapter 2](templates.md); +- an empty file (no templates after comments and blank lines); +- a file that is not valid UTF-8; +- a named slot inside a slot-free role (`.entity` / `.voc` / `.blacklist`); +- templates within one **`.intent`** or **`.dialog`** declaring **different + slot sets** — every line of one definition MUST declare the same `{slots}` so + the engine captures, or the caller fills, the same slots whichever line + matched or was chosen (OVOS-INTENT-1 §5.5; OVOS-INTENT-2 §4.1/§4.2; + OVOS-INTENT-3 §5.1); +- a base name outside the allowed charset (lowercase letters, digits, + underscores); +- an `.entity` whose base name — which names a slot — begins with a digit; +- the same `(role, base name)` appearing twice in one language tree; +- a `` reference to a vocabulary that does not exist. + +**Warnings** — suspicious but not fatal: + +- a resource file sitting outside any language directory; +- a language directory not named like a BCP-47 tag; +- a language directory with no resource files; +- a legacy file type (`.rx`, `.value`, `.list`, …) — not one of the six + OVOS-INTENT-2 roles; +- a `.blacklist` with no matching `.intent` to suppress; +- a file name that is not lowercase. + +A `.prompt` is checked too, but not as a template — it is plain text, so only +its naming and non-emptiness are checked, never template syntax. + +### Slot-set consistency + +OVOS-INTENT-1 §5.5 says every template of one definition MUST declare the +identical slot set, and a tool "MUST reject" one that does not. This is restated +normatively for each slot-bearing role — `.intent` by OVOS-INTENT-2 §4.1 and +OVOS-INTENT-3 §5.1 ("Every template in one intent MUST declare the same set of +named slots"), `.dialog` by OVOS-INTENT-2 §4.2 — so the linter treats a +divergent slot set as an **error** for both. A phrasing that genuinely needs a +different slot, such as: + +``` +(play|put on) {query} +(play|put on) {query} (on|using) {engine} +``` + +is **two** intents, not one: §5.5 says "place them in separate files and handle +them individually". (The two-line block above appears as a *worked example* in +OVOS-INTENT-3 §5.3, but that example is illustrative and itself violates the +§5.5 MUST it does not restate; the normative rule governs.) + +## Targeting an older spec version + +A skill may need to run on a device that has not been updated. `--spec-version` +flags any feature **newer than a target version**, so you learn before +shipping that a skill will not work there: + +| Version | Adds | +|---------|------| +| `0` | the legacy, undocumented Mycroft/OVOS de-facto behaviour | +| `1` | the formalized specs — and the `.blacklist` role | +| `2` | `` inline vocabulary references | +| `3` | the `.prompt` role *(the default)* | + +```bash +ovos-spec-lint my-skill/locale --spec-version 1 +``` + +With `--spec-version 1`, a template using `` is an **error** — a +version-1 runtime cannot expand it. With `--spec-version 2`, a `.prompt` file +is a **warning**, and with `--spec-version 0` a `.blacklist` file likewise — an +older runtime silently ignores a role it does not know, so that resource just +will not take effect. The default, `3`, flags nothing extra. + +## Using it as a library + +The CLI is a thin wrapper over `lint_locale`, which returns the findings so a +tool can process them: + +```python +from ovos_spec_tools import lint_locale + +findings = lint_locale("my-skill/locale") +for finding in findings: + print(finding.severity, finding.path, finding.message) + +errors = [f for f in findings if f.severity == "error"] +``` + +Each `Finding` has `.severity` (`"error"` or `"warning"`), `.path`, and +`.message`. + +## In CI + +Add a step that lints the locale folder of any skill you maintain: + +```yaml +- run: pip install ovos-spec-tools +- run: ovos-spec-lint locale +``` + +A malformed template now fails the build instead of failing a user's device. + +## Next + +[API reference](api-reference.md) — every public name at a glance. diff --git a/docs/locale-resources.md b/docs/locale-resources.md new file mode 100644 index 00000000..100e2faa --- /dev/null +++ b/docs/locale-resources.md @@ -0,0 +1,156 @@ +# 3. Locale resources + +> **Spec coverage.** This chapter is the reference for **OVOS-INTENT-2 — Locale +> Resource Formats** (`resources.py`). `LocaleResources` is the *conformant +> loader* of OVOS-INTENT-2 §5: it discovers languages (§5.1 / §2), locates a +> file under the override precedence (§5.2 / §2.1), applies the common reader +> (§5.3 / §3), applies the per-format rule (§5.4 / §4), and rejects an empty +> file (§5.5 / §5). Template expansion is delegated to the OVOS-INTENT-1 +> Expander ([chapter 2](templates.md)). + +A skill ships its localized text as plain-text files under a `locale/` folder. +This chapter covers what those files are and how `LocaleResources` loads them. +It is the implementation of OVOS-INTENT-2. + +## The folder layout + +``` +my-skill/ +└── locale/ + ├── en-US/ + │ ├── play.intent + │ ├── confirm.dialog + │ ├── media.entity + │ ├── yes.voc + │ └── trailers.blacklist + ├── pt-PT/ + │ └── … + └── de-DE/ + └── … +``` + +One subdirectory per language, named with a **BCP-47 tag** (`en-US`, `pt-PT`, +`zh-Hans`). A language directory may itself contain subdirectories — they are +an authoring convenience and carry no meaning; a resource is found by a +recursive search. A resource is identified by its **`(role, base name)`** pair, +so `confirm.intent` and `confirm.dialog` are two distinct resources. + +## The six roles + +The file extension is a resource's **role**: + +| Role | Extension | Slots? | What it is | +|------|-----------|--------|------------| +| Intent | `.intent` | yes | training samples for one skill action | +| Dialog | `.dialog` | yes | phrases the assistant speaks back | +| Entity | `.entity` | no | example values for a slot | +| Vocabulary | `.voc` | no | a named keyword / phrase set | +| Blacklist | `.blacklist` | no | phrases that suppress an intent | +| Prompt | `.prompt` | yes | a whole-file prompt for a language model | + +The first five are **lists of templates** ([chapter 2](templates.md)) — one +per line, with `#` comment lines and blank lines ignored. `.intent` and +`.dialog` are **slot-bearing** (they may use `{name}`); `.entity`, `.voc` and +`.blacklist` are **slot-free**. + +`.prompt` is the exception: not a template list but **one whole-file document** +read verbatim — every line, including `#` and blank lines, is kept. It carries +`{name}` substitution points but is otherwise plain text ([chapter 4](dialog.md)). + +Legacy OVOS file types (`.rx`, `.value`, `.list`, …) are deliberately *not* +roles here — the linter flags them ([chapter 7](linting.md)). + +## Loading with `LocaleResources` + +```python +from ovos_spec_tools import LocaleResources + +res = LocaleResources("my-skill/locale") +``` + +The **language is given per call**, not at construction. A locale folder is a +skill's multilingual unit, so one `LocaleResources` serves every language: + +```python +res.load_intent("play", "en-US") +res.load_intent("play", "pt-PT") # same instance +``` + +The load methods: + +| Method | Returns | +|--------|---------| +| `load_intent(name, lang)` | the union of every template's sample set, slots intact | +| `load_entity(name, lang)` | the expanded value set | +| `load_vocabulary(name, lang)` | the expanded phrase set | +| `load_blacklist(name, lang)` | the expanded phrase set | +| `load_dialog(name, lang)` | the raw phrase strings — **not** expanded (see below) | +| `load_prompt(name, lang)` | the whole `.prompt` file as one string | +| `vocabularies(lang)` | every `.voc`, as a `name → templates` dict | +| `entities(lang)` | every `.entity`, as a `name → values` dict | + +`.intent`, `.entity`, `.voc`, and `.blacklist` are expanded at load time. A +`.dialog` is **not** — its phrases are returned verbatim, because expansion +happens once per spoken response, on the single phrase chosen +([chapter 4](dialog.md)). A `.prompt` is returned as the whole file, for a +prompt renderer to fill ([chapter 4](dialog.md)). + +`` references inside an `.intent` resolve automatically against the +`.voc` files of the same language — you do not pass vocabularies yourself. + +A missing resource raises `FileNotFoundError`. A malformed one raises +`MalformedResource`, each case tied to a spec MUST: + +- an **empty file** — OVOS-INTENT-2 §5 ("every file MUST contribute at least + one template"); +- a **duplicate `(role, base name)`** in one language tree — §2 ("MUST NOT share + a base name anywhere within one language directory tree"); +- a **named slot in a slot-free role** — §4.3 (`.entity`/`.voc`/`.blacklist` + are the slot-free format). + +## Override precedence + +A resource may come from three places (OVOS-INTENT-2 §2.1), highest priority +first; **first match wins**, and an override replaces the whole lower file: + +1. **user** — per-user overrides, under a path the assistant decides; +2. **skill** — the files bundled with the skill; +3. **core** — fallback files shipped by the assistant framework. + +```python +res = LocaleResources( + skill_locale="my-skill/locale", + core_locale="/opt/ovos/locale", + user_locale="/home/me/.local/share/ovos/my-skill/locale", +) +``` + +A file at a higher level replaces the whole lower-level file with the same +`(role, base name)`. The user-data path is **assistant-defined** — this package +takes it as a parameter and imports no configuration of its own. + +## Smart language fallback + +OVOS-INTENT-2 §2.2 is explicit that fallback is **non-normative**: a loader +*SHOULD* prefer an exact match and *MAY* fall back to the nearest available +language. `LocaleResources` implements that suggested fallback. When a skill has +no directory for the requested language, it resolves to the **nearest +available** language instead — a request for `en-AU` loads `en-US`. This is +re-evaluated on every call, so the same instance can serve an exact language and +a fallback one side by side. + +```python +res.load_intent("play", "en-AU") # finds en-US/ if there is no en-AU/ +``` + +The nearness logic is [language matching](language-matching.md). Two knobs on +the constructor: + +- `max_language_distance` — how far a fallback may reach (default 10); `0` + disables the fallback, leaving exact matches only; +- `lang_resolver` — a `(target, available, max_distance) -> str | None` + callable, defaulting to `closest_lang`; replace it to change the policy. + +## Next + +[Dialog](dialog.md) — turning a loaded `.dialog` into a spoken sentence. diff --git a/docs/message.md b/docs/message.md new file mode 100644 index 00000000..67b33a87 --- /dev/null +++ b/docs/message.md @@ -0,0 +1,219 @@ +# Bus Messages + +The `Message` class is the on-the-wire envelope that OVOS components +exchange — utterances coming in, intent matches going out, skills +announcing handler outcomes, hosts dispatching to skills. The shape is +defined by [OVOS-MSG-1][spec] and this module is the reference +implementation. + +[spec]: https://github.com/OpenVoiceOS/architecture/blob/dev/message-object.md + +## What is in a Message + +Every Message is a JSON object with exactly three top-level fields: + +| Field | Type | Meaning | +|---|---|---| +| `type` | string | The topic — `ovos.intent.matched`, `speak`, `:`, … | +| `data` | object | Topic-specific payload; shape fixed by whichever spec defines `type` | +| `context` | object | Routing keys, the session carrier, and any layer-2 metadata | + +```python +from ovos_spec_tools import Message + +m = Message("speak", {"utterance": "hello"}, {"source": "skill.id"}) +``` + +The constructor rejects malformed input: an empty `type`, a non-string +`type`, or a non-`dict` `data` / `context` raise +`MalformedMessage`. Pass the dicts by reference — they are stored +as-is; if you need a detached copy use the derivations below. + +## Routing keys + +Two `context` keys mark the OVOS/handler-code boundary: + +- `source` — opaque identifier of the producer; +- `destination` — opaque identifier (or list of identifiers) of the + intended consumer(s). Absent or empty means **broadcast**. + +The envelope treats both as opaque strings — no parsing, no validation +beyond "is it there". How identifiers are minted is a deployment +concern. Hivemind, for example, uses `source` / `destination` to thread +remote peers through the bus without OVOS itself learning about them. + +## The session carrier + +`context.session` is the carrier for one conversational session — the +unit of "this wake-word interaction" or "this HiveMind client +connection". OVOS-MSG-1 makes two of its keys normative: + +- `session.session_id` — string identifier; the value `"default"` is + **reserved** and means *originates from the device itself* (used by + `ovos-audio` to keep TTS local). +- `session.lang` — BCP-47 tag for the user's preferred output language + (distinct from `data.lang` which describes the payload). + +The rest of `session`'s shape is opaque to this Message — full Session +structure is the job of the consumer (typically `ovos-bus-client`'s +`Session` class or a future session spec). + +**Absent session = `session_id: "default"`.** A Message with no +`session` in its context is treated the same as one carrying the +reserved value. Producers don't have to emit a session for +device-local messages. + +## The three derivations + +Each derivation returns a new Message with the right routing/session +fields for its role. The runtime class is preserved — subclasses of +`Message` get back instances of their own subclass. + +### `forward(type, data)` — relay under a new topic + +Keeps `context` (including `source`, `destination`, and `session`) +exactly as it was. The forwarder does **not** become the new `source`; +the original producer remains named. + +```python +ack = m.forward("ovos.utterance.handled", {"id": "u-7"}) +# ack.context == m.context (deep-copied) +``` + +### `reply(type, data, context=None)` — send back to the asker + +Copies `context`, then swaps the routing keys so the new Message is +addressed back to the original producer: + +- the new `destination` is the old `source`; +- the new `source` is the old `destination` (the first entry if it was + an array — exact choice is implementation-defined). + +Other `context` keys, including `session`, pass through unchanged. The +optional `context` argument is overlaid before the swap, matching +`ovos-bus-client.Message.reply` behaviour: + +```python +ack = m.reply("speak", {"utterance": "got it"}) +# ack.context["source"] == m.context["destination"] +# ack.context["destination"] == m.context["source"] +``` + +### `response(data, context=None)` — sugar for `.response`-suffixed reply + +A response is a `reply` whose topic is the source topic with +`.response` appended. Used by request/response chains so an observer +can recognize the answer: + +```python +res = Message("ovos.intent.list").response({"intents": ["..."]}) +res.msg_type # 'ovos.intent.list.response' +``` + +## Serialization + +`serialize()` produces a single UTF-8 JSON object per [OVOS-MSG-1 +§6][spec]: + +```python +wire = m.serialize() +recovered = Message.deserialize(wire) +assert recovered == m +``` + +Nested objects in `data` / `context` that expose a `.serialize()` +method (the duck-typed protocol used by `ovos-bus-client.Session` and +similar carriers) are converted before JSON encoding — `Message` +itself doesn't know about Session or any specific carrier type, but +its `.serialize()` walks containers and calls the method when present. + +For a JSON-decoded view without the string intermediate, `m.as_dict` +returns the same envelope as a dictionary: + +```python +m.as_dict # {'type': 'ovos.test', 'data': {...}, 'context': {...}} +``` + +`deserialize` rejects malformed payloads as `MalformedMessage`: +unparsable JSON, non-object root, unknown top-level keys, missing +`type`, wrong value types. Both `str` and `bytes` are accepted, and an +already-parsed `dict` short-circuits the JSON step. + +## A note on correlation + +There is **no** per-message identifier, no in-reply-to chain, no +host-managed request/response bookkeeping. Messages on the bus are +fully asynchronous. The `.response` suffix and the preserved `session` +give you enough raw material to do your own correlation if you need +it — match an incoming `.response` against an outstanding +request in the same `session_id`. + +Components that need per-conversation state track it themselves keyed +on `session.session_id`. A Message on the bus is **self-contained**; +any state a later consumer needs is either inside the Message or kept +by some component out of band — never recovered by a hidden host-side +correlation index. + +## What this module does not do + +By design, no transport (websocket, queue, …), no encryption, no +authentication, no delivery guarantees, no ordering guarantees, no +session lifecycle, no identifier-assignment policy. Each of those is +out of scope for the envelope and belongs in the layer that consumes +it — `ovos-bus-client` for the websocket transport, layer-2 systems +(HiveMind) for multi-tenant routing, individual subsystems for their +own per-session state. + +## Subclassing + +`Message` is designed to be subclassed. Both `deserialize` (a +classmethod) and `forward` / `reply` / `response` return instances of +the runtime class, so a transport-layer subclass propagates through +derivation chains without losing identity: + +```python +class TransportMessage(Message): + def serialize(self): + body = super().serialize() + # add framing, encryption, … here + return body + +m = TransportMessage("ovos.test") +forwarded = m.forward("ovos.next") +assert isinstance(forwarded, TransportMessage) +``` + +This is exactly how `ovos-bus-client.Message` is expected to extend +the spec primitive once it adopts this class. + +## Spec section map + +Every part of the `Message` API traces to a specific OVOS-MSG-1 +clause. The module's docstrings cite these inline; this table is the +quick index. + +| API surface | OVOS-MSG-1 § | What it implements | +|---|---|---| +| `Message(type, data, context)` | §2, §2.1–§2.3 | the three-field envelope; constructor type checks | +| `MalformedMessage` (constructor) | §2.1–§2.3 | reject non-string `type`, non-dict `data`/`context` | +| `msg_type` / `data` / `context` | §2 | the wire fields `type` / `data` / `context` | +| `context["source"]` / `["destination"]` | §3, §3.2–§3.4 | opaque routing keys, carried not parsed | +| `context["session"]` | §4 | session carrier (inner shape owned by OVOS-SESSION-1) | +| `DEFAULT_SESSION_ID` | §4.1 | the reserved `"default"` session id | +| `forward(type, data)` | §5.1 | relay, context preserved unchanged (deep-copied) | +| `reply(type, data, context)` | §5.2 | swap routing keys, address back to the asker | +| `response(data, context)` | §5.3 | `.response`-suffixed reply | +| (no correlation API) | §5.4 | messages are fully async; no host-side correlation | +| `serialize()` | §6 | single UTF-8 JSON object, finite numbers only | +| `as_dict` | §2, §6 | JSON-decoded envelope view | +| `deserialize(payload)` | §6, §7 | parse + reject malformed payloads | +| `MalformedMessage` (deserialize) | §7 | the consumer "MUST reject" rules | + +## See also + +- [Bus namespaces](bus-namespaces.md) — `SpecMessage`, `MIGRATION_MAP`, + and the transparent legacy↔`ovos.*` bridge that rides on top of these + derivations (`forward` for the handler-lifecycle trio, `reply` for the + stop pong, and so on). +- [Spec traceability](spec-traceability.md) — every public symbol in + `ovos-spec-tools` mapped to its authoritative spec section. diff --git a/docs/spec-traceability.md b/docs/spec-traceability.md new file mode 100644 index 00000000..6fbda30c --- /dev/null +++ b/docs/spec-traceability.md @@ -0,0 +1,217 @@ +# Spec traceability + +`ovos-spec-tools` exists for one reason: to be the **reference +implementation of the OVOS formal specifications**. Every public symbol in +the package implements a specific clause of a specific spec — there is no +"general-purpose" code here. This page is the proof: every name exported +from `ovos_spec_tools`, mapped to its authoritative spec section. + +The specs live in the +[OpenVoiceOS/architecture](https://github.com/OpenVoiceOS/architecture) +repository. Section numbers (`§N`) refer to the spec named in the same row. +Where a symbol is a *shared primitive* used across several spec clauses, the +"primary" owning clause is cited and the dependents are noted. + +## Bus / message domain + +The bus domain (`message.py`, `messages.py`) is the most heavily +spec-annotated; its module and per-symbol docstrings cite OVOS-MSG-1 inline. +See [Bus messages](message.md) and [Bus namespaces](bus-namespaces.md) for +the prose. + +### `message.py` — OVOS-MSG-1 (Bus Message) + +| Symbol | Spec | § | +|---|---|---| +| `Message` | OVOS-MSG-1 | §2 (envelope) | +| `Message.msg_type` / `.data` / `.context` | OVOS-MSG-1 | §2.1 / §2.2 / §2.3 | +| `Message.__init__` (type checks) | OVOS-MSG-1 | §2.1–§2.3, §7 | +| `Message.forward` | OVOS-MSG-1 | §5.1 | +| `Message.reply` | OVOS-MSG-1 | §5.2 | +| `Message.response` | OVOS-MSG-1 | §5.3 | +| `Message.serialize` | OVOS-MSG-1 | §6 | +| `Message.deserialize` | OVOS-MSG-1 | §6, §7 | +| `Message.as_dict` | OVOS-MSG-1 | §2, §6 | +| `MalformedMessage` | OVOS-MSG-1 | §2, §6, §7 (consumer "MUST reject") | +| `DEFAULT_SESSION_ID` | OVOS-MSG-1 | §4.1 (reserved `"default"`) | +| `RoutingValue` (type alias) | OVOS-MSG-1 | §3 / §3.3 (routing key shape) | + +The routing keys (`context.source` / `context.destination`, §3) and the +session carrier (`context.session`, §4) are carried by `Message` but their +*inner* semantics are owned elsewhere: §3 by MSG-1 itself (opaque), §4's +inner shape by **OVOS-SESSION-1**. + +### `messages.py` — the spec topic vocabulary and the migration bridge + +`SpecMessage` members each name a topic owned by another spec; the bridge +(`MIGRATION_MAP` / `NamespaceTranslator`) is the cross-cutting machinery that +moves the bus onto those topics. + +| Symbol | Spec | § | +|---|---|---| +| `SpecMessage.UTTERANCE` | OVOS-PIPELINE-1 | §9.1 | +| `SpecMessage.SPEAK` | OVOS-PIPELINE-1 | §9.6 | +| `SpecMessage.UTTERANCE_HANDLED` | OVOS-PIPELINE-1 | §9.5 | +| `SpecMessage.UTTERANCE_CANCELLED` | OVOS-TRANSFORM-1 | §8.2 (defined here; PIPELINE-1 §6.4 references it) | +| `SpecMessage.INTENT_MATCHED` | OVOS-PIPELINE-1 | §9.2 | +| `SpecMessage.INTENT_UNMATCHED` | OVOS-PIPELINE-1 | §9.3 | +| `SpecMessage.INTENT_HANDLER_START` / `_COMPLETE` / `_ERROR` | OVOS-PIPELINE-1 | §8.1 | +| `SpecMessage.INTENT_REGISTER_KEYWORD` | OVOS-INTENT-4 | §5 | +| `SpecMessage.INTENT_REGISTER_TEMPLATE` | OVOS-INTENT-4 | §6 | +| `SpecMessage.ENTITY_REGISTER` | OVOS-INTENT-4 | §7 | +| `SpecMessage.INTENT_DEREGISTER` | OVOS-INTENT-4 | §8.2 | +| `SpecMessage.ENTITY_DEREGISTER` | OVOS-INTENT-4 | §8.3 | +| `SpecMessage.SKILL_DEREGISTER` | OVOS-INTENT-4 | §8.4 | +| `SpecMessage.INTENT_ENABLE` / `INTENT_DISABLE` | OVOS-INTENT-4 | §8.5 | +| `SpecMessage.INTENT_LIST` / `_LIST_RESPONSE` | OVOS-INTENT-4 | §10.1 | +| `SpecMessage.INTENT_DESCRIBE` / `_DESCRIBE_RESPONSE` | OVOS-INTENT-4 | §10.2 | +| `SpecMessage.STOP_PING` / `STOP_PONG` | OVOS-STOP-1 | §4.2 | +| `SpecMessage.STOP` | OVOS-STOP-1 | §5.3 | +| `SpecMessage.LISTENER_RECORD_STARTED` / `_RECORD_ENDED` / `LISTENER_SLEEP` / `LISTENER_AWOKEN` | OVOS-AUDIO-IN-1 | §6.1 / §6.2 / §6.3 / §6.4 | +| `SpecMessage.SPEAK_B64` | OVOS-AUDIO-1 | §3.4 | +| `SpecMessage.AUDIO_SPEECH` | OVOS-AUDIO-1 | §4.3 | +| `SpecMessage.AUDIO_QUEUE` / `AUDIO_PLAY_SOUND` | OVOS-AUDIO-1 | §4.1 / §4.2 | +| `SpecMessage.AUDIO_STOP` | OVOS-AUDIO-1 | §6 | +| `SpecMessage.AUDIO_IS_SPEAKING` | OVOS-AUDIO-1 | §5.3 | +| `SpecMessage.AUDIO_OUTPUT_STARTED` / `_OUTPUT_ENDED` | OVOS-AUDIO-1 | §5.1 / §5.2 | +| `SpecMessage.MIC_LISTEN` | OVOS-AUDIO-1 | §4.4 | +| `SpecMessage.SESSION_SYNC` | OVOS-SESSION-2 | §2.7 (bus table §7) | +| `SpecMessage.CONVERSE_ACTIVE_LIST` / `_ACTIVE_LIST_RESPONSE` | OVOS-CONVERSE-1 | §6.1 | +| `SpecMessage.PERSONA_QUERY` / `_ANSWER` | OVOS-PERSONA-1 | §8.5 (§11 bus surface) | +| `SpecMessage.PERSONA_LIST` / `_LIST_RESPONSE` | OVOS-PERSONA-1 | §8.7 (§11) | +| `SpecMessage.PERSONA_REGISTER` / `_DEREGISTER` | OVOS-PERSONA-1 | §9 (§11) | +| `SpecMessage.PERSONA_ACTIVATED` / `_DISMISSED` | OVOS-PERSONA-1 | §11 | +| `SpecMessage.FALLBACK_REGISTER` / `_DEREGISTER` | OVOS-FALLBACK-1 | §3.1 / §3.2 (§9 bus surface) | +| `SpecMessage.COMMON_QUERY_PING` / `_PONG` | OVOS-COMMON-QUERY-1 | §6.1 / §6.2 (§13 bus surface) | +| `SpecMessage.TRANSFORMER_{AUDIO,UTTERANCE,METADATA,INTENT,DIALOG,TTS}_LIST` / `_LIST_RESPONSE` | OVOS-TRANSFORM-1 | §6 (six static query/response pairs) | +| `SpecMessage.COMMON_PLAY_{PLAY,SEARCH}` | OVOS-OCP-1 | §4.2 | +| `SpecMessage.COMMON_PLAY_{PAUSE,RESUME,STOP,NEXT,PREVIOUS,SEEK}` | OVOS-OCP-1 | §4.3 | +| `SpecMessage.COMMON_PLAY_{PLAYER,MEDIA,TRACK}_STATE` | OVOS-OCP-1 | §4.4 | +| `MIGRATION_MAP` | (bridge) | encodes the renames defined by PIPELINE-1 §8/§9, INTENT-4 §8, STOP-1 §4/§5, AUDIO-IN-1 | +| `SPEC_TO_LEGACY` | (bridge) | reverse of `MIGRATION_MAP` | +| `migration_counterpart` | (bridge) | symmetric counterpart lookup | +| `NamespaceTranslator` | (bridge) | dual-emit + receive-dedup; mirrors `ovos-bus-client` / `FakeBus` | +| `NamespaceTranslator.counterpart_topics` | (bridge) | send-side dual-emit | +| `NamespaceTranslator.is_migrated` | (bridge) | participation pre-check | +| `NamespaceTranslator.new_mirror_guard` | (bridge) | receive-side mirror-window dedup | + +The bridge symbols are not owned by one numbered spec clause — they are the +*mechanism* that carries the bus from the legacy topic names to the `ovos.*` +topics the specs above define, while honouring OVOS-MSG-1 §6 (payload key +order is not significant, used in the dedup fingerprint). The two topics the +bridge deliberately does **not** map trace to OVOS-INTENT-4 §5 (N→1 +registration consolidation) and OVOS-MSG-1 §2.1.1 (runtime-assembled +placeholder topics); see [Bus namespaces](bus-namespaces.md). + +## Session domain — OVOS-SESSION-1 + +`session.py` is the reference implementation of the §4 session carrier's +*inner* shape (which OVOS-MSG-1 §4 defers to OVOS-SESSION-1), plus the +handler-list helpers that PIPELINE-1 and CONVERSE-1 keep in the session. + +| Symbol | Spec | § | +|---|---|---| +| `Session` | OVOS-SESSION-1 | §3 (registered field set) | +| `SESSION1_REGISTERED_FIELDS` | OVOS-SESSION-1 | §3 | +| `SESSION1_OWNED_FIELDS` | OVOS-SESSION-1 | §3 (fields SESSION-1 itself owns) | +| `DEFAULT_CONVERSE_HANDLERS_CAP` | OVOS-CONVERSE-1 | §2.1 (handler-list cap) | +| `MalformedSession` | OVOS-SESSION-1 | §3 (malformed wire shape) | + +(`session.py` is owned by an in-flight PR; the citations above describe the +symbols it exports as imported by the package `__init__`.) + +## Intent template / locale domain + +### `expansion.py` — OVOS-INTENT-1 (Sentence Template Grammar) + +| Symbol | Spec | § | +|---|---|---| +| `expand` | OVOS-INTENT-1 | §3 (grammar), §3.7 (inline vocab), §4 (sample set) | +| `inline_keywords` | OVOS-INTENT-1 | §3.7 (`` inline vocabulary references) | +| `MalformedTemplate` | OVOS-INTENT-1 | §3.6 (malformed templates) | + +### `resources.py` — OVOS-INTENT-2 (Locale Resource Formats) + +| Symbol | Spec | § | +|---|---|---| +| `LocaleResources` | OVOS-INTENT-2 | §2 (layout / override precedence), §4 (six roles) | +| `read_resource_file` | OVOS-INTENT-2 | §3 (common reader) | +| `read_prompt_file` | OVOS-INTENT-2 | §3, §4.4 (`.prompt` whole-file read) | +| `iter_locale_dirs` | OVOS-INTENT-2 | §2 (locale layout) + §2.2 distance fallback | +| `find_lang_dir` | OVOS-INTENT-2 | §2.2 (closest-language directory resolution) | +| `keyword_form` | OVOS-INTENT-2 | §4.3 (slot-free template grouping) | +| `normalize_for_match` | OVOS-INTENT-2 | §4.3 (match-comparison normalization) | +| `utterance_contains` | OVOS-INTENT-2 | §4.3 (vocabulary / keyword match) | +| `strip_samples` | OVOS-INTENT-2 | §4.3 (keyword removal) | +| `MalformedResource` | OVOS-INTENT-2 | §3, §4 (malformed file / layout) | + +### `dialog.py` — OVOS-INTENT-2 §4.2 (Dialog) + +| Symbol | Spec | § | +|---|---|---| +| `render` | OVOS-INTENT-2 | §4.2 (select + fill a `.dialog` phrase) | +| `DialogRenderer` | OVOS-INTENT-2 | §4.2 (stateful, multilingual renderer) | +| `UnfilledSlot` | OVOS-INTENT-2 | §4.2 (a chosen phrase has an unfilled slot) | +| `verify_slot_consistency` | OVOS-INTENT-1 | §7 + §5.5 (Dialog renderer MUST verify all phrases declare the same slot set) | + +### `prompt.py` — OVOS-INTENT-2 §4.4 (`.prompt`) + +| Symbol | Spec | § | +|---|---|---| +| `render_prompt` | OVOS-INTENT-2 | §4.4 (conservative `{name}` substitution) | +| `PromptRenderer` | OVOS-INTENT-2 | §4.4 (stateful, multilingual prompt renderer) | + +## Language matching — shared OVOS-INTENT-2 §2.2 primitive + +`language.py` is the single implementation of "closest available language", +the smart-fallback logic OVOS-INTENT-2 §2.2 relies on for resource +resolution (and which TTS/STT plugins reuse). + +| Symbol | Spec | § | +|---|---|---| +| `standardize_lang` | OVOS-INTENT-2 | §2.2 (BCP-47 tag normalization) | +| `lang_distance` | OVOS-INTENT-2 | §2.2 (regional-match distance, `<10` usable) | +| `lang_matches` | OVOS-INTENT-2 | §2.2 (match predicate) | +| `closest_lang` | OVOS-INTENT-2 | §2.2 (closest-match resolution) | + +## Linting — OVOS-INTENT-1 / OVOS-INTENT-2 + +`lint.py` validates resource *syntax* (INTENT-1) and *naming/layout* +(INTENT-2); the `--spec-version` switch flags features newer than a target +spec version. + +| Symbol | Spec | § | +|---|---|---| +| `lint_locale` | OVOS-INTENT-1 + OVOS-INTENT-2 | INTENT-1 §3 (syntax) + INTENT-2 §2/§4 (layout/roles) | +| `Finding` | (tooling) | a single lint result (`severity`/`path`/`message`) | +| `ovos-spec-lint` (CLI) | (tooling) | command wrapper over `lint_locale` | + +## Package + +| Symbol | Spec | § | +|---|---|---| +| `__version__` | (packaging) | installed package version string | + +## Coverage + +Every name in `ovos_spec_tools.__all__` appears in a table above. The only +rows whose "spec" column is not a numbered OVOS spec clause are: + +- the **bridge** machinery (`MIGRATION_MAP`, `SPEC_TO_LEGACY`, + `migration_counterpart`, `NamespaceTranslator` and its methods) — these + *carry* the bus onto spec topics rather than implementing one clause, and + each entry/mapping is itself traced to the spec rename it encodes; +- **tooling/packaging** surface (`Finding`, the `ovos-spec-lint` command, + `__version__`). + +Everything else is a direct reference implementation of a cited OVOS-MSG-1, +OVOS-SESSION-1, OVOS-SESSION-2, OVOS-INTENT-1, OVOS-INTENT-2, OVOS-INTENT-4, +OVOS-PIPELINE-1, OVOS-STOP-1, OVOS-AUDIO-IN-1, OVOS-AUDIO-1, OVOS-CONVERSE-1, +OVOS-PERSONA-1, OVOS-FALLBACK-1, OVOS-COMMON-QUERY-1, OVOS-TRANSFORM-1, or +OVOS-OCP-1 clause. + +`SpecMessage` is the static `ovos.*` topic vocabulary: it carries one member +per spec-defined fixed-string topic and deliberately omits runtime-templated +topics (MSG-1 §2.1.1 — the `:…`, `…`, per-skill ping +placeholders) and topics used by `ovos-bus-client` that no spec defines +(`ovos.session.update_default`, `ovos.session.start`, `ovos.context.set`). diff --git a/docs/templates.md b/docs/templates.md new file mode 100644 index 00000000..feac7fc5 --- /dev/null +++ b/docs/templates.md @@ -0,0 +1,180 @@ +# 2. Sentence templates + +> **Spec coverage.** This chapter is the reference for **OVOS-INTENT-1 — the +> Sentence Template Grammar** (`expansion.py`). `expand()` is the *Expander* +> conformance role of OVOS-INTENT-1 §7: it implements the token set of §3, the +> enumeration algorithm of §4.1, and the malformed-form rejection of §3.6. + +A **sentence template** is a compact string that stands for a set of +sentences. It is the grammar of OVOS-INTENT-1, and it is the foundation of this +package: resource files are lists of templates, and the dialog renderer +renders them. + +`expand()` is the whole interface: + +```python +from ovos_spec_tools import expand, MalformedTemplate +``` + +`expand(template)` returns the **sample set** — the finite list of sentences +the template denotes. + +## The four tokens + +A template is literal words interspersed with four kinds of token. + +### Literal words + +Anything that is not a token is literal text, matched verbatim: + +```python +expand("turn on the lights") +# ['turn on the lights'] +``` + +### Alternatives — `(a|b|c)` + +Parentheses hold branches separated by `|`; each combination takes one branch: + +```python +expand("(turn on|switch on|enable) it") +# ['turn on it', 'switch on it', 'enable it'] +``` + +A branch may be empty — `(please|)` means "please, or nothing": + +```python +expand("(please|) help me") +# ['please help me', 'help me'] +``` + +### Optionals — `[x]` + +`[x]` is exactly `(x|)` — the segment is included or omitted: + +```python +expand("turn on [the] lights") +# ['turn on the lights', 'turn on lights'] +``` + +Groups nest freely: + +```python +expand("turn on [(all|every) ]light[s]") +# ['turn on all lights', 'turn on lights', 'turn on every lights', +# 'turn on all light', 'turn on light', 'turn on every light'] +``` + +### Named slots — `{name}` + +A `{name}` slot is a placeholder for a value that varies — a song title, a +city. It is **opaque**: expansion carries it through untouched and never +enumerates it. + +```python +expand("(play|put on) {query}") +# ['play {query}', 'put on {query}'] +``` + +Who fills a slot, and when, depends on the file role: an intent engine fills it +from speech; a skill fills a dialog slot before the phrase is spoken. Expansion +itself never does. + +### Vocabulary references — `` + +`` (OVOS-INTENT-1 §3.7) pulls in a named **vocabulary** — a reusable set +of phrasings — and expands it in place. Pass the vocabularies as a dict: + +```python +expand(" [there]", {"greeting": ["hello", "hi", "hey"]}) +# ['hello there', 'hi there', 'hey there', 'hello', 'hi', 'hey'] +``` + +This is how you avoid repeating the same `(hello|hi|hey)` group across many +templates: define it once as a vocabulary, reference it as ``. A +vocabulary may itself contain `` references; they resolve recursively +(§4.1 step 1). A reference must resolve to a **slot-free** set — a vocabulary +may not introduce a `{slot}` — and a single-member vocabulary substitutes its +bare member rather than a one-branch group. + +## How expansion works — the §4.1 algorithm + +`expand()` follows OVOS-INTENT-1 §4.1 exactly, in order: + +1. **Resolve `` references** to alternative groups (recursively). +2. **Rewrite `[x]` as `(x|)`** — an optional is sugar for an empty branch. +3. **Cartesian product** of the innermost `(...)` groups, repeated until no + parenthesis remains. +4. **Normalize whitespace** — collapse runs of spaces, strip the ends (this is + what removes the double space an empty branch leaves behind). +5. **De-duplicate**, preserving first-seen order. + +Worked through on `(turn|switch) [the] (light|fan)` — three 2-branch groups +once `[the]` becomes `(the|)`, so `2×2×2 = 8` combinations collapse (after +whitespace normalization) to the 8-sentence sample set of §4.2: + +```python +expand("(turn|switch) [the] (light|fan)") +# ['turn the light', 'switch the light', 'turn light', 'switch light', +# 'turn the fan', 'switch the fan', 'turn fan', 'switch fan'] +``` + +The **sample set is a set** — §4 defines its membership, not an ordering — so +the eight sentences are exactly those of §4.2; the sequence above is just the +order `expand()` happens to emit them in. Throughout, `{name}` slots are +**opaque** — carried through and never enumerated (§4.1 final note). + +## The input model + +The grammar is built for **voice**. Templates and the utterances matched +against them are assumed already *ASR-normalized*: lowercase, alphanumeric +words, single spaces, no punctuation. This package **expands; it does not +normalize** — normalization happens upstream. + +One consequence: the metacharacters `( ) [ ] { } < > |` never occur as literal +spoken text, so there is no escaping mechanism and none is needed. + +(`.dialog` phrases are the exception — they are spoken *output*, so they may +carry mixed case and punctuation. See [Dialog](dialog.md).) + +## Malformed templates + +A template that cannot mean anything sensible raises `MalformedTemplate`: + +```python +try: + expand("turn (on|off the lights") +except MalformedTemplate as error: + print(error) # unbalanced metacharacters ... +``` + +The malformed forms (OVOS-INTENT-1 §3.6): + +| Form | Example | Why | +|------|---------|-----| +| Unbalanced metacharacters | `(a|b` | a bracket is never closed | +| Single-branch group | `(word)`, `()` | a group must offer a *choice* | +| Empty sample | `[hello]`, `(|)` | a sample with no words trains nothing | +| Slot-only template | `{name}` | a template needs anchoring literal text | +| Adjacent slots | `{a} {b}` | no word between two slots to delimit them | +| Repeated slot name | `{x} and {x}` | a slot is defined once per sample | +| Undefined vocabulary | `` | no such vocabulary was supplied | +| Cyclic vocabulary | ``→``→`` | resolution would not terminate | + +Two are checked against the **expanded samples**, not the raw template: +`{a} [x] {b}` is malformed because the empty-`x` branch yields the adjacent +pair `{a} {b}`; and a template is malformed if *any* branch combination +produces an empty sentence. + +## Templates as training data + +Expansion produces the *shape of the training data* a template contributes — +nothing more. A capable intent engine **generalizes beyond** the sample set: it +recognizes phrasings that were never enumerated. So keep templates focused and +readable; you are not obliged to spell out every wording. Matching and +generalization are the engine's job and are out of scope here. + +## Next + +[Locale resources](locale-resources.md) — where templates live on disk, and +how to load them. diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000..6c74d8f2 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,94 @@ +# Examples + +## Runnable scripts + +Each script is self-contained and prints its output. Install the package first +(`pip install -e .` from the repo root), then run e.g. `python examples/expand.py`. + +| Script | Demonstrates | +|--------|--------------| +| [`expand.py`](expand.py) | `expand()` — alternatives, optionals, opaque `{slot}`s, `` references, and rejection of malformed templates (OVOS-INTENT-1) | +| [`load_resources.py`](load_resources.py) | `LocaleResources` — loading `.intent` / `.voc` / `.entity` / `.dialog` from a locale tree (OVOS-INTENT-2) | +| [`render_dialog.py`](render_dialog.py) | `render()` and `DialogRenderer` — phrase selection, repetition avoidance, default slots, `.entity` fallback (OVOS-INTENT-2 §4.2) | +| [`render_prompt.py`](render_prompt.py) | `render_prompt()` and `PromptRenderer` — whole-file prompts, conservative `{name}` substitution, fenced-code exemption (OVOS-INTENT-2 §4.4) | +| [`lint.py`](lint.py) | `lint_locale()` — the linter used as a library | + +The loader, dialog and prompt scripts read [`skill-locale/`](skill-locale), a +small valid skill locale; `lint.py` reads `dirty-locale/` (below). + +## `dirty-locale` — the linter against a broken locale + +`dirty-locale/` is a deliberately broken skill locale. Every file in it trips +at least one `ovos-spec-lint` check, so it doubles as a tour of what the linter +catches. Two files (`en-US/good.intent`, `en-US/good.voc`) are valid and +produce no findings. + +### The locale + +``` +dirty-locale/locale/ +├── stray.intent misplaced — not inside a language directory +├── english/ directory name is not a BCP-47 tag +│ └── hello.intent (valid content) +└── en-US/ + ├── good.intent valid — no findings + ├── good.voc valid — no findings + ├── unbalanced.intent unbalanced metacharacters + ├── single_branch.intent a group with only one branch + ├── adjacent.intent two slots with no word between them + ├── repeat.intent the same slot name twice in a sample + ├── empty.voc no templates after comments are dropped + ├── colors.voc a named slot in a slot-free role + ├── Bad-Name.intent base name outside the allowed charset + ├── 2nd.entity .entity base name begins with a digit + ├── refs.intent reference to a vocabulary that does not exist + ├── old.rx a legacy file type, not an OVOS-INTENT-2 role + ├── sub1/menu.intent ┐ same (role, base name) twice in one + └── sub2/menu.intent ┘ language tree +``` + +### Running the linter + +```console +$ ovos-spec-lint examples/dirty-locale/locale +examples/dirty-locale/locale/stray.intent: warning: resource file is not inside a language directory +examples/dirty-locale/locale/en-US/old.rx: warning: .rx is a legacy file type, not an OVOS-INTENT-2 resource role +examples/dirty-locale/locale/en-US/sub2/menu.intent: error: duplicate .intent resource 'menu' — also at examples/dirty-locale/locale/en-US/sub1/menu.intent +examples/dirty-locale/locale/en-US/2nd.entity: error: .entity base name '2nd' names a slot and must not begin with a digit +examples/dirty-locale/locale/en-US/Bad-Name.intent: error: base name 'Bad-Name' must be lowercase ASCII letters, digits and underscores only +examples/dirty-locale/locale/en-US/Bad-Name.intent: warning: file name should be lowercase +examples/dirty-locale/locale/en-US/adjacent.intent: error: adjacent slots in sample 'set {hours}{minutes}' of template 'set {hours}{minutes}': a literal word must separate any two slots [in: 'set {hours}{minutes}'] +examples/dirty-locale/locale/en-US/colors.voc: error: .voc is slot-free but a template contains a named slot [in: 'the color {shade}'] +examples/dirty-locale/locale/en-US/empty.voc: error: empty file — every resource file must contribute at least one template (OVOS-INTENT-2 §5) +examples/dirty-locale/locale/en-US/refs.intent: error: undefined vocabulary reference [in: ' {name}'] +examples/dirty-locale/locale/en-US/repeat.intent: error: repeated slot name in sample '{x} and {x}' of template '{x} and {x}': each slot is defined once per sample [in: '{x} and {x}'] +examples/dirty-locale/locale/en-US/single_branch.intent: error: single-branch group (button): a group must offer a choice between at least two branches [in: 'press (button)'] +examples/dirty-locale/locale/en-US/unbalanced.intent: error: unbalanced metacharacters in template 'turn (on|off the lights' [in: 'turn (on|off the lights'] +examples/dirty-locale/locale/english: warning: directory name 'english' is not a BCP-47 language tag + +10 error(s), 4 warning(s) +$ echo $? +1 +``` + +The command exits non-zero because there are errors. With `--strict` it would +also exit non-zero on the warnings alone. + +### Findings, by rule + +| File | Severity | Rule | +|------|----------|------| +| `stray.intent` | warning | a resource file must live inside a `/` directory (OVOS-INTENT-2 §2) | +| `english/` | warning | a language directory is named with a BCP-47 tag (§2) | +| `old.rx` | warning | `.rx` is a legacy type, not one of the five OVOS-INTENT-2 roles | +| `Bad-Name.intent` | warning | file names are lowercase | +| `unbalanced.intent` | error | unbalanced metacharacters (OVOS-INTENT-1 §3.6) | +| `single_branch.intent` | error | a group must have at least two branches (§3.6) | +| `adjacent.intent` | error | a literal word must separate two slots (§3.6) | +| `repeat.intent` | error | a slot name appears once per sample (§3.6) | +| `empty.voc` | error | every file contributes at least one template (OVOS-INTENT-2 §5) | +| `colors.voc` | error | `.voc` is slot-free — no named slots (§4.3) | +| `Bad-Name.intent` | error | base name charset: lowercase letters, digits, underscores (§2) | +| `2nd.entity` | error | an `.entity` base name names a slot — no leading digit (§3.4) | +| `refs.intent` | error | `` must resolve to a known vocabulary (§3.7) | +| `sub1`+`sub2/menu.intent` | error | `(role, base name)` is unique per language tree (§2) | diff --git a/examples/dirty-locale/locale/en-US/2nd.entity b/examples/dirty-locale/locale/en-US/2nd.entity new file mode 100644 index 00000000..6fa9bf82 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/2nd.entity @@ -0,0 +1,2 @@ +second +third diff --git a/examples/dirty-locale/locale/en-US/Bad-Name.intent b/examples/dirty-locale/locale/en-US/Bad-Name.intent new file mode 100644 index 00000000..982793c3 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/Bad-Name.intent @@ -0,0 +1 @@ +whatever diff --git a/examples/dirty-locale/locale/en-US/adjacent.intent b/examples/dirty-locale/locale/en-US/adjacent.intent new file mode 100644 index 00000000..d1ede4c5 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/adjacent.intent @@ -0,0 +1 @@ +set {hours}{minutes} diff --git a/examples/dirty-locale/locale/en-US/colors.voc b/examples/dirty-locale/locale/en-US/colors.voc new file mode 100644 index 00000000..db8889d1 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/colors.voc @@ -0,0 +1 @@ +the color {shade} diff --git a/examples/dirty-locale/locale/en-US/empty.voc b/examples/dirty-locale/locale/en-US/empty.voc new file mode 100644 index 00000000..25094d7c --- /dev/null +++ b/examples/dirty-locale/locale/en-US/empty.voc @@ -0,0 +1,2 @@ +# this file has no templates + diff --git a/examples/dirty-locale/locale/en-US/good.intent b/examples/dirty-locale/locale/en-US/good.intent new file mode 100644 index 00000000..7e237e80 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/good.intent @@ -0,0 +1,2 @@ +# clean intent +(play|put on) {query} diff --git a/examples/dirty-locale/locale/en-US/good.voc b/examples/dirty-locale/locale/en-US/good.voc new file mode 100644 index 00000000..c6fb3993 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/good.voc @@ -0,0 +1,3 @@ +yes +yeah +(sure|of course) diff --git a/examples/dirty-locale/locale/en-US/old.rx b/examples/dirty-locale/locale/en-US/old.rx new file mode 100644 index 00000000..52e47c97 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/old.rx @@ -0,0 +1 @@ +(?P.*) diff --git a/examples/dirty-locale/locale/en-US/refs.intent b/examples/dirty-locale/locale/en-US/refs.intent new file mode 100644 index 00000000..665e6abe --- /dev/null +++ b/examples/dirty-locale/locale/en-US/refs.intent @@ -0,0 +1 @@ + {name} diff --git a/examples/dirty-locale/locale/en-US/repeat.intent b/examples/dirty-locale/locale/en-US/repeat.intent new file mode 100644 index 00000000..b240eb6a --- /dev/null +++ b/examples/dirty-locale/locale/en-US/repeat.intent @@ -0,0 +1 @@ +{x} and {x} diff --git a/examples/dirty-locale/locale/en-US/single_branch.intent b/examples/dirty-locale/locale/en-US/single_branch.intent new file mode 100644 index 00000000..c42d3762 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/single_branch.intent @@ -0,0 +1 @@ +press (button) diff --git a/examples/dirty-locale/locale/en-US/sub1/menu.intent b/examples/dirty-locale/locale/en-US/sub1/menu.intent new file mode 100644 index 00000000..1f8ae2ed --- /dev/null +++ b/examples/dirty-locale/locale/en-US/sub1/menu.intent @@ -0,0 +1 @@ +open the menu diff --git a/examples/dirty-locale/locale/en-US/sub2/menu.intent b/examples/dirty-locale/locale/en-US/sub2/menu.intent new file mode 100644 index 00000000..262a1e01 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/sub2/menu.intent @@ -0,0 +1 @@ +show menu diff --git a/examples/dirty-locale/locale/en-US/unbalanced.intent b/examples/dirty-locale/locale/en-US/unbalanced.intent new file mode 100644 index 00000000..2b5c77e5 --- /dev/null +++ b/examples/dirty-locale/locale/en-US/unbalanced.intent @@ -0,0 +1 @@ +turn (on|off the lights diff --git a/examples/dirty-locale/locale/english/hello.intent b/examples/dirty-locale/locale/english/hello.intent new file mode 100644 index 00000000..37d4e6c5 --- /dev/null +++ b/examples/dirty-locale/locale/english/hello.intent @@ -0,0 +1 @@ +hi there diff --git a/examples/dirty-locale/locale/stray.intent b/examples/dirty-locale/locale/stray.intent new file mode 100644 index 00000000..3b18e512 --- /dev/null +++ b/examples/dirty-locale/locale/stray.intent @@ -0,0 +1 @@ +hello world diff --git a/examples/expand.py b/examples/expand.py new file mode 100644 index 00000000..122bc768 --- /dev/null +++ b/examples/expand.py @@ -0,0 +1,27 @@ +"""Expanding sentence templates — OVOS-INTENT-1. + +`expand()` turns a template into its sample set. Run: `python examples/expand.py` +""" +from ovos_spec_tools import MalformedTemplate, expand + +# Alternatives and optionals expand to every combination. +print("worked example:") +for sample in sorted(expand("(turn|switch) [the] (light|fan)")): + print(" ", sample) + +# Named slots are opaque — carried through, never expanded. +print("\nnamed slots are opaque:") +print(" ", expand("(play|put on) {query}")) + +# `` references expand a named vocabulary in place. +print("\ninline vocabulary reference:") +vocabularies = {"greeting": ["hello", "hi", "hey"]} +print(" ", expand(" [there] {name}", vocabularies)) + +# Malformed templates (OVOS-INTENT-1 §3.6) are rejected. +print("\nmalformed templates are rejected:") +for bad in ["turn (on|off the lights", "press (button)", "{a}{b}"]: + try: + expand(bad) + except MalformedTemplate as error: + print(f" {bad!r} -> {error}") diff --git a/examples/lint.py b/examples/lint.py new file mode 100644 index 00000000..ad3d3cff --- /dev/null +++ b/examples/lint.py @@ -0,0 +1,20 @@ +"""Linting a locale folder programmatically. + +`lint_locale()` is what the `ovos-spec-lint` command wraps; it returns every +`Finding` so a tool can process them. Run: `python examples/lint.py` + +See dirty-locale/ and README.md for the locale being linted here. +""" +from pathlib import Path + +from ovos_spec_tools import lint_locale + +dirty_locale = Path(__file__).parent / "dirty-locale" / "locale" +findings = lint_locale(dirty_locale) + +for finding in findings: + print(finding) + +errors = sum(1 for f in findings if f.severity == "error") +warnings = len(findings) - errors +print(f"\n{errors} error(s), {warnings} warning(s)") diff --git a/examples/load_resources.py b/examples/load_resources.py new file mode 100644 index 00000000..7cd323e5 --- /dev/null +++ b/examples/load_resources.py @@ -0,0 +1,34 @@ +"""Loading a skill's locale resources — OVOS-INTENT-2. + +`LocaleResources` discovers and loads the five resource roles from a +`locale//` tree. One instance serves every language: the language is a +parameter of each load call. Run: `python examples/load_resources.py` +""" +from pathlib import Path + +from ovos_spec_tools import LocaleResources + +locale = Path(__file__).parent / "skill-locale" / "locale" +resources = LocaleResources(str(locale)) + +# `.intent` — loaded as its sample set, named slots intact. +print("play.intent samples (en-US):") +for sample in resources.load_intent("play", "en-US"): + print(" ", sample) + +# `.intent` using a `` reference — the .voc is resolved automatically. +print("\ngreet.intent samples (uses ):") +for sample in resources.load_intent("greet", "en-US"): + print(" ", sample) + +# `.voc` — a slot-free vocabulary, loaded as its expanded phrase set. +print("\ngreeting.voc:", resources.load_vocabulary("greeting", "en-US")) + +# `.entity` — a value set. +print("weekday.entity:", resources.load_entity("weekday", "en-US")) + +# `.dialog` — loaded as raw phrase strings, NOT expanded (expansion is +# per-render; see render_dialog.py). +print("\nweather.dialog phrases:") +for phrase in resources.load_dialog("weather", "en-US"): + print(" ", phrase) diff --git a/examples/render_dialog.py b/examples/render_dialog.py new file mode 100644 index 00000000..0d09abe7 --- /dev/null +++ b/examples/render_dialog.py @@ -0,0 +1,31 @@ +"""Rendering dialog — OVOS-INTENT-2 §4.2. + +Shows the stateless `render()` function and the stateful, multilingual +`DialogRenderer`. A seeded `random.Random` makes the output reproducible. +Run: `python examples/render_dialog.py` +""" +import random +from pathlib import Path + +from ovos_spec_tools import DialogRenderer, LocaleResources, render + +locale = Path(__file__).parent / "skill-locale" / "locale" +resources = LocaleResources(str(locale)) + +# The stateless function: pick a phrase, expand its variety, fill the slots. +phrases = resources.load_dialog("weather", "en-US") +print("render() — stateless:") +print(" ", render(phrases, slots={"temperature": 21}, rng=random.Random(1))) + +# The stateful renderer is multilingual — the language is given per render() +# call — and avoids repeating the phrase it chose last (per language). +print("\nDialogRenderer — repetition avoidance:") +renderer = DialogRenderer(resources, "weather", rng=random.Random(1)) +for _ in range(3): + print(" ", renderer.render("en-US", {"temperature": 21})) + +# Default slots are set once and reused; `.entity` fills anything left over. +# `agenda.dialog` has a {weekday} slot and weekday.entity supplies the values. +print("\nDialogRenderer — .entity fallback:") +agenda = DialogRenderer(resources, "agenda", rng=random.Random(2)) +print(" no slots passed:", agenda.render("en-US")) # {weekday} from .entity diff --git a/examples/render_prompt.py b/examples/render_prompt.py new file mode 100644 index 00000000..dbaff51c --- /dev/null +++ b/examples/render_prompt.py @@ -0,0 +1,27 @@ +"""Rendering prompts — OVOS-INTENT-2 §4.4. + +Shows the stateless `render_prompt()` and the resource-backed `PromptRenderer`. +A `.prompt` is whole-file plain text: only the double-brace `{{name}}` form is +a substitution point. Single-brace `{name}`, literal JSON braces, unfilled +slots, and fenced code blocks are all left untouched. +Run: `python examples/render_prompt.py` +""" +from pathlib import Path + +from ovos_spec_tools import LocaleResources, PromptRenderer, render_prompt + +locale = Path(__file__).parent / "skill-locale" / "locale" +resources = LocaleResources(str(locale)) + +# The stateless function over an explicit prompt string. Only `{{assistant}}` +# and `{{query}}` are supplied — the literal JSON single braces and the +# `{{example}}` inside the fenced code block are left exactly as written. +text = resources.load_prompt("system", "en-US") +print("render_prompt() — only the supplied slots are filled:\n") +print(render_prompt(text, {"assistant": "OVOS", "query": "what time is it"})) + +# The resource-backed renderer: the language is given per call, and default +# slots set once are reused on every call. +print("\nPromptRenderer — default {{assistant}}, {{query}} per call:\n") +renderer = PromptRenderer(resources, "system", slots={"assistant": "OVOS"}) +print(renderer.render("en-US", {"query": "set a five minute timer"})) diff --git a/examples/skill-locale/locale/en-US/agenda.dialog b/examples/skill-locale/locale/en-US/agenda.dialog new file mode 100644 index 00000000..f3a83cf4 --- /dev/null +++ b/examples/skill-locale/locale/en-US/agenda.dialog @@ -0,0 +1 @@ +Your next event is on {weekday}. diff --git a/examples/skill-locale/locale/en-US/greet.intent b/examples/skill-locale/locale/en-US/greet.intent new file mode 100644 index 00000000..789351ef --- /dev/null +++ b/examples/skill-locale/locale/en-US/greet.intent @@ -0,0 +1 @@ + [there] diff --git a/examples/skill-locale/locale/en-US/greeting.voc b/examples/skill-locale/locale/en-US/greeting.voc new file mode 100644 index 00000000..77ebb4f3 --- /dev/null +++ b/examples/skill-locale/locale/en-US/greeting.voc @@ -0,0 +1,4 @@ +# greeting words +hello +hi +hey diff --git a/examples/skill-locale/locale/en-US/play.intent b/examples/skill-locale/locale/en-US/play.intent new file mode 100644 index 00000000..b7923e4e --- /dev/null +++ b/examples/skill-locale/locale/en-US/play.intent @@ -0,0 +1,2 @@ +(play|put on) {query} +i want to (hear|listen to) {query} diff --git a/examples/skill-locale/locale/en-US/system.prompt b/examples/skill-locale/locale/en-US/system.prompt new file mode 100644 index 00000000..70651fd0 --- /dev/null +++ b/examples/skill-locale/locale/en-US/system.prompt @@ -0,0 +1,12 @@ +# Assistant system prompt + +You are {{assistant}}, a helpful voice assistant. Keep your answers short. + +The user asked: {{query}} + +When you reply with structured data, use a JSON object like {"answer": "..."}. +The single-brace placeholder below is illustrative and is left untouched: + +``` +{"answer": "{{example}}"} +``` diff --git a/examples/skill-locale/locale/en-US/weather.dialog b/examples/skill-locale/locale/en-US/weather.dialog new file mode 100644 index 00000000..5232043a --- /dev/null +++ b/examples/skill-locale/locale/en-US/weather.dialog @@ -0,0 +1,3 @@ +It is {temperature} degrees. +Right now it's {temperature} degrees out. +(Currently|At the moment) {temperature} degrees. diff --git a/examples/skill-locale/locale/en-US/weekday.entity b/examples/skill-locale/locale/en-US/weekday.entity new file mode 100644 index 00000000..89c0fa07 --- /dev/null +++ b/examples/skill-locale/locale/en-US/weekday.entity @@ -0,0 +1,2 @@ +monday +(tues|wednes|thurs|fri)day diff --git a/ngi.png b/ngi.png new file mode 100644 index 00000000..bfd401af Binary files /dev/null and b/ngi.png differ diff --git a/ovos_intent_primitives/__init__.py b/ovos_intent_primitives/__init__.py deleted file mode 100644 index 3e4e56e5..00000000 --- a/ovos_intent_primitives/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Reference implementation of the OVOS intent specifications. - -`ovos-intent-primitives` provides the low-level, dependency-light primitives -the OVOS intent specifications describe: - -- :func:`~ovos_intent_primitives.expansion.expand` — the OVOS-INTENT-1 - sentence template expander. - -The resource-file loader (OVOS-INTENT-2) and the dialog renderer -(OVOS-INTENT-2 §4.2) are added in a later release. -""" -from ovos_intent_primitives.expansion import MalformedTemplate, expand -from ovos_intent_primitives.version import __version__ - -__all__ = ["expand", "MalformedTemplate", "__version__"] diff --git a/ovos_spec_tools/__init__.py b/ovos_spec_tools/__init__.py new file mode 100644 index 00000000..1ea2022c --- /dev/null +++ b/ovos_spec_tools/__init__.py @@ -0,0 +1,149 @@ +"""Reference implementation of the OVOS formal specifications. + +`ovos-spec-tools` provides the low-level, dependency-light primitives the OVOS +formal specifications describe: + +- :func:`~ovos_spec_tools.expansion.expand` — the OVOS-INTENT-1 sentence + template expander; +- :class:`~ovos_spec_tools.resources.LocaleResources` — the OVOS-INTENT-2 + locale resource-file loader; +- :func:`~ovos_spec_tools.dialog.render` / :class:`~ovos_spec_tools.dialog.DialogRenderer` + — the OVOS-INTENT-2 §4.2 dialog renderer; +- :func:`~ovos_spec_tools.prompt.render_prompt` / :class:`~ovos_spec_tools.prompt.PromptRenderer` + — the OVOS-INTENT-2 §4.4 ``.prompt`` renderer; +- :func:`~ovos_spec_tools.language.standardize_lang`, + :func:`~ovos_spec_tools.language.lang_distance`, + :func:`~ovos_spec_tools.language.lang_matches`, and + :func:`~ovos_spec_tools.language.closest_lang` — language-tag normalization, + distance, match checking, and closest-match resolution; +- :func:`~ovos_spec_tools.resources.iter_locale_dirs` — locale subdirectory + discovery with optional native-language filtering; +- :func:`~ovos_spec_tools.resources.keyword_form`, + :func:`~ovos_spec_tools.resources.utterance_contains`, + :func:`~ovos_spec_tools.resources.strip_samples` — slot-free template + grouping (§4.3) and utterance match / strip primitives; +- :class:`~ovos_spec_tools.message.Message` — the OVOS-MSG-1 bus message + envelope with the ``forward`` / ``reply`` / ``response`` derivations; +- :class:`~ovos_spec_tools.session.Session` — the OVOS-SESSION-1 session + carrier reference implementation, carrying the full §3 registered field + set with omission-not-null serialization and the + OVOS-PIPELINE-1 §7.1 / OVOS-CONVERSE-1 §2.1 / §2.2 handler-list helpers; +- :class:`~ovos_spec_tools.intent.IntentBuilder` / + :class:`~ovos_spec_tools.intent.Intent` — the adapt-free, plugin-agnostic + keyword intent-definition primitives mapping to the OVOS-INTENT-4 §5 keyword + registration model, with :func:`~ovos_spec_tools.intent.open_intent_envelope` + and the :func:`~ovos_spec_tools.intent.voc_match` ``.voc`` matching helper; +- :func:`~ovos_spec_tools.lint.lint_locale` — a locale resource linter, also + exposed as the ``ovos-spec-lint`` command. +""" +from ovos_spec_tools.dialog import ( + DialogRenderer, + UnfilledSlot, + render, + verify_slot_consistency, +) +from ovos_spec_tools.expansion import MalformedTemplate, expand, inline_keywords +from ovos_spec_tools.message import ( + DEFAULT_SESSION_ID, + MalformedMessage, + Message, +) +from ovos_spec_tools.language import ( + closest_lang, + lang_distance, + lang_matches, + standardize_lang, +) +from ovos_spec_tools.intent import ( + Intent, + IntentBuilder, + MalformedIntent, + open_intent_envelope, + voc_match, +) +from ovos_spec_tools.lint import ( + Finding, + declared_slots, + lint_locale, + lint_required_slots, + validate_required_slots, +) +from ovos_spec_tools.messages import ( + MIGRATION_MAP, + MIGRATION_PAYLOAD_TRANSFORMS, + SPEC_TO_LEGACY, + NamespaceTranslator, + SpecMessage, + migration_counterpart, +) +from ovos_spec_tools.prompt import PromptRenderer, render_prompt +from ovos_spec_tools.session import ( + DEFAULT_CONVERSE_HANDLERS_CAP, + MalformedSession, + SESSION1_OWNED_FIELDS, + SESSION1_REGISTERED_FIELDS, + Session, +) +from ovos_spec_tools.resources import ( + LocaleResources, + MalformedResource, + find_lang_dir, + iter_locale_dirs, + keyword_form, + normalize_for_match, + read_prompt_file, + read_resource_file, + strip_samples, + utterance_contains, +) +from ovos_spec_tools.version import __version__ + +__all__ = [ + "Message", + "MalformedMessage", + "DEFAULT_SESSION_ID", + "Session", + "MalformedSession", + "SESSION1_OWNED_FIELDS", + "SESSION1_REGISTERED_FIELDS", + "DEFAULT_CONVERSE_HANDLERS_CAP", + "Intent", + "IntentBuilder", + "open_intent_envelope", + "voc_match", + "expand", + "inline_keywords", + "MalformedTemplate", + "LocaleResources", + "MalformedResource", + "find_lang_dir", + "iter_locale_dirs", + "keyword_form", + "normalize_for_match", + "read_resource_file", + "read_prompt_file", + "strip_samples", + "utterance_contains", + "render", + "DialogRenderer", + "verify_slot_consistency", + "UnfilledSlot", + "render_prompt", + "PromptRenderer", + "standardize_lang", + "lang_distance", + "lang_matches", + "closest_lang", + "lint_locale", + "declared_slots", + "validate_required_slots", + "lint_required_slots", + "Finding", + "SpecMessage", + "MIGRATION_MAP", + "MIGRATION_PAYLOAD_TRANSFORMS", + "SPEC_TO_LEGACY", + "migration_counterpart", + "NamespaceTranslator", + "__version__", +] diff --git a/ovos_spec_tools/dialog.py b/ovos_spec_tools/dialog.py new file mode 100644 index 00000000..e2809ca2 --- /dev/null +++ b/ovos_spec_tools/dialog.py @@ -0,0 +1,245 @@ +"""Reference dialog renderer for OVOS-INTENT-2 §4.2. + +This is the reference implementation of the **Dialog renderer** conformance +role of OVOS-INTENT-1 §7. Rendering a dialog means: verify all phrases declare +the same slot set (§5.5), select one phrase from a loaded ``.dialog``, expand +its ``(a|b)`` / ``[x]`` variety to a single variant, and fill every ``{name}`` +slot with a value. The §7 role MUST "verify that all phrases in a dialog +definition declare the same slot set (§5.5)"; :func:`verify_slot_consistency` +performs that check and both render paths call it before rendering. + +Two interfaces are provided: + +- :func:`render` — a stateless one-shot function over explicit phrases; +- :class:`DialogRenderer` — a stateful, **multilingual** object backed by a + :class:`~ovos_spec_tools.resources.LocaleResources`: the language is given + per :meth:`DialogRenderer.render` call, and the renderer avoids repeating + the phrase it chose last (independently per language). + +A slot may be written in either equivalent spelling — single-brace ``{name}`` +or double-brace ``{{name}}`` (OVOS-INTENT-1 §3.4). The two are folded to the +same canonical ``{name}`` slot by :func:`~ovos_spec_tools.expansion.expand` +during expansion, so this renderer fills both transparently. A slot with no +value raises :class:`UnfilledSlot`. +""" +from __future__ import annotations + +import random as _random +import re +from typing import Dict, Optional, Protocol, Sequence + +from ovos_spec_tools.expansion import MalformedTemplate, expand + +__all__ = [ + "render", + "DialogRenderer", + "Chooser", + "UnfilledSlot", + "verify_slot_consistency", +] + +_SLOT_TOKEN_RE = re.compile(r"\{([a-z][a-z0-9_]*)\}") + + +def verify_slot_consistency(phrases: Sequence[str]) -> None: + """Verify every phrase of a dialog declares the identical slot set (§5.5). + + OVOS-INTENT-1 §7 makes this a **MUST** for the *Dialog renderer* role: it + "verify[s] that all phrases in a dialog definition declare the same slot + set (§5.5)". A definition MUST NOT mix templates that declare different + slots, so the caller supplies the same values whichever phrase is chosen + (§5.1, OVOS-INTENT-2 §4.2). + + A phrase *declares* a slot name if that name appears anywhere in it; + optionality does not change this (§5.5), so ``say [{x}]`` and ``say {x}`` + declare the same slot set ``{x}`` and may coexist. + + Args: + phrases: the phrase strings of one ``.dialog`` definition. + + Raises: + MalformedTemplate: two phrases declare different slot sets (§5.5). + """ + slot_sets = {frozenset(_SLOT_TOKEN_RE.findall(phrase)) for phrase in phrases} + if len(slot_sets) > 1: + rendered = sorted( + "{" + ", ".join(sorted(s)) + "}" for s in slot_sets) + raise MalformedTemplate( + "dialog phrases declare different slot sets " + f"({', '.join(rendered)}): every phrase in one dialog definition " + "MUST declare the same {slots} (OVOS-INTENT-1 §5.5); a phrasing " + "that needs different slots belongs in a separate dialog") + + +class Chooser(Protocol): + """A source of random selection: anything with a ``choice`` method. + + The :mod:`random` module and a :class:`random.Random` instance both + satisfy it. Inject a seeded ``random.Random`` for deterministic output. + """ + + def choice(self, seq: Sequence[str]) -> str: + ... + + +class UnfilledSlot(ValueError): + """A chosen dialog phrase has a named slot with no value. + + Per OVOS-INTENT-1 §5.1 the caller must supply a value for every slot in the + chosen phrase; a phrase with an unfilled slot must not be sent to TTS. + """ + + +def render(phrases: Sequence[str], + slots: Optional[Dict[str, object]] = None, + vocabularies: Optional[Dict[str, Sequence[str]]] = None, + rng: Optional[Chooser] = None) -> str: + """Render one phrase from a list of dialog phrases (stateless). + + This is the language-agnostic primitive: the caller has already chosen and + loaded the phrases. For a multilingual, resource-backed renderer use + :class:`DialogRenderer`. + + Args: + phrases: the phrase strings of a ``.dialog``. + slots: caller-supplied values for the phrase's named slots, keyed by + slot name. Values are converted to text. + vocabularies: vocabularies for any ```` references in the phrase. + rng: an object with a ``choice`` method, for phrase and variant + selection; defaults to the :mod:`random` module. + + Returns: + One rendered phrase, ready for text-to-speech. + + Raises: + UnfilledSlot: a slot in the chosen phrase has no value. + ValueError: ``phrases`` is empty. + MalformedTemplate: the chosen phrase is not a valid template, or the + phrases declare divergent slot sets (OVOS-INTENT-1 §5.5). + """ + if not phrases: + raise ValueError("no dialog phrases to render") + # §7 Dialog-renderer MUST: all phrases in a dialog declare the same slot + # set (§5.5). Enforce before choosing, so a divergent definition is + # rejected regardless of which phrase the chooser would pick. + verify_slot_consistency(phrases) + chooser = rng if rng is not None else _random + phrase = chooser.choice(list(phrases)) + return _render_phrase(phrase, slots or {}, vocabularies, chooser) + + +class DialogRenderer: + """A stateful, multilingual renderer for one named ``.dialog``. + + Backed by a :class:`~ovos_spec_tools.resources.LocaleResources`: the dialog + name is fixed at construction, but the **language is given per** + :meth:`render` **call**, so one renderer serves every language the dialog + is shipped in. Each call loads that language's phrases, vocabularies, and + ``.entity`` value sets afresh. + + Unlike the bare :func:`render`, it **avoids repeating the phrase it chose + on the previous call** — tracked independently per language — so a + repeatedly-spoken response does not sound mechanical. Repetition avoidance + is an implementation choice the spec explicitly allows (§4.2). + """ + + def __init__(self, resources, name: str, + rng: Optional[Chooser] = None, + slots: Optional[Dict[str, object]] = None): + """ + Args: + resources: a :class:`~ovos_spec_tools.resources.LocaleResources` + (or anything with ``load_dialog``, ``vocabularies`` and + ``entities`` methods taking a language). + name: the base name of the ``.dialog`` to render. + rng: an object with a ``choice`` method; defaults to :mod:`random`. + slots: **default** slot values, held for the renderer's lifetime + and reused on every :meth:`render` call. A per-call value + overrides a default. + """ + self.resources = resources + self.name = name + self.rng = rng if rng is not None else _random + self.default_slots: Dict[str, object] = dict(slots or {}) + self._last: Dict[str, str] = {} # last phrase chosen, per language + + def render(self, lang: str, + slots: Optional[Dict[str, object]] = None) -> str: + """Render one phrase of the dialog in ``lang``. + + A slot is filled, in order of precedence, from: the per-call ``slots``; + then the renderer's default slots; then a random value from the slot's + ``.entity`` set for ``lang``. A slot none of these supply raises + :class:`UnfilledSlot`. + + The phrase chosen on the previous call **for the same language** is + avoided when the dialog has more than one phrase. + + Args: + lang: the BCP-47 language tag to render in. + slots: per-call slot values; each overrides a default. + + Returns: + One rendered phrase, ready for text-to-speech. + + Raises: + UnfilledSlot: a slot in the chosen phrase has no value. + FileNotFoundError: the dialog does not exist for ``lang``. + MalformedTemplate: the chosen phrase is not a valid template, or + the dialog's phrases declare divergent slot sets + (OVOS-INTENT-1 §5.5). + """ + phrases = self.resources.load_dialog(self.name, lang) + # §7 Dialog-renderer MUST verify all phrases declare the same slot set + # (§5.5) before rendering. + verify_slot_consistency(phrases) + effective = dict(self.default_slots) + if slots: + effective.update(slots) + + last = self._last.get(lang) + choices = [p for p in phrases if p != last] or phrases + phrase = self.rng.choice(choices) + self._last[lang] = phrase + + return _render_phrase(phrase, effective, + self.resources.vocabularies(lang), self.rng, + self.resources.entities(lang)) + + +def _render_phrase(phrase: str, + slots: Dict[str, object], + vocabularies: Optional[Dict[str, Sequence[str]]], + chooser: Chooser, + entities: Optional[Dict[str, Sequence[str]]] = None) -> str: + """Expand one phrase to a variant, then fill its slots. + + Expansion keeps slots opaque (OVOS-INTENT-1 §4), so it runs first and a + slot value can never be parsed as grammar. + """ + variant = chooser.choice(expand(phrase, vocabularies)) + return _fill_slots(variant, slots, entities, chooser) + + +def _fill_slots(variant: str, + slots: Dict[str, object], + entities: Optional[Dict[str, Sequence[str]]], + chooser: Chooser) -> str: + """Replace every ``{name}`` in ``variant`` with a value. + + A value is taken from ``slots``; failing that, a random valid value from + the slot's ``.entity`` set in ``entities``; failing that, the slot has no + value and :class:`UnfilledSlot` is raised. + """ + + def replace(match: "re.Match") -> str: + name = match.group(1) + if name in slots: + return str(slots[name]) + if entities and entities.get(name): + return str(chooser.choice(list(entities[name]))) + raise UnfilledSlot( + f"dialog slot {{{name}}} has no value: it was not filled by the " + f"caller and has no .entity fallback") + + return _SLOT_TOKEN_RE.sub(replace, variant) diff --git a/ovos_intent_primitives/expansion.py b/ovos_spec_tools/expansion.py similarity index 57% rename from ovos_intent_primitives/expansion.py rename to ovos_spec_tools/expansion.py index 4acf9022..30279631 100644 --- a/ovos_intent_primitives/expansion.py +++ b/ovos_spec_tools/expansion.py @@ -9,7 +9,11 @@ - literal words; - ``(a|b|c)`` alternatives; - ``[x]`` optional segments, equivalent to ``(x|)``; -- ``{name}`` named slots — opaque, carried through unchanged, never expanded; +- ``{name}`` named slots — opaque, carried through unchanged, never expanded. + The double-brace form ``{{name}}`` is an **equivalent** spelling of the same + named slot (OVOS-INTENT-1 §3.4): ``{name}`` and ``{{name}}`` denote the same + slot, so a template may use either spelling and the resulting sample set is + identical; - ```` inline vocabulary references — replaced, before expansion, by a named slot-free vocabulary (OVOS-INTENT-1 §3.7). @@ -24,7 +28,7 @@ import re from typing import Dict, List, Optional, Sequence, Tuple -__all__ = ["expand", "MalformedTemplate"] +__all__ = ["expand", "fold_double_braces", "MalformedTemplate"] # A slot or vocabulary name: lowercase ASCII letters, digits, underscores; # never beginning with a digit (OVOS-INTENT-1 §3.4). @@ -33,6 +37,13 @@ # forbid the matching bracket so a malformed/nested token cannot be matched. _SLOT_TOKEN_RE = re.compile(r"\{([^{}]*)\}") _VOC_TOKEN_RE = re.compile(r"<([^<>]*)>") +# The double-brace slot spelling ``{{name}}`` (OVOS-INTENT-1 §3.4) — an +# equivalent form of the single-brace named slot. It is folded to ``{name}`` +# before any other parsing. The interior forbids braces so the token is flat, +# and the form is matched here (and folded) **before** the single-brace token +# is ever considered, so ``{{x}}`` is read as one slot and never mis-parsed as +# ``{`` + ``{x}`` + ``}``. +_DOUBLE_SLOT_TOKEN_RE = re.compile(r"\{\{([^{}]*)\}\}") # Two named slots in a sample with only whitespace between them. _ADJACENT_SLOTS_RE = re.compile(r"\}\s*\{") @@ -75,6 +86,14 @@ def _expand(template: str, if not isinstance(template, str): raise MalformedTemplate(f"template must be a string, got {type(template)!r}") + # Fold the double-brace slot spelling ``{{name}}`` to the canonical + # single-brace ``{name}`` (OVOS-INTENT-1 §3.4) before any other parsing, + # so the two spellings are exactly equivalent and every downstream check + # (balance, names, slot-only, adjacency, repetition) sees one slot form. + # Done first — and matching the double form before the single — so + # ``{{x}}`` is never mis-read as ``{`` + ``{x}`` + ``}``. + template = fold_double_braces(template) + _check_balanced(template) _check_names(template) @@ -99,6 +118,18 @@ def _expand(template: str, return samples +def fold_double_braces(template: str) -> str: + """Fold every ``{{name}}`` to the equivalent ``{name}`` (§3.4). + + The double-brace spelling is matched and replaced **before** the + single-brace token is ever examined, so ``{{x}}`` collapses to ``{x}`` as + a single slot rather than being read as ``{`` + ``{x}`` + ``}``. The + interior is carried through verbatim, so an ill-formed interior survives + to be rejected by the same §3.4 name check that guards ``{name}``. + """ + return _DOUBLE_SLOT_TOKEN_RE.sub(lambda m: "{" + m.group(1) + "}", template) + + def _check_balanced(template: str) -> None: """Reject unbalanced or malformed metacharacters (§3.6).""" # `{name}` and `` are flat tokens — they do not nest. Strip every @@ -226,3 +257,117 @@ def _check_sample(sentence: str, template: str) -> None: raise MalformedTemplate( f"repeated slot name in sample {sentence!r} of template " f"{template!r}: each slot is defined once per sample") + + +def inline_keywords( + template: str, + vocabularies: Dict[str, Sequence[str]] | None = None, + *, + max_values: Optional[int] = None, +) -> str: + """Inline ```` references as ``(v1|v2|…)`` alternation groups. + + Partial application of OVOS-INTENT-1 §4.1 step 1 (resolve ```` + references) that stops **before** the rest of expansion: §3.7 / §4.1 define + a ```` reference as equivalent to the alternative group of the + vocabulary's members written in its place, and this returns exactly that + rewritten *template* rather than the fully enumerated sample set. It exists + because engines like Padatious do not look up ``.voc`` files at runtime — + they need keywords baked into the template body as standard ``(a|b|c)`` + alternations, then expand the result themselves. + + Unlike :func:`expand`, this is intentionally lenient and **not** a + conformant expander: it does no §3.6 validation and leaves an unknown + keyword's angle brackets stripped as literal text rather than raising + :class:`MalformedTemplate` for an undefined reference. Feed its output to + :func:`expand` for the validated sample set. + + By default **every** value of a keyword is inlined (no silent truncation). + Resolution recurses through nested references — ```` inside ```` — and + a reference cycle raises :class:`MalformedTemplate` per OVOS-INTENT-1 §4.1, + rather than being cut off at an arbitrary depth. + + Parameters + ---------- + template + Template string with ```` references. + vocabularies + Flat ``{keyword: [values]}`` mapping. If ``None`` or empty the + template is returned unchanged. + max_values + Optional bound on the number of values inlined per keyword. ``None`` + (the default) inlines **all** values. OVOS-INTENT-1 §4.3 permits an + expander to *refuse* (not silently drop) a template whose expansion + exceeds a documented limit; accordingly, when a keyword has more than + ``max_values`` members this **raises** :class:`MalformedTemplate`. It + never silently truncates the value list. + + Returns + ------- + str + Template with all ```` references inlined as ``(a|b|c)`` + groups. Keywords not found in ``vocabularies`` have their angle + brackets stripped and become literal text. + + Raises + ------ + MalformedTemplate + On a reference cycle (OVOS-INTENT-1 §4.1), or when a keyword's value + count exceeds an explicit ``max_values`` bound (OVOS-INTENT-1 §4.3, + refuse-and-document). + + Example + ------- + >>> inline_keywords(" [the] {name}", + ... {"turn_on": ["turn on", "switch on"]}) + '(turn on|switch on) [the] {name}' + """ + if not vocabularies: + return template + return _inline_keywords(template, vocabularies, max_values, ()) + + +def _inline_keywords(template: str, + vocabularies: Dict[str, Sequence[str]], + max_values: Optional[int], + stack: Tuple[str, ...]) -> str: + """Resolve ```` references recursively, leftmost first, with cycle + detection (OVOS-INTENT-1 §4.1). ``stack`` is the chain of references being + resolved; a name already on it is a cycle and is rejected.""" + while True: + match = _VOC_TOKEN_RE.search(template) + if match is None: + return template + name = match.group(1) + + if name in stack: + raise MalformedTemplate( + f"cyclic vocabulary reference <{name}> " + f"(chain: {' -> '.join(stack + (name,))})") + + vals = vocabularies.get(name) + if not vals: + # Lenient: an unknown keyword is left as literal text (brackets + # stripped) rather than raising. Skip past it so the leftmost-first + # scan continues and an undefined reference cannot loop forever. + template = (template[:match.start()] + name + + template[match.end():]) + continue + + if max_values is not None and len(vals) > max_values: + # §4.3 permits a documented limit enforced by REFUSING — never by + # silently dropping values (which would be data loss). + raise MalformedTemplate( + f"vocabulary <{name}> has {len(vals)} values, exceeding the " + f"max_values bound of {max_values} (OVOS-INTENT-1 §4.3: a limit " + f"is enforced by refusing, not by truncating)") + + # Resolve any nested references inside each value before substituting, + # carrying this name on the stack for cycle detection. + resolved_vals = [ + _inline_keywords(v, vocabularies, max_values, stack + (name,)) + for v in vals + ] + substitution = "(" + "|".join(resolved_vals) + ")" + template = (template[:match.start()] + substitution + + template[match.end():]) diff --git a/ovos_spec_tools/intent.py b/ovos_spec_tools/intent.py new file mode 100644 index 00000000..9b65815e --- /dev/null +++ b/ovos_spec_tools/intent.py @@ -0,0 +1,510 @@ +"""Plugin-agnostic intent-definition primitives for the OVOS-INTENT-4 keyword model. + +This module is a clean, dependency-light ``IntentBuilder`` / ``Intent`` pair — +the plugin-agnostic form of the intent-definition classes ``ovos-workshop`` +exposes to skills. It carries **no** ``adapt`` dependency: it is pure data +describing the **structure** of a keyword intent — +which vocabularies are *required*, *optional*, *one_of*, or *excluded* — exactly +as OVOS-INTENT-4 §5 defines the ``ovos.intent.register.keyword`` payload. + +The split of responsibility, per OVOS-INTENT-4 §5.1, is: + +- the **builder** captures the intent *structure* (vocabulary **names** under + each role) — that is all a skill expresses when it writes + ``IntentBuilder("Foo").require("Set").one_of("Up", "Down").build()``; +- a **producer** (the skill loader) later inlines each vocabulary's expanded + ``samples`` to form the wire :meth:`Intent.to_keyword_payload` descriptors — + file paths never cross the bus (§5.1). + +Because samples are inlined downstream, the descriptors this module emits carry +``name`` only by default. A producer that already has the expanded samples may +supply them through the ``samples`` argument of :meth:`Intent.to_keyword_payload`. + +The public API is **source-compatible** with the ``ovos-workshop`` classes it +replaces, so skills and intent engines can re-point their imports here without +code changes: + +- ``IntentBuilder(name).require(t, attribute_name=None, optional=False)``, + ``.optionally(t, attribute_name=None)``, ``.one_of(*args)``, + ``.exclude(t)``, ``.build()``, ``.name``; +- ``Intent(name, requires, at_least_one, optional, excludes)`` exposing + ``.name``, ``.requires``, ``.at_least_one``, ``.optional``, ``.excludes``; +- :func:`open_intent_envelope` reconstructing an :class:`Intent` from a Message. +""" +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional, Sequence, Tuple + +_log = logging.getLogger(__name__) + +__all__ = [ + "Intent", + "IntentBuilder", + "MalformedIntent", + "open_intent_envelope", + "voc_match", +] + +# A keyword role entry as the legacy adapt/workshop classes stored it: +# ``(entity_type, attribute_name)`` for required/optional, a bare entity type +# for excludes, and a tuple of entity types for each one_of group. +RoleEntry = Tuple[str, str] + + +class MalformedIntent(ValueError): + """A keyword intent that violates an OVOS-INTENT-3 §4.2 structural MUST. + + Raised when a built / emitted intent breaks one of the two §4.2 + well-formedness MUSTs: + + - it declares **no** ``required`` and **no** ``one-of`` constraint, so + nothing must be present for it to match ("an intent with only optional + and excluded constraints has nothing that must be present and is + malformed"); + - it lists the **same vocabulary under two different roles** ("a vocabulary + MUST appear under at most one role within a single intent. Listing the + same vocabulary under two roles … is contradictory and malformed"). + + These are the data-model counterparts of the checks the locale linter + already performs; raising at build/emit time means an invalid keyword + intent is rejected before it can be registered on the bus. + """ + + +class Intent: + """A built keyword-intent **definition** — name plus the four role lists. + + This mirrors the attribute surface of the legacy ``ovos-workshop`` / + ``adapt`` ``Intent`` so existing readers keep working: + + - ``name`` (str) — the intent name (skills munge it to + ``:`` before registration); + - ``requires`` — ``list[(entity_type, attribute_name)]``; every entry MUST + occur (OVOS-INTENT-4 §5.2 ``required``); + - ``at_least_one`` — ``list[tuple[entity_type, ...]]``; each inner tuple is + a group, one member of which MUST occur (§5.2 ``one_of``); + - ``optional`` — ``list[(entity_type, attribute_name)]``; captured if + present (§5.2 ``optional``); + - ``excludes`` — ``list[entity_type]``; any occurrence suppresses the + match (§5.2 ``excluded``). + + Unlike the legacy class this carries **no matching logic** — matching is a + pipeline-plugin concern (OVOS-PIPELINE-1). It is pure data: the structural + half of a ``ovos.intent.register.keyword`` registration, ready for a + producer to inline vocabulary samples into (§5.1). + """ + + def __init__(self, name: str = "", + requires: Optional[Sequence] = None, + at_least_one: Optional[Sequence] = None, + optional: Optional[Sequence] = None, + excludes: Optional[Sequence] = None): + """ + Args: + name: the intent name. + requires: required entries, each ``(entity_type, attribute_name)`` + or a bare ``entity_type`` (normalized to a pair). + at_least_one: ``one_of`` groups, each a sequence of entity types. + optional: optional entries, same shape as ``requires``. + excludes: excluded entity types, each a bare ``entity_type`` (a + ``(type, attr)`` pair is accepted and reduced to its type). + """ + self.name = name + self.requires: List[RoleEntry] = [self._as_pair(r) + for r in (requires or [])] + self.at_least_one: List[Tuple[str, ...]] = [tuple(self._as_name(e) + for e in group) + for group in + (at_least_one or [])] + self.optional: List[RoleEntry] = [self._as_pair(o) + for o in (optional or [])] + self.excludes: List[str] = [self._as_name(e) for e in (excludes or [])] + + @staticmethod + def _as_pair(entry) -> RoleEntry: + """Normalize an entry to ``(entity_type, attribute_name)``. + + Accepts a bare string (attribute defaults to the type) or any 2-tuple. + """ + if isinstance(entry, str): + return (entry, entry) + entity_type, attribute_name = entry[0], entry[1] + return (entity_type, attribute_name or entity_type) + + @staticmethod + def _as_name(entry) -> str: + """Reduce an entry to its bare entity-type name.""" + if isinstance(entry, str): + return entry + return entry[0] + + # -- OVOS-INTENT-3 §4.2 well-formedness ---------------------------------- + + def validate(self) -> "Intent": + """Reject an intent that violates an OVOS-INTENT-3 §4.2 structural MUST. + + Enforces the two §4.2 well-formedness rules the spec calls ``MUST``: + + - **(a)** a keyword intent **MUST** declare at least one ``required`` + or ``one-of`` constraint — "an intent with only optional and + excluded constraints has nothing that must be present and is + malformed"; + - **(b)** a vocabulary **MUST** appear under at most one role — listing + the same vocabulary under two roles (e.g. both required and excluded) + "is contradictory and malformed". + + Returns ``self`` so it can be used inline (``Intent(...).validate()``). + + Raises: + MalformedIntent: if either §4.2 rule is violated. + """ + # (a) at least one required or one-of must be present. + if not self.requires and not self.at_least_one: + raise MalformedIntent( + f"keyword intent {self.name!r} declares no required and no " + "one-of constraint — it has nothing that must be present " + "(OVOS-INTENT-3 §4.2)") + + # (b) a vocabulary appears under at most one role. Collect every + # (vocabulary -> roles it appears in) and reject any vocabulary that + # spans more than one role. one-of vocabularies count once per name + # regardless of how many groups list them. + roles: Dict[str, set] = {} + for name, _ in self.requires: + roles.setdefault(name, set()).add("required") + for name, _ in self.optional: + roles.setdefault(name, set()).add("optional") + for group in self.at_least_one: + for name in group: + roles.setdefault(name, set()).add("one_of") + for name in self.excludes: + roles.setdefault(name, set()).add("excluded") + clashes = {name: sorted(r) for name, r in roles.items() if len(r) > 1} + if clashes: + detail = "; ".join(f"{name!r} under {roles}" + for name, roles in sorted(clashes.items())) + raise MalformedIntent( + f"keyword intent {self.name!r} lists a vocabulary under more " + f"than one role — {detail} (OVOS-INTENT-3 §4.2)") + return self + + # -- OVOS-INTENT-4 §5 emission ------------------------------------------- + + @staticmethod + def _descriptor(name: str, + samples: Optional[Dict[str, List[str]]]) -> Dict[str, Any]: + """Build one §5.1 vocabulary descriptor for ``name``. + + ``samples`` is an optional ``vocab_name -> samples`` map a producer may + supply when it has already expanded the vocabularies. When absent — the + usual builder-side case — only ``name`` is emitted and the producer is + expected to inline ``samples`` before the payload crosses the bus + (§5.1). + """ + descriptor: Dict[str, Any] = {"name": name} + if samples and name in samples: + descriptor["samples"] = list(samples[name]) + return descriptor + + def to_keyword_payload(self, skill_id: Optional[str] = None, + lang: Optional[str] = None, + samples: Optional[Dict[str, List[str]]] = None + ) -> Dict[str, Any]: + """Emit the OVOS-INTENT-4 §5.2 keyword-registration structure. + + Returns a dict with the four shape-stable role keys ``required``, + ``optional``, ``one_of``, ``excluded`` (§5.2 mandates all four are + present, even when empty), each a list of vocabulary descriptors (§5.1). + ``one_of`` is a list of groups, each a list of descriptors. + + Identity fields (``skill_id``, ``intent_name``, ``lang``; §3.2) are + included when provided — ``intent_name`` is always set from ``name``. + Vocabulary ``samples`` are inlined per descriptor only when the + ``samples`` map carries them; otherwise the producer inlines them + before emission (§5.1). + + Args: + skill_id: optional ``skill_id`` identity field (§3.2). + lang: optional ``lang`` identity field (§3.2). + samples: optional ``vocab_name -> expanded samples`` map; when a + name is present its descriptor gains a ``samples`` entry. + + Returns: + the §5.2 keyword payload structure. + + A malformed intent (OVOS-INTENT-3 §4.2 — no required/one-of + constraint, or a vocabulary listed under two roles) is **logged as a + warning** rather than raised, so emitting stays backward-compatible; + call :meth:`validate` explicitly to reject before emit. + """ + try: + self.validate() + except MalformedIntent as err: + _log.warning("emitting register payload for malformed intent %r " + "(OVOS-INTENT-3 §4.2): %s", self.name, err) + payload: Dict[str, Any] = {} + if skill_id is not None: + payload["skill_id"] = skill_id + payload["intent_name"] = self.name + if lang is not None: + payload["lang"] = lang + payload["required"] = [self._descriptor(t, samples) + for t, _ in self.requires] + payload["optional"] = [self._descriptor(t, samples) + for t, _ in self.optional] + payload["one_of"] = [[self._descriptor(t, samples) for t in group] + for group in self.at_least_one] + payload["excluded"] = [self._descriptor(t, samples) + for t in self.excludes] + return payload + + def __repr__(self) -> str: + return (f"Intent(name={self.name!r}, requires={self.requires!r}, " + f"at_least_one={self.at_least_one!r}, " + f"optional={self.optional!r}, excludes={self.excludes!r})") + + def __eq__(self, other) -> bool: + if not isinstance(other, Intent): + return NotImplemented + return (self.name == other.name and + self.requires == other.requires and + self.at_least_one == other.at_least_one and + self.optional == other.optional and + self.excludes == other.excludes) + + +class IntentBuilder: + """Fluent builder for a keyword :class:`Intent` — adapt-free. + + Source-compatible with the ``ovos-workshop`` / ``adapt`` ``IntentBuilder``: + it accumulates vocabulary **names** under the four OVOS-INTENT-4 §5 roles + and :meth:`build` freezes them into an :class:`Intent`. It captures only the + intent *structure* (§5.1) — no samples, no matching. + + Example: + >>> intent = (IntentBuilder("SetBrightness") + ... .require("Set") + ... .require("Brightness") + ... .one_of("Up", "Down") + ... .optionally("Politely") + ... .exclude("Question") + ... .build()) + >>> intent.to_keyword_payload()["required"] + [{'name': 'Set'}, {'name': 'Brightness'}] + """ + + def __init__(self, intent_name: str): + """ + Args: + intent_name: the name of the intent being built. + """ + self.name = intent_name + self.requires: List[RoleEntry] = [] + self.at_least_one: List[Tuple[str, ...]] = [] + self.optional: List[RoleEntry] = [] + self.excludes: List[str] = [] + + def require(self, entity_type: str, attribute_name: Optional[str] = None, + optional: bool = False) -> "IntentBuilder": + """Require (or, with ``optional=True``, optionally capture) a vocabulary. + + Args: + entity_type: the vocabulary name. + attribute_name: name of the captured attribute on the match result; + defaults to ``entity_type``. + optional: when True, behaves like :meth:`optionally` — kept for + source-compatibility with the legacy signature. + + Returns: + self, for chaining. + """ + if not attribute_name: + attribute_name = entity_type + if optional: + self.optional.append((entity_type, attribute_name)) + else: + self.requires.append((entity_type, attribute_name)) + return self + + def optionally(self, entity_type: str, + attribute_name: Optional[str] = None) -> "IntentBuilder": + """Optionally capture a vocabulary (OVOS-INTENT-4 §5.2 ``optional``). + + Args: + entity_type: the vocabulary name. + attribute_name: captured-attribute name; defaults to ``entity_type``. + + Returns: + self, for chaining. + """ + if not attribute_name: + attribute_name = entity_type + self.optional.append((entity_type, attribute_name)) + return self + + def one_of(self, *args: str) -> "IntentBuilder": + """Require at least one of the given vocabularies (§5.2 ``one_of``). + + Each call adds one **group**; at least one member of each group must + occur. Separate calls express ``one_of(A, B)`` *and* ``one_of(C, D)``. + + Args: + *args: vocabulary names forming one group. + + Returns: + self, for chaining. + """ + self.at_least_one.append(tuple(args)) + return self + + def exclude(self, entity_type: str) -> "IntentBuilder": + """Forbid a vocabulary (§5.2 ``excluded``): its presence suppresses the + match. + + Args: + entity_type: the vocabulary name to exclude. + + Returns: + self, for chaining. + """ + self.excludes.append(entity_type) + return self + + def build(self) -> Intent: + """Freeze the accumulated roles into an :class:`Intent`. + + A built intent should satisfy the OVOS-INTENT-3 §4.2 well-formedness + MUSTs (:meth:`Intent.validate`): declare at least one required or + one-of constraint, and not list a vocabulary under two roles. A + malformed builder state is **logged as a warning** rather than raised, + so ``build()`` stays backward-compatible — call :meth:`Intent.validate` + explicitly (or rely on the locale linter) to enforce §4.2 where you + want to reject. + """ + intent = Intent(self.name, self.requires, self.at_least_one, + self.optional, self.excludes) + try: + intent.validate() + except MalformedIntent as err: + _log.warning("built intent %r is malformed per OVOS-INTENT-3 " + "§4.2: %s", self.name, err) + return intent + + +def open_intent_envelope(message) -> Intent: + """Reconstruct an :class:`Intent` from a Message payload. + + Mirrors the legacy ``ovos-workshop`` helper: it reads an intent definition + out of ``message.data``. Both the legacy serialization keys + (``name`` / ``requires`` / ``at_least_one`` / ``optional`` / ``excludes``, + as produced by ``Intent.__dict__``) and the OVOS-INTENT-4 §5.2 wire keys + (``intent_name`` / ``required`` / ``one_of`` / ``optional`` / ``excluded``, + whose role entries are vocabulary descriptors) are accepted, so the helper + round-trips a definition serialized by either generation. + + For §5.2 descriptors only the ``name`` is read into the role list — the + inlined ``samples`` are not part of the structural :class:`Intent`. + + Args: + message: a Message-like object exposing a ``data`` mapping. + + Returns: + the reconstructed :class:`Intent`. + """ + data = getattr(message, "data", None) + if data is None: + data = message # tolerate being handed a raw dict + + name = data.get("name") or data.get("intent_name") or "" + + def _names(role) -> List[str]: + # Accept §5.2 descriptors ({"name": ...}), bare strings, or legacy + # (type, attr) pairs — reduce each to its vocabulary name. + out = [] + for entry in role or []: + if isinstance(entry, dict): + out.append(entry.get("name")) + elif isinstance(entry, str): + out.append(entry) + else: # (entity_type, attribute_name) + out.append(entry[0]) + return out + + requires = data.get("requires") + if requires is None: + requires = _names(data.get("required")) + + optional = data.get("optional") + # legacy `optional` is already pairs; §5.2 `optional` is descriptors + if optional and isinstance(optional[0], dict): + optional = _names(optional) + + excludes = data.get("excludes") + if excludes is None: + excludes = _names(data.get("excluded")) + + at_least_one = data.get("at_least_one") + if at_least_one is None: + # §5.2 `one_of`: list of groups, each a list of descriptors. + at_least_one = [_names(group) for group in data.get("one_of") or []] + + return Intent(name, requires, at_least_one, optional, excludes) + + +def voc_match(utterance: str, voc_name: str, lang: str, + locale, *, + exact: bool = False, + strip_diacritics: bool = True, + strip_punct: bool = True) -> bool: + """Load a named ``.voc`` and test whether ``utterance`` matches it. + + This is the plugin-agnostic equivalent of + ``OVOSAbstractApplication.voc_match`` / the skill ``voc_match`` — the + helper common-query, OCP, and other pipelines use without depending on + ``ovos-workshop``. It loads the ``.voc`` for ``lang`` and + matches with whole-word OVOS-INTENT-2 §4.3 semantics — identical to the + skill helper (so pipelines behave the same): a sample ``yes`` matches + ``"yes, please"`` but not ``"yesterday"``. + + Args: + utterance: the (ASR-normalized) text to test. + voc_name: the ``.voc`` base name (no extension). + lang: BCP-47 language tag of the resource to load. + locale: either a :class:`~ovos_spec_tools.resources.LocaleResources` + instance, or a ``locale/`` directory path (``str`` / ``Path``), or + a sequence of such paths searched in override-precedence order + (user, skill, core — see + :class:`~ovos_spec_tools.resources.LocaleResources`). + exact: require equality after normalization rather than whole-word + substring containment. + strip_diacritics: forwarded to the matcher. + strip_punct: forwarded to the matcher. + + Returns: + ``True`` iff any ``.voc`` sample matches; ``False`` when the resource + does not exist for the language. + """ + from ovos_spec_tools.resources import LocaleResources + + if isinstance(locale, LocaleResources): + resources = locale + elif isinstance(locale, (str, bytes)) or hasattr(locale, "__fspath__"): + resources = LocaleResources(str(locale)) + else: # a sequence of locale dirs, highest precedence first + dirs = [str(p) for p in locale] + if not dirs: + return False + # Map the precedence-ordered dirs onto the user/skill/core slots, which + # LocaleResources searches in that same order. A single dir is the + # skill locale; the spare slots stay empty. + if len(dirs) == 1: + resources = LocaleResources(skill_locale=dirs[0]) + else: + resources = LocaleResources( + user_locale=dirs[0], + skill_locale=dirs[1], + core_locale=dirs[2] if len(dirs) > 2 else None) + return resources.voc_match( + utterance, voc_name, lang, exact=exact, + strip_diacritics=strip_diacritics, strip_punct=strip_punct) diff --git a/ovos_spec_tools/language.py b/ovos_spec_tools/language.py new file mode 100644 index 00000000..38f89bb2 --- /dev/null +++ b/ovos_spec_tools/language.py @@ -0,0 +1,284 @@ +"""Language-tag utilities — standardization, distance, and closest match. + +This module is the cross-spec implementation of OVOS's BCP-47 language-matching +and locale-resolution rules. It is referenced by — but not owned by — a single +spec; the rules it encodes are drawn from: + +- **OVOS-INTENT-2 §2** — locale directories are named with BCP-47 tags and tag + comparison is **case-insensitive** (``en-us`` and ``en-US`` denote the same + language). :func:`standardize_lang` is the canonicalization that makes that + comparison total, and :func:`lang_distance` returns ``0`` for tags that differ + only in case. +- **OVOS-INTENT-2 §2.2** — the *language fallback* suggestion: a loader MAY fall + back to the nearest available language, and the spec names ``langcodes``' + ``tag_distance()`` with a "distance below 10 is a usable regional match" + threshold. :data:`DEFAULT_MAX_LANGUAGE_DISTANCE` is exactly that ``10``; + :func:`closest_lang` is exactly that "nearest available" selection. §2.2 is + explicitly **non-normative** ("an implementation choice, not a requirement"), + so everything here is a reference policy, not a conformance obligation. +- the **SESSION-1 language family** — a session carries a ``lang`` tag; the same + matching rules decide whether a session's language is served by an available + resource/voice/model. + +OVOS resolves "the closest available language for a request" in many places — +locale resources, TTS voices, STT models — and the logic has been +reimplemented repeatedly (``ovos_utils.lang.get_language_dir``, +``phoonnx.match_lang``, …) with subtle drift between the copies. This module is +intended to be the **single implementation** behind all of them. + +It is built on one distance function, :func:`lang_distance`; :func:`closest_lang` +is simply "the candidate with the smallest distance". All the policy — tag +standardization, the norm-region preference, the behaviour when ``langcodes`` +is absent — lives inside :func:`lang_distance`, not in branchy callers. + +**Dialect-fallback semantics**, precisely. A request resolves to a candidate iff +their :func:`lang_distance` is below the threshold (an exact match — distance +``0`` — always resolves). The distance ladder is, from nearest to farthest: +identical tag (``0``) < a bare tag vs its own norm region (``0``, see +:func:`_with_norm_region`) < two regions of one language (a small regional +distance) < the bare/generic form vs a region of the same language < a different +primary language (``>= 100`` with the coarse measure — never a usable match). + +The norm-region preference is **non-normative implementation policy**, not a +spec rule: a bare ``pt`` is measured **from ``pt-PT``**, not from the +most-populous ``pt-BR``, so ``pt`` falls back to ``pt-PT`` before ``pt-BR``. +This deliberately *diverges* from ``langcodes``' population default — which +§2.2 endorses — and is a choice this package makes, not a requirement of any +spec (see :data:`_NORM_REGION`). + +``langcodes`` is an optional dependency (OVOS-INTENT-2 §2.2 names it). Without +it, :func:`lang_distance` falls back to a coarse same-language / +different-language measure that still honours the same ordering for the cases +locale resolution actually depends on. That fallback, and its distance values, +are likewise this package's own implementation policy — §2.2 is silent on the +no-``langcodes`` case. +""" +from __future__ import annotations + +from typing import Optional, Sequence + +__all__ = [ + "standardize_lang", + "lang_distance", + "lang_matches", + "closest_lang", + "DEFAULT_MAX_LANGUAGE_DISTANCE", +] + +# A language distance below 10 is a usable regional match (OVOS-INTENT-2 §2.2; +# see the langcodes distance-values documentation). +DEFAULT_MAX_LANGUAGE_DISTANCE = 10 + +# NON-NORMATIVE IMPLEMENTATION POLICY (no spec backing). `langcodes` resolves a +# bare tag to its *most-populous* region — for "pt" that is "pt-BR". This map +# deliberately overrides that for languages whose unmarked form OVOS prefers to +# resolve to a reference variety instead: a bare "pt" is measured from "pt-PT". +# OVOS-INTENT-2 §2.2 endorses `langcodes`; this preference diverges from it on +# purpose and is this package's policy, not a requirement of any spec. Add a +# language here only when its bare tag has a clear reference region distinct +# from the populous one. +_NORM_REGION = { + "pt": "PT", +} + + +def standardize_lang(tag: str) -> str: + """Normalize a BCP-47 language tag for comparison. + + Implements the canonicalization presupposed by OVOS-INTENT-2 §2's + "tag comparison is case-insensitive" rule: it folds underscores to hyphens, + lowercases the primary subtag, uppercases the region, and (via ``langcodes``) + resolves canonical script/region forms — so that two on-disk spellings of + one language (``en_us``, ``en-US``) reduce to a single comparable string. + + Args: + tag: a BCP-47-ish language tag, possibly underscore-separated or + mixed-case (as locale directory names and config values often are). + + Returns: + The standardized tag. With ``langcodes`` installed this is + ``langcodes.standardize_tag``'s output; without it, a primary-lowercase, + region-uppercase normalization. + + Uses ``langcodes.standardize_tag`` when available — handling underscores, + case, and script/region forms — and falls back to a simple normalization + otherwise. + """ + if tag.lower() in ("tl", "tgl"): + # NON-NORMATIVE IMPLEMENTATION POLICY (no spec backing). langcodes folds + # Tagalog into the `fil` macrolanguage; OVOS keeps `tl` distinct, so + # this one tag is normalized by hand. This is a deliberate divergence + # from langcodes (which §2.2 endorses), not a spec requirement. + return "tl" + try: + from langcodes import standardize_tag + return str(standardize_tag(tag)) + except Exception: + normalized = tag.replace("_", "-") + if "-" in normalized: + primary, rest = normalized.split("-", 1) + return f"{primary.lower()}-{rest.upper()}" + return normalized.lower() + + +def _with_norm_region(tag: str) -> str: + """Give a bare language tag its norm region (``pt`` -> ``pt-PT``). + + A regioned tag, or a language with no norm region, is returned unchanged. + The norm-region map is non-normative implementation policy (see + :data:`_NORM_REGION`); it diverges from ``langcodes`` on purpose. + """ + if "-" in tag: + return tag + region = _NORM_REGION.get(tag.lower()) + return f"{tag.lower()}-{region.upper()}" if region else tag + + +def _langcodes_distance(desired: str, supported: str) -> Optional[int]: + """``langcodes.tag_distance``, retried on the primary subtag if the full + tag is unparseable. ``None`` if no distance can be computed — including + when ``langcodes`` is not installed.""" + try: + from langcodes import tag_distance + except ImportError: + return None + for candidate in (supported, supported.split("-")[0]): + try: + return int(tag_distance(desired, candidate)) + except Exception: + continue + return None + + +def _coarse_distance(desired: str, supported: str) -> int: + """A ``langcodes``-free distance. + + The two tags are already standardized and norm-expanded. A shared primary + subtag is near, a differing one is far; the generic (region-less) form of + a language counts as nearer than a sibling region of it. + + The values 3 / 5 / 100 are **arbitrary, ordering-only** numbers with no + spec basis: OVOS-INTENT-2 §2.2 names ``langcodes``' ``tag_distance`` and is + silent on the no-``langcodes`` case, so this fallback exists purely to + preserve the same nearest-to-farthest *ordering* that locale resolution + depends on. Only the relative order (and being below/above the threshold of + 10) is meaningful; the magnitudes are not. + """ + if desired.split("-")[0].lower() != supported.split("-")[0].lower(): + return 100 # a different language — beyond any usable threshold + if "-" not in desired or "-" not in supported: + return 3 # one side is the generic language tag + return 5 # two different regions of the same language + + +def lang_distance(desired: str, supported: str) -> int: + """The distance between two BCP-47 language tags. + + The numeric backbone of the OVOS-INTENT-2 §2.2 fallback: §2.2 names + ``langcodes``' ``tag_distance()`` and its "below 10 is a usable regional + match" reading, which this function adopts (with the norm-region correction + below) so every OVOS component ranks dialects identically. + + ``0`` is identical; a larger number is further apart; a value of 10 or + more is not a usable match. Both tags are standardized, and a bare tag is + measured **from its norm region** — so ``lang_distance("pt", "pt-PT")`` is + ``0`` while ``lang_distance("pt", "pt-BR")`` is a regional difference. The + norm-region step is non-normative implementation policy that diverges from + ``langcodes``' population-based default on purpose (see :data:`_NORM_REGION`), + not a §2.2 requirement. + + Args: + desired: the requested BCP-47 tag. + supported: the candidate BCP-47 tag to measure against. + + Returns: + A non-negative distance: ``0`` for identical (after standardization and + norm-region expansion), a small value for a regional difference, and a + large value (``>= 100`` under the coarse measure) for a different + primary language. + + Uses ``langcodes.tag_distance`` when available, and a coarse same-language + measure otherwise. + """ + a = standardize_lang(desired) + b = standardize_lang(supported) + if a.lower() == b.lower(): + return 0 + a, b = _with_norm_region(a), _with_norm_region(b) + if a.lower() == b.lower(): + return 0 + distance = _langcodes_distance(a, b) + return distance if distance is not None else _coarse_distance(a, b) + + +def lang_matches(a: str, b: str, + max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE) -> bool: + """Return ``True`` when two BCP-47 tags are close enough to interchange. + + Convenience wrapper around :func:`lang_distance` for the common + ``if score < threshold`` check that cross-component boundaries (intent + engines, TTS/STT plugin routing, locale lookup) reimplement by hand. The + default threshold matches :data:`DEFAULT_MAX_LANGUAGE_DISTANCE` — the + OVOS-INTENT-2 §2.2 "distance below 10 is a usable regional match" line. + + Args: + a: one BCP-47 tag. + b: the other BCP-47 tag. The relation is symmetric. + max_distance: the exclusive upper bound on an acceptable distance. An + exact match (distance ``0``) always matches regardless; pass + ``max_distance=0`` to require an exact match (OVOS-INTENT-2 §2.2's + "SHOULD prefer an exact match", with fallback disabled). + + Returns: + ``True`` iff the tags are identical or within ``max_distance``. + """ + distance = lang_distance(a, b) + if distance == 0: + return True + return max_distance > 0 and distance < max_distance + + +def closest_lang(target: str, available: Sequence[str], + max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE + ) -> Optional[str]: + """Return the entry of ``available`` closest to ``target``. + + This is the reference implementation of the OVOS-INTENT-2 §2.2 language + fallback ("a loader MAY fall back to the nearest available language"). §2.2 + is non-normative, so this selection is the package's recommended policy, not + a conformance requirement; a loader is free to disable it (``max_distance=0``, + yielding §2.2's "SHOULD prefer an exact match" with no fallback). + + The candidate with the smallest :func:`lang_distance` wins. An exact match + always resolves; any other match resolves only if its distance is **below** + ``max_distance`` (so ``max_distance=0`` accepts exact matches only). + ``None`` is returned when nothing qualifies. + + Args: + target: the requested BCP-47 tag (e.g. a session/skill language). + available: the tags actually on hand — locale directory names, voice + ids, model languages. Iterated once; ties resolve to the first + smallest-distance candidate in iteration order. + max_distance: exclusive upper bound on an acceptable non-exact match + (default :data:`DEFAULT_MAX_LANGUAGE_DISTANCE` == §2.2's ``10``). + + Returns: + The original string from ``available`` that best matches ``target``, or + ``None`` when nothing qualifies. The value is returned **verbatim** (not + standardized) so a caller can map it straight back to a directory, a + voice, a model, and so on. + """ + best: Optional[str] = None + best_distance: Optional[int] = None + for candidate in available: + distance = lang_distance(target, candidate) + if best_distance is None or distance < best_distance: + best, best_distance = candidate, distance + + if best is None: + return None + if best_distance == 0: + return best + if max_distance > 0 and best_distance < max_distance: + return best + return None diff --git a/ovos_spec_tools/lint.py b/ovos_spec_tools/lint.py new file mode 100644 index 00000000..9313d957 --- /dev/null +++ b/ovos_spec_tools/lint.py @@ -0,0 +1,518 @@ +"""Locale resource linter for the OVOS specifications. + +Validates the **syntax** (OVOS-INTENT-1) and the **naming and layout** +(OVOS-INTENT-2) of every resource file under a locale directory, and reports +every problem found rather than stopping at the first. Each finding is phrased +to point at the exact spec clause it enforces, so a failing lint tells the +author which MUST they violated. + +Clause map (which spec rule each rule enforces): + +- *empty file* → OVOS-INTENT-2 §5 step 5 — "Reject an empty file … every file + MUST contribute at least one template" (and for ``.prompt``, content). +- *duplicate (role, base name)* → OVOS-INTENT-2 §2 — "Two files with the same + extension MUST NOT share a base name anywhere within one language directory + tree". +- *base-name / ``.entity`` charset* → OVOS-INTENT-2 §2 + OVOS-INTENT-1 §3.4 — + base names are lowercase letters/digits/underscores; an ``.entity`` name also + obeys the slot-name rule (not beginning with a digit). +- *not-a-BCP-47 directory* → OVOS-INTENT-2 §2 — "Language directories are named + with BCP-47 language tags". +- *legacy extension / unknown role* → OVOS-INTENT-2 §1 + §5 — only the six + defined roles exist; a loader "MUST NOT introduce additional resource file + roles". Legacy Mycroft types are flagged, not parsed. +- *named slot in a slot-free role* → OVOS-INTENT-2 §4.3 — ``.entity`` / ``.voc`` + / ``.blacklist`` are slot-free. +- *template syntax* → OVOS-INTENT-1 §3.6 (delegated to + :func:`~ovos_spec_tools.expansion.expand`). +- *slot-set consistency* → OVOS-INTENT-2 §4.2 (``.dialog`` only): a ``.dialog`` + whose phrases declare different slot sets is an ERROR (the caller fills the + slots of the rendered phrase, so all phrases must expose the same slots). + ``.intent`` templates MAY declare different slot sets — their union is the + intent's slot set (OVOS-INTENT-2 §4.1, OVOS-INTENT-3 §5.1) — and are NOT + flagged. +- *blacklist with no matching ``.intent``* → OVOS-INTENT-2 §4.3 — a + ``.blacklist`` "is paired by base name with exactly one ``.intent``". +- *required slot declared by no template* → OVOS-INTENT-3 §5.3 — "A required + slot MUST be declared by at least one template in the intent … a tool MUST + reject the definition at registration time." This is an intent-**definition** + rule, not a single-file rule: ``required_slots`` lives above the raw + ``.intent`` file. The locale linter (which sees only files) cannot enforce + it; the check is exposed as :func:`validate_required_slots` (raises) and + :func:`lint_required_slots` (returns a :class:`Finding`) for the + registration/loading path that has both the ``required_slots`` list and the + templates in hand. + +Exposed as the ``ovos-spec-lint`` command:: + + ovos-spec-lint path/to/locale + +The argument may be a ``locale/`` directory (every language subdirectory is +checked) or a single ``/`` directory. + +The ``--spec-version`` option additionally flags features newer than a target +OVOS spec version, for skills that must run on older deployments: + +- **0** — the legacy, undocumented Mycroft/OVOS de-facto behaviour; +- **1** — the formalized specs; adds the ``.blacklist`` role; +- **2** — adds ```` inline vocabulary references; +- **3** — adds the ``.prompt`` role. +""" +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Sequence + +from ovos_spec_tools.expansion import ( + MalformedTemplate, + expand, + fold_double_braces, +) +from ovos_spec_tools.resources import ( + PROMPT_ROLE, + SLOT_BEARING_ROLES, + SLOT_FREE_ROLES, + read_prompt_file, + read_resource_file, +) + +__all__ = [ + "Finding", + "lint_locale", + "declared_slots", + "validate_required_slots", + "lint_required_slots", + "main", +] + +# The six OVOS-INTENT-2 resource roles. +ROLE_EXTENSIONS = SLOT_BEARING_ROLES + SLOT_FREE_ROLES + (PROMPT_ROLE,) +# File types OVOS-INTENT-2 deliberately does not define — flagged, not parsed. +LEGACY_EXTENSIONS = (".rx", ".value", ".list", ".word", ".template", ".qml") + +# The OVOS spec version that introduced each feature. This ladder is a *tooling* +# concept, not a clause in any single spec: it lets a skill target an older +# deployment by flagging roles/tokens that a runtime predating their +# introduction will silently ignore (a forward-compatibility lint, not a +# conformance check). Mapping: V1 = the formalized OVOS-INTENT-1/2 (adds the +# `.blacklist` role over the legacy V0 set), V2 = the `` inline vocabulary +# reference (OVOS-INTENT-1 §3.7), V3 = the `.prompt` role (OVOS-INTENT-2 §4.4). +DEFAULT_SPEC_VERSION = 3 +_BLACKLIST_SINCE = 1 # the `.blacklist` role +_VOCABULARY_REFERENCE_SINCE = 2 # the `` inline vocabulary reference +_PROMPT_ROLE_SINCE = 3 # the `.prompt` role +# Roles introduced after V0, by the spec version that added them. +_ROLE_SINCE = {".blacklist": _BLACKLIST_SINCE, PROMPT_ROLE: _PROMPT_ROLE_SINCE} + +_BASE_NAME_RE = re.compile(r"[a-z0-9_]+") +_SLOT_NAME_RE = re.compile(r"[a-z][a-z0-9_]*") +_LANG_TAG_RE = re.compile(r"[a-z]{2,3}(-[A-Za-z0-9]+)*") +_SLOT_RE = re.compile(r"\{([a-z][a-z0-9_]*)\}") + +ERROR = "error" +WARNING = "warning" + + +@dataclass +class Finding: + """One problem found by the linter. + + Severity is :data:`ERROR` for a spec **MUST** violation (a malformed + template, a duplicate ``(role, base name)``, an empty file, a slot in a + slot-free role) and :data:`WARNING` for a **SHOULD**/advisory issue (a + legacy file type, a non-BCP-47 directory name, an unpaired ``.blacklist``). + Only errors fail a non-``--strict`` run. + + Attributes: + severity: :data:`ERROR` or :data:`WARNING`. + path: the offending file or directory, as a string. + message: a human-readable description, citing the spec clause violated. + """ + + severity: str # ERROR or WARNING + path: str + message: str + + def __str__(self) -> str: + return f"{self.path}: {self.severity}: {self.message}" + + +def declared_slots(templates: Sequence[str]) -> frozenset: + """The union of the named slots declared across ``templates``. + + An intent's slot set is the **union** of the slots declared by its + templates (OVOS-INTENT-2 §4.1, OVOS-INTENT-3 §5.1): the engine extracts + only the slots of whichever template matched, so a slot the intent can + ever fill is one declared by *some* template. Both equivalent spellings + ``{name}`` and ``{{name}}`` (OVOS-INTENT-1 §3.4) count, so each template is + folded to the single-brace canonical form before its slots are read. + + Args: + templates: the template lines of one ``.intent`` definition. + + Returns: + The frozen union of every named slot any template declares. + """ + slots: set = set() + for template in templates: + slots.update(_SLOT_RE.findall(fold_double_braces(template))) + return frozenset(slots) + + +def validate_required_slots( + required_slots: Sequence[str], + templates: Sequence[str]) -> None: + """Verify every required slot is declared by at least one template (§5.3). + + OVOS-INTENT-3 §5.3 makes this a **MUST** at registration time: "A required + slot MUST be declared by at least one template in the intent. Declaring a + required slot that no template mentions is malformed: the intent can never + match, and a tool MUST reject the definition at registration time." + + ``required_slots`` is an intent-**definition** field — it lives above the + raw ``.intent`` file, alongside the templates. This validator is the place + that sees both together; call it when registering or loading an intent + definition so a malformed one (a required slot no template can fill) is + rejected before it can silently never fire. + + Args: + required_slots: the slot names the intent declares as required. + templates: the intent's template lines (its ``.intent`` samples). + + Raises: + MalformedTemplate: a required slot is declared by no template, so the + intent can never match (OVOS-INTENT-3 §5.3). + """ + available = declared_slots(templates) + missing = [name for name in required_slots if name not in available] + if missing: + have = "{" + ", ".join(sorted(available)) + "}" if available else "{}" + raise MalformedTemplate( + f"required slot(s) {{{', '.join(missing)}}} declared by no " + f"template — the intent's templates declare only {have}, so the " + "intent can never match. A required slot MUST be declared by at " + "least one template (OVOS-INTENT-3 §5.3)") + + +def lint_required_slots( + path: str, + required_slots: Sequence[str], + templates: Sequence[str]) -> List[Finding]: + """Lint an intent's ``required_slots`` against its templates (§5.3). + + A :class:`Finding`-returning wrapper over :func:`validate_required_slots`, + for callers that lint intent definitions (where ``required_slots`` metadata + is available alongside the templates) and want findings rather than an + exception. Each undeclared required slot is an :data:`ERROR` (a §5.3 MUST). + + Args: + path: the offending intent's identifier (file path or name), reported + on the finding. + required_slots: the slot names the intent declares as required. + templates: the intent's template lines. + + Returns: + One :data:`ERROR` finding if any required slot is undeclared, else an + empty list. + """ + try: + validate_required_slots(required_slots, templates) + except MalformedTemplate as exc: + return [Finding(ERROR, path, str(exc))] + return [] + + +def lint_locale(path, spec_version: int = DEFAULT_SPEC_VERSION) -> List[Finding]: + """Lint a locale directory, or a single language directory. + + The argument is dispatched by its name: a directory whose name matches the + BCP-47 shape (OVOS-INTENT-2 §2) is treated as a single ``/`` tree; + anything else is treated as the ``locale/`` root and each immediate + subdirectory is linted as a language tree. This lets the linter accept + either ``locale`` or ``locale/en-US`` as its target. + + Args: + path: a ``locale/`` directory or a single ``/`` directory. + spec_version: the target OVOS spec version; a resource using a feature + newer than it is flagged (see the module docstring's version ladder). + + Returns: + Every :class:`Finding`, in file order — never short-circuiting, so one + run surfaces all problems. A non-directory ``path`` yields a single + :data:`ERROR` finding. + """ + root = Path(path) + if not root.is_dir(): + return [Finding(ERROR, str(root), "not a directory")] + + if _LANG_TAG_RE.fullmatch(root.name): + return _lint_language_tree(root, spec_version) + + findings: List[Finding] = [] + for child in sorted(root.iterdir()): + if child.is_file() and child.suffix in ROLE_EXTENSIONS: + # §2: resources live under locale//, never loose at the root. + findings.append(Finding( + WARNING, str(child), + "resource file is not inside a language directory " + "(OVOS-INTENT-2 §2)")) + language_dirs = [c for c in sorted(root.iterdir()) if c.is_dir()] + if not language_dirs: + findings.append(Finding( + WARNING, str(root), + "no language directories found (OVOS-INTENT-2 §2)")) + for language_dir in language_dirs: + findings.extend(_lint_language_tree(language_dir, spec_version)) + return findings + + +def _lint_language_tree(language_dir: Path, spec_version: int) -> List[Finding]: + """Lint one ``/`` directory and all its subdirectories.""" + findings: List[Finding] = [] + if not _LANG_TAG_RE.fullmatch(language_dir.name): + # §2: "Language directories are named with BCP-47 language tags." + findings.append(Finding( + WARNING, str(language_dir), + f"directory name {language_dir.name!r} is not a BCP-47 " + f"language tag (OVOS-INTENT-2 §2)")) + + role_files: List[Path] = [] + for path in sorted(language_dir.rglob("*")): + if not path.is_file(): + continue + if path.suffix in ROLE_EXTENSIONS: + role_files.append(path) + elif path.suffix in LEGACY_EXTENSIONS: + # §1 defines exactly six roles; §5 forbids a loader inventing more. + # Legacy Mycroft types are flagged (not parsed) so an author knows + # the file will be silently ignored by a conformant loader. + findings.append(Finding( + WARNING, str(path), + f"{path.suffix} is a legacy file type, not an " + f"OVOS-INTENT-2 resource role (OVOS-INTENT-2 §1)")) + + if not role_files: + findings.append(Finding( + WARNING, str(language_dir), + "language directory contains no resource files " + "(OVOS-INTENT-2 §2)")) + + # Duplicate (role, base name) within one language tree (OVOS-INTENT-2 §2). + first_seen: Dict[tuple, Path] = {} + for path in role_files: + key = (path.suffix, path.stem) + if key in first_seen: + findings.append(Finding( + ERROR, str(path), + f"duplicate {path.suffix} resource {path.stem!r} — also at " + f"{first_seen[key]} (OVOS-INTENT-2 §2: a (role, base name) " + f"must be unique per language tree)")) + else: + first_seen[key] = path + + # Per-role spec-version gate (a role newer than the target is flagged), + # plus the `.blacklist` pairing check (§4.3). + intent_names = {stem for (ext, stem) in first_seen if ext == ".intent"} + for path in role_files: + since = _ROLE_SINCE.get(path.suffix) + if since is not None and spec_version < since: + findings.append(Finding( + WARNING, str(path), + f"the {path.suffix} role requires spec version {since}; a " + f"version-{spec_version} runtime ignores it")) + if path.suffix == ".blacklist" and path.stem not in intent_names: + # §4.3: a .blacklist "is paired by base name with exactly one + # .intent" whose match it suppresses; an unpaired one is inert. + findings.append(Finding( + WARNING, str(path), + f"blacklist {path.stem!r} has no matching " + f"{path.stem}.intent to suppress (OVOS-INTENT-2 §4.3)")) + + # Vocabularies, for resolving references during expansion. + vocabularies: Dict[str, List[str]] = {} + for path in role_files: + if path.suffix == ".voc": + try: + vocabularies[path.stem] = read_resource_file(path) + except (OSError, UnicodeError): + pass # _lint_file reports the read error below + + for path in role_files: + findings.extend(_lint_file(path, vocabularies, spec_version)) + return findings + + +def _lint_file(path: Path, + vocabularies: Dict[str, Sequence[str]], + spec_version: int) -> List[Finding]: + """Lint one resource file: naming, then the syntax of every template. + + Order mirrors the loader steps of OVOS-INTENT-2 §5: naming (§2), then the + common reader (§3), then the per-format rule (§4) — for templated roles via + a conformant expander (OVOS-INTENT-1), for ``.prompt`` as a whole-file + non-empty check (§4.4). + + Args: + path: the resource file to lint. + vocabularies: every ``.voc`` in this language tree, so a ```` + reference (OVOS-INTENT-1 §3.7) resolves during expansion instead of + being mis-reported as undefined. + spec_version: the target spec version, for the feature gates. + + Returns: + Every :class:`Finding` for this file. + """ + findings: List[Finding] = [] + extension = path.suffix + base_name = path.stem + + # --- naming (OVOS-INTENT-2 §2) ------------------------------------------ + if not _BASE_NAME_RE.fullmatch(base_name): + findings.append(Finding( + ERROR, str(path), + f"base name {base_name!r} must be lowercase ASCII letters, " + f"digits and underscores only (OVOS-INTENT-2 §2)")) + if extension == ".entity" and not _SLOT_NAME_RE.fullmatch(base_name): + # §2: where a base name names a slot (an .entity names its {slot}), it + # additionally obeys the slot-name rule of OVOS-INTENT-1 §3.4. + findings.append(Finding( + ERROR, str(path), + f".entity base name {base_name!r} names a slot and must not " + f"begin with a digit (OVOS-INTENT-2 §2 / OVOS-INTENT-1 §3.4)")) + if path.name != path.name.lower(): + # §2: base names and extensions are lowercase. Reported as a warning + # (not an error) because the casefold is recoverable on case-insensitive + # filesystems; on case-sensitive ones it will fail lookup. + findings.append(Finding( + WARNING, str(path), + "file name should be lowercase (OVOS-INTENT-2 §2)")) + + # --- `.prompt` — a whole-file document, not a template list (§4.4) ------ + if extension == PROMPT_ROLE: + try: + text = read_prompt_file(path) + except (OSError, UnicodeError) as exc: + findings.append(Finding( + ERROR, str(path), f"cannot read file: {exc}")) + else: + if not text.strip(): + findings.append(Finding( + ERROR, str(path), + "empty file — every resource file must contribute " + "content (OVOS-INTENT-2 §5)")) + return findings + + # --- read (OVOS-INTENT-2 §3) -------------------------------------------- + try: + templates = read_resource_file(path) + except (OSError, UnicodeError) as exc: + findings.append(Finding(ERROR, str(path), f"cannot read file: {exc}")) + return findings + if not templates: + findings.append(Finding( + ERROR, str(path), + "empty file — every resource file must contribute at least one " + "template (OVOS-INTENT-2 §5)")) + return findings + + # --- syntax (OVOS-INTENT-1) --------------------------------------------- + slot_free = extension in SLOT_FREE_ROLES + slot_bearing = extension in SLOT_BEARING_ROLES + slot_sets: List[frozenset] = [] + for template in templates: + if spec_version < _VOCABULARY_REFERENCE_SINCE and "<" in template: + findings.append(Finding( + ERROR, str(path), + f"an inline vocabulary reference <…> requires spec version " + f"{_VOCABULARY_REFERENCE_SINCE}; a version-{spec_version} " + f"runtime will not expand this template [in: {template!r}]")) + try: + samples = expand(template, vocabularies) + except MalformedTemplate as exc: + findings.append(Finding( + ERROR, str(path), f"{exc} [in: {template!r}]")) + continue + if slot_free and any("{" in sample for sample in samples): + # §4.3: .entity/.voc/.blacklist are the slot-free format — no {name}. + findings.append(Finding( + ERROR, str(path), + f"{extension} is slot-free but a template contains a named " + f"slot (OVOS-INTENT-2 §4.3) [in: {template!r}]")) + if slot_bearing: + # Fold ``{{name}}`` to ``{name}`` first (OVOS-INTENT-1 §3.4) so the + # two equivalent slot spellings yield the identical slot set and a + # template mixing them is not mis-flagged as slot-inconsistent. + slot_sets.append( + frozenset(_SLOT_RE.findall(fold_double_braces(template)))) + + # --- slot consistency: `.dialog` ONLY ----------------------------------- + # `.dialog` phrases MUST all declare the same slot set: the caller fills the + # slots of whichever phrase is rendered, so every phrase must expose the + # identical slots (OVOS-INTENT-2 §4.2; OVOS-INTENT-1 §5.5). A `.dialog` + # whose phrases diverge is malformed — ERROR. + # + # `.intent` is the deliberate OPPOSITE: its templates MAY declare different + # slot sets, and the intent's slot set is their union — the engine extracts + # only the slots of the template that matched (OVOS-INTENT-2 §4.1, + # OVOS-INTENT-3 §5.1). A tool MUST NOT reject a `.intent` for divergent + # slots, so divergence is NOT flagged for the `.intent` role. + if extension == ".dialog" and len(set(slot_sets)) > 1: + findings.append(Finding( + ERROR, str(path), + "a .dialog's phrases declare different slot sets — every phrase in " + "one .dialog MUST declare the same {slots} (OVOS-INTENT-2 §4.2; " + "OVOS-INTENT-1 §5.5)")) + + return findings + + +def main(argv: Optional[Sequence[str]] = None) -> int: + """Entry point for the ``ovos-spec-lint`` command. + + Lints the target, prints every :class:`Finding`, then a count summary. + + Args: + argv: command-line arguments (defaults to ``sys.argv[1:]``). + + Returns: + ``0`` on success; ``1`` if any error was found, or — with ``--strict`` + — if any warning was found (so a CI step can gate on locale health). + """ + parser = argparse.ArgumentParser( + prog="ovos-spec-lint", + description="Validate OVOS locale resource files against " + "OVOS-INTENT-1 (syntax) and OVOS-INTENT-2 (naming).") + parser.add_argument( + "locale", help="path to a locale/ directory, or a / directory") + parser.add_argument( + "--strict", action="store_true", + help="exit non-zero if there are warnings as well as errors") + parser.add_argument( + "--spec-version", type=int, choices=(0, 1, 2, 3), + default=DEFAULT_SPEC_VERSION, + help="target OVOS spec version; flags features newer than it " + "(0: legacy, 1: adds .blacklist, 2: adds references, " + f"3: adds the .prompt role). Default {DEFAULT_SPEC_VERSION}.") + args = parser.parse_args(argv) + + findings = lint_locale(args.locale, spec_version=args.spec_version) + errors = [f for f in findings if f.severity == ERROR] + warnings = [f for f in findings if f.severity == WARNING] + + for finding in findings: + print(finding) + if findings: + print() + print(f"{len(errors)} error(s), {len(warnings)} warning(s)") + + if errors or (args.strict and warnings): + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/ovos_spec_tools/message.py b/ovos_spec_tools/message.py new file mode 100644 index 00000000..ef7c5161 --- /dev/null +++ b/ovos_spec_tools/message.py @@ -0,0 +1,528 @@ +"""Reference implementation of OVOS-MSG-1 — the bus :class:`Message` envelope. + +Spec implemented +---------------- +**OVOS-MSG-1** (*Bus Message Specification*, Version 1, Draft). This module +is the reference implementation of every normative clause the spec assigns +to the Message *value* (as opposed to the transport): the JSON envelope of +§2, the routing keys of §3 (carried but never interpreted), the session +carrier of §4 (carried as an opaque dict), the three derivations of §5 +(:meth:`Message.forward`, :meth:`Message.reply`, :meth:`Message.response`), +and the serialization/conformance rules of §6/§7. + +Conformance surface +------------------- +This class is simultaneously a conformant **producer** and **consumer** in +the §7 sense: + +- *producer* — :meth:`serialize` emits a single UTF-8 JSON object with only + ``type`` / ``data`` / ``context`` (§7 producer MUSTs, §6 serialization); + :meth:`forward` / :meth:`reply` / :meth:`response` follow §5 exactly. +- *consumer* — :meth:`deserialize` rejects every payload the §7 consumer + rules call malformed, and treats absent ``data`` / ``context`` as ``{}`` + (§2). ``source`` / ``destination`` / ``session`` are treated as opaque + and optional (§3.4, §4, §7 consumer MUSTs). + +Non-goals (§7 / §1) +------------------- +This module deliberately does **not** own: transport (websocket, queue, …), +encryption, authentication, authorization, delivery/ordering guarantees, +retry, session lifecycle (start/end/expiry/resumption), the *internal* shape +of ``session`` (owned by OVOS-SESSION-1), identifier-assignment policy, and +multi-tenant routing semantics beyond the opaque layer-2 substrate of §3.4 / +§4.2. Those concerns live in the layers that consume this primitive — +``ovos-bus-client`` for the websocket transport, HiveMind for multi-tenant +routing, ``ovos-audio`` / ``ovos-core`` for the per-topic policy decisions +keyed on ``session_id`` (§4) and ``lang``. + +The implementation has **no dependencies** outside the standard library; +it is the foundation other bus-layer code (notably ``ovos-bus-client``'s +transport-layer ``Message`` subclass) builds on. +""" +from __future__ import annotations + +import json +import re +from copy import deepcopy +from typing import Any, Dict, List, Optional, Union + +__all__ = ["Message", "MalformedMessage", "DEFAULT_SESSION_ID"] + +#: The OVOS-MSG-1 §2.1 ``type`` syntax: a non-empty run of ASCII letters, +#: digits, and the four punctuation characters the spec permits — ``.`` ``:`` +#: ``_`` ``-``. §2.1 forbids whitespace and any other character outright +#: ("ASCII letters, digits, ``.``, ``:``, ``_``, ``-``; no whitespace"), so +#: anything outside this set — an embedded space, a slash, a non-ASCII letter — +#: is a malformed topic the producer MUST NOT emit. Lowercase is only +#: RECOMMENDED (§2.1), so uppercase is accepted. +_TYPE_SYNTAX_RE = re.compile(r"[A-Za-z0-9.:_-]+") + +#: A routing key value as it may appear on the wire (OVOS-MSG-1 §3): a single +#: opaque identifier string, or — for ``destination`` only — an array of them +#: (§3.3). ``None`` models the absent/broadcast case (§3.3). The envelope +#: never parses these beyond string equality (§3.4). +RoutingValue = Union[str, List[str], None] + + +#: The reserved ``session_id`` meaning "the Message originates from the +#: device itself" (OVOS-MSG-1 §4.1). Used by ``ovos-audio`` to decide +#: that synthesized TTS plays out of the device's own speakers, and as +#: the implicit value for any Message that arrives without a ``session`` +#: in its context (§4.3). +DEFAULT_SESSION_ID = "default" + + +def _stamp_live_session(message: "Message") -> "Message": + """Refresh a derived message's ``session`` to the live value for its id. + + ``forward`` / ``reply`` deep-copy the originating message's ``session`` + snapshot (§5.1/§5.2). That snapshot may predate the current handler's + mutations, so this re-stamps the derived message with the live session for + its id via :meth:`SessionManager.sync_message_session` — see that class for + why this is always either a meaningful refresh or a no-op, never a discard. + + Best-effort and non-invasive: a message with **no** session is left as-is + (§5 carries no session forward), and a session id the registry never folded + is left untouched — so this is transparent to pure spec usage that never + wires a registry (the session is carried over verbatim, as §5 mandates). + """ + try: + ctx = getattr(message, "context", None) or {} + if not ctx.get("session"): + return message + from ovos_spec_tools.session import SessionManager + return SessionManager.sync_message_session(message) + except Exception: # never let session bookkeeping break message derivation + return message + + +def _freeze(value: Any) -> Any: + """Recursively convert ``value`` into a deterministic, hashable form. + + Used to derive ``__hash__`` for :class:`Message` and + :class:`~ovos_spec_tools.session.Session` from the same nested + dict/list payloads their ``__eq__`` compares. The mapping preserves + Python's value-equality invariant: anything that compares equal + freezes to an equal (and therefore equally-hashing) form. In + particular ``{"x": 1}`` and ``{"x": 1.0}`` freeze equal because the + underlying ``int``/``float`` are themselves equal and hash equal — + unlike a ``json.dumps`` digest, which would diverge on ``"1"`` vs + ``"1.0"``. + + - ``dict`` → ``frozenset`` of ``(key, _freeze(val))`` pairs + (order-independent, mirroring dict equality); + - ``list`` / ``tuple`` → ``tuple`` of frozen items (order-preserving); + - ``set`` / ``frozenset`` → ``frozenset`` of frozen items; + - already-hashable scalars (``str``, ``int``, ``float``, ``bool``, + ``None``, …) → returned unchanged; + - any remaining exotic unhashable → its ``repr`` (deterministic + last resort). + """ + if isinstance(value, dict): + return frozenset((k, _freeze(v)) for k, v in value.items()) + if isinstance(value, (list, tuple)): + return tuple(_freeze(v) for v in value) + if isinstance(value, (set, frozenset)): + return frozenset(_freeze(v) for v in value) + try: + hash(value) + except TypeError: + return repr(value) + return value + + +class MalformedMessage(ValueError, AssertionError): + """A serialized payload that does not conform to OVOS-MSG-1 §2 / §6. + + Raised by :meth:`Message.deserialize` when the payload fails any + structural rule the spec calls out as ``MUST``: unknown top-level + keys (§2), missing ``type`` (§2), wrong value types, or unparsable + JSON (§6). + + Inherits from both :class:`ValueError` (the correctly-typed + exception class) **and** :class:`AssertionError` (the type raised by + ``ovos_bus_client.Message``'s bare ``assert`` constructor checks), so + ``except AssertionError`` handlers in downstream code also catch the + same conditions. + """ + + +class Message: + """OVOS-MSG-1 envelope. + + Three top-level fields per §2: + + - :attr:`msg_type` (the wire field ``type``) — non-empty topic string + matching the §2.1 syntax (ASCII letters/digits/``.``/``:``/``_``/``-``, + no whitespace); + - :attr:`data` — JSON-object payload; shape fixed by whichever + specification defines :attr:`msg_type`; + - :attr:`context` — JSON-object metadata, carrying the §3 routing + keys (``source`` / ``destination``), the §4 session carrier, and + any layer-2 metadata higher systems attach. + + The :attr:`data` and :attr:`context` dicts are stored by reference; + callers that mutate them after constructing a Message see those + mutations reflected in the Message. :meth:`forward` / :meth:`reply` + / :meth:`response` deep-copy the context before deriving so the + derived Message is independent of the source. + """ + + def __init__(self, msg_type: str, + data: Optional[Dict[str, Any]] = None, + context: Optional[Dict[str, Any]] = None) -> None: + """Construct a Message envelope (OVOS-MSG-1 §2). + + Args: + msg_type: the wire ``type`` field — the topic string (§2.1). + Stored as :attr:`msg_type`. May be empty at construction + time (see the inline note); :meth:`serialize` is the §7 + conformance gate for emitted output. + data: the §2.2 payload object, or ``None`` for the empty + default (``{}``). Stored **by reference**. + context: the §2.3 metadata object (routing keys §3, session + carrier §4, layer-2 metadata), or ``None`` for ``{}``. + Stored **by reference**. + + Raises: + MalformedMessage: if ``msg_type`` is not a ``str`` (§2.1), or + ``data`` / ``context`` are given but not ``dict`` (§2.2 / + §2.3). These mirror the §7 consumer "wrong types" rule at + the constructor boundary. + """ + # §2.1 requires the wire ``type`` to be non-empty, but it is a + # spec rule for **emitted** Messages — the construct-then-forward + # pattern (``Message("").forward(real_type, data)``) is widely + # used to build a routing scaffold before the real topic is + # known, so the constructor accepts an empty string here and + # :meth:`serialize` is the gate that *rejects* non-conformant wire + # output (an empty ``type`` raises there, per §7 producer rules). + if not isinstance(msg_type, str): + raise MalformedMessage("msg_type must be a string (§2.1)") + if data is not None and not isinstance(data, dict): + raise MalformedMessage("data must be a dict (§2.2)") + if context is not None and not isinstance(context, dict): + raise MalformedMessage("context must be a dict (§2.3)") + self.msg_type: str = msg_type + # §2.2 / §2.3: absent data/context are equivalent to ``{}``. Stored + # by reference per the class docstring (derivations deep-copy). + self.data: Dict[str, Any] = data if data is not None else {} + self.context: Dict[str, Any] = context if context is not None else {} + + def __eq__(self, other: object) -> bool: + # Value equality over the full §2 envelope (type + data + context). + # Two Messages are equal iff every wire field is equal; routing + # keys and session live inside ``context`` so they participate. + return (isinstance(other, Message) + and other.msg_type == self.msg_type + and other.data == self.data + and other.context == self.context) + + def __hash__(self) -> int: + # Hash over the same §2 fields ``__eq__`` compares (type + data + + # context), frozen into a deterministic hashable form so equal + # Messages always hash equal (the hash/eq contract). NOTE: ``data`` + # and ``context`` are mutable dicts stored by reference, so this is + # a *point-in-time snapshot* — safe for ``functools.lru_cache`` keys + # and short-lived set/dict membership, but do not mutate a Message + # while it is live as a dict key. + return hash((self.msg_type, + _freeze(self.data), + _freeze(self.context))) + + def __repr__(self) -> str: + # Eval-friendly debug form; not the wire format (use serialize()). + return (f"{self.__class__.__name__}(" + f"msg_type={self.msg_type!r}, " + f"data={self.data!r}, context={self.context!r})") + + # --- §6 serialization --------------------------------------------------- + + @property + def as_dict(self) -> Dict[str, Any]: + """The Message as a JSON-decoded ``dict`` (OVOS-MSG-1 §2 envelope). + + Returns ``{"type": ..., "data": ..., "context": ...}`` — the §2 + envelope keyed by the **wire** field names (``type``, not + ``msg_type``). Equivalent to ``json.loads(self.serialize())`` and + round-trips through :meth:`deserialize`; offered as a property for + callers that want a one-shot dict view without the JSON-string + intermediate. Carriers exposing ``.serialize()`` are converted the + same way they would be on the wire (see :meth:`_to_jsonable`). + + Returns: + The §2 envelope as a plain JSON-decoded dictionary. + """ + return json.loads(self.serialize()) + + @staticmethod + def _to_jsonable(value: Any) -> Any: + """Convert ``value`` to a JSON-friendly form for §6 serialization. + + Recursively walks containers and converts any object exposing a + ``.serialize()`` method (the duck-typed protocol used by OVOS + carrier objects like the OVOS-SESSION-1 ``Session``) by calling it. + Plain JSON types pass through unchanged. + + This keeps :class:`Message` an honest pure-envelope class — it + doesn't know about ``Session`` or any other carrier type, honouring + OVOS-MSG-1's separation of the envelope from the §4 session shape + (owned by OVOS-SESSION-1) — while letting callers stuff such objects + directly into ``data`` / ``context`` and serialize the result. + + Args: + value: any value found inside ``data`` / ``context``. + + Returns: + A JSON-encodable structure: the result of ``value.serialize()`` + for carrier objects, a recursively-converted ``dict`` / ``list`` + for containers, or ``value`` itself for plain JSON scalars. + """ + # Direct .serialize() — Session, nested Message, etc. + ser = getattr(value, "serialize", None) + if callable(ser) and not isinstance(value, (dict, list, tuple)): + try: + value = ser() + except Exception: + pass + if isinstance(value, dict): + return {k: Message._to_jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [Message._to_jsonable(v) for v in value] + return value + + def serialize(self) -> str: + """Render the Message as a single UTF-8 JSON object (OVOS-MSG-1 §6). + + The output is exactly one top-level JSON object — never an array or + a stream (§6) — with the §2 wire keys ``type`` / ``data`` / + ``context``. ``NaN`` / ``±Infinity`` are forbidden, so + ``allow_nan=False`` makes a non-finite number raise rather than emit + invalid JSON (§6: "Numbers MUST be finite"). ``ensure_ascii=False`` + emits valid UTF-8 directly (§6: Unicode strings). Object key order + is **not** significant (§6), so no ordering is imposed. Nested + objects in ``data`` / ``context`` exposing ``.serialize()`` (e.g. + the OVOS-SESSION-1 ``Session`` carrier) are converted first via + :meth:`_to_jsonable`. + + Subclasses MAY override to add the transport-layer concerns §6/§7 + leave out — framing, encryption, alternative encoders — without + touching this envelope contract. + + This method is the §7 producer conformance gate: §2.1 requires + ``type`` to be a **non-empty** string and §7 makes a non-empty + ``type`` a producer ``MUST``. The constructor tolerates an empty + ``msg_type`` to allow the construct-then-``forward`` scaffold + pattern, but emitting such a Message would put a malformed + envelope on the wire, so ``serialize`` **refuses** it. + + Returns: + A single UTF-8 JSON object string conforming to §6. + + Raises: + MalformedMessage: when ``msg_type`` is empty — §2.1/§7 require a + producer to emit a non-empty ``type``; an empty-``type`` + scaffold Message must be ``forward``/``reply``-derived into + a real topic before it can be serialized. Also when + ``msg_type`` violates the §2.1 syntax (a character outside + ASCII letters/digits/``.``/``:``/``_``/``-``, e.g. an embedded + space): such a topic must not reach the wire. + ValueError: from :func:`json.dumps` when ``data`` / ``context`` + contain a non-finite number (``allow_nan=False``), enforcing + the §6 "Numbers MUST be finite" rule at emit time. + """ + # §2.1 / §7 producer MUST: the emitted ``type`` must be a non-empty + # string. The constructor accepts "" for the construct-then-forward + # scaffold; this is the gate that refuses to put it on the wire. + if not self.msg_type: + raise MalformedMessage( + "cannot serialize a Message with an empty 'type' — §2.1/§7 " + "require a producer to emit a non-empty topic; derive a real " + "topic via forward()/reply() before serializing") + # §2.1 ``type`` syntax: ASCII letters, digits, ``.`` ``:`` ``_`` ``-``; + # no whitespace. The non-emptiness check above is the §7 producer MUST + # for an empty topic; this is the §2.1 producer MUST for the charset. + # An embedded space (``Message("a b")``) or any other character must + # not reach the wire, so ``serialize`` refuses it here. + if not _TYPE_SYNTAX_RE.fullmatch(self.msg_type): + raise MalformedMessage( + f"'type' {self.msg_type!r} violates OVOS-MSG-1 §2.1 syntax — a " + "topic may contain only ASCII letters, digits, '.', ':', '_', " + "'-' and no whitespace") + return json.dumps( + {"type": self.msg_type, + "data": self._to_jsonable(self.data), + "context": self._to_jsonable(self.context)}, + ensure_ascii=False, allow_nan=False) + + @classmethod + def deserialize(cls, + payload: Union[str, bytes, bytearray, Dict[str, Any]] + ) -> Message: + """Construct a Message from a serialized JSON object (OVOS-MSG-1 §6). + + Args: + payload: a UTF-8 ``bytes`` / ``bytearray`` (decoded first), a + ``str`` (parsed as JSON), or an already-parsed ``dict`` + (the JSON step is skipped). + + Returns: + An instance of ``cls`` — subclasses (e.g. ``ovos-bus-client``'s + transport-layer Message) deserialize to their own type — with + absent ``data`` / ``context`` defaulted to ``{}`` (§2). + + Raises :class:`MalformedMessage` per the §7 ``MUST reject`` + conformance rules: unparsable JSON, non-object root, unknown + top-level keys, missing ``type``, or wrong value types — including + a present-but-non-object ``data`` / ``context`` (``[]``, ``0``, + ``false``, …), which §6 forbids silently coercing to ``{}``. + """ + if isinstance(payload, (bytes, bytearray)): + payload = payload.decode("utf-8") + if isinstance(payload, str): + try: + obj = json.loads(payload) + except json.JSONDecodeError as exc: + raise MalformedMessage( + f"payload is not valid JSON: {exc}") from exc + else: + obj = payload + if not isinstance(obj, dict): + raise MalformedMessage( + "Message payload must be a JSON object (§2)") + # §2: "Other top-level keys MUST NOT appear; consumers MUST + # reject any Message with unknown top-level keys." + unknown = set(obj.keys()) - {"type", "data", "context"} + if unknown: + raise MalformedMessage( + f"unknown top-level keys {sorted(unknown)!r} — §2 " + "forbids any key other than 'type', 'data', 'context'") + if "type" not in obj: + raise MalformedMessage("missing required key 'type' (§2)") + # §2.2/§2.3: when present, ``data`` and ``context`` MUST be JSON + # objects. An ABSENT key defaults to ``{}`` ("MAY be empty"), but a + # *present* wrong-typed value ([], 0, false, "", null) is malformed + # and §6 forbids silently coercing it — reject per the §7 consumer + # "wrong types" MUST instead of papering over it with ``or {}``. + for key in ("data", "context"): + if key in obj and obj[key] is not None \ + and not isinstance(obj[key], dict): + raise MalformedMessage( + f"'{key}' must be a JSON object when present, got " + f"{type(obj[key]).__name__} (§2.{2 if key == 'data' else 3}/§7)") + # An absent (or explicit null) data/context defaults to {} per + # §2.2/§2.3 ("an absent data/context is equivalent to {}"). + return cls(obj["type"], + obj["data"] if obj.get("data") is not None else {}, + obj["context"] if obj.get("context") is not None else {}) + + # --- §5 derivations ----------------------------------------------------- + + def forward(self, msg_type: str, + data: Optional[Dict[str, Any]] = None) -> Message: + """Implement OVOS-MSG-1 §5.1 (``forward``) — relay under a new topic. + + Produces ``{type: msg_type, data: data, context: deepcopy(C)}``: + ``context`` (the §3 routing keys **and** the §4 ``session``) is + carried over **unchanged**, so the forwarder does **not** become + the new ``source`` — the original producer stays named (§5.1, §3.2). + This is the derivation the spec mandates for relay/notification + topics that must preserve the asker's routing — e.g. the + PIPELINE-1 §8 handler-lifecycle trio and SESSION-2 §2.7 + ``ovos.session.sync`` are both required to be ``forward``-derived. + + The context is **deep-copied** so the derived Message is independent + of the source (a §5.1 "MUST NOT modify a session already present" + guarantee — mutating either Message's context cannot leak into the + other). + + Args: + msg_type: ``type`` of the relayed Message (``T'`` in §5.1). + data: payload of the relayed Message (``D'``); ``None`` → ``{}``. + + Returns: + A new Message of ``self``'s runtime class (subclasses propagate, + see :ref:`Subclassing`). + """ + derived = self.__class__( + msg_type, data or {}, deepcopy(self.context)) + return _stamp_live_session(derived) + + def reply(self, msg_type: str, + data: Optional[Dict[str, Any]] = None, + context: Optional[Dict[str, Any]] = None) -> Message: + """Implement OVOS-MSG-1 §5.2 (``reply``) — address back to the asker. + + Deep-copies :attr:`context` and **reverses** the §3 routing keys so + the result is addressed back to the source Message's producer: + + - new ``destination`` := old ``source`` (§5.2 step 1, when set); + - new ``source`` := old ``destination`` — if the old + ``destination`` was an array, the **first** entry is chosen + (§5.2 step 2: the choice is implementation-defined and consumers + MUST NOT rely on a particular member); + - every other context key, including ``session`` (§4), is preserved + unchanged (§5.2 step 3). + + Any keys supplied via ``context`` are overlaid on the copied context + **before** the swap, matching ``ovos_bus_client.Message.reply`` + behaviour: passing + ``context={"source": "C", "destination": "D"}`` yields + ``source=D, destination=C`` because the §5.2 swap is the final step. + Non-routing keys overlaid this way (a custom ``session`` shape, a + tracing identifier, …) pass through untouched — the envelope never + ascribes meaning to them (§2.3). + + Args: + msg_type: ``type`` of the reply Message (``T'`` in §5.2). + data: payload of the reply (``D'``); ``None`` → ``{}``. + context: optional context keys overlaid before the §5.2 swap. + + Returns: + A new Message of ``self``'s runtime class with the §5.2 routing. + + Note: + A producer that maintains no ``source`` / ``destination`` at all + gets a context with neither key set — i.e. a broadcast (§3.3), + which §5.2 names as the only well-defined behaviour absent + addressing information. + """ + new_context = deepcopy(self.context) + if context: + new_context.update(context) + # §5.2 swap. Read both sides BEFORE writing to avoid clobbering. + src = new_context.get("source") + dst = new_context.get("destination") + if dst is not None: + # array-of-strings form: producer chooses one; consumers + # MUST NOT rely on a particular member being chosen (§5.2) + new_context["source"] = ( + dst[0] if isinstance(dst, list) and dst else dst) + if src is not None: + new_context["destination"] = src + return _stamp_live_session( + self.__class__(msg_type, data or {}, new_context)) + + def response(self, data: Optional[Dict[str, Any]] = None, + context: Optional[Dict[str, Any]] = None) -> Message: + """Implement OVOS-MSG-1 §5.3 (``response``) — a ``.response``-suffixed reply. + + Equivalent to ``reply(self.msg_type + ".response", data, context)`` + (§5.3). Topics defined in other specifications MAY rely on the + ``.response`` suffix convention to mark a Message as the answer to a + prior request — e.g. INTENT-4 §10's ``ovos.intent.list.response`` / + ``ovos.intent.describe.response`` and PIPELINE-1 §10's + ``ovos.pipeline..intents.list.response``. Because it + delegates to :meth:`reply`, the §5.2 routing reversal and the §4 + session preservation both apply, which is exactly what lets an asker + correlate ``.response`` against an outstanding request in + the same ``session`` (§5.4). + + Args: + data: payload of the response (``D'``); ``None`` → ``{}``. + context: optional context keys overlaid before the §5.2 swap. + + Returns: + A new Message whose ``type`` is ``self.msg_type + ".response"``. + """ + return self.reply(self.msg_type + ".response", data, context) diff --git a/ovos_spec_tools/messages.py b/ovos_spec_tools/messages.py new file mode 100644 index 00000000..e2020292 --- /dev/null +++ b/ovos_spec_tools/messages.py @@ -0,0 +1,783 @@ +"""Canonical OVOS-spec bus-message topics and the legacy↔``ovos.*`` bridge. + +Specs implemented +----------------- +This module is the vocabulary and migration map for the OVOS bus-namespace +move from the Mycroft topics to the ``ovos.*`` namespace. Each +:class:`SpecMessage` member names a topic an OVOS specification defines, and +is tied to its owning spec section in the per-member comments below: + +- **OVOS-PIPELINE-1** — §8 handler-lifecycle trio + (``ovos.intent.handler.{start,complete,error}``) and §9 utterance-layer + topics (``ovos.utterance.{handle,speak,handled,cancelled}``, + ``ovos.intent.{matched,unmatched}``); +- **OVOS-INTENT-4** — registration / deregistration / enable-disable / + introspection topics (§§5–8, §10); +- **OVOS-STOP-1** — §4.2 ping/pong and §5 global-stop broadcast + (``ovos.stop.*``); +- **OVOS-AUDIO-IN-1** — listener lifecycle signals (``ovos.listener.*``). + AUDIO-IN-1 §6.1–§6.4 mandates ``ovos.listener.record.started`` / + ``.record.ended`` / ``ovos.listener.sleep`` / ``ovos.listener.awoken``; +- **OVOS-SESSION-2** — the out-of-utterance session sync topic + (``ovos.session.sync`` §2.7); +- **OVOS-CONVERSE-1** — the active-handler introspection pair + (``ovos.converse.active.list`` / ``.response`` §6.1); +- **OVOS-PERSONA-1** — the §11 bus surface (``ovos.persona.{query,answer,list, + list.response,register,deregister,activated,dismissed}``); +- **OVOS-FALLBACK-1** — fallback registry topics (``ovos.fallback.{register, + deregister}`` §3); +- **OVOS-COMMON-QUERY-1** — the wants-to-answer poll + (``ovos.common_query.{ping,pong}`` §6); +- **OVOS-TRANSFORM-1** — §6 introspection: six static query/response pairs + ``ovos.transformer..list`` / ``.response`` (audio/utterance/metadata/ + intent/dialog/tts), plus the cancellation terminal ``ovos.utterance.cancelled`` + (§8.2, "defined here"); +- **OVOS-OCP-1** — the §4 Virtual Media Player bus surface under the reserved + ``ovos.common_play.*`` prefix (play/search, the transport controls, and the + player/media/track state reports); +- **OVOS-AUDIO-1** — the audio **output** service bus surface (§7), + mandated. Two rendering modes (``ovos.utterance.speak.b64`` + §3.4 → ``ovos.audio.speech`` §4.3 for remote clients); the playback model + (``ovos.audio.queue`` §4.1 scheduled sounds, ``ovos.audio.play_sound`` §4.2 + instant sounds); the output-lifecycle signals (``ovos.audio.output.started`` + / ``.output.ended`` §5.1/§5.2); the speaking-status query + (``ovos.audio.is_speaking`` §5.3); stop integration (``ovos.audio.stop`` + §6); and the listen-flag mic re-open (``ovos.mic.listen`` §4.4). + +Why an enum +----------- +Referencing ``SpecMessage.SPEAK`` instead of the raw +``"ovos.utterance.speak"`` makes downstream code self-documenting: a +``SpecMessage`` member is provably spec-defined, while a bare string is +visibly legacy or implementation-specific. The enum is the *vocabulary* half +of the migration; :data:`MIGRATION_MAP` is the *rename* half. + +The legacy↔``ovos.*`` bridge (``MIGRATION_MAP`` / :class:`NamespaceTranslator`) +------------------------------------------------------------------------------- +``MIGRATION_MAP`` maps each legacy topic to the ``SpecMessage`` that replaces +it. The bus dual-emit (``ovos-bus-client``'s ``MessageBusClient`` and +``ovos_utils.fakebus.FakeBus``, both driven by :class:`NamespaceTranslator`) +mirrors an emission onto its counterpart topic so a consumer still subscribed +to the legacy name during the migration window keeps receiving the event. The +receive side deduplicates the mirror (see +:meth:`NamespaceTranslator.new_mirror_guard`) so a handler subscribed to both +names runs once. + +Payload translation, not verbatim mirroring. Some renames are *payload +compatible* (the legacy and spec topics carry the same ``data`` shape) and the +mirror forwards the payload unchanged. Others are *shape-changing* renames +(the handler trio, the INTENT-4 management topics): for those, forwarding the +producer's payload verbatim would hand a legacy-only consumer spec-shaped +``data`` it cannot read. :data:`MIGRATION_PAYLOAD_TRANSFORMS` therefore pairs +each shape-changing topic with two pure functions and +:meth:`NamespaceTranslator.translate_payload` applies the right one per +direction, so the mirrored payload is in the **recipient's** shape. Several +transforms are **best-effort / lossy** (a field that does not exist in the +other shape is synthesized or dropped); each lossy case is documented at +:data:`MIGRATION_PAYLOAD_TRANSFORMS`. The "legacy consumers keep working +without code changes" guarantee is therefore scoped: it holds outright for +payload-compatible renames, and holds *best-effort* (with documented loss) for +the shape-changing ones. + +Two renames stay out of the map entirely because no per-topic payload +transform can bridge them: the OVOS-INTENT-4 *registration* consolidation (an +N→1 restructure the stateless bus cannot synthesize) and the per-skill +placeholder ``ovos.stop.*`` ping topics (not static strings). Both are +documented at :data:`MIGRATION_MAP`. + +Note on dual-emit and MSG-1 §5.4. The dual-emit/mirror-window behaviour is a +non-normative **implementation policy** of this migration tooling, not a +spec-mandated mechanism. No OVOS specification mandates dual-emit, and +OVOS-MSG-1 §5.4 explicitly disavows any host-side correlation/bookkeeping of +the kind a mirror window resembles; the window here is a pragmatic dedup +heuristic for the migration period, scoped to :class:`NamespaceTranslator`. + +Out of enum / map scope (OVOS-MSG-1 §2.1.1 runtime-assembled topics) +-------------------------------------------------------------------- +Topics whose ``type`` is assembled at runtime from identifiers are neither +enum members nor static-mappable: ``ovos.pipeline..intents.list`` +(PIPELINE-1 §10), the ``:`` dispatch topic +(PIPELINE-1 §7), the per-skill ``.stop.ping`` / ``.stop`` +placeholders (STOP-1), the CONVERSE-1 §6.2/§6.3 +``.converse.{ping,pong}`` / ``:{converse,response}`` poll +and dispatch, the FALLBACK-1 §6/§7 ``.fallback.{ping,pong}`` / +``:fallback``, and the COMMON-QUERY-1 §7/§3 +``:common_query`` / ``.common_query.response`` / +``:common_query``. + +Spec-referenced but NOT spec-defined (kept OUT of the enum) +---------------------------------------------------------- +``ovos.session.update_default`` and ``ovos.session.start`` are used by +``ovos-bus-client`` but no specification defines them — SESSION-2 §1 explicitly +defers session-lifecycle observability topics — so they are implementation +internals and legitimately remain bare strings in the bus client. +``ovos.context.set`` / ``.unset`` / ``.clear`` appear only as a stray reference +in OVOS-TRANSFORM-1 (mis-citing "OVOS-CONTEXT-1 §5"); OVOS-CONTEXT-1 §5 in fact +defines exactly three mutation pathways and the bus one is ``ovos.session.sync`` +(§5.3), so no ``ovos.context.*`` topic is spec-defined. +""" +import json as _json +import time +from enum import Enum +from typing import Any, Callable, Dict, List, Optional, Tuple + + +class SpecMessage(str, Enum): + """Canonical ``ovos.*`` bus topics defined by the OVOS specifications. + + Every member's value is a literal topic string a specification assigns to + the ``ovos.*`` namespace; the per-member comments cite the owning spec + section. Members subclass ``str`` (OVOS-MSG-1 §2.1 topics are strings) so + they can be used directly as topics:: + + bus.on(SpecMessage.SPEAK, handler) + bus.emit(Message(SpecMessage.UTTERANCE, {...})) + + Absence of a topic here does not imply it is not spec-defined; topics from + specs not yet catalogued are flagged *provisional* in the comments. + """ + + def __str__(self) -> str: + # Behave like py3.11 ``StrEnum``: ``str()`` / f-string yield the bare + # topic ("ovos.utterance.speak"), not "SpecMessage.SPEAK", so a member + # is interchangeable with the OVOS-MSG-1 §2.1 wire ``type`` string. + return self.value + + # --- OVOS-PIPELINE-1 §9 utterance-layer topics --- + #: §9.1 — utterance-layer entry point; producer→orchestrator (also the + #: AUDIO-IN-1 §5 emission target). + UTTERANCE = "ovos.utterance.handle" + #: §9.6 — natural-language output exit point; handler→output layer. + SPEAK = "ovos.utterance.speak" + #: §9.5 — universal end-marker, exactly one per entry Message. + UTTERANCE_HANDLED = "ovos.utterance.handled" + #: OVOS-TRANSFORM-1 §8.2 — terminal event, utterance cancelled by a + #: transformer ("new; defined here"); MUST be followed by UTTERANCE_HANDLED. + #: PIPELINE-1 §6.4 references it as a terminal outcome but does not own it. + UTTERANCE_CANCELLED = "ovos.utterance.cancelled" + #: §9.2 — match notification (NOT a dispatch), broadcast. + INTENT_MATCHED = "ovos.intent.matched" + #: §9.3 — emitted when pipeline iteration claimed no match, broadcast. + INTENT_UNMATCHED = "ovos.intent.unmatched" + # §8 handler-lifecycle trio — orchestrator-emitted around each dispatch, + # each ``forward``-derived from the dispatch Message (§8, MSG-1 §5.1). + INTENT_HANDLER_START = "ovos.intent.handler.start" # §8.1 before invoke + INTENT_HANDLER_COMPLETE = "ovos.intent.handler.complete" # §8.1 normal return + INTENT_HANDLER_ERROR = "ovos.intent.handler.error" # §8.1 raised/timeout + + # --- OVOS-INTENT-4 registration / management / introspection topics --- + #: §5 — register a keyword intent (vocab descriptors inlined, §5.1/§5.2). + INTENT_REGISTER_KEYWORD = "ovos.intent.register.keyword" + #: §6 — register a template intent. + INTENT_REGISTER_TEMPLATE = "ovos.intent.register.template" + #: §8.2 — remove one intent (the (skill_id, intent_name[, lang]) triple). + INTENT_DEREGISTER = "ovos.intent.deregister" + #: §8.5 — re-arm a previously disabled intent. + INTENT_ENABLE = "ovos.intent.enable" + #: §8.5 — suppress an intent without removing its definition. + INTENT_DISABLE = "ovos.intent.disable" + #: §7 — register an ``.entity`` value-set hint. + ENTITY_REGISTER = "ovos.entity.register" + #: §8.3 — remove one entity. + ENTITY_DEREGISTER = "ovos.entity.deregister" + #: §8.4 — remove everything owned by a skill_id. + SKILL_DEREGISTER = "ovos.skill.deregister" + #: §10.1 — query the orchestrator-owned manifest (observer→orchestrator). + INTENT_LIST = "ovos.intent.list" + #: §10.1 — the ``.response`` reply to ``ovos.intent.list`` (MSG-1 §5.3). + INTENT_LIST_RESPONSE = "ovos.intent.list.response" + #: §10.2 — query the full definition of one intent. + INTENT_DESCRIBE = "ovos.intent.describe" + #: §10.2 — the ``.response`` reply to ``ovos.intent.describe`` (MSG-1 §5.3). + INTENT_DESCRIBE_RESPONSE = "ovos.intent.describe.response" + + # --- OVOS-STOP-1 stop cascade topics --- + #: §4.2 — broadcast stoppability query. The per-skill ``.stop.ping`` + #: placeholder form is NOT static-mappable (see MIGRATION_MAP). + STOP_PING = "ovos.stop.ping" + #: §4.2 — shared stoppability reply topic; pong is ``reply``-derived (MSG-1 §5.2). + STOP_PONG = "ovos.stop.pong" + #: §5.3 — universal stop broadcast; ``ovos.stop.*`` namespace is reserved by STOP-1. + STOP = "ovos.stop" + + # --- OVOS-AUDIO-1 audio-OUTPUT service bus surface (§7, mandated) --- + # Defined by OVOS-AUDIO-1 (audio-out.md), NOT by AUDIO-IN-1. + #: §3.4 — remote-client rendering mode; same TTS pipeline as ``SPEAK`` but + #: emits ``ovos.audio.speech`` (b64) instead of enqueueing local playback. + SPEAK_B64 = "ovos.utterance.speak.b64" + #: §4.3 — synthesised audio (base64) emitted for a ``SPEAK_B64`` Message; a + #: bridge relays it to the remote client; audio-output → broadcast. + AUDIO_SPEECH = "ovos.audio.speech" + #: §4.1 — queue a sound for sequential (FIFO) scheduled playback; any → audio. + AUDIO_QUEUE = "ovos.audio.queue" + #: §4.2 — fire-and-forget instant sound, played immediately over the queue; + #: any → audio. + AUDIO_PLAY_SOUND = "ovos.audio.play_sound" + #: §6 — stop audio output: clear the scheduled queue and terminate playback; + #: any → audio. + AUDIO_STOP = "ovos.audio.stop" + #: §5.3 — query whether the audio output service is currently speaking; the + #: service answers ``{"speaking": bool}`` (§7 names no distinct response + #: topic, so the reply is ``reply``-derived, MSG-1 §5.2); any → audio. + AUDIO_IS_SPEAKING = "ovos.audio.is_speaking" + #: §5.1 — playback session started (idle→active); audio-output → broadcast. + AUDIO_OUTPUT_STARTED = "ovos.audio.output.started" + #: §5.2 — playback session ended (queue empty, last item done); audio-output + #: → broadcast. + AUDIO_OUTPUT_ENDED = "ovos.audio.output.ended" + #: §4.4 — re-open the mic after a ``listen: true`` item; audio-output → + #: broadcast. + MIC_LISTEN = "ovos.mic.listen" + + # --- OVOS-AUDIO-IN-1 listener lifecycle signals (§6, mandated) --- + #: §6.1 — voice-command capture began; audio-input → broadcast. + LISTENER_RECORD_STARTED = "ovos.listener.record.started" + #: §6.2 — voice-command capture ended; audio-input → broadcast. + LISTENER_RECORD_ENDED = "ovos.listener.record.ended" + #: §6.3 — controller → audio-input: enter sleep mode and suspend capture. + LISTENER_SLEEP = "ovos.listener.sleep" + #: §6.4 — left sleep mode (sleep→awake transition); audio-input → broadcast. + LISTENER_AWOKEN = "ovos.listener.awoken" + + # --- OVOS-SESSION-2 out-of-utterance session sync (§2.7, bus table §7) --- + #: SESSION-2 §2.7 — broadcast an explicit session update OUTSIDE the + #: utterance lifecycle; the updated snapshot rides in ``Message.data.session`` + #: and the orchestrator MUST merge it. (``ovos.session.update_default`` and + #: ``ovos.session.start`` are NOT spec-defined — see module note — and stay + #: out of this enum; SESSION-2 §1 explicitly defers lifecycle topics.) + SESSION_SYNC = "ovos.session.sync" + + # --- OVOS-CONVERSE-1 §6.1 active-handler introspection --- + # The §6.2 ``.converse.{ping,pong}`` poll and the §6.3 + # ``:{converse,response}`` dispatches are runtime-templated + # (MSG-1 §2.1.1) and so are NOT static enum members. + #: §6.1 — observer→orchestrator: snapshot ``session.converse_handlers``. + CONVERSE_ACTIVE_LIST = "ovos.converse.active.list" + #: §6.1 — the ``.response`` reply carrying ``{converse_handlers}`` (MSG-1 §5.3). + CONVERSE_ACTIVE_LIST_RESPONSE = "ovos.converse.active.list.response" + + # --- OVOS-PERSONA-1 §11 bus surface --- + #: §8.5 — out-of-band query to the active persona; any → persona. + PERSONA_QUERY = "ovos.persona.query" + #: §8.5 — the query response; persona → any component. + PERSONA_ANSWER = "ovos.persona.answer" + #: §8.7 — enumerate supported persona identities; any → persona. + PERSONA_LIST = "ovos.persona.list" + #: §8.7 — the ``.response`` supported-identity listing (MSG-1 §5.3). + PERSONA_LIST_RESPONSE = "ovos.persona.list.response" + #: §9 — register a persona at runtime; any → persona. + PERSONA_REGISTER = "ovos.persona.register" + #: §9 — deregister a persona at runtime; any → persona. + PERSONA_DEREGISTER = "ovos.persona.deregister" + #: §11 — a persona became active for a session (best-effort); persona → broadcast. + PERSONA_ACTIVATED = "ovos.persona.activated" + #: §11 — a persona was dismissed from a session (best-effort); persona → broadcast. + PERSONA_DISMISSED = "ovos.persona.dismissed" + + # --- OVOS-FALLBACK-1 §9 bus surface --- + # The §6.1 ``.fallback.{ping,pong}`` poll and the §7 + # ``:fallback`` dispatch are runtime-templated (MSG-1 §2.1.1) + # and so are NOT static enum members. + #: §3.1 — a skill registers itself as a fallback handler; skill → broadcast. + FALLBACK_REGISTER = "ovos.fallback.register" + #: §3.2 — a skill removes itself from the fallback registry; skill → broadcast. + FALLBACK_DEREGISTER = "ovos.fallback.deregister" + + # --- OVOS-COMMON-QUERY-1 §13 bus surface --- + # The §7.1 ``:common_query`` / ``.common_query.response`` + # and the §3/§10 ``:common_query`` dispatch are runtime-templated + # (MSG-1 §2.1.1) and so are NOT static enum members. + #: §6.1 — the wants-to-answer poll broadcast; plugin → all skills. + COMMON_QUERY_PING = "ovos.common_query.ping" + #: §6.2 — a skill claims it can answer, ``reply``-derived (MSG-1 §5.2); + #: skill → plugin. + COMMON_QUERY_PONG = "ovos.common_query.pong" + + # --- OVOS-TRANSFORM-1 §6 introspection (broadcast query / scatter response) --- + # Six concrete static query/response pairs (one per §3 transformer chain); + # the spec enumerates them, it does NOT define a templated + # ``ovos.transformer..list`` pattern. + #: §6/§3.1 — list loaded audio transformers; any → broadcast. + TRANSFORMER_AUDIO_LIST = "ovos.transformer.audio.list" + #: §6/§3.1 — scatter ``.response`` of loaded audio transformers (MSG-1 §5.3). + TRANSFORMER_AUDIO_LIST_RESPONSE = "ovos.transformer.audio.list.response" + #: §6/§3.2 — list loaded utterance transformers; any → broadcast. + TRANSFORMER_UTTERANCE_LIST = "ovos.transformer.utterance.list" + #: §6/§3.2 — scatter ``.response`` of loaded utterance transformers. + TRANSFORMER_UTTERANCE_LIST_RESPONSE = "ovos.transformer.utterance.list.response" + #: §6/§3.3 — list loaded metadata transformers; any → broadcast. + TRANSFORMER_METADATA_LIST = "ovos.transformer.metadata.list" + #: §6/§3.3 — scatter ``.response`` of loaded metadata transformers. + TRANSFORMER_METADATA_LIST_RESPONSE = "ovos.transformer.metadata.list.response" + #: §6/§3.4 — list loaded intent transformers; any → broadcast. + TRANSFORMER_INTENT_LIST = "ovos.transformer.intent.list" + #: §6/§3.4 — scatter ``.response`` of loaded intent transformers. + TRANSFORMER_INTENT_LIST_RESPONSE = "ovos.transformer.intent.list.response" + #: §6/§3.5 — list loaded dialog transformers; any → broadcast. + TRANSFORMER_DIALOG_LIST = "ovos.transformer.dialog.list" + #: §6/§3.5 — scatter ``.response`` of loaded dialog transformers. + TRANSFORMER_DIALOG_LIST_RESPONSE = "ovos.transformer.dialog.list.response" + #: §6/§3.6 — list loaded TTS transformers; any → broadcast. + TRANSFORMER_TTS_LIST = "ovos.transformer.tts.list" + #: §6/§3.6 — scatter ``.response`` of loaded TTS transformers. + TRANSFORMER_TTS_LIST_RESPONSE = "ovos.transformer.tts.list.response" + + # --- OVOS-OCP-1 §4 Virtual Media Player bus surface --- + # The ``ovos.common_play.*`` prefix is reserved by OCP-1 §4.1. The inline + # ``…search.start`` / ``…search.end`` brackets (§4.2 prose) are not given + # their own normative Message rows, so they are flagged, not enumerated. + #: §4.2 — begin playback of a resolved result / queue. + COMMON_PLAY_PLAY = "ovos.common_play.play" + #: §4.2 — acquire candidate media for a phrase (pipeline discovery step). + COMMON_PLAY_SEARCH = "ovos.common_play.search" + #: §4.3 — now-playing → PAUSED. + COMMON_PLAY_PAUSE = "ovos.common_play.pause" + #: §4.3 — now-playing → PLAYING. + COMMON_PLAY_RESUME = "ovos.common_play.resume" + #: §4.3 — now-playing → STOPPED (an OVOS-STOP-1 subscriber, §7). + COMMON_PLAY_STOP = "ovos.common_play.stop" + #: §4.3 — advance the queue. + COMMON_PLAY_NEXT = "ovos.common_play.next" + #: §4.3 — retreat the queue. + COMMON_PLAY_PREVIOUS = "ovos.common_play.previous" + #: §4.3 — move the position within now-playing. + COMMON_PLAY_SEEK = "ovos.common_play.seek" + #: §4.4 — announce the §3.1 player-state value; player → broadcast. + COMMON_PLAY_PLAYER_STATE = "ovos.common_play.player.state" + #: §4.4 — announce the §3.2 media-state value; player → broadcast. + COMMON_PLAY_MEDIA_STATE = "ovos.common_play.media.state" + #: §4.4 — announce now-playing track transitions; player → broadcast. + COMMON_PLAY_TRACK_STATE = "ovos.common_play.track.state" + + +#: Legacy (Mycroft-era) topic -> the :class:`SpecMessage` that supersedes it. +#: This is the single source of truth for the legacy↔``ovos.*`` renames that +#: the bus bridges. ``NamespaceTranslator`` reads it (and its reverse, +#: :data:`SPEC_TO_LEGACY`) to pick the counterpart topic; it then translates the +#: payload into the recipient's shape via :data:`MIGRATION_PAYLOAD_TRANSFORMS` +#: (identity for payload-compatible renames), so a consumer still on the legacy +#: topic during the migration window receives ``data`` it can read. +#: +#: Two kinds of entry live here: +#: +#: * **Payload-compatible renames** — legacy and spec topics carry the same +#: ``data`` shape; the mirror forwards the payload unchanged (identity +#: transform). Most entries are of this kind. +#: * **Shape-changing renames** — the INTENT-4 management topics +#: (``detach_intent``, ``enable_intent``/``disable_intent``) change the +#: payload shape across the rename. For these the bridge does NOT forward the +#: payload verbatim: :data:`MIGRATION_PAYLOAD_TRANSFORMS` carries a +#: best-effort, sometimes lossy transform pair (documented there) that +#: reshapes the payload per direction. +#: +#: Deliberate exclusions (registration consolidation, per-skill stop ping +#: placeholders) are documented at the bottom — no per-topic payload transform +#: can bridge them. The PIPELINE-1 §8 handler-lifecycle trio is also excluded: +#: it is orchestrator-owned (the orchestrator emits the spec topics directly; +#: the skill framework keeps the legacy ones as a private done-signal), so it is +#: deliberately NOT migrated. +MIGRATION_MAP: Dict[str, SpecMessage] = { + # --- PIPELINE-1 §9 utterance layer (payload-compatible 1:1 renames) --- + "recognizer_loop:utterance": SpecMessage.UTTERANCE, # PIPELINE-1 §9.1 + "speak": SpecMessage.SPEAK, # PIPELINE-1 §9.6 + # --- AUDIO-1 §5.1/§5.2/§4.4 audio-output signals (payload-compatible + # 1:1 renames) --- + "recognizer_loop:audio_output_start": SpecMessage.AUDIO_OUTPUT_STARTED, # AUDIO-1 §5.1 + "recognizer_loop:audio_output_end": SpecMessage.AUDIO_OUTPUT_ENDED, # AUDIO-1 §5.2 + "mycroft.mic.listen": SpecMessage.MIC_LISTEN, # AUDIO-1 §4.4 + # --- AUDIO-1 §3.4/§4.1/§4.2/§5.3/§6 audio-output bus surface + # (payload-compatible 1:1 renames; legacy handler names verified against + # ovos-audio register_handlers) --- + "speak:b64_audio": SpecMessage.SPEAK_B64, # AUDIO-1 §3.4 {utterance, listen} + "speak:b64_audio.response": SpecMessage.AUDIO_SPEECH, # AUDIO-1 §4.3 {audio, listen, ...} + "mycroft.audio.queue": SpecMessage.AUDIO_QUEUE, # AUDIO-1 §4.1 {uri} + "mycroft.audio.play_sound": SpecMessage.AUDIO_PLAY_SOUND, # AUDIO-1 §4.2 {uri} + "mycroft.audio.speak.status": SpecMessage.AUDIO_IS_SPEAKING, # AUDIO-1 §5.3 (query, empty) + "mycroft.audio.speech.stop": SpecMessage.AUDIO_STOP, # AUDIO-1 §6 (empty) + # --- AUDIO-IN-1 §6 listener lifecycle (payload-compatible renames) --- + "recognizer_loop:record_begin": SpecMessage.LISTENER_RECORD_STARTED, # AUDIO-IN-1 §6.1 + "recognizer_loop:record_end": SpecMessage.LISTENER_RECORD_ENDED, # AUDIO-IN-1 §6.2 + "recognizer_loop:sleep": SpecMessage.LISTENER_SLEEP, # AUDIO-IN-1 §6.3 + "mycroft.awoken": SpecMessage.LISTENER_AWOKEN, # AUDIO-IN-1 §6.4 + # NOTE: the PIPELINE-1 §8 handler-lifecycle trio is intentionally NOT + # migrated. It is orchestrator-owned — the orchestrator emits the spec + # ``ovos.intent.handler.*`` directly, while the skill framework keeps + # emitting the legacy ``mycroft.skill.handler.*`` as a private done-signal. + # It is also a shape-changing (not payload-compatible) event, so bridging it + # would both double-emit the spec trio and reshape the payload. The two + # namespaces are kept separate by design (PIPELINE-1 §8 / §11). + # --- STOP-1 (1:1 renames) --- + "skill.stop.pong": SpecMessage.STOP_PONG, # STOP-1 §4.2 stoppability reply + "mycroft.stop": SpecMessage.STOP, # STOP-1 §5.3 universal stop broadcast + # --- PIPELINE-1 §9.3 intent outcome (1:1 rename) --- + "complete_intent_failure": SpecMessage.INTENT_UNMATCHED, + # --- INTENT-4 §8 intent management --- + # detach_intent / enable_intent / disable_intent are SHAPE-CHANGING renames + # (legacy munged/partial payload vs spec {skill_id, intent_name, lang}) — + # reshaped per direction by MIGRATION_PAYLOAD_TRANSFORMS (best-effort). + "detach_intent": SpecMessage.INTENT_DEREGISTER, # §8.2 (shape-changing) + "detach_skill": SpecMessage.SKILL_DEREGISTER, # §8.4 (payload-compatible: {skill_id}) + "mycroft.skill.enable_intent": SpecMessage.INTENT_ENABLE, # §8.5 (shape-changing) + "mycroft.skill.disable_intent": SpecMessage.INTENT_DISABLE, # §8.5 (shape-changing) + # + # ---- DELIBERATE EXCLUSION 1: INTENT-4 §5–§7 *registration* ---- + # ovos.intent.register.keyword / .register.template / ovos.entity.register + # are NOT mappable, because the rename is not 1:1. Adapt emits N legacy + # `register_vocab` messages + one `register_intent` (the intent references + # its vocab by name); INTENT-4 §5.2 consolidates all of that into ONE + # register.keyword Message with the vocab descriptors INLINED (§5.1). A + # transparent bus bridge cannot synthesize that N→1 consolidation — it would + # have to buffer and join messages it has no schema for — so registration + # requires INTENT-4 *adoption* in the producer (ovos_workshop/intents.py) and + # consumers (the pipeline plugins), not a static topic rename here. + # + # ---- DELIBERATE EXCLUSION 2: STOP-1 per-skill ping placeholders ---- + # The legacy stoppability handshake used per-skill topics shaped + # `{skill_id}.stop.ping` / `{skill_id}.stop` (OVOS-MSG-1 §2.1.1 runtime- + # assembled topics). STOP-1 §4.2/§5.3 replace them with the single broadcast + # topics ovos.stop.ping / ovos.stop. A `{skill_id}.*` placeholder is not a + # static string, so it cannot be a dict key; the migration there is handled by + # producers/consumers subscribing on both forms, not by this map. +} + +#: Reverse index: spec topic (plain ``str``) -> the legacy topic it replaces. +#: Derived from :data:`MIGRATION_MAP`; keys are ``SpecMessage.value`` strings so +#: lookups work with either an enum member (via its ``str`` value) or a raw topic. +SPEC_TO_LEGACY: Dict[str, str] = {v.value: k for k, v in MIGRATION_MAP.items()} + + +# --------------------------------------------------------------------------- +# Per-topic payload translation +# --------------------------------------------------------------------------- +# A migrating topic that is a *payload-compatible* rename carries the same +# ``data`` shape on both names, so the mirror forwards the payload unchanged. +# A *shape-changing* rename does not: the legacy and spec topics describe the +# same event with different fields, so forwarding the producer's payload +# verbatim would hand the recipient ``data`` in the wrong shape. For those, the +# bridge reshapes the payload with a pair of pure functions. + +#: One ``(legacy_to_spec, spec_to_legacy)`` transform pair. Each function takes +#: a payload ``dict`` and returns a NEW payload ``dict`` in the other shape; it +#: must never mutate its input. The pair is keyed by the **legacy** topic in +#: :data:`MIGRATION_PAYLOAD_TRANSFORMS`. +PayloadTransform = Callable[[Dict[str, Any]], Dict[str, Any]] + + +def _identity(data: Dict[str, Any]) -> Dict[str, Any]: + """Payload-compatible default: copy the payload through unchanged.""" + return dict(data) + + +# --- detach_intent ↔ ovos.intent.deregister (INTENT-4 §8.2) --- +# Legacy: {"intent_name": ":"} (the munged form — see +# ovos_workshop/intents.py::detach_intent). +# Spec: {"skill_id", "intent_name", "lang"?} (§8.2; lang optional). +# CLEANLY BIDIRECTIONAL on skill_id/intent_name: +# spec->legacy joins ":"; legacy->spec splits on the +# FIRST ":" (MSG-1 §2.1.1: skill_id must not contain ":"). +# LOSSY: legacy carries no ``lang`` -> legacy->spec omits it (spec §8.2 allows +# an omitted lang = "all languages"); spec->legacy drops ``lang``. + +def _detach_legacy_to_spec(data: Dict[str, Any]) -> Dict[str, Any]: + munged = data.get("intent_name", "") + if ":" in munged: + skill_id, intent_name = munged.split(":", 1) + return {"skill_id": skill_id, "intent_name": intent_name} + # No separator -> cannot split; best-effort pass the whole as intent_name. + return {"intent_name": munged} if munged else {} + + +def _detach_spec_to_legacy(data: Dict[str, Any]) -> Dict[str, Any]: + skill_id = data.get("skill_id", "") + intent_name = data.get("intent_name", "") + if skill_id: + return {"intent_name": f"{skill_id}:{intent_name}"} + # No skill_id to join -> emit the bare intent_name (best-effort). + return {"intent_name": intent_name} + + +# --- mycroft.skill.{enable,disable}_intent ↔ ovos.intent.{enable,disable} +# (INTENT-4 §8.5) --- +# Legacy: {"intent_name": ""} (no skill_id, no lang) +# Spec: {"skill_id", "intent_name", "lang"?} (§8.5) +# LOSSY: legacy->spec CANNOT recover ``skill_id`` (absent in the legacy +# payload) -> it is OMITTED, and ``lang`` likewise (spec treats an omitted +# lang as "all languages", §8.5). This is the documented limitation: a +# legacy-sourced enable/disable bridged to the spec topic targets the intent +# name without a skill scope; a consumer that needs skill scoping must obtain +# it elsewhere (e.g. Message.context["skill_id"]). spec->legacy drops +# skill_id/lang, keeping only intent_name. + +def _toggle_legacy_to_spec(data: Dict[str, Any]) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if "intent_name" in data: + out["intent_name"] = data["intent_name"] + # skill_id / lang not recoverable from the legacy payload -> omitted. + return out + + +def _toggle_spec_to_legacy(data: Dict[str, Any]) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if "intent_name" in data: + out["intent_name"] = data["intent_name"] + # spec skill_id / lang have no legacy field -> dropped. + return out + + +#: **legacy topic** -> ``(legacy_to_spec, spec_to_legacy)`` payload transforms. +#: +#: Only the SHAPE-CHANGING renames need an entry; every other migrating topic +#: is payload-compatible and uses the IDENTITY transform implicitly (see +#: :meth:`NamespaceTranslator.translate_payload`). Keyed by the legacy topic so +#: both :data:`MIGRATION_MAP` (legacy->spec) and :data:`SPEC_TO_LEGACY` +#: (spec->legacy) resolve to the same pair. +#: +#: Lossy cases (each documented at the transform above): +#: +#: * **detach_intent** — cleanly bidirectional on ``skill_id``/``intent_name``; +#: ``lang`` is not present in the legacy payload. +#: * **enable/disable** — legacy carries neither ``skill_id`` nor ``lang``, so +#: legacy->spec omits both (a legacy-sourced toggle has no skill scope in +#: ``data``). +MIGRATION_PAYLOAD_TRANSFORMS: Dict[str, Tuple[PayloadTransform, PayloadTransform]] = { + "detach_intent": (_detach_legacy_to_spec, _detach_spec_to_legacy), + "mycroft.skill.enable_intent": (_toggle_legacy_to_spec, _toggle_spec_to_legacy), + "mycroft.skill.disable_intent": (_toggle_legacy_to_spec, _toggle_spec_to_legacy), +} + + +def migration_counterpart(topic: str) -> Optional[str]: + """Return the other-namespace counterpart of a migrating topic, else ``None``. + + The bridge is symmetric: a legacy topic maps to its ``ovos.*`` replacement, + and a spec topic maps back to the legacy name it replaces. This is the + lookup the dual-emit (:meth:`NamespaceTranslator.counterpart_topics`) and + the receive-side dedup (:meth:`NamespaceTranslator.new_mirror_guard`) both + rest on. + + Args: + topic: a bus topic string (legacy or ``ovos.*``). + + Returns: + The counterpart topic as a plain ``str`` (a ``SpecMessage.value``, never + the enum member itself), or ``None`` if ``topic`` does not migrate. + """ + if topic in MIGRATION_MAP: + # legacy -> spec; ``.value`` so the return is a plain str, not a member + return MIGRATION_MAP[topic].value + return SPEC_TO_LEGACY.get(topic) # spec -> legacy, or None when unmapped + + +class NamespaceTranslator: + """Shared legacy↔``ovos.*`` bus-namespace migration logic (OVOS bus bridge). + + This is the shared **migration tooling** that ``ovos-bus-client``'s + ``MessageBusClient`` and ``ovos_utils``' ``FakeBus`` both delegate to, so + the real websocket bus and the test/satellite double behave identically. It + carries no I/O and no config: the two direction flags are passed in (the + caller reads env/config), keeping ``ovos-spec-tools`` dependency-free. + + .. note:: + + **Non-normative implementation policy.** The dual-emit + mirror-window + dedup behaviour described below is a pragmatic migration *policy* of + this tooling, **not** a spec-mandated mechanism. No OVOS specification + mandates dual-emit, and OVOS-MSG-1 §5.4 explicitly disavows any + host-side correlation/bookkeeping of the kind a mirror window resembles. + The window is a best-effort dedup heuristic scoped to the migration + period and to this class; it is not part of any conformance surface. + + The bridge has two halves, matching the two halves of a dual-emit bus: + + - **send side** — :meth:`counterpart_topics` tells the bus which extra + topic to mirror an emission onto, so a producer that emits only the spec + topic still reaches legacy subscribers (and vice-versa); the bus calls + :meth:`translate_payload` to reshape the mirrored payload into the + recipient's shape (identity for payload-compatible renames, a best-effort + transform for shape-changing ones — see + :data:`MIGRATION_PAYLOAD_TRANSFORMS`). + - **receive side** — :meth:`new_mirror_guard` lets a handler subscribed to + *both* names run exactly once, by recognising the mirror re-delivery and + dropping it. :meth:`is_migrated` is the cheap pre-check ("does this topic + participate at all?"). + + Args: + modernize: when ``True``, emitting a **legacy** topic also emits its + ``ovos.*`` spec counterpart (carries the migration forward). + emit_legacy: when ``True``, emitting an ``ovos.*`` **spec** topic also + emits the legacy counterpart (keeps un-migrated consumers working). + window: length, in ``clock`` seconds, of the **mirror window** — the + interval within which a counterpart re-delivery of the same + payload+context is treated as the mirror of a just-seen Message + (and so suppressed) rather than a genuine second event. + """ + + def __init__(self, modernize: bool = True, emit_legacy: bool = True, + window: float = 1.0) -> None: + self.modernize: bool = modernize + self.emit_legacy: bool = emit_legacy + self.window: float = window + + def counterpart_topics(self, msg_type: str) -> List[str]: + """Extra topic(s) the bus should ALSO emit ``msg_type`` on (0 or 1). + + The result is the **send-side** half of the dual-emit: the bus emits the + original Message, then re-emits — on each returned topic — the payload + produced by :meth:`translate_payload` (reshaped into the counterpart + topic's shape, identity for payload-compatible renames). + + Args: + msg_type: the topic the producer asked to emit. + + Returns: + ``[spec_topic]`` when ``msg_type`` is a legacy topic and + ``modernize`` is set; ``[legacy_topic]`` when ``msg_type`` is a spec + topic and ``emit_legacy`` is set; ``[]`` otherwise (non-migrating + topic, or the relevant direction disabled). At most one entry — a + topic has exactly one counterpart. + """ + if self.modernize and msg_type in MIGRATION_MAP: + return [MIGRATION_MAP[msg_type].value] + if self.emit_legacy and msg_type in SPEC_TO_LEGACY: + return [SPEC_TO_LEGACY[msg_type]] + return [] + + def translate_payload(self, from_topic: str, to_topic: str, + data: Dict[str, Any]) -> Dict[str, Any]: + """Reshape ``data`` from ``from_topic``'s shape into ``to_topic``'s shape. + + This is the **payload half** of the bridge, the companion to + :meth:`counterpart_topics`: when the bus mirrors an emission onto the + counterpart topic, it calls this to translate the payload so a consumer + on the counterpart topic receives ``data`` in *its* shape rather than + the producer's. + + Direction is inferred from ``from_topic``: + + - ``from_topic`` is a **legacy** topic (in :data:`MIGRATION_MAP`) → + apply the ``legacy_to_spec`` transform; + - ``from_topic`` is a **spec** topic (in :data:`SPEC_TO_LEGACY`) → + apply the ``spec_to_legacy`` transform. + + If the migrating topic has no entry in + :data:`MIGRATION_PAYLOAD_TRANSFORMS` (a payload-compatible rename) the + transform is the **identity** — a shallow copy of ``data`` unchanged. + The same identity copy is returned when ``from_topic``/``to_topic`` are + not a migrating pair at all, so the method is always safe to call. + + The transforms are pure: ``data`` is never mutated, and a NEW dict is + always returned. Some transforms are **best-effort / lossy** — see + :data:`MIGRATION_PAYLOAD_TRANSFORMS` for the per-topic loss notes. + + Args: + from_topic: the topic the Message was emitted on (legacy or spec). + to_topic: the counterpart topic the mirror is being emitted on. Used + only to confirm direction; the transform is keyed off the legacy + topic of the pair. + data: the source Message's payload. + + Returns: + A new payload ``dict`` in ``to_topic``'s shape. + """ + data = data or {} + # Resolve the legacy topic of this migrating pair (the transform key), + # and which direction we are translating. + if from_topic in MIGRATION_MAP: + legacy_topic, direction = from_topic, 0 # legacy -> spec + elif from_topic in SPEC_TO_LEGACY: + legacy_topic, direction = SPEC_TO_LEGACY[from_topic], 1 # spec -> legacy + else: + # Not a migrating topic — nothing to translate, return a copy. + return dict(data) + transform = MIGRATION_PAYLOAD_TRANSFORMS.get(legacy_topic) + if transform is None: + # Payload-compatible rename: identity. + return dict(data) + return transform[direction](data) + + def is_migrated(self, topic: str) -> bool: + """Whether ``topic`` participates in the migration (so needs dedup). + + Args: + topic: a bus topic string (legacy or ``ovos.*``). + + Returns: + ``True`` if ``topic`` appears on either side of :data:`MIGRATION_MAP` + — i.e. it has a counterpart and a handler subscribed to both names + should run it through :meth:`new_mirror_guard`. + """ + return topic in MIGRATION_MAP or topic in SPEC_TO_LEGACY + + def new_mirror_guard(self, + clock: Optional[Callable[[], float]] = None + ) -> Callable[[object], bool]: + """Build a stateful ``is_mirror(message) -> bool`` for receive-side dedup. + + **Non-normative implementation policy** (migration tooling): the + mirror-window dedup below is a pragmatic heuristic for the migration + period, not a spec-mandated behaviour. OVOS-MSG-1 §5.4 disavows + host-side correlation; this window is deliberately narrow and scoped to + :class:`NamespaceTranslator` so it cannot be mistaken for one. + + Each call returns a **fresh closure with its own private ``seen`` state**, + so one guard is created per handler (or per subscription) and guards do + not cross-talk. The returned predicate answers: *is this Message the + dual-emit mirror of one this guard already let through?* — and a bus + that subscribes a handler to both namespaces calls it to run the handler + exactly once. + + Mirror-window semantics: + + - A Message is a **mirror** iff a previously-seen Message had the same + payload+context fingerprint, a **different** ``msg_type``, and that + earlier type's :func:`migration_counterpart` equals this one — i.e. + it is the same event re-delivered on the counterpart topic. Mirrors + return ``True`` (drop). + - Two genuine events on the **same** topic are never suppressed + (``prev[0] != mtype`` guards this): identical repeats on one topic + are real repeats, not a mirror. + - Entries older than ``window`` (per ``clock``) are evicted before each + check, so a counterpart that arrives *after* the window is treated as + a new event — the bridge only ever collapses a near-simultaneous + dual-emit pair, never two deliberate emissions spaced apart. + + Args: + clock: a monotonic seconds source; defaults to ``time.monotonic``, + resolved **per call** so it stays monkeypatchable in tests. + + Returns: + A predicate ``is_mirror(message) -> bool``. It is defensive: any + message lacking ``msg_type`` / ``data`` / ``context``, or whose + payload is not JSON-fingerprintable, returns ``False`` (never drop + what we cannot prove is a mirror). + """ + # Per-guard private state: fingerprint -> (msg_type, timestamp). Scoped + # to this closure so distinct handlers never share dedup history. + seen: Dict[str, Tuple[str, float]] = {} + window = self.window + + def is_mirror(message: object) -> bool: + try: + mtype = message.msg_type # type: ignore[attr-defined] + # Fingerprint payload+context, order-independent (MSG-1 §6: key + # order is not significant). ``default=str`` tolerates carrier + # objects (e.g. Session) that are not natively JSON-encodable. + fingerprint = _json.dumps([message.data, message.context], # type: ignore[attr-defined] + sort_keys=True, default=str) + except Exception: + # Not a Message-shaped object — cannot prove a mirror, so keep it. + return False + now = (clock or time.monotonic)() + # Evict entries past the mirror window before matching. + for key in [k for k, (_, ts) in seen.items() if now - ts >= window]: + seen.pop(key, None) + prev = seen.get(fingerprint) + # Mirror iff same payload, DIFFERENT topic, and the topics are + # registered counterparts of each other (a true dual-emit pair). + if prev is not None and prev[0] != mtype \ + and migration_counterpart(prev[0]) == mtype: + return True + seen[fingerprint] = (mtype, now) + return False + + return is_mirror diff --git a/ovos_spec_tools/prompt.py b/ovos_spec_tools/prompt.py new file mode 100644 index 00000000..e4459980 --- /dev/null +++ b/ovos_spec_tools/prompt.py @@ -0,0 +1,172 @@ +"""Reference prompt renderer for OVOS-INTENT-2 §4.4. + +A ``.prompt`` is the localized, whole-file plain-text prompt a skill feeds to a +language model. Unlike a ``.dialog`` it is **not** a sentence template: none of +the OVOS-INTENT-1 grammar applies, so ``(``, ``[``, ``|`` and the rest are +literal. The one special construct is the **double-brace** ``{{name}}`` +substitution point, and it is applied **conservatively** — a prompt is +free-form text that routinely embeds code and JSON, and rendering it must +never corrupt text the author did not write as a slot. + +A ``.prompt`` uses the double-brace form **only** (OVOS-INTENT-2 §4.4): a +single ``{name}`` is **literal text** and is never substituted, as are any +lone ``{`` or ``}``. This is the rationale for the double-brace requirement — +prompts are natural-language LLM text that routinely contains literal single +braces (JSON such as ``{"key": 1}``, code), so a single brace must never +trigger substitution. The single-brace ``{name}`` form (OVOS-INTENT-1 §3.4) +remains the slot token of the *template* roles (``.intent``/``.dialog``); it +has no special meaning in a ``.prompt``. + +A ``{{name}}`` is replaced by a caller-supplied value only when all three hold: + +1. it is a well-formed double-brace token ``{{name}}`` whose name is lowercase + ASCII letters, digits and underscores, not beginning with a digit (so + ``{{}}``, ``{{ }}``, a single ``{name}``, and JSON such as ``{"key": 1}`` + are left untouched). The name charset is identical to the slot-name rule of + OVOS-INTENT-1 §3.4; +2. the caller supplied a value for that name — an **unfilled** ``{{name}}`` is + left as literal text, not an error (the deliberate opposite of ``.dialog``, + where §4.2/OVOS-INTENT-1 §5.1 require **every** slot be filled before TTS); +3. it does not lie inside a ```` ``` ```` fenced code block. Fence detection + here is a simple heuristic (counting triple backticks): a line whose first + non-whitespace content is three or more backticks toggles the fence, and an + unterminated fence extends to end-of-file. Nested/indented fences are + implementation-defined. + +Two interfaces are provided: + +- :func:`render_prompt` — a stateless function over an explicit prompt string; +- :class:`PromptRenderer` — a stateful, **multilingual** object backed by a + :class:`~ovos_spec_tools.resources.LocaleResources`. +""" +from __future__ import annotations + +import re +from typing import Dict, List, Optional + +__all__ = ["render_prompt", "PromptRenderer"] + +# A {{name}} substitution point — the OVOS-INTENT-2 §4.4 double-brace form, +# the ONLY substitution token in a .prompt. The name uses the §3.4 charset. +# A single-brace {name} is deliberately NOT matched: in a prompt it is literal +# text. The interior of the braces forbids ``{``/``}`` so the token is flat and +# a single ``{name}`` nested inside ``{{ }}`` cannot accidentally extend it. +_SLOT_RE = re.compile(r"\{\{([a-z][a-z0-9_]*)\}\}") + + +def _is_fence(line: str) -> bool: + """Whether a line opens or closes a ```` ``` ```` fenced code block. + + Implements the OVOS-INTENT-2 §4.4 fence test in its permitted simplified + form: a line whose first non-whitespace content is three backticks toggles + the fenced-block state. ``lstrip(" \\t")`` mirrors §4.4's "first + non-whitespace content"; only the opening delimiter is checked (info + strings and longer fences are treated the same), which §4.4 allows as a + "simpler heuristic" so long as well-formed prompts render identically. + + Args: + line: one line of the prompt, with or without its trailing newline. + + Returns: + ``True`` if this line is a fence delimiter, else ``False``. + """ + return line.lstrip(" \t").startswith("```") + + +def render_prompt(text: str, + slots: Optional[Dict[str, object]] = None) -> str: + """Render a ``.prompt`` string (stateless). + + The whole ``text`` is the prompt; every character is significant. + ``{{name}}`` substitution points are filled per the rules in the module + docstring — conservatively, and leaving an unfilled slot as literal text. + A single-brace ``{name}`` and any literal brace are passed through + unchanged. + + Args: + text: the whole-file content of a ``.prompt`` (§4.4 "the whole file, + verbatim, is one prompt"). + slots: caller-supplied values, keyed by slot name. Values are + converted to text via ``str()``. A name absent here, any + malformed ``{{…}}`` (per §4.4 condition 1), and every single-brace + ``{name}`` are left as literal text — §4.4's "slots are optional" + and the double-brace-only rule. + + Returns: + The prompt with its supplied slots substituted, otherwise verbatim — + byte-for-byte unchanged outside the substituted ``{{name}}`` points + (``splitlines(keepends=True)`` preserves the original line endings, and + text inside a fenced block is reproduced untouched). An empty ``text`` + returns ``""``. + """ + values = slots or {} + + def replace(match: "re.Match") -> str: + name = match.group(1) + if name in values: + return str(values[name]) + return match.group(0) # an unfilled {{name}} stays literal (§4.4) + + rendered: List[str] = [] + in_fence = False + # keepends=True so the file is reproduced verbatim apart from substitution. + for line in text.splitlines(keepends=True): + if _is_fence(line): + in_fence = not in_fence + rendered.append(line) + elif in_fence: + rendered.append(line) + else: + rendered.append(_SLOT_RE.sub(replace, line)) + return "".join(rendered) + + +class PromptRenderer: + """A stateful, multilingual renderer for one named ``.prompt``. + + Backed by a :class:`~ovos_spec_tools.resources.LocaleResources`: the prompt + name is fixed at construction, but the **language is given per** + :meth:`render` **call**, so one renderer serves every language the prompt + is shipped in. + + Unlike :class:`~ovos_spec_tools.dialog.DialogRenderer` it carries no + randomness and no ``.entity`` fallback — a prompt has a single whole-file + body and its slots are optional (§4.4). It does hold **default** slot + values, reused on every call. + """ + + def __init__(self, resources, name: str, + slots: Optional[Dict[str, object]] = None): + """ + Args: + resources: a :class:`~ovos_spec_tools.resources.LocaleResources` + (or anything with a ``load_prompt(name, lang)`` method). + name: the base name of the ``.prompt`` to render. + slots: **default** slot values, held for the renderer's lifetime + and reused on every :meth:`render` call. A per-call value + overrides a default. + """ + self.resources = resources + self.name = name + self.default_slots: Dict[str, object] = dict(slots or {}) + + def render(self, lang: str, + slots: Optional[Dict[str, object]] = None) -> str: + """Render the prompt in ``lang``. + + Args: + lang: the BCP-47 language tag to render in. + slots: per-call slot values; each overrides a default. + + Returns: + The rendered prompt string. + + Raises: + FileNotFoundError: the prompt does not exist for ``lang``. + MalformedResource: the resolved ``.prompt`` file is empty. + """ + text = self.resources.load_prompt(self.name, lang) + effective = dict(self.default_slots) + if slots: + effective.update(slots) + return render_prompt(text, effective) diff --git a/ovos_spec_tools/resources.py b/ovos_spec_tools/resources.py new file mode 100644 index 00000000..d3d845af --- /dev/null +++ b/ovos_spec_tools/resources.py @@ -0,0 +1,699 @@ +"""Reference loader for OVOS-INTENT-2 — Locale Resource Formats. + +This module is the reference implementation of the loader of OVOS-INTENT-2: it +discovers a skill's locale resource files, reads them with the common reader +(§3), and loads each of the six resource roles per its format (§4). + +The six roles, by extension: + +- ``.intent`` — slot-bearing intent training samples; +- ``.dialog`` — slot-bearing spoken-response phrases; +- ``.entity`` — slot-free example values for a slot; +- ``.voc`` — slot-free vocabulary; +- ``.blacklist`` — slot-free intent-suppression phrases; +- ``.prompt`` — a whole-file plain-text language-model prompt (§4.4). + +The user-data path of the override precedence (§2.1) is **assistant-defined**; +this module takes it as a parameter and imports no configuration. + +The **language is given per query**, not fixed at construction: a locale +folder is the multilingual unit of a skill, so one :class:`LocaleResources` +serves every language the skill ships. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union + +from ovos_spec_tools.expansion import expand +from ovos_spec_tools.language import ( + DEFAULT_MAX_LANGUAGE_DISTANCE, + closest_lang, + standardize_lang, +) + +__all__ = [ + "LocaleResources", + "MalformedResource", + "find_lang_dir", + "iter_locale_dirs", + "keyword_form", + "normalize_for_match", + "read_resource_file", + "read_prompt_file", + "strip_samples", + "utterance_contains", + "SLOT_BEARING_ROLES", + "SLOT_FREE_ROLES", + "PROMPT_ROLE", +] + +# Resource roles, by file extension (OVOS-INTENT-2 §1). The five template +# roles are line-oriented; `.prompt` is a single whole-file document (§4.4). +SLOT_BEARING_ROLES = (".intent", ".dialog") +SLOT_FREE_ROLES = (".entity", ".voc", ".blacklist") +PROMPT_ROLE = ".prompt" + +# A resolver maps a requested language and the available language tags to the +# best one, or None — the signature of `ovos_spec_tools.language.closest_lang`. +LanguageResolver = Callable[[str, Sequence[str], int], Optional[str]] + + +class MalformedResource(ValueError): + """A resource file or skill layout that violates OVOS-INTENT-2. + + Raised for an empty resource file (§5), a duplicate ``(role, base name)`` + within one language tree (§2), and a named slot in a slot-free role (§4.3). + """ + + +def read_resource_file(path: Path) -> List[str]: + """Apply the OVOS-INTENT-2 §3 common reader to one file. + + The file is read as UTF-8 (§3: "the file is UTF-8 … a reader that encounters + [a BOM] MUST discard it"), a leading byte-order mark is discarded, and both + ``LF`` and ``CRLF`` line endings are accepted (§3: "a reader MUST accept + both"). Each line is stripped; blank lines and ``#``-comment lines are + dropped (§3: "a blank line is skipped … a line whose first character is + ``#`` is a comment"). There are no inline comments — a ``#`` mid-line is + literal — so only a line *beginning* with ``#`` is dropped. The surviving + lines — each one template (OVOS-INTENT-1) — are returned in order. + + Args: + path: the resource file to read (a line-oriented role, not ``.prompt``; + see :func:`read_prompt_file` for the whole-file role). + + Returns: + The template lines, in file order, with blanks and comments removed. An + all-blank/all-comment file yields ``[]`` — the caller treats that as the + §5 "empty file" fault. + """ + text = path.read_text(encoding="utf-8-sig") # utf-8-sig discards a BOM + templates: List[str] = [] + for raw_line in text.splitlines(): # splitlines() accepts LF and CRLF + line = raw_line.strip() + if not line or line.startswith("#"): + continue + templates.append(line) + return templates + + +def iter_locale_dirs(root: Path, + native_langs: Optional[Sequence[str]] = None, + max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE + ) -> Iterator[Tuple[str, Path]]: + """Iterate ``/locale//`` subdirs as ``(lang, path)`` pairs. + + Implements the "discover languages" loader step (OVOS-INTENT-2 §5 step 1, + §2 layout): each immediate subdirectory of ``/locale/`` is one + language tree, named with a BCP-47 tag. + + Each immediate subdirectory of ``/locale/`` is treated as a locale + tree; its name is normalized with :func:`standardize_lang` and yielded as + the first item of the pair. The second item is the directory ``Path``. + + When ``native_langs`` is given, each subdir is matched against the natives + via :func:`closest_lang` and yielded only when its closest native is within + ``max_distance`` — useful for "a skill declares ``en``, the locale tree has + ``en-US``, accept it" without forcing the caller to repeat that walk. + + Resource loaders (``.rx``, ``.dialog``, ``.voc``, ``.intent``, ``.json``) + reinvent this walk by hand and disagree on the macro/full-tag policy. Use + this and the disagreement goes away — locales are always discovered as + full-tag directories, ``closest_lang`` reconciles at query time. + + Args: + root: the skill root containing a ``locale/`` directory. + native_langs: if given, only locales whose closest native is within + ``max_distance`` are yielded (§2.2 nearness, applied as a filter). + max_distance: the §2.2 distance threshold for that filter. + + Yields: + ``(standardized_lang, dir_path)`` for each accepted locale directory, + in sorted directory order. A non-tag or unparseable subdir name is + skipped. ``root`` without a ``locale/`` child yields nothing. + """ + root = Path(root) + locales_root = root / "locale" + if not locales_root.is_dir(): + return + natives_norm = ([standardize_lang(lang) for lang in native_langs] + if native_langs is not None else None) + for entry in sorted(locales_root.iterdir()): + if not entry.is_dir(): + continue + try: + lang_norm = standardize_lang(entry.name) + except Exception: + continue + if natives_norm is not None: + if closest_lang(lang_norm, natives_norm, + max_distance=max_distance) is None: + continue + yield lang_norm, entry + + +def find_lang_dir(base_path: Union[str, Path], lang: str, + lang_resolver: Optional["LanguageResolver"] = None, + max_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE + ) -> Optional[Path]: + """Resolve the best ``//`` subdirectory for *lang*. + + The immediate subdirectories of ``base_path`` are taken as the set of + available languages; the resolver (default :func:`closest_lang`) + picks the closest match within ``max_distance`` (OVOS-INTENT-2 §2.2 + smart fallback). Case-mismatch (``en-US`` vs ``en-us``) and + minor regional fallback (``en-AU`` to ``en-US``) both resolve + through :func:`closest_lang` — no separate exact-match short-circuit + is needed. + + Returns the resolved :class:`Path`, or ``None`` if ``base_path`` is + not a directory or no available subdir is close enough. + + This is the standalone primitive backing + :meth:`LocaleResources._lang_dir` — use this when you want one + language-aware directory lookup without constructing a full + :class:`LocaleResources`. Skills typically use ``LocaleResources`` + (which wires the override-precedence search); tools that walk a + single locale tree (``locate this lang's resource root``) call this. + """ + base = Path(base_path) + if not base.is_dir(): + return None + names = [c.name for c in base.iterdir() if c.is_dir()] + resolver = lang_resolver if lang_resolver is not None else closest_lang + match = resolver(lang, names, max_distance) + if match is None: + return None + # `match` is one of the candidate names normalized by the resolver; + # find the original directory entry that standardizes to it so we + # return the on-disk casing. + for name in names: + if standardize_lang(name) == standardize_lang(match): + return base / name + return None + + +def keyword_form(template_line: str, + vocabularies: Optional[Dict[str, List[str]]] = None + ) -> Tuple[str, List[str]]: + """Split one slot-free template line into ``(entity, aliases)``. + + The line is expanded via :func:`expand` (with ``vocabularies`` available + for ```` references), lowercased, deduplicated and sorted. The + first item is the canonical **entity**; the rest are **aliases** that + canonicalize to it. + + A ``.voc`` line like ``(hi|hello|hey)`` becomes one keyword whose entity + value is ``"hi"`` and whose aliases are ``["hello", "hey"]`` — any + consumer that distinguishes a canonical form from synonyms can use this + grouping directly (OVOS-INTENT-2 §4.3 slot-free roles). + + The canonical/alias split is a tooling convention layered *on top of* the + spec's unordered sample set (OVOS-INTENT-1 §4): the spec defines no + "canonical" member, so "first after case-fold + sort" is chosen here purely + to be deterministic. + + Args: + template_line: one slot-free template line (a ``.voc``/``.entity`` line). + vocabularies: vocabularies for any ```` reference in the line + (OVOS-INTENT-1 §3.7). + + Returns: + ``(entity, aliases)`` — the canonical form and its synonyms. An empty or + whitespace-only line, or a line that fails to expand, yields + ``("", [])`` (a malformed line is dropped, not raised, so one bad line + does not poison a batch keyword load). + + This leniency is **deliberate and confined to this low-level helper**: it + serves only the best-effort keyword extractors (:meth:`vocabulary_keywords`, + :meth:`entity_keywords`), which group ``.voc``/``.entity`` lines into + canonical/alias pairs and skip a line that will not expand. It is **not** a + conformant load path — the conformant loaders (:meth:`_load_expanded` and + everything built on it: :meth:`load_intent`, :meth:`load_vocabulary`, …) + call :func:`expand` directly and **raise** on a malformed template or a + slot in a slot-free role per OVOS-INTENT-2 §5 / OVOS-INTENT-1 §3.6. No + conformant load relies on this function's silent drop. + """ + if not template_line.strip(): + return "", [] + try: + samples = expand(template_line, vocabularies) + except Exception: + # A malformed line yields no keyword rather than poisoning the batch. + # Lenient by design — see the note in the docstring; conformant loaders + # call expand() directly and raise instead. + return "", [] + options = sorted({s.lower() for s in samples}) + if not options: + return "", [] + return options[0], options[1:] + + +def normalize_for_match(text: str, *, + strip_diacritics: bool = True, + strip_punct: bool = True) -> str: + """Lowercase, strip, and optionally fold diacritics / ASCII punctuation. + + Used as the comparison normalization for :func:`utterance_contains` and + :func:`strip_samples`. The two folding steps are independent: + + - ``strip_diacritics=True`` (default) decomposes combining marks (NFD) + and drops them — ``"olá"`` becomes ``"ola"``, ``"über"`` becomes + ``"uber"`` — so the comparison is accent-insensitive. + - ``strip_punct=True`` (default) removes ASCII punctuation. Curly + braces ``{`` and ``}`` are preserved so slot markers survive a + pre-render pass. + + Set either flag to ``False`` for languages where the distinction is + semantic (e.g. French ``ou``/``où``). + + This is a *match-time* normalization the resource consumers apply, distinct + from the upstream ASR normalization OVOS-INTENT-1 §2 presumes; it exists + because real utterances and authored ``.voc`` lines drift in accent and + punctuation. ``{``/``}`` are preserved so a slot marker survives a + pre-render pass. + + Args: + text: the string to normalize. + strip_diacritics: fold combining marks via NFD decomposition. + strip_punct: drop ASCII punctuation except ``{`` and ``}``. + + Returns: + The normalized, lowercased, whitespace-trimmed string. + """ + import unicodedata + text = text.strip().lower() + if strip_diacritics: + text = "".join( + c for c in unicodedata.normalize("NFD", text) + if not unicodedata.combining(c) + ) + if strip_punct: + import string + rm_chars = set(c for c in string.punctuation if c not in ("{", "}")) + text = "".join(c for c in text if c not in rm_chars) + return text + + +def utterance_contains(utterance: str, samples: Sequence[str], + *, + exact: bool = False, + strip_diacritics: bool = True, + strip_punct: bool = True) -> bool: + """True iff ``utterance`` matches any item in ``samples``. + + With ``exact=True`` the utterance must equal a sample after + normalization. With ``exact=False`` (the default) a sample is found + when it appears in the utterance as a whole-word substring — so a + sample ``yes`` matches ``"yes, please"`` but not ``"yesterday"``. + + Both sides are normalized via :func:`normalize_for_match`. The + ``strip_diacritics`` and ``strip_punct`` flags are independent — + forward each one as required by the target language. + + The default whole-word (non-``exact``) mode implements the + OVOS-INTENT-2 §4.3 occurrence rule used for ``.voc``/``.blacklist`` testing: + a phrase "occurs" when its words appear "as a contiguous sequence of whole + words … not a raw substring" — which is why ``art`` does not match within + ``start``. + + Args: + utterance: the (ASR-normalized) text to test. + samples: the phrase set to look for, e.g. an expanded ``.voc``. + exact: require equality after normalization rather than substring. + strip_diacritics: forwarded to :func:`normalize_for_match`. + strip_punct: forwarded to :func:`normalize_for_match`. + + Returns: + ``True`` iff any sample matches. An empty utterance or empty sample set + returns ``False``. + """ + if not utterance or not samples: + return False + norm = lambda s: normalize_for_match( + s, strip_diacritics=strip_diacritics, strip_punct=strip_punct) + utt = norm(utterance) + norm_samples = [norm(s) for s in samples] + if exact: + return any(s and s == utt for s in norm_samples) + import re + # `(? str: + """Return ``utterance`` with every whole-word occurrence of any sample + removed. + + Samples are stripped longest first so that a composite match is + consumed before any of its shorter constituents (``"give up"`` before + ``"up"``). The match is whole-word and case-insensitive; the utterance + is otherwise returned with original casing and punctuation. The whole-word + anchoring is the same OVOS-INTENT-2 §4.3 "contiguous whole words" rule as + :func:`utterance_contains`. + + Args: + utterance: the text to strip from. + samples: phrases to remove (e.g. an expanded ``.voc`` of filler words). + + Returns: + ``utterance`` with every whole-word sample occurrence removed; double + spaces left behind are **not** collapsed (the caller normalizes if it + needs to). + """ + import re + for s in sorted({s for s in samples if s and s.strip()}, + key=len, reverse=True): + # `(? str: + """Read a ``.prompt`` whole and verbatim (OVOS-INTENT-2 §3, §4.4). + + A ``.prompt`` is **not** line-oriented: it is read whole, with no line + stripping and no blank- or ``#``-comment-line filtering, because §4.4 states + "every character is part of the prompt" (``#`` lines are ordinary prompt + text, unlike in the line-oriented roles of §3). The file is UTF-8 and a + leading byte-order mark is discarded (§3's "a reader … MUST discard" a BOM). + + Args: + path: the resolved ``.prompt`` file. + + A ``.prompt`` has **no** special syntax beyond ``{{name}}`` substitution + (OVOS-INTENT-2 §4.4): a single ``{name}``, any lone brace, and an + ```` HTML comment are all **literal pass-through text** — they + reach the model unchanged. The whole file is returned verbatim. + + Args: + path: the resolved ``.prompt`` file. + + Returns: + The file's whole content as a single string, byte-for-byte except the + stripped BOM and Python's universal-newline decoding. + """ + return path.read_text(encoding="utf-8-sig") # utf-8-sig discards a BOM (§3) + + +class LocaleResources: + """Loads OVOS-INTENT-2 resource files for one skill, across languages. + + One instance serves **every language** a skill ships: the language is a + parameter of each load call, not of the constructor, because a locale + folder is the multilingual unit of a skill. + + A resource is resolved through the override precedence of §2.1 — user + overrides, then skill resources, then core resources — searching each + source's ``/`` directory and all its subdirectories recursively. + + When the requested language has no directory, a **smart language fallback** + (OVOS-INTENT-2 §2.2, non-normative) selects the nearest available language. + Resolution is done by :func:`ovos_spec_tools.language.closest_lang` and is + re-run on every load, so one instance serves different languages — and + different fallbacks. The fallback needs the optional ``langcodes`` + dependency; without it, only exact tags resolve. + """ + + def __init__(self, skill_locale: str, + core_locale: Optional[str] = None, + user_locale: Optional[str] = None, + lang_resolver: Optional[LanguageResolver] = None, + max_language_distance: int = DEFAULT_MAX_LANGUAGE_DISTANCE): + """ + Args: + skill_locale: path to the skill's ``locale/`` directory. + core_locale: path to the assistant's core ``locale/`` directory. + user_locale: path to the user-override ``locale/`` directory + (its root is assistant-defined, §2.1). + lang_resolver: ``(target, available, max_distance) -> best | None``; + resolves a requested language against the available ones. + Defaults to :func:`ovos_spec_tools.language.closest_lang`. + max_language_distance: passed to the resolver — the fallback + accepts a language whose tag distance is **below** this + (default 10, §2.2). ``0`` disables the fallback. + """ + # Highest precedence first (§2.1): user, skill, core. + self._sources: List[Path] = [ + Path(p) for p in (user_locale, skill_locale, core_locale) + if p is not None + ] + self._lang_resolver: LanguageResolver = ( + lang_resolver if lang_resolver is not None else closest_lang) + self.max_language_distance = max_language_distance + + def _lang_dir(self, source: Path, lang: str) -> Optional[Path]: + """The ``/`` directory for ``lang`` under one source. + + The language is resolved against the available subdirectories by the + ``lang_resolver`` — an exact tag, or the smart fallback of §2.2. + """ + return find_lang_dir(source, lang, + lang_resolver=self._lang_resolver, + max_distance=self.max_language_distance) + + def find(self, base_name: str, extension: str, + lang: str) -> Optional[Path]: + """Locate a resource file by ``(base name, extension)`` in ``lang``. + + Walks the override precedence (§2.1) — user, then skill, then core — + and inside each source's ``/`` directory descends recursively + looking for ````. The first match wins. + + ``lang`` is resolved against each source's available subdirectories + via the language resolver, so a request for ``en`` matches an + ``en-US/`` tree (and vice versa) within the configured distance. + + Raises :class:`MalformedResource` if more than one file with the + same ``(role, base name)`` exists within one language tree — + OVOS-INTENT-2 §2 requires uniqueness. + + Returns the resolved path, or ``None`` if no source has it. + """ + filename = base_name + extension + for source in self._sources: + lang_dir = self._lang_dir(source, lang) + if lang_dir is None: + continue + matches = sorted(p for p in lang_dir.rglob(filename) if p.is_file()) + if len(matches) > 1: + raise MalformedResource( + f"duplicate resource {filename!r} within {lang_dir} — " + f"a (role, base name) must be unique per language tree") + if matches: + return matches[0] + return None + + def vocabularies(self, lang: str) -> Dict[str, List[str]]: + """Every ``.voc`` reachable for ``lang``, as a name→templates map + suitable for resolving ```` references during expansion.""" + vocs: Dict[str, List[str]] = {} + # Lowest precedence first, so a higher-precedence file overrides. + for source in reversed(self._sources): + lang_dir = self._lang_dir(source, lang) + if lang_dir is None: + continue + for path in lang_dir.rglob("*.voc"): + if path.is_file(): + vocs[path.stem] = read_resource_file(path) + return vocs + + def entities(self, lang: str) -> Dict[str, List[str]]: + """Every ``.entity`` reachable for ``lang``, as a name→value-set map — + each value set expanded, with override precedence applied.""" + names = set() + for source in self._sources: + lang_dir = self._lang_dir(source, lang) + if lang_dir is None: + continue + for path in lang_dir.rglob("*.entity"): + if path.is_file(): + names.add(path.stem) + return {name: self.load_entity(name, lang) for name in sorted(names)} + + def _keywords_for(self, extension: str, lang: str + ) -> Iterator[Tuple[str, str, List[str]]]: + """Walk every slot-free resource of one role and group expansions + line-by-line. Yields ``(resource_name, entity, aliases)`` triples + — one yield per non-empty template line, with the OVOS-INTENT-2 + §4.3 ``(entity, aliases)`` convention applied via + :func:`keyword_form`.""" + vocabularies = self.vocabularies(lang) + for source in self._sources: + lang_dir = self._lang_dir(source, lang) + if lang_dir is None: + continue + for path in sorted(lang_dir.rglob(f"*{extension}")): + if not path.is_file(): + continue + for template in read_resource_file(path): + entity, aliases = keyword_form(template, vocabularies) + if entity: + yield path.stem, entity, aliases + + def vocabulary_keywords(self, lang: str + ) -> Iterator[Tuple[str, str, List[str]]]: + """Yield ``(voc_name, entity, aliases)`` for every line of every + ``.voc`` reachable for ``lang``. + + One yield per template line: the line's expansion is split into a + canonical entity (first sorted alternative) and its aliases via + :func:`keyword_form`. Suits any consumer that registers keyword + sets with a primary/alias distinction (OVOS-INTENT-2 §4.3). + """ + return self._keywords_for(".voc", lang) + + def entity_keywords(self, lang: str + ) -> Iterator[Tuple[str, str, List[str]]]: + """Yield ``(entity_name, entity, aliases)`` for every line of every + ``.entity`` reachable for ``lang``. Same shape as + :meth:`vocabulary_keywords`; ``.entity`` and ``.voc`` share the + slot-free template format (§4.3).""" + return self._keywords_for(".entity", lang) + + def voc_list(self, base_name: str, lang: str) -> List[str]: + """Load a ``.voc`` as its flat sample list — the same content + :meth:`load_vocabulary` returns, named consistently with + :meth:`voc_match` and :meth:`remove_voc` for callers that want + to inspect or cache the phrase set independently. + + Returns ``[]`` when the resource does not exist for the language. + """ + try: + return self.load_vocabulary(base_name, lang) + except FileNotFoundError: + return [] + + def voc_match(self, utterance: str, base_name: str, lang: str, *, + exact: bool = False, + strip_diacritics: bool = True, + strip_punct: bool = True) -> bool: + """Convenience: load a ``.voc`` for ``lang`` and check whether + ``utterance`` matches any of its samples. + + Equivalent to ``utterance_contains(utterance, + self.load_vocabulary(base_name, lang), ...)`` with the same flag + semantics, including the ``(? str: + """Convenience: load a ``.voc`` for ``lang`` and strip every + whole-word occurrence of any sample from ``utterance``. + + Equivalent to ``strip_samples(utterance, + self.load_vocabulary(base_name, lang))``; samples are stripped + longest-first so a composite phrase consumes its parts before + any shorter fallback. Returns ``utterance`` unchanged if the + resource does not exist for the language. + """ + if not utterance: + return utterance + try: + samples = self.load_vocabulary(base_name, lang) + except FileNotFoundError: + return utterance + return strip_samples(utterance, samples) + + def _load_expanded(self, base_name: str, extension: str, + lang: str) -> List[str]: + """Load a resource and expand it to its sample set.""" + path = self.find(base_name, extension, lang) + if path is None: + raise FileNotFoundError( + f"no {extension} resource named {base_name!r} for " + f"language {lang!r}") + templates = read_resource_file(path) + if not templates: + raise MalformedResource( + f"empty resource file {path} — every file must contribute at " + f"least one template (§5)") + vocabularies = self.vocabularies(lang) + slot_free = extension in SLOT_FREE_ROLES + samples: List[str] = [] + for template in templates: + for sample in expand(template, vocabularies): + if slot_free and "{" in sample: + raise MalformedResource( + f"{extension} resource {path} is slot-free but " + f"template {template!r} contains a named slot") + if sample not in samples: + samples.append(sample) + return samples + + def load_intent(self, base_name: str, lang: str) -> List[str]: + """Load an ``.intent`` as its sample set, named slots intact (§4.1).""" + return self._load_expanded(base_name, ".intent", lang) + + def load_entity(self, base_name: str, lang: str) -> List[str]: + """Load an ``.entity`` value set (§4.3).""" + return self._load_expanded(base_name, ".entity", lang) + + def load_vocabulary(self, base_name: str, lang: str) -> List[str]: + """Load a ``.voc`` phrase set (§4.3).""" + return self._load_expanded(base_name, ".voc", lang) + + def load_blacklist(self, base_name: str, lang: str) -> List[str]: + """Load a ``.blacklist`` phrase set (§4.3).""" + return self._load_expanded(base_name, ".blacklist", lang) + + def load_dialog(self, base_name: str, lang: str) -> List[str]: + """Load a ``.dialog`` as its list of phrase strings. + + Unlike the other roles a ``.dialog`` is **not** expanded at load time — + expansion happens per render, on the one phrase chosen (§4.2). The + phrase strings are returned verbatim, for a dialog renderer to consume. + """ + path = self.find(base_name, ".dialog", lang) + if path is None: + raise FileNotFoundError( + f"no .dialog resource named {base_name!r} for " + f"language {lang!r}") + phrases = read_resource_file(path) + if not phrases: + raise MalformedResource( + f"empty resource file {path} — every file must contribute at " + f"least one template (§5)") + return phrases + + def load_prompt(self, base_name: str, lang: str) -> str: + """Load a ``.prompt`` as its whole-file string (§4.4). + + A ``.prompt`` is read whole and verbatim — not split into templates, + not line-filtered — and is returned for a prompt renderer to fill. See + :class:`ovos_spec_tools.prompt.PromptRenderer`. + + Raises: + FileNotFoundError: no such ``.prompt`` for ``lang``. + MalformedResource: the file is empty or only whitespace (§5). + """ + path = self.find(base_name, PROMPT_ROLE, lang) + if path is None: + raise FileNotFoundError( + f"no .prompt resource named {base_name!r} for " + f"language {lang!r}") + text = read_prompt_file(path) + if not text.strip(): + raise MalformedResource( + f"empty resource file {path} — every file must contribute " + f"content (§5)") + return text diff --git a/ovos_spec_tools/session.py b/ovos_spec_tools/session.py new file mode 100644 index 00000000..bd11dec5 --- /dev/null +++ b/ovos_spec_tools/session.py @@ -0,0 +1,904 @@ +"""Reference implementation of OVOS-SESSION-1 — the session carrier. + +This module defines the **wire shape** of the JSON object that travels +inside ``Message.context.session``, and the rules consumers follow when +reading and propagating it. It is the canonical reference implementation +of the field set OVOS-SESSION-1 §3 claims, carrying every registered +field with spec-correct serialize / deserialize and the recency / cap / +prune helpers the field owners (OVOS-PIPELINE-1 §7.1, OVOS-CONVERSE-1 +§2.1 / §2.2) describe. + +Scope per OVOS-SESSION-1 §1: + +- the JSON shape (§2), including the *omissible-but-never-nullable* + rule (§2 / §2.1); +- the closed set of fields claimed in this version (§3), each carried + as a first-class attribute; +- the reserved ``"default"`` ``session_id`` value (§3.1); +- propagation across the OVOS-MSG-1 §5 derivations (§4) — modelled + here as round-trip equality; +- serialization per OVOS-MSG-1 §6 (§5). + +Design constraints — this is **pure data + stdlib only**. It does not +import ``ovos-bus-client``, ``ovos-config``, the bus, or any heavy +``ovos-utils`` machinery. Deployment defaults (the pipeline ordering, +the converse-handler cap, the configured language) are **injected** by +the caller, never read from a config singleton here. The lifecycle +surface that bus-client layers on top — ``SessionManager``, +``dig_for_message``, the ``active_skills`` / ``utterance_states`` +back-compat projections — lives in bus-client's subclass, not here. + +Out of scope (§1 non-goals): session lifecycle, a session store, +authentication / authorization, encryption, and the *semantics* of any +field whose owner is not OVOS-SESSION-1 — those are owned by the citing +specification. This module fixes only the wire contract, the §3.1 / +§3.2 / §3.3 fields whose meaning OVOS-SESSION-1 itself owns, and the +mechanical recency / cap / prune behaviour the owners describe for the +handler-list fields. +""" +from __future__ import annotations + +import json +import logging +import time +from copy import deepcopy +from threading import Lock +from typing import Any, Dict, List, Optional, Union + +from ovos_spec_tools.message import DEFAULT_SESSION_ID, _freeze + +__all__ = [ + "Session", + "SessionManager", + "MalformedSession", + "DEFAULT_SESSION_ID", + "DEFAULT_CONVERSE_HANDLERS_CAP", + "SESSION1_OWNED_FIELDS", + "SESSION1_REGISTERED_FIELDS", +] + + +_log = logging.getLogger(__name__) + + +#: OVOS-CONVERSE-1 §2.1 default cap for the converse-handler recency +#: stack. A deployment MAY raise, lower, or set it to "unbounded" +#: (a value ``<= 0``). The cap is **not** session state: it is a +#: deployment value the orchestrator applies at insertion time, passed +#: as the ``cap`` argument to :meth:`Session.add_converse_handler`. This +#: constant is the spec's documented §2.1 default for that argument. +DEFAULT_CONVERSE_HANDLERS_CAP = 64 + + +#: The field names whose semantics OVOS-SESSION-1 itself owns +#: (§3.1 ``session_id`` + §3.2 language signals + §3.3 ``site_id``). +SESSION1_OWNED_FIELDS = frozenset({ + "session_id", "site_id", "lang", + "secondary_langs", "output_lang", + "stt_lang", "request_lang", "detected_lang", +}) + + +#: The six OVOS-TRANSFORM-1 §5 transformer-chain fields, each an ordered +#: array of plugin ids. +_TRANSFORMER_FIELDS = ( + "audio_transformers", "utterance_transformers", + "metadata_transformers", "intent_transformers", + "dialog_transformers", "tts_transformers", +) + +#: The list-valued denylist / override fields claimed by other specs +#: (OVOS-PIPELINE-1 §5, OVOS-TRANSFORM-1 §5.2, OVOS-FALLBACK-1 §4). All are +#: arrays of string for which an empty array is wire-equivalent to omission +#: (§3.4). ``fallback_handlers`` is registered here per the SESSION-1 §3 +#: field table (owner OVOS-FALLBACK-1 §4); see the constructor docstring. +_LIST_OVERRIDE_FIELDS = ( + "pipeline", + "blacklisted_skills", "blacklisted_intents", "blacklisted_pipelines", + "blacklisted_audio_transformers", "blacklisted_utterance_transformers", + "blacklisted_metadata_transformers", "blacklisted_intent_transformers", + "blacklisted_dialog_transformers", "blacklisted_tts_transformers", + "fallback_handlers", +) + _TRANSFORMER_FIELDS + +#: The object-valued override field (OVOS-CONTEXT-1 §2 ``intent_context``). +_OBJECT_OVERRIDE_FIELDS = ("intent_context",) + +#: The handler-list / response-mode fields whose recency / cap / prune +#: behaviour this module reproduces (OVOS-PIPELINE-1 §7.1, +#: OVOS-CONVERSE-1 §2.1 / §2.2). +_HANDLER_FIELDS = ("active_handlers", "converse_handlers", "response_mode") + +#: String fields claimed by other specs. ``persona_id`` is registered by +#: OVOS-PERSONA-1 and recognized here per the OVOS-SESSION-1 §2.2 field +#: registry; OVOS-SESSION-1 carries it opaquely (its semantics are owned +#: by OVOS-PERSONA-1). An empty / unset value is wire-equivalent to +#: omission (§2.1). +_STRING_OVERRIDE_FIELDS = ("persona_id",) + + +#: The full closed set of fields OVOS-SESSION-1 §3 recognizes in this +#: version. A consumer that recognizes any of these interprets it per +#: its owner specification; everything else is unknown-field passthrough +#: (§2.4) carried opaquely in :attr:`Session.extras`. +SESSION1_REGISTERED_FIELDS = frozenset(SESSION1_OWNED_FIELDS).union( + _LIST_OVERRIDE_FIELDS, _OBJECT_OVERRIDE_FIELDS, _HANDLER_FIELDS, + _STRING_OVERRIDE_FIELDS) + + +class MalformedSession(ValueError): + """A serialized session payload violates OVOS-SESSION-1 §2 / §5. + + Raised only for structural failures that the spec calls a hard + error: the carrier is not a JSON object, or the JSON is unparsable. + Per §2 an explicit ``null`` on a field is **not** a Message-level + rejection — it is logged and treated as omitted; see + :meth:`Session.from_dict`. + """ + + +def _is_bcp47(value: Any) -> bool: + """Cheap shape check: a BCP-47 tag is a non-empty string with no + whitespace. The full grammar is owned by language tools; this + module rejects only obvious type errors.""" + return isinstance(value, str) and bool(value) and not any( + c.isspace() for c in value) + + +class Session: + """OVOS-SESSION-1 carrier — the canonical reference implementation. + + Every field is **optional** on the wire (§2). The constructor and + :meth:`to_dict` honour the omissible-but-never-nullable rule: a + field whose value is ``None`` / empty is **absent** from the + serialized object, never emitted as ``null`` (§2.1, §3.4). Unknown + fields supplied via ``extras`` round-trip unchanged (§2.4). + + The Python-level default for ``session_id`` is ``None`` (omitted on + the wire). :meth:`resolved_session_id` returns the value a consumer + would fill in — ``"default"`` when omitted (§3.1, §2.1). + + :param session_id: §3.1 — opaque identity string. ``None`` ⇒ omitted + ⇒ resolves to ``"default"`` at consumption. + :param lang: §3.2.1 — user's preferred language (BCP-47). + :param secondary_langs: §3.2.2 — additional BCP-47 tags, ordered. + Empty list and ``None`` are equivalent on the wire (both omitted). + :param output_lang: §3.2.3 — preferred response language (BCP-47). + :param stt_lang: §3.2.4 — language STT actually transcribed in. + :param request_lang: §3.2.5 — language the emitter reported (hint). + :param detected_lang: §3.2.6 — language a detector classified. + :param site_id: §3.3 — opaque group / location identifier. + :param pipeline: OVOS-PIPELINE-1 §5 — ordered pipeline-plugin ids. + :param intent_context: OVOS-CONTEXT-1 §2 — the declarative intent + context object (opaque to this module). + :param blacklisted_skills: OVOS-PIPELINE-1 §5 — skill ids to ignore. + :param blacklisted_intents: OVOS-PIPELINE-1 §5 — intent ids to ignore. + :param blacklisted_pipelines: OVOS-PIPELINE-1 §5 — pipeline ids to skip. + :param audio_transformers ... tts_transformers: OVOS-TRANSFORM-1 §5 — + the six ordered transformer chains. + :param blacklisted_*_transformers: OVOS-TRANSFORM-1 §5.2 — per-chain + denylists. + :param fallback_handlers: OVOS-FALLBACK-1 §4 — an ordered array of + skill-id strings registered as a session field by the SESSION-1 §3 + field table (owner OVOS-FALLBACK-1 §4). Carried opaquely here; its + semantics are owned by OVOS-FALLBACK-1. Empty list and ``None`` are + wire-equivalent (both omitted, §3.4 / §2.1). Like + ``converse_handlers``, OVOS-FALLBACK-1 is a **forward reference** + (it cites this field but is not yet merged); the field is carried + regardless, on the strength of the SESSION-1 §3 registration. + :param active_handlers: OVOS-PIPELINE-1 §7.1 dispatch-recency record — + a head-first, deduplicated list of ``{skill_id, activated_at}``. + :param converse_handlers: OVOS-CONVERSE-1 §2.1 converse-eligibility + list — head-first, deduplicated, tail-dropped at the + orchestrator-supplied cap. The cap itself is **not** a session + field (not serialized, not in :data:`SESSION1_REGISTERED_FIELDS`); + it is a deployment value the orchestrator applies at insertion + time via the ``cap`` argument to :meth:`add_converse_handler`. + :param response_mode: OVOS-CONVERSE-1 §2.2 pending-response window — + a single ``{skill_id, expires_at}`` object, or ``None``. + :param persona_id: OVOS-PERSONA-1 — opaque identifier of the persona + bound to this session. Registered as a session field by + OVOS-PERSONA-1 (recognized here per the OVOS-SESSION-1 §2.2 field + registry); ``None`` ⇒ omitted on the wire (§2.1). + :param extras: passthrough mapping for fields claimed by future + specifications (anything outside :data:`SESSION1_REGISTERED_FIELDS`). + Treated opaquely per §2.4. + """ + + def __init__(self, + session_id: Optional[str] = None, + lang: Optional[str] = None, + secondary_langs: Optional[List[str]] = None, + output_lang: Optional[str] = None, + stt_lang: Optional[str] = None, + request_lang: Optional[str] = None, + detected_lang: Optional[str] = None, + site_id: Optional[str] = None, + pipeline: Optional[List[str]] = None, + intent_context: Optional[Dict[str, Any]] = None, + blacklisted_skills: Optional[List[str]] = None, + blacklisted_intents: Optional[List[str]] = None, + blacklisted_pipelines: Optional[List[str]] = None, + audio_transformers: Optional[List[str]] = None, + utterance_transformers: Optional[List[str]] = None, + metadata_transformers: Optional[List[str]] = None, + intent_transformers: Optional[List[str]] = None, + dialog_transformers: Optional[List[str]] = None, + tts_transformers: Optional[List[str]] = None, + blacklisted_audio_transformers: Optional[List[str]] = None, + blacklisted_utterance_transformers: Optional[List[str]] = None, + blacklisted_metadata_transformers: Optional[List[str]] = None, + blacklisted_intent_transformers: Optional[List[str]] = None, + blacklisted_dialog_transformers: Optional[List[str]] = None, + blacklisted_tts_transformers: Optional[List[str]] = None, + fallback_handlers: Optional[List[str]] = None, + active_handlers: Optional[List[Dict[str, Any]]] = None, + converse_handlers: Optional[List[Dict[str, Any]]] = None, + response_mode: Optional[Dict[str, Any]] = None, + persona_id: Optional[str] = None, + extras: Optional[Dict[str, Any]] = None): + if session_id is not None and (not isinstance(session_id, str) + or not session_id): + # §6 producer: non-empty string when set. + raise MalformedSession( + "session_id must be a non-empty string when set (§6)") + if site_id is not None and (not isinstance(site_id, str) + or not site_id): + raise MalformedSession( + "site_id must be a non-empty string when set (§3.3)") + if persona_id is not None and (not isinstance(persona_id, str) + or not persona_id): + # OVOS-PERSONA-1 registered field: non-empty string when set. + raise MalformedSession( + "persona_id must be a non-empty string when set " + "(OVOS-PERSONA-1)") + for name, value in (("lang", lang), ("output_lang", output_lang), + ("stt_lang", stt_lang), + ("request_lang", request_lang), + ("detected_lang", detected_lang)): + if value is not None and not _is_bcp47(value): + raise MalformedSession( + f"{name} must be a non-empty BCP-47 string (§3.2)") + if secondary_langs is not None: + if not isinstance(secondary_langs, list): + raise MalformedSession( + "secondary_langs must be an array (§3.2.2)") + seen = set() + for tag in secondary_langs: + if not _is_bcp47(tag): + raise MalformedSession( + "secondary_langs entries must be BCP-47 strings " + "(§3.2.2)") + if tag in seen: + raise MalformedSession( + "secondary_langs MUST NOT contain duplicates " + "(§3.2.2)") + seen.add(tag) + # §3.2.2: MUST NOT contain `lang`. + if lang is not None and lang in seen: + raise MalformedSession( + "secondary_langs MUST NOT contain `lang` (§3.2.2)") + if extras is not None and not isinstance(extras, dict): + raise MalformedSession("extras must be a dict") + + # --- §3.1 / §3.2 / §3.3 owned scalars and lists --------------------- + self.session_id = session_id + self.lang = lang + self.secondary_langs = list(secondary_langs) if secondary_langs else None + self.output_lang = output_lang + self.stt_lang = stt_lang + self.request_lang = request_lang + self.detected_lang = detected_lang + self.site_id = site_id + + # --- other-spec list/object override fields (carried opaquely) ------ + self.pipeline = list(pipeline) if pipeline else None + self.intent_context = dict(intent_context) if intent_context else None + # OVOS-PERSONA-1 registered scalar (carried opaquely) + self.persona_id = persona_id + self.blacklisted_skills = self._as_str_list(blacklisted_skills) + self.blacklisted_intents = self._as_str_list(blacklisted_intents) + self.blacklisted_pipelines = self._as_str_list(blacklisted_pipelines) + self.audio_transformers = self._as_str_list(audio_transformers) + self.utterance_transformers = self._as_str_list(utterance_transformers) + self.metadata_transformers = self._as_str_list(metadata_transformers) + self.intent_transformers = self._as_str_list(intent_transformers) + self.dialog_transformers = self._as_str_list(dialog_transformers) + self.tts_transformers = self._as_str_list(tts_transformers) + self.blacklisted_audio_transformers = self._as_str_list( + blacklisted_audio_transformers) + self.blacklisted_utterance_transformers = self._as_str_list( + blacklisted_utterance_transformers) + self.blacklisted_metadata_transformers = self._as_str_list( + blacklisted_metadata_transformers) + self.blacklisted_intent_transformers = self._as_str_list( + blacklisted_intent_transformers) + self.blacklisted_dialog_transformers = self._as_str_list( + blacklisted_dialog_transformers) + self.blacklisted_tts_transformers = self._as_str_list( + blacklisted_tts_transformers) + # OVOS-FALLBACK-1 §4 registered array-of-string (carried opaquely) + self.fallback_handlers = self._as_str_list(fallback_handlers) + + # --- PIPELINE-1 §7.1 / CONVERSE-1 §2.1 / §2.2 handler state --------- + # The §2.1 converse-handler cap is NOT session state: a constructed + # or deserialized session is never capped on load. The cap is a + # deployment value the orchestrator supplies at insertion time + # (see :meth:`add_converse_handler`). + self.active_handlers: List[Dict[str, Any]] = self._coerce_handlers( + active_handlers) + self.converse_handlers: List[Dict[str, Any]] = self._coerce_handlers( + converse_handlers) + self.response_mode: Optional[Dict[str, Any]] = self._coerce_response_mode( + response_mode) + + # --- §2.4 unknown-field passthrough -------------------------------- + self.extras: Dict[str, Any] = dict(extras) if extras else {} + + # --- helpers ------------------------------------------------------------ + + @staticmethod + def _as_str_list(value: Optional[List[str]]) -> Optional[List[str]]: + """Normalize a list-valued override into a copied list, or + ``None`` when empty / unset. An empty array is wire-equivalent + to omission (§3.4), so both collapse to ``None``.""" + if not value: + return None + if not isinstance(value, list): + raise MalformedSession("expected an array of string (§3)") + return list(value) + + # --- §3.1 reserved-default resolution ----------------------------------- + + @property + def is_default(self) -> bool: + """``True`` when this session resolves to the reserved + ``"default"`` identity (§3.1): an omitted, empty, or explicit + ``"default"`` ``session_id`` all map here.""" + return self.session_id in (None, DEFAULT_SESSION_ID) + + def resolved_session_id(self) -> str: + """The ``session_id`` a consumer would key per-session state on + — ``"default"`` when omitted (§3.1 + §2.1).""" + return self.session_id or DEFAULT_SESSION_ID + + # ------------------------------------------------------------------ + # OVOS-PIPELINE-1 §7.1 / OVOS-CONVERSE-1 §2.1 / §2.2 handler helpers + # ------------------------------------------------------------------ + @staticmethod + def _coerce_handlers( + handlers: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: + """Normalize a handler list into the spec ``{skill_id, + activated_at}`` object shape (PIPELINE-1 §7.1 / CONVERSE-1 §2.1). + + Accepts either the spec object shape (list of dicts) or the + legacy ``[skill_id, activated_at]`` pair shape (tolerated on + deserialization). Entries are deduplicated by ``skill_id`` + (head wins) and kept head-first by recency.""" + out: List[Dict[str, Any]] = [] + seen = set() + for entry in handlers or []: + if isinstance(entry, dict): + skill_id = entry.get("skill_id") + activated_at = entry.get("activated_at", time.time()) + elif isinstance(entry, (list, tuple)) and entry: + skill_id = entry[0] + activated_at = entry[1] if len(entry) > 1 else time.time() + else: + continue + if not skill_id or skill_id in seen: + continue + seen.add(skill_id) + out.append({"skill_id": skill_id, "activated_at": activated_at}) + return out + + @staticmethod + def _coerce_response_mode( + response_mode: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Normalize a ``response_mode`` value into the spec + ``{skill_id, expires_at}`` shape, or ``None`` when there is no + holder. Malformed values resolve to ``None`` (SESSION-1 §2.1, + CONVERSE-1 §2.2).""" + if not isinstance(response_mode, dict): + return None + skill_id = response_mode.get("skill_id") + if not skill_id: + return None + return {"skill_id": skill_id, + "expires_at": response_mode.get("expires_at", -1)} + + @staticmethod + def _promote_handler(handlers: List[Dict[str, Any]], skill_id: str, + activated_at: Optional[float] = None + ) -> List[Dict[str, Any]]: + """Dedup-and-promote ``skill_id`` to the head of ``handlers`` + (in place). + + Removes any existing entry with the same ``skill_id`` then + inserts a fresh ``{skill_id, activated_at}`` at index 0 — the + recency-stack rule shared by OVOS-PIPELINE-1 §7.1 and + OVOS-CONVERSE-1 §3.1.""" + if activated_at is None: + activated_at = time.time() + handlers[:] = [h for h in handlers if h.get("skill_id") != skill_id] + handlers.insert(0, {"skill_id": skill_id, "activated_at": activated_at}) + return handlers + + @staticmethod + def _cap_handlers(handlers: List[Dict[str, Any]], + cap: Optional[int]) -> List[Dict[str, Any]]: + """Tail-drop ``handlers`` down to ``cap`` entries (in place). + + ``cap`` is the orchestrator-supplied per-insertion limit + (OVOS-CONVERSE-1 §2.1), not session state. A ``cap`` of ``None`` + or ``<= 0`` means "unbounded". The least-recent surviving owners + (the tail) are dropped.""" + if cap and cap > 0 and len(handlers) > cap: + del handlers[cap:] + return handlers + + @property + def active(self) -> bool: + """``True`` when any handler is active in ``active_handlers`` + (OVOS-PIPELINE-1 §7.1).""" + return len(self.active_handlers) > 0 + + # active_handlers (OVOS-PIPELINE-1 §7.1) + def add_active_handler(self, skill_id: str, + activated_at: Optional[float] = None): + """Push a handler onto ``active_handlers``, dedup-and-promoting + it to the head (OVOS-PIPELINE-1 §7.1). The list is head-first by + recency; any prior entry with the same ``skill_id`` is evicted.""" + self._promote_handler(self.active_handlers, skill_id, activated_at) + + def remove_active_handler(self, skill_id: str): + """Remove ``skill_id`` from ``active_handlers`` (e.g. a STOP-1 + drain).""" + self.active_handlers[:] = [h for h in self.active_handlers + if h.get("skill_id") != skill_id] + + # converse_handlers (OVOS-CONVERSE-1 §2.1 / §3.1) + def add_converse_handler(self, skill_id: str, + activated_at: Optional[float] = None, + cap: Optional[int] = DEFAULT_CONVERSE_HANDLERS_CAP): + """Stamp a handler onto ``converse_handlers``: dedup-promote to + head, then tail-drop at ``cap`` (OVOS-CONVERSE-1 §2.1 / §3.1). + + ``cap`` is the orchestrator-supplied per-insertion limit — a + deployment value applied here, never stored on the session. It + defaults to the spec's documented §2.1 default + (:data:`DEFAULT_CONVERSE_HANDLERS_CAP`); ``None`` or ``<= 0`` + means "unbounded".""" + self._promote_handler(self.converse_handlers, skill_id, activated_at) + self._cap_handlers(self.converse_handlers, cap) + + def remove_converse_handler(self, skill_id: str): + """Remove ``skill_id`` from ``converse_handlers``.""" + self.converse_handlers[:] = [h for h in self.converse_handlers + if h.get("skill_id") != skill_id] + + def prune_converse_handlers(self, ttl: float, + now: Optional[float] = None): + """Drop ``converse_handlers`` entries older than ``ttl`` seconds + (OVOS-CONVERSE-1 §3.2). + + ``now - activated_at > ttl`` is dropped. A non-positive ``ttl`` + disables time-based pruning. The caller (the orchestrator) + invokes this at the pre-converse and pre-list-emission + boundaries.""" + if not ttl or ttl <= 0: + return + now = now if now is not None else time.time() + self.converse_handlers[:] = [ + h for h in self.converse_handlers + if now - h.get("activated_at", now) <= ttl + ] + + # response_mode (OVOS-CONVERSE-1 §2.2) — single-holder + def set_response_mode(self, skill_id: str, expires_at: float): + """Set the single-holder response window (OVOS-CONVERSE-1 §2.2). + + Overwrites any existing holder silently (single-holder + invariant).""" + self.response_mode = {"skill_id": skill_id, "expires_at": expires_at} + + def clear_response_mode(self, skill_id: Optional[str] = None): + """Clear the response window. + + When ``skill_id`` is given, clears it only if that skill + currently holds the window (a skill MUST NOT clear another's + hold); otherwise clears unconditionally.""" + if self.response_mode is None: + return + if skill_id is None or self.response_mode.get("skill_id") == skill_id: + self.response_mode = None + + # --- §2 / §5 wire shape ------------------------------------------------- + + def to_dict(self) -> Dict[str, Any]: + """Render the session as a JSON-friendly dict per §2 + §5. + + Fields whose Python value is ``None`` / empty are **omitted**, + never emitted as ``null`` (§2.1). An empty list-valued override + is wire-equivalent to omission and is dropped (§3.4). ``extras`` + are merged in last and round-trip unchanged (§2.4).""" + out: Dict[str, Any] = {} + + # §3.1 / §3.2 / §3.3 owned scalars + if self.session_id is not None: + out["session_id"] = self.session_id + if self.lang is not None: + out["lang"] = self.lang + if self.secondary_langs: + out["secondary_langs"] = list(self.secondary_langs) + if self.output_lang is not None: + out["output_lang"] = self.output_lang + if self.stt_lang is not None: + out["stt_lang"] = self.stt_lang + if self.request_lang is not None: + out["request_lang"] = self.request_lang + if self.detected_lang is not None: + out["detected_lang"] = self.detected_lang + if self.site_id is not None: + out["site_id"] = self.site_id + + # other-spec list/object override fields (omit-when-empty, §3.4) + for name in _LIST_OVERRIDE_FIELDS: + value = getattr(self, name) + if value: + out[name] = list(value) + for name in _OBJECT_OVERRIDE_FIELDS: + value = getattr(self, name) + if value: + out[name] = deepcopy(value) + + # OVOS-PERSONA-1 registered scalar(s) (omit-when-empty, §2.1) + for name in _STRING_OVERRIDE_FIELDS: + value = getattr(self, name) + if value is not None: + out[name] = value + + # PIPELINE-1 §7.1 / CONVERSE-1 §2.1 / §2.2 — omit-when-empty + if self.active_handlers: + out["active_handlers"] = deepcopy(self.active_handlers) + if self.converse_handlers: + out["converse_handlers"] = deepcopy(self.converse_handlers) + if self.response_mode: + out["response_mode"] = dict(self.response_mode) + + # §2.4 unknown-field tolerance — passthrough. + for k, v in self.extras.items(): + if k in out: + # An owned field set both ways is a programming error; + # owned wins (single source of truth). + continue + out[k] = deepcopy(v) + return out + + def serialize(self) -> str: + """Emit a JSON string per §5 (no NaN, UTF-8, single object).""" + return json.dumps(self.to_dict(), ensure_ascii=False, + allow_nan=False) + + # --- construction from the wire ----------------------------------------- + + @classmethod + def from_dict(cls, payload: Optional[Dict[str, Any]]) -> "Session": + """Construct a Session from a JSON-decoded dict per §2. + + Tolerant of the §2.1 deferral surface: an explicit ``null`` on + any registered field is logged and treated as omitted, not as a + rejection. Unknown fields are preserved verbatim in + :attr:`extras` (§2.4). Passing ``None`` (no ``session`` key in + ``context``) yields the same well-formed empty session as ``{}`` + (§2.1).""" + if payload is None: + return cls() + if not isinstance(payload, dict): + raise MalformedSession( + "session must be a JSON object (§2 / §5)") + + kwargs: Dict[str, Any] = {} + extras: Dict[str, Any] = {} + for key, value in payload.items(): + if key not in SESSION1_REGISTERED_FIELDS: + extras[key] = value + continue + if value is None: + # §2.1: explicit null is malformed — log, treat as omitted. + _log.warning( + "OVOS-SESSION-1 §2.1: explicit null on `%s` is " + "malformed; treating as omitted", key) + continue + kwargs[key] = value + if extras: + kwargs["extras"] = extras + return cls(**kwargs) + + @classmethod + def deserialize(cls, + payload: Union[str, bytes, bytearray, + Dict[str, Any], None]) -> "Session": + """Parse a serialized session per §5 and §2. + + Accepts bytes/str JSON, an already-parsed dict, or ``None``. + Raises :class:`MalformedSession` only for the structural + failures §5 calls hard errors (unparsable JSON, non-object + root). Field-level malformedness (an explicit ``null``) is + absorbed by :meth:`from_dict`.""" + if payload is None: + return cls() + if isinstance(payload, (bytes, bytearray)): + payload = payload.decode("utf-8") + if isinstance(payload, str): + try: + payload = json.loads(payload) + except json.JSONDecodeError as exc: + raise MalformedSession( + f"session payload is not valid JSON: {exc}") from exc + return cls.from_dict(payload) + + # --- §4 propagation ----------------------------------------------------- + + def propagate(self) -> "Session": + """Return a deep copy suitable for attaching to a derived + Message (§4). Every field — known and unknown — rides along + unchanged.""" + return Session.from_dict(self.to_dict()) + + @classmethod + def materialize_default(cls) -> "Session": + """Materialize the default session per §4.1. + + Sets ``session_id`` to ``"default"`` and leaves every other + field omitted; the §4.1 rule forbids populating per-component + overrides on a materialized default.""" + return cls(session_id=DEFAULT_SESSION_ID) + + def update_from(self, other: "Session") -> "Session": + """Fold ``other``'s state onto this session in place, per §2 semantics. + + Applied with full OVOS-SESSION-1 §2 deserialization rather than a raw + ``__dict__`` merge: the incoming state is round-tripped through + :meth:`serialize` / :meth:`deserialize`, so a key present on the wire + overrides this session's value (even when empty) and a null / omitted + key resolves to the spec default — exactly as a freshly received message + would parse. Round-tripping also rebuilds nested state, so the live + object never aliases ``other``'s mutable sub-objects. + + ``deserialize`` is resolved through ``type(self)`` so a subclass (e.g. + the ovos-bus-client ``Session`` with its legacy projections) rebuilds as + itself. The ``session_id`` is preserved — it is the registry key under + the singleton :class:`SessionManager` and must not drift even if + ``other`` carries a different one. + """ + if other is self: + return self + rebuilt = type(self).deserialize(other.serialize()) + rebuilt.session_id = self.session_id + self.__dict__ = rebuilt.__dict__ + return self + + # --- value semantics ---------------------------------------------------- + + def __eq__(self, other: object) -> bool: + return (isinstance(other, Session) + and self.to_dict() == other.to_dict()) + + def __hash__(self) -> int: + # Hash over the same canonical ``to_dict()`` view ``__eq__`` + # compares, frozen into a deterministic hashable form so equal + # Sessions always hash equal (the hash/eq contract). NOTE: a + # Session is mutable, so this is a *point-in-time snapshot* — safe + # for ``functools.lru_cache`` keys and short-lived set/dict + # membership, but do not mutate a Session while it is live as a + # dict key. + return hash(_freeze(self.to_dict())) + + def __repr__(self) -> str: + return f"Session({self.to_dict()!r})" + + +class SessionManager: + """Process-wide registry of live :class:`Session` objects — one per id. + + .. note:: + ``SessionManager`` is an **implementation detail, not part of any + spec**. OVOS-SESSION-1 only mandates the *wire contract* (§4): every + Message carries a serialized session snapshot, and a derived Message + (``forward`` / ``reply`` / ``response``) carries the session of the + Message it was derived from. ``SessionManager`` is one powerful *way* + to honour that contract inside a single process; nothing on the wire + depends on it. + + Why a singleton per id + ---------------------- + The bus is value-passing: state travels as the serialized ``session`` on + each Message, and any component can hold a reference to "the session" while + a flow runs. If every ``get()`` rebuilt a fresh ``Session`` from the + incoming snapshot, a reference taken early would silently go stale the + moment a later message flipped a field. Keeping exactly **one live object + per ``session_id``** — and *folding* each incoming snapshot onto it + (:meth:`update` / :meth:`get`, last-writer-wins per §2) — means a held + reference always observes the latest state. That is the whole benefit. + + Why stamping ``forward`` / ``reply`` is always correct (the key insight) + ----------------------------------------------------------------------- + ``Message.forward`` / ``Message.reply`` deep-copy the *originating* + message's context, **including its ``session`` snapshot** (§5.1/§5.2 carry + the session over). But that snapshot was frozen when the originating + message was built — possibly *before* the current handler mutated the + session. So at derivation time the registry re-stamps the new Message with + the live session for its id (:meth:`sync_message_session`). This is sound, + by exhaustive cases: + + 1. **An intervening message updated this session** between handler-start + and the ``forward``. SESSION-1 §4 says that message carried the new + session, so a correct receiver folded it onto the singleton — the live + session *is* the update. Stamping propagates it. (Failing to propagate + it — shipping the stale pre-mutation copy — would be the implementation + bug; the spec predicts the update exists and must not be discarded.) + 2. **No message arrived in that window.** The singleton equals what the + handler started with plus its own in-place mutations (which go through + the singleton). Stamping is then a **no-op** with respect to intent — + it re-serializes the same state. + + Either way the derived Message leaves with the freshest session for its id; + the stamp is *either a meaningful refresh or a no-op, never a discard*. + This is what lets downstream consumers trust that the ``session`` on any + message they receive is current, without re-querying anything. + + Canonical example + ----------------- + The universal handler shape — read, mutate, derive, emit:: + + sess = SessionManager.get(message) # fold incoming snapshot, get the singleton + sess.activate_skill("my.skill") # mutate the one live object for this id + reply = message.forward("my.skill.activate") + # forward() deep-copied message.context["session"] — the PRE-activation + # snapshot. sync_message_session re-stamps reply with the live (now + # skill-active) session, so the activation rides out on the wire. + bus.emit(reply) # consumers fold the fresh session back + + Without the stamp, ``reply`` would carry a session with ``my.skill`` *not* + active; a consumer folding it would regress the singleton — the exact + desync this guards against. + + Pluggability + ------------ + ``session_cls`` lets a downstream layer (e.g. ovos-bus-client, whose + ``Session`` adds legacy projections) point the registry at a subclass so + folds/stamps build the richer object. Construction goes through + ``deserialize`` to stay agnostic of subclass ``__init__`` signatures. + """ + + #: Session class the registry builds; override downstream to a subclass. + session_cls = Session + #: the singleton for the reserved "default" id (lazily materialized). + default_session: Optional["Session"] = None + #: id -> the one live Session object for that id. + sessions: Dict[str, "Session"] = {} + _lock = Lock() + + @classmethod + def _wire_dict(cls, sess: "Session") -> Dict[str, Any]: + """Return ``sess`` as a wire dict. + + ``Session.serialize`` returns a JSON string here, but a subclass + (ovos-bus-client) overrides it to return a dict carrying extra legacy + projection keys. Normalise both to a dict so the richer shape, when + present, survives onto ``message.context["session"]``. + """ + data = sess.serialize() + if isinstance(data, (str, bytes, bytearray)): + data = json.loads(data) + return data + + @classmethod + def get_default_session(cls) -> "Session": + """Return (materializing once) the singleton for the reserved default id. + + The shared ``sessions`` dict is the single source of truth — keyed off + ``sessions[DEFAULT_SESSION_ID]``, never a per-class ``default_session`` + mirror. This matters when a subclass (ovos-bus-client) and this base + both reach the registry: a class-attribute mirror would shadow per class + and diverge, whereas the dict is one shared object. ``default_session`` + is kept as a convenience mirror but is always re-synced from the dict. + """ + sess = cls.sessions.get(DEFAULT_SESSION_ID) + if sess is None: + sess = cls.session_cls.deserialize({"session_id": DEFAULT_SESSION_ID}) + cls.sessions[DEFAULT_SESSION_ID] = sess + cls.default_session = sess + return sess + + @classmethod + def _store(cls, sess: "Session") -> "Session": + """Register ``sess`` and return the one live object for its id. + + When the id is already known the incoming snapshot is *folded* onto the + existing instance (:meth:`Session.update_from`, §2 last-writer-wins) and + that object — never the snapshot — is returned, so object identity per + id is stable for the process lifetime. + """ + with cls._lock: + existing = cls.sessions.get(sess.session_id) + if existing is not None and existing is not sess: + existing.update_from(sess) + sess = existing + cls.sessions[sess.session_id] = sess + if sess.session_id == DEFAULT_SESSION_ID: + cls.default_session = sess + return sess + + @classmethod + def update(cls, sess: "Session") -> "Session": + """Fold ``sess`` onto the singleton for its id; return the live object.""" + if not sess: + raise ValueError("Expected Session and got None") + return cls._store(sess) + + @classmethod + def get(cls, message: Optional["object"] = None) -> "Session": + """Return the live singleton for ``message``'s session (folding its snapshot). + + The inbound half of the contract: a received message's ``session`` + snapshot is folded onto the singleton for its id and that live object is + returned. With no message (or no session on it) the default session is + returned. The default session is a normal session per §4 — it folds like + any other (there is no owner-only reservation; the wire is value-passing). + """ + if message is None: + return cls.get_default_session() + ctx = getattr(message, "context", None) or {} + snap = ctx.get("session") + if snap is None: + return cls.get_default_session() + msg_sess = cls.session_cls.deserialize(snap) + if not msg_sess.session_id: + # a session naming no id IS the default (§4.1/§4.3) + msg_sess.session_id = DEFAULT_SESSION_ID + return cls._store(msg_sess) + + @classmethod + def sync_message_session(cls, message: "object", + default_session_id: str = DEFAULT_SESSION_ID + ) -> "object": + """Stamp an outgoing/derived message with the live session for its own id. + + The outbound half of the contract (see the class docstring for *why* + this is always either a refresh or a no-op). Keyed off the message's + **own** ``session_id`` so a derived message addressed to a different + session is stamped from *that* session, not the caller's. A session id + this process does not hold is left untouched — we never overwrite a + session we never folded (e.g. a relay forwarding a remote one). A + message with no session at all gets the default-session stamp. + """ + ctx = message.context + snap = ctx.get("session") + if snap is None: + # §4.3: a Message without a session implies the default session — + # stamp it (also the emit inject-when-missing path). + ctx["session"] = cls._wire_dict(cls.get_default_session()) + return message + if isinstance(snap, dict): + # §4.1/§4.3: a session that names no id IS the default session, so + # resolve an absent session_id to the reserved default id and stamp + # it. A NAMED id this process never folded (e.g. a relay's remote + # session) is carried verbatim — we never overwrite one we don't own. + sid = snap.get("session_id") or DEFAULT_SESSION_ID + if sid == DEFAULT_SESSION_ID: + ctx["session"] = cls._wire_dict(cls.get_default_session()) + else: + live = cls.sessions.get(sid) + if live is not None: + ctx["session"] = cls._wire_dict(live) + return message + + @classmethod + def reset_default_session(cls) -> "Session": + """Replace the default session with a fresh empty one and return it.""" + with cls._lock: + sess = cls.session_cls.deserialize({"session_id": DEFAULT_SESSION_ID}) + cls.default_session = cls.sessions[DEFAULT_SESSION_ID] = sess + return sess diff --git a/ovos_intent_primitives/version.py b/ovos_spec_tools/version.py similarity index 76% rename from ovos_intent_primitives/version.py rename to ovos_spec_tools/version.py index 85b4624a..ee3daae7 100644 --- a/ovos_intent_primitives/version.py +++ b/ovos_spec_tools/version.py @@ -1,7 +1,7 @@ # START_VERSION_BLOCK -VERSION_MAJOR = 0 -VERSION_MINOR = 0 -VERSION_BUILD = 1 +VERSION_MAJOR = 1 +VERSION_MINOR = 2 +VERSION_BUILD = 2 VERSION_ALPHA = 1 # END_VERSION_BLOCK diff --git a/pyproject.toml b/pyproject.toml index b06cc6f8..101172ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,28 +3,44 @@ requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] -name = "ovos-intent-primitives" +name = "ovos-spec-tools" dynamic = ["version"] -description = "Reference implementation of the OVOS intent specifications" +description = "Reference implementation of the OVOS formal specifications" readme = "README.md" license = { text = "Apache-2.0" } authors = [{ name = "jarbasai", email = "jarbasai@mailfence.com" }] -requires-python = ">=3.8" -dependencies = [ - "langcodes", +requires-python = ">=3.10" +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] +dependencies = [] [project.optional-dependencies] +# `langcodes` powers the smart language fallback of LocaleResources; without +# it, language resolution is exact-match only. +langcodes = [ + "langcodes", +] test = [ "pytest", + "langcodes", ] +[project.scripts] +ovos-spec-lint = "ovos_spec_tools.lint:main" + [project.urls] -Homepage = "https://github.com/OpenVoiceOS/ovos-intent-primitives" +Homepage = "https://github.com/OpenVoiceOS/ovos-spec-tools" [tool.setuptools.dynamic] -version = { attr = "ovos_intent_primitives.version.__version__" } +version = { attr = "ovos_spec_tools.version.__version__" } [tool.setuptools.packages.find] where = ["."] -include = ["ovos_intent_primitives*"] +include = ["ovos_spec_tools*"] diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..5db72dd6 --- /dev/null +++ b/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ] +} diff --git a/test/test_dialog.py b/test/test_dialog.py new file mode 100644 index 00000000..72e55bfa --- /dev/null +++ b/test/test_dialog.py @@ -0,0 +1,245 @@ +"""Conformance tests for the OVOS-INTENT-2 §4.2 reference dialog renderer.""" +import pytest + +from ovos_spec_tools import ( + DialogRenderer, + LocaleResources, + MalformedTemplate, + UnfilledSlot, + render, + verify_slot_consistency, +) + + +class _FixedRng: + """A deterministic stand-in for `random`: always picks a chosen index.""" + + def __init__(self, index=0): + self.index = index + + def choice(self, seq): + seq = list(seq) + return seq[self.index % len(seq)] + + +def _resources(tmp_path, files): + """Build a LocaleResources over a locale with the given {relpath: text}.""" + locale = tmp_path / "locale" + for relpath, text in files.items(): + path = locale / relpath + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return LocaleResources(str(locale)) + + +# --- render() — the stateless function --------------------------------------- + +def test_render_fills_slots(): + out = render(["It is {temperature} degrees."], + slots={"temperature": 21}) + assert out == "It is 21 degrees." + + +def test_render_fills_double_brace_slots(): + """OVOS-INTENT-1 §3.4 — ``{{name}}`` is an equivalent slot spelling.""" + out = render(["It is {{temperature}} degrees."], + slots={"temperature": 21}) + assert out == "It is 21 degrees." + + +def test_render_mixed_brace_spellings_share_values(): + out = render(["{{a}} and {b}"], slots={"a": "X", "b": "Y"}) + assert out == "X and Y" + + +def test_render_selects_a_phrase(): + phrases = ["first phrase", "second phrase"] + assert render(phrases, rng=_FixedRng(0)) == "first phrase" + assert render(phrases, rng=_FixedRng(1)) == "second phrase" + + +def test_render_expands_variety_then_fills(): + out = render(["(Currently|At the moment) it is {t} degrees"], + slots={"t": 5}, rng=_FixedRng(0)) + assert out == "Currently it is 5 degrees" + + +def test_unfilled_slot_raises(): + with pytest.raises(UnfilledSlot): + render(["It is {temperature} degrees."], slots={}) + + +def test_slot_value_is_never_parsed_as_grammar(): + """Expansion happens before fill, so a value with metacharacters is safe.""" + out = render(["you said {echo}"], slots={"echo": "(a|b)"}) + assert out == "you said (a|b)" + + +def test_render_resolves_vocabulary_references(): + out = render([" {name}"], slots={"name": "Sam"}, + vocabularies={"greet": ["hello"]}, rng=_FixedRng(0)) + assert out == "hello Sam" + + +def test_empty_phrase_list_raises(): + with pytest.raises(ValueError): + render([]) + + +# --- DialogRenderer — stateful, multilingual --------------------------------- + +def test_renderer_fills_slots(tmp_path): + res = _resources(tmp_path, {"en-US/t.dialog": "It is {temperature} degrees."}) + renderer = DialogRenderer(res, "t") + assert renderer.render("en-US", {"temperature": 21}) == "It is 21 degrees." + + +def test_renderer_serves_multiple_languages(tmp_path): + res = _resources(tmp_path, { + "en-US/hi.dialog": "Hello.", + "pt-BR/hi.dialog": "Ola.", + }) + renderer = DialogRenderer(res, "hi") + assert renderer.render("en-US") == "Hello." + assert renderer.render("pt-BR") == "Ola." + + +def test_renderer_avoids_repeating_the_last_phrase(tmp_path): + """With two phrases, consecutive renders must alternate (§4.2).""" + res = _resources(tmp_path, {"en-US/c.dialog": "alpha\nbeta\n"}) + renderer = DialogRenderer(res, "c") + first = renderer.render("en-US") + second = renderer.render("en-US") + third = renderer.render("en-US") + assert first != second + assert second != third + assert {first, second} == {"alpha", "beta"} + + +def test_repetition_avoidance_is_per_language(tmp_path): + """The 'last phrase' is tracked separately for each language.""" + res = _resources(tmp_path, { + "en-US/c.dialog": "alpha\nbeta\n", + "pt-BR/c.dialog": "um\ndois\n", + }) + renderer = DialogRenderer(res, "c") + renderer.render("en-US") + # the pt-BR render is unconstrained by the en-US history + assert renderer.render("pt-BR") in {"um", "dois"} + + +def test_renderer_unfilled_slot_raises(tmp_path): + res = _resources(tmp_path, {"en-US/s.dialog": "say {name}"}) + renderer = DialogRenderer(res, "s") + with pytest.raises(UnfilledSlot): + renderer.render("en-US") + + +def test_renderer_missing_dialog_for_language_raises(tmp_path): + res = _resources(tmp_path, {"en-US/hi.dialog": "Hello."}) + renderer = DialogRenderer(res, "hi") + with pytest.raises(FileNotFoundError): + renderer.render("de-DE") + + +# --- default slots and .entity fallback -------------------------------------- + +def test_renderer_default_slots_are_reused(tmp_path): + res = _resources(tmp_path, {"en-US/g.dialog": "Hello {name}."}) + renderer = DialogRenderer(res, "g", slots={"name": "Sam"}) + assert renderer.render("en-US") == "Hello Sam." + assert renderer.render("en-US") == "Hello Sam." + + +def test_per_call_slot_overrides_a_default(tmp_path): + res = _resources(tmp_path, {"en-US/g.dialog": "Hello {name}."}) + renderer = DialogRenderer(res, "g", slots={"name": "Sam"}) + assert renderer.render("en-US", {"name": "Max"}) == "Hello Max." + + +def test_unfilled_slot_falls_back_to_entity(tmp_path): + res = _resources(tmp_path, { + "en-US/today.dialog": "today is {day}", + "en-US/day.entity": "monday\n", + }) + renderer = DialogRenderer(res, "today") + assert renderer.render("en-US") == "today is monday" + + +def test_slot_precedence_call_then_default_then_entity(tmp_path): + res = _resources(tmp_path, { + "en-US/v.dialog": "value {x}", + "en-US/x.entity": "from_entity\n", + }) + renderer = DialogRenderer(res, "v", slots={"x": "from_default"}) + # default beats the .entity fallback + assert renderer.render("en-US") == "value from_default" + # a per-call value beats both + assert renderer.render("en-US", {"x": "from_call"}) == "value from_call" + + +# --- edge cases -------------------------------------------------------------- + +def test_renderer_with_one_phrase_repeats_it(tmp_path): + res = _resources(tmp_path, {"en-US/one.dialog": "the only phrase"}) + renderer = DialogRenderer(res, "one") + assert renderer.render("en-US") == "the only phrase" + assert renderer.render("en-US") == "the only phrase" + + +def test_render_phrase_with_no_slots(): + assert render(["a fixed phrase"]) == "a fixed phrase" + + +def test_render_slot_value_with_braces_stays_literal(): + """The filled value is inserted verbatim — never re-parsed as a slot.""" + out = render(["you said {echo}"], slots={"echo": "{not a slot}"}) + assert out == "you said {not a slot}" + + +def test_render_numeric_slot_value_is_stringified(): + assert render(["count {n}"], slots={"n": 0}) == "count 0" + + +def test_renderer_missing_dialog_raises(tmp_path): + res = _resources(tmp_path, {"en-US/x.dialog": "hello"}) + renderer = DialogRenderer(res, "absent") + with pytest.raises(FileNotFoundError): + renderer.render("en-US") + + +# --- §5.5 slot-set consistency (the §7 Dialog-renderer MUST) ----------------- + +def test_verify_slot_consistency_accepts_identical_slot_sets(): + # Optionality does not change a declared slot set (§5.5): `say [{x}]` and + # `say {x}` both declare {x}. + verify_slot_consistency(["say [{x}]", "say {x}"]) # no raise + + +def test_verify_slot_consistency_rejects_divergent_slot_sets(): + with pytest.raises(MalformedTemplate): + verify_slot_consistency(["hello {name}", "hi {name} and {title}"]) + + +def test_render_rejects_divergent_slot_sets(): + # §7: the renderer MUST verify all phrases declare the same slot set (§5.5), + # regardless of which phrase the chooser would pick. + phrases = ["hello {name}", "hi {name} and {title}"] + with pytest.raises(MalformedTemplate): + render(phrases, slots={"name": "Sam", "title": "Dr"}) + + +def test_renderer_rejects_divergent_slot_sets(tmp_path): + res = _resources(tmp_path, { + "en-US/g.dialog": "hello {name}\nhi {name} and {title}\n"}) + renderer = DialogRenderer(res, "g") + with pytest.raises(MalformedTemplate): + renderer.render("en-US", {"name": "Sam", "title": "Dr"}) + + +def test_render_consistent_dialog_still_works(tmp_path): + res = _resources(tmp_path, { + "en-US/g.dialog": "hello {name}\nhi there {name}\n"}) + renderer = DialogRenderer(res, "g") + assert renderer.render("en-US", {"name": "Sam"}) in ( + "hello Sam", "hi there Sam") diff --git a/test/test_expansion.py b/test/test_expansion.py index bf4c3770..bc403a13 100644 --- a/test/test_expansion.py +++ b/test/test_expansion.py @@ -5,7 +5,7 @@ """ import pytest -from ovos_intent_primitives import MalformedTemplate, expand +from ovos_spec_tools import MalformedTemplate, expand, inline_keywords # --- §4.2 the worked example ------------------------------------------------- @@ -76,6 +76,60 @@ def test_digit_in_slot_name(): assert expand("set channel {channel_2}") == ["set channel {channel_2}"] +# --- §3.4 dual-brace slot spelling: {name} == {{name}} ---------------------- + +def test_double_brace_slot_is_equivalent_to_single_brace(): + """§3.4 — ``{{name}}`` is an equivalent spelling of ``{name}``.""" + assert expand("(buy|sell) {{item}}") == expand("(buy|sell) {item}") + assert sorted(expand("(buy|sell) {{item}}")) == ["buy {item}", "sell {item}"] + + +def test_double_brace_folds_to_single_brace_in_samples(): + """A ``{{name}}`` slot is emitted as the canonical single-brace ``{name}``.""" + assert expand("it is currently {{temperature}} degrees") == [ + "it is currently {temperature} degrees", + ] + + +def test_double_brace_inside_groups(): + assert sorted(expand("(buy {{item}}|sell {{item}})")) == [ + "buy {item}", "sell {item}", + ] + assert sorted(expand("[really ]want {{item}}")) == [ + "really want {item}", "want {item}", + ] + + +def test_mixed_single_and_double_brace_slots(): + """Both spellings may appear in one template and mean distinct slots.""" + assert expand("move {{from}} to {to}") == ["move {from} to {to}"] + + +def test_double_brace_slot_only_template_is_malformed(): + """§3.6 — a bare slot is malformed in either spelling.""" + with pytest.raises(MalformedTemplate): + expand("{{name}}") + + +def test_double_brace_invalid_name_is_malformed(): + """§3.4 — the folded interior is name-checked exactly like a single slot.""" + for tpl in ("{{Name}} here", "{{1st}} here", "{{bad name}} here"): + with pytest.raises(MalformedTemplate): + expand(tpl) + + +def test_double_brace_repeated_slot_is_malformed(): + """§3.6 — repeating a slot is malformed across spellings too.""" + with pytest.raises(MalformedTemplate): + expand("{{x}} and {x}") + + +def test_double_brace_adjacent_slots_is_malformed(): + """§3.6 — adjacent slots remain malformed after folding.""" + with pytest.raises(MalformedTemplate): + expand("{{a}} {b}") + + # --- §3.7 inline vocabulary references -------------------------------------- VOCAB = {"greeting": ["hello", "hi", "good morning"]} @@ -177,3 +231,124 @@ def test_repeated_slot_name(): def test_invalid_names(template): with pytest.raises(MalformedTemplate): expand(template) + + +# --- edge cases -------------------------------------------------------------- + +def test_leading_trailing_and_repeated_whitespace_is_normalized(): + assert expand(" turn on the lights ") == ["turn on the lights"] + + +def test_tabs_count_as_whitespace(): + assert expand("turn\ton") == ["turn on"] + + +def test_vocabularies_argument_is_optional(): + assert expand("plain literal template") == ["plain literal template"] + + +def test_a_bare_vocabulary_reference_is_a_valid_template(): + assert sorted(expand("", {"g": ["hello", "hi"]})) == ["hello", "hi"] + + +def test_vocabulary_member_may_be_multiple_words(): + assert expand(" there", {"g": ["good morning"]}) == ["good morning there"] + + +def test_empty_vocabulary_is_malformed(): + with pytest.raises(MalformedTemplate): + expand("say ", {"x": []}) + + +def test_expand_does_not_enforce_slot_consistency(): + """expand() works one template at a time; the cross-template slot rule + (OVOS-INTENT-1 §5.5) is enforced elsewhere, not here.""" + assert sorted(expand("(buy {item}|leave)")) == ["buy {item}", "leave"] + + +def test_optional_around_a_slot(): + assert sorted(expand("[really ]want {item}")) == [ + "really want {item}", "want {item}", + ] + + +def test_unicode_literal_words_pass_through(): + assert expand("ligar a televisão") == ["ligar a televisão"] + + +def test_duplicate_branches_collapse_to_one_sample(): + assert expand("(go|go) home") == ["go home"] + + +# --- inline_keywords ---------------------------------------------------------- + + +def test_inline_keywords_basic(): + tpl = " [the] {name}" + vocab = {"turn_on": ["turn on", "switch on"]} + assert inline_keywords(tpl, vocab) == "(turn on|switch on) [the] {name}" + + +def test_inline_keywords_nested(): + tpl = " " + vocab = { + "broadcast": ["ذع", "بلغ"], + "a": ["آ", "أ"], + "everywhere": ["كل مكان"], + } + result = inline_keywords(tpl, vocab) + assert "بلغ" in result + assert "كل مكان" in result + assert "(آ|أ)" in result + + +def test_inline_keywords_empty_vocab(): + assert inline_keywords(" lights", {}) == " lights" + + +def test_inline_keywords_none_vocab(): + assert inline_keywords(" lights", None) == " lights" + + +def test_inline_keywords_strips_unresolved(): + # An unknown keyword is left as literal text with its angle brackets + # stripped (the documented lenient behaviour), not raised on. + vocab = {"known": ["yes"]} + result = inline_keywords(" ", vocab) + assert result == "unknown (yes)" + + +def test_inline_keywords_inlines_all_values_by_default(): + # No silent truncation: every value is inlined (OVOS-INTENT-1 §4.3 — a + # limit is enforced by refusing, not by dropping). + vocab = {"x": [str(i) for i in range(20)]} + result = inline_keywords("", vocab) + for i in range(20): + assert f"|{i}|" in f"|{result[1:-1]}|" + assert result.count("|") == 19 + + +def test_inline_keywords_max_values_refuses(): + # §4.3: an explicit bound, when exceeded, REFUSES (raises) — it never + # silently truncates the value list. + vocab = {"x": [str(i) for i in range(20)]} + with pytest.raises(MalformedTemplate): + inline_keywords("", vocab, max_values=5) + + +def test_inline_keywords_max_values_within_bound(): + vocab = {"x": ["a", "b", "c"]} + assert inline_keywords("", vocab, max_values=5) == "(a|b|c)" + + +def test_inline_keywords_cycle_raises(): + # A reference cycle is rejected (OVOS-INTENT-1 §4.1), not cut off at an + # arbitrary recursion depth. + vocab = {"a": [""], "b": [""]} + with pytest.raises(MalformedTemplate): + inline_keywords("", vocab) + + +def test_inline_keywords_no_refs(): + assert inline_keywords("hello world", {"x": ["y"]}) == "hello world" + diff --git a/test/test_find_lang_dir.py b/test/test_find_lang_dir.py new file mode 100644 index 00000000..f28c19a5 --- /dev/null +++ b/test/test_find_lang_dir.py @@ -0,0 +1,100 @@ +"""Tests for :func:`ovos_spec_tools.find_lang_dir`.""" +import os +import tempfile +import unittest +from pathlib import Path + +from ovos_spec_tools import find_lang_dir + + +def _mkdirs(root, *names): + for name in names: + os.makedirs(os.path.join(root, name)) + + +class TestFindLangDir(unittest.TestCase): + + def setUp(self): + self.root = tempfile.mkdtemp(prefix="findlang_") + + def tearDown(self): + import shutil + shutil.rmtree(self.root, ignore_errors=True) + + def test_exact_uppercase_match(self): + _mkdirs(self.root, "en-US", "de-DE") + self.assertEqual(find_lang_dir(self.root, "en-US"), + Path(self.root) / "en-US") + + def test_exact_lowercase_match(self): + _mkdirs(self.root, "en-us") + self.assertEqual(find_lang_dir(self.root, "en-us"), + Path(self.root) / "en-us") + + def test_case_mismatch_resolves(self): + """Request uppercase, dir is lowercase — closest_lang resolves + through standardize_lang on both sides.""" + _mkdirs(self.root, "en-us") + self.assertEqual(find_lang_dir(self.root, "en-US"), + Path(self.root) / "en-us") + + def test_subdialect_fallback(self): + """Request en-AU, only en-US available — minor regional + difference (langcodes distance ~4) is below the default max + of 10.""" + _mkdirs(self.root, "en-US", "fr-FR") + self.assertEqual(find_lang_dir(self.root, "en-AU"), + Path(self.root) / "en-US") + + def test_macro_fallback(self): + """Request bare ``en``, en-US directory available — resolves.""" + _mkdirs(self.root, "en-US") + self.assertEqual(find_lang_dir(self.root, "en"), + Path(self.root) / "en-US") + + def test_no_match_returns_none(self): + _mkdirs(self.root, "pt-PT", "ja-JP") + self.assertIsNone(find_lang_dir(self.root, "ko-KR")) + + def test_missing_base_path_returns_none(self): + self.assertIsNone( + find_lang_dir(os.path.join(self.root, "nope"), "en-US")) + + def test_max_distance_zero_requires_exact_normalized_match(self): + """``max_distance=0`` disables smart fallback; only same-tag + matches resolve.""" + _mkdirs(self.root, "en-US") + self.assertEqual( + find_lang_dir(self.root, "en-US", max_distance=0), + Path(self.root) / "en-US") + self.assertIsNone( + find_lang_dir(self.root, "en-AU", max_distance=0)) + + def test_custom_resolver_is_honoured(self): + """A caller can swap the resolver — useful for tests or + deployments that want a different fallback policy.""" + _mkdirs(self.root, "en-US", "de-DE") + + def _always_de(target, available, max_distance): + return "de-DE" if "de-DE" in available else None + + self.assertEqual( + find_lang_dir(self.root, "en-US", lang_resolver=_always_de), + Path(self.root) / "de-DE") + + def test_ignores_non_directory_entries(self): + """A regular file alongside lang dirs must not be considered a + candidate.""" + _mkdirs(self.root, "en-US") + Path(self.root, "stray.txt").write_text("noise") + self.assertEqual(find_lang_dir(self.root, "en-US"), + Path(self.root) / "en-US") + + def test_accepts_path_or_str_base(self): + _mkdirs(self.root, "en-US") + self.assertEqual(find_lang_dir(Path(self.root), "en-US"), + Path(self.root) / "en-US") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_intent.py b/test/test_intent.py new file mode 100644 index 00000000..6ae3d7fe --- /dev/null +++ b/test/test_intent.py @@ -0,0 +1,251 @@ +"""Conformance tests for the OVOS-INTENT-4 keyword intent primitives.""" +import pytest + +from ovos_spec_tools import ( + Intent, + IntentBuilder, + LocaleResources, + MalformedIntent, + open_intent_envelope, + voc_match, +) +from ovos_spec_tools.message import Message + + +def _write(path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +# --- IntentBuilder API surface (workshop/adapt-compatible) ------------------- + +def test_builder_accumulates_roles(): + b = (IntentBuilder("SetBrightness") + .require("Set") + .require("Brightness") + .one_of("Up", "Down") + .optionally("Politely") + .exclude("Question")) + assert b.name == "SetBrightness" + assert b.requires == [("Set", "Set"), ("Brightness", "Brightness")] + assert b.at_least_one == [("Up", "Down")] + assert b.optional == [("Politely", "Politely")] + assert b.excludes == ["Question"] + + +def test_builder_require_attribute_name(): + b = IntentBuilder("X").require("Color", "favourite_colour") + assert b.requires == [("Color", "favourite_colour")] + + +def test_builder_require_optional_flag(): + # legacy signature: require(..., optional=True) routes to optional + b = IntentBuilder("X").require("Y", optional=True) + assert b.requires == [] + assert b.optional == [("Y", "Y")] + + +def test_builder_chaining_returns_self(): + b = IntentBuilder("X") + assert b.require("A") is b + assert b.optionally("B") is b + assert b.one_of("C", "D") is b + assert b.exclude("E") is b + + +def test_build_returns_intent_with_attributes(): + intent = IntentBuilder("Name").require("XKeyword").optionally("Y").build() + assert isinstance(intent, Intent) + assert intent.name == "Name" + assert intent.requires == [("XKeyword", "XKeyword")] + assert intent.optional == [("Y", "Y")] + assert intent.at_least_one == [] + assert intent.excludes == [] + + +# --- OVOS-INTENT-4 §5.2 keyword payload mapping ------------------------------ + +def test_to_keyword_payload_structure(): + intent = (IntentBuilder("SetBrightness") + .require("Set") + .require("Brightness") + .one_of("Up", "Down") + .optionally("Politely") + .exclude("Question") + .build()) + payload = intent.to_keyword_payload(skill_id="lighting.skill", + lang="en-US") + assert payload["skill_id"] == "lighting.skill" + assert payload["intent_name"] == "SetBrightness" + assert payload["lang"] == "en-US" + assert payload["required"] == [{"name": "Set"}, {"name": "Brightness"}] + assert payload["optional"] == [{"name": "Politely"}] + assert payload["one_of"] == [[{"name": "Up"}, {"name": "Down"}]] + assert payload["excluded"] == [{"name": "Question"}] + + +def test_to_keyword_payload_all_four_keys_present_when_empty(): + # §5.2: a producer MUST include all four role keys, even when empty. + intent = IntentBuilder("Bare").require("Only").build() + payload = intent.to_keyword_payload() + for key in ("required", "optional", "one_of", "excluded"): + assert key in payload + assert payload["optional"] == [] + assert payload["one_of"] == [] + assert payload["excluded"] == [] + assert payload["intent_name"] == "Bare" + # identity fields omitted when not provided + assert "skill_id" not in payload + assert "lang" not in payload + + +def test_to_keyword_payload_inlines_samples_when_supplied(): + intent = IntentBuilder("X").require("Set").one_of("Up", "Down").build() + samples = {"Set": ["set", "change"], "Up": ["up", "higher"]} + payload = intent.to_keyword_payload(samples=samples) + assert payload["required"] == [{"name": "Set", "samples": ["set", "change"]}] + assert payload["one_of"] == [[ + {"name": "Up", "samples": ["up", "higher"]}, + {"name": "Down"}, # no samples supplied -> name-only + ]] + + +# --- open_intent_envelope round-trip ----------------------------------------- + +def test_open_intent_envelope_legacy_keys(): + intent = (IntentBuilder("Foo").require("A").optionally("B") + .one_of("C", "D").exclude("E").build()) + # legacy serialization == Intent.__dict__ shape + msg = Message("register_intent", { + "name": intent.name, + "requires": intent.requires, + "at_least_one": [list(g) for g in intent.at_least_one], + "optional": intent.optional, + "excludes": intent.excludes, + }) + rebuilt = open_intent_envelope(msg) + assert rebuilt == intent + + +def test_open_intent_envelope_intent4_descriptors(): + intent = (IntentBuilder("Foo").require("A").optionally("B") + .one_of("C", "D").exclude("E").build()) + payload = intent.to_keyword_payload(skill_id="s", lang="en-US") + msg = Message("ovos.intent.register.keyword", payload) + rebuilt = open_intent_envelope(msg) + assert rebuilt == intent + + +def test_open_intent_envelope_accepts_raw_dict(): + rebuilt = open_intent_envelope({"intent_name": "Z", + "required": [{"name": "A"}], + "optional": [], "one_of": [], + "excluded": []}) + assert rebuilt.name == "Z" + assert rebuilt.requires == [("A", "A")] + + +# --- voc_match convenience --------------------------------------------------- + +@pytest.fixture +def locale(tmp_path): + _write(tmp_path / "locale" / "en-US" / "yes.voc", + "yes\nyeah\nyep\nof course\n") + return tmp_path / "locale" + + +def test_voc_match_whole_word_hit(locale): + res = LocaleResources(str(locale)) + assert voc_match("yes, please", "yes", "en-US", res) is True + + +def test_voc_match_whole_word_no_substring(locale): + res = LocaleResources(str(locale)) + # 'yes' must NOT match inside 'yesterday' (whole-word §4.3 semantics) + assert voc_match("yesterday is gone", "yes", "en-US", res) is False + + +def test_voc_match_multiword_entry(locale): + res = LocaleResources(str(locale)) + assert voc_match("well of course i will", "yes", "en-US", res) is True + + +def test_voc_match_accepts_path(locale): + assert voc_match("yeah right", "yes", "en-US", str(locale)) is True + + +def test_voc_match_accepts_sequence_of_dirs(locale): + assert voc_match("yep indeed", "yes", "en-US", [str(locale)]) is True + + +def test_voc_match_missing_voc_returns_false(locale): + assert voc_match("anything", "nonexistent", "en-US", str(locale)) is False + + +# --- OVOS-INTENT-3 §4.2 well-formedness (validate raises; build/emit warn) --- + +def test_validate_rejects_intent_with_no_required_and_no_one_of(): + """§4.2: a keyword intent MUST declare at least one required or one-of + constraint — only optional + excluded "has nothing that must be present + and is malformed". Explicit validate() enforces this.""" + intent = Intent("Bad", optional=[("Politely", "Politely")], + excludes=["Question"]) + with pytest.raises(MalformedIntent): + intent.validate() + + +def test_build_warns_but_does_not_raise_on_malformed_intent(caplog): + """build() stays backward-compatible: a §4.2-malformed builder logs a + warning rather than raising. Enforcement is via explicit validate / lint.""" + builder = IntentBuilder("Bad").optionally("Politely").exclude("Question") + with caplog.at_level("WARNING"): + intent = builder.build() # must NOT raise + assert any("malformed" in r.message.lower() for r in caplog.records) + with pytest.raises(MalformedIntent): # explicit validation still rejects + intent.validate() + + +def test_build_accepts_intent_with_only_one_of(): + """A single one-of group satisfies §4.2 (at least one of required/one-of).""" + intent = IntentBuilder("OK").one_of("Up", "Down").build() + assert intent.at_least_one == [("Up", "Down")] + + +def test_validate_rejects_same_vocab_under_two_roles(): + """§4.2: a vocabulary MUST appear under at most one role; required + excluded + of the same vocab is contradictory and malformed.""" + intent = Intent("Bad", requires=[("Light", "Light")], excludes=["Light"]) + with pytest.raises(MalformedIntent): + intent.validate() + + +def test_validate_rejects_vocab_required_and_one_of(): + intent = Intent("Bad", requires=[("Set", "Set")], + at_least_one=[("Set", "Down")]) + with pytest.raises(MalformedIntent): + intent.validate() + + +def test_validate_returns_self_for_well_formed_intent(): + intent = Intent("Good", requires=[("Set", "Set")]) + assert intent.validate() is intent + + +def test_to_keyword_payload_warns_but_emits_malformed_intent(caplog): + """Emitting a §4.2-malformed register payload warns rather than raising, + keeping the producer backward-compatible.""" + intent = Intent("Bad", optional=[("Politely", "Politely")], + excludes=["Question"]) + with caplog.at_level("WARNING"): + payload = intent.to_keyword_payload() # must NOT raise + assert payload["intent_name"] == "Bad" + assert any("malformed" in r.message.lower() for r in caplog.records) + + +def test_raw_intent_construction_does_not_validate(): + """Raw ``Intent(...)`` construction stays permissive (the scaffold / wire + round-trip path); validation happens at build()/emit. The empty-default + Intent and open_intent_envelope reconstruction must not raise.""" + Intent() # scaffold default, no raise + rebuilt = open_intent_envelope({"intent_name": "Z"}) + assert rebuilt.name == "Z" diff --git a/test/test_iter_locale_dirs.py b/test/test_iter_locale_dirs.py new file mode 100644 index 00000000..0c6dfeb9 --- /dev/null +++ b/test/test_iter_locale_dirs.py @@ -0,0 +1,69 @@ +"""Tests for `iter_locale_dirs` — the canonical locale subdir walker.""" +import pytest + +from ovos_spec_tools import iter_locale_dirs + + +def _make_skill(tmp_path, *lang_dirs): + """Create ``/locale//`` for each given lang.""" + for lang in lang_dirs: + (tmp_path / "locale" / lang).mkdir(parents=True, exist_ok=True) + return tmp_path + + +def test_no_locale_dir_yields_nothing(tmp_path): + assert list(iter_locale_dirs(tmp_path)) == [] + + +def test_each_locale_subdir_is_yielded(tmp_path): + root = _make_skill(tmp_path, "en-US", "pt-PT", "de-DE") + langs = sorted(lang for lang, _ in iter_locale_dirs(root)) + assert langs == ["de-DE", "en-US", "pt-PT"] + + +def test_yields_subdir_path_as_second_item(tmp_path): + root = _make_skill(tmp_path, "en-US") + pairs = list(iter_locale_dirs(root)) + assert len(pairs) == 1 + lang, path = pairs[0] + assert lang == "en-US" + assert path == root / "locale" / "en-US" + + +def test_subdir_name_is_normalized(tmp_path): + """``en_us/`` and ``EN-us/`` resolve to the same canonical tag.""" + root = _make_skill(tmp_path, "en_us") + [(lang, _)] = list(iter_locale_dirs(root)) + assert lang.lower() == "en-us" + + +def test_files_in_locale_root_are_ignored(tmp_path): + root = _make_skill(tmp_path, "en-US") + (root / "locale" / "stray.txt").write_text("not a locale dir") + langs = [lang for lang, _ in iter_locale_dirs(root)] + assert langs == ["en-US"] + + +def test_native_filter_keeps_a_regional_subdir(tmp_path): + """A skill that declares ``en`` as native still discovers ``en-US/``.""" + pytest.importorskip("langcodes") + root = _make_skill(tmp_path, "en-US", "fr-FR") + langs = sorted(lang for lang, _ in + iter_locale_dirs(root, native_langs=["en"])) + assert langs == ["en-US"] + + +def test_native_filter_drops_unrelated_languages(tmp_path): + pytest.importorskip("langcodes") + root = _make_skill(tmp_path, "en-US", "fr-FR", "ja-JP") + langs = sorted(lang for lang, _ in + iter_locale_dirs(root, native_langs=["en-GB", "fr"])) + assert langs == ["en-US", "fr-FR"] + + +def test_native_filter_is_skipped_when_natives_none(tmp_path): + """``native_langs=None`` (the default) yields every subdir, even ones + no native would match — the caller asked for the full walk.""" + root = _make_skill(tmp_path, "en-US", "ja-JP") + langs = sorted(lang for lang, _ in iter_locale_dirs(root)) + assert langs == ["en-US", "ja-JP"] diff --git a/test/test_language.py b/test/test_language.py new file mode 100644 index 00000000..67f70c2b --- /dev/null +++ b/test/test_language.py @@ -0,0 +1,148 @@ +"""Tests for the language-tag utilities.""" +import pytest + +import ovos_spec_tools.language as language +from ovos_spec_tools import closest_lang, lang_distance, lang_matches, standardize_lang + + +# --- standardize_lang -------------------------------------------------------- + +def test_standardize_normalizes_underscores_and_case(): + assert standardize_lang("en_us").lower() == "en-us" + + +def test_standardize_keeps_tagalog_as_tl(): + """langcodes folds `tl` into `fil`; OVOS keeps it distinct.""" + assert standardize_lang("tl") == "tl" + assert standardize_lang("tgl") == "tl" + + +# --- lang_distance ----------------------------------------------------------- + +def test_distance_is_zero_for_identical_tags(): + assert lang_distance("en-US", "en_us") == 0 + + +def test_distance_is_large_for_different_languages(): + assert lang_distance("en-US", "fr-FR") >= 10 + + +def test_distance_measures_a_bare_tag_from_its_norm_region(): + """`pt` is identical to `pt-PT` and merely regional to `pt-BR`.""" + assert lang_distance("pt", "pt-PT") == 0 + assert lang_distance("pt", "pt-BR") > 0 + + +# --- closest_lang ------------------------------------------------------------ + +def test_closest_lang_exact_match(): + assert closest_lang("en-US", ["pt-BR", "en-US", "de-DE"]) == "en-US" + + +def test_closest_lang_exact_match_after_standardization(): + assert closest_lang("en_US", ["en-US"]) == "en-US" + + +def test_closest_lang_returns_the_original_string(): + """The returned value is the entry from `available`, verbatim.""" + assert closest_lang("EN-us", ["en-US"]) == "en-US" + + +def test_closest_lang_no_match_returns_none(): + assert closest_lang("zz-ZZ", ["en-US", "pt-BR"]) is None + + +def test_closest_lang_disabled_with_zero_distance(): + assert closest_lang("en-AU", ["en-US"], max_distance=0) is None + + +def test_closest_lang_regional_fallback(): + pytest.importorskip("langcodes") + assert closest_lang("en-AU", ["pt-BR", "en-US"]) == "en-US" + + +def test_closest_lang_skips_a_distant_language(): + pytest.importorskip("langcodes") + # fr-FR is a different language and beyond the distance cap; en-GB is near. + assert closest_lang("en-AU", ["fr-FR", "en-GB"]) == "en-GB" + + +def test_closest_lang_prefers_the_norm_region_for_a_bare_tag(): + """langcodes makes bare `pt` closest to pt-BR; the norm region pt-PT wins.""" + assert closest_lang("pt", ["pt-BR", "pt-PT"]) == "pt-PT" + assert closest_lang("pt", ["pt-PT", "pt-BR"]) == "pt-PT" # order-independent + + +def test_explicit_region_is_respected_over_the_norm_preference(): + assert closest_lang("pt-BR", ["pt-PT", "pt-BR"]) == "pt-BR" + + +def test_bare_tag_falls_back_when_norm_region_absent(): + pytest.importorskip("langcodes") + assert closest_lang("pt", ["pt-BR"]) == "pt-BR" + + +# --- the langcodes-absent path ---------------------------------------------- + +def test_distance_without_langcodes_shares_primary_subtag(monkeypatch): + """With no langcodes distance, a shared primary subtag still resolves.""" + monkeypatch.setattr(language, "_langcodes_distance", lambda a, b: None) + assert closest_lang("en-AU", ["pt-BR", "en-GB", "fr-FR"]) == "en-GB" + + +def test_distance_without_langcodes_prefers_the_generic_tag(monkeypatch): + monkeypatch.setattr(language, "_langcodes_distance", lambda a, b: None) + assert closest_lang("en-AU", ["en-US", "en", "en-GB"]) == "en" + + +def test_distance_without_langcodes_no_shared_subtag(monkeypatch): + monkeypatch.setattr(language, "_langcodes_distance", lambda a, b: None) + assert closest_lang("en-AU", ["pt-BR", "fr-FR"]) is None + + +# --- edge cases -------------------------------------------------------------- + +def test_closest_lang_empty_available_is_none(): + assert closest_lang("en-US", []) is None + + +def test_standardize_lang_empty_string(): + assert standardize_lang("") == "" + + +def test_lang_distance_is_symmetric_enough_for_identity(): + assert lang_distance("en-US", "en-US") == 0 + assert lang_distance("EN_us", "en-US") == 0 + + +def test_lang_distance_regional_versus_different_language(): + pytest.importorskip("langcodes") + assert lang_distance("de-DE", "de-AT") < 10 # a usable regional match + assert lang_distance("de-DE", "nl-NL") >= 10 # a different language + + +def test_closest_lang_regioned_request_falls_back_to_a_sibling(tmp_path): + pytest.importorskip("langcodes") + assert closest_lang("pt-MZ", ["en-US", "pt-BR"]) == "pt-BR" + + +# --- lang_matches ------------------------------------------------------------ + +def test_lang_matches_identical_tags(): + assert lang_matches("en-US", "en_us") is True + + +def test_lang_matches_different_languages(): + pytest.importorskip("langcodes") + assert lang_matches("en-US", "fr-FR") is False + + +def test_lang_matches_regional_sibling_within_default_threshold(): + pytest.importorskip("langcodes") + assert lang_matches("de-DE", "de-AT") is True + + +def test_lang_matches_exact_only_when_max_distance_zero(): + from ovos_spec_tools import lang_matches as lm + assert lm("en-US", "en-US", max_distance=0) is True + assert lm("en-US", "en-GB", max_distance=0) is False diff --git a/test/test_lint.py b/test/test_lint.py new file mode 100644 index 00000000..cae02aea --- /dev/null +++ b/test/test_lint.py @@ -0,0 +1,379 @@ +"""Tests for the locale resource linter (`ovos-spec-lint`).""" +import pytest + +from ovos_spec_tools.expansion import MalformedTemplate +from ovos_spec_tools.lint import ( + ERROR, + WARNING, + declared_slots, + lint_locale, + lint_required_slots, + main, + validate_required_slots, +) + + +def _write(path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _errors(findings): + return [f for f in findings if f.severity == ERROR] + + +def _warnings(findings): + return [f for f in findings if f.severity == WARNING] + + +def test_clean_locale_has_no_findings(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "(play|put on) {query}\n") + _write(locale / "en-US" / "yes.voc", "yes\nyeah\n") + _write(locale / "en-US" / "greet.dialog", "Hello {name}.\n") + assert lint_locale(locale) == [] + + +def test_malformed_template_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "bad.intent", "turn (on|off the lights\n") + errors = _errors(lint_locale(locale)) + assert len(errors) == 1 + assert "bad.intent" in errors[0].path + + +def test_empty_file_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "empty.voc", "# just a comment\n") + assert any("empty" in f.message for f in _errors(lint_locale(locale))) + + +def test_slot_in_slot_free_role_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "thing.voc", "the {slot}\n") + assert any("slot-free" in f.message for f in _errors(lint_locale(locale))) + + +def test_bad_base_name_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "Bad-Name.intent", "hello world\n") + assert any("base name" in f.message for f in _errors(lint_locale(locale))) + + +def test_entity_base_name_starting_with_digit_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "2nd.entity", "second\n") + assert any("digit" in f.message for f in _errors(lint_locale(locale))) + + +def test_duplicate_resource_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "a" / "dup.intent", "first\n") + _write(locale / "en-US" / "b" / "dup.intent", "second\n") + assert any("duplicate" in f.message for f in _errors(lint_locale(locale))) + + +def test_legacy_extension_is_a_warning(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.voc", "yes\n") + _write(locale / "en-US" / "old.rx", ".*\n") + findings = lint_locale(locale) + assert _errors(findings) == [] + assert any(".rx" in f.message for f in _warnings(findings)) + + +def test_file_outside_a_language_directory_is_a_warning(tmp_path): + locale = tmp_path / "locale" + _write(locale / "stray.intent", "hello world\n") + assert any("language directory" in f.message + for f in _warnings(lint_locale(locale))) + + +def test_non_bcp47_language_directory_is_a_warning(tmp_path): + locale = tmp_path / "locale" + _write(locale / "english" / "x.intent", "hello world\n") + assert any("BCP-47" in f.message for f in _warnings(lint_locale(locale))) + + +def test_unresolved_vocabulary_reference_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.intent", " {name}\n") + assert _errors(lint_locale(locale)) + + +def test_can_point_at_a_single_language_directory(tmp_path): + lang = tmp_path / "en-US" + _write(lang / "play.intent", "(play|stop) {query}\n") + assert lint_locale(lang) == [] + + +# --- the CLI ----------------------------------------------------------------- + +def test_main_returns_zero_for_a_clean_locale(tmp_path, capsys): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "play {query}\n") + assert main([str(locale)]) == 0 + + +def test_main_returns_one_on_errors(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "bad.intent", "{a}{b}\n") + assert main([str(locale)]) == 1 + + +def test_main_strict_fails_on_warnings(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "old.list", "a\n") + assert main([str(locale)]) == 0 + assert main([str(locale), "--strict"]) == 1 + + +# --- edge cases -------------------------------------------------------------- + +def test_lint_nonexistent_path_is_an_error(): + findings = lint_locale("/no/such/locale/path") + assert any(f.severity == ERROR for f in findings) + + +def test_lint_empty_locale_warns(tmp_path): + locale = tmp_path / "locale" + locale.mkdir() + assert any("no language" in f.message for f in lint_locale(locale)) + + +def test_unknown_extension_is_ignored(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "notes.txt", "ignore me\n") + _write(locale / "en-US" / "ok.intent", "hello world\n") + assert lint_locale(locale) == [] + + +def test_lint_accepts_a_single_language_directory(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "ok.intent", "hello world\n") + assert lint_locale(locale / "en-US") == [] + + +# --- slot consistency: .dialog ONLY (OVOS-INTENT-2 §4.2) --------------------- + +# .intent templates MAY declare different slot sets — the engine extracts only +# the matched template's slots and the intent's slot set is their union +# (OVOS-INTENT-2 §4.1, OVOS-INTENT-3 §5.1). A tool MUST NOT reject .intent for +# divergent slots, so divergence is NOT flagged for the .intent role. + +def test_divergent_slots_in_one_intent_is_allowed(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "p.intent", "play {query}\nstop {engine}\n") + assert not any("slot sets" in f.message for f in _errors(lint_locale(locale))) + + +def test_mixing_slotted_and_slotless_lines_in_intent_is_allowed(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "p.intent", "play {query}\njust stop\n") + assert not any("slot sets" in f.message for f in _errors(lint_locale(locale))) + + +def test_consistent_slots_across_an_intent_is_clean(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "p.intent", + "(play|put on) {query}\ni want {query}\n") + assert lint_locale(locale) == [] + + +# .dialog still requires identical slot sets. + +def test_inconsistent_slots_in_one_dialog_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.dialog", + "Hello {name}.\nNice to meet you, {title} and {surname}.\n") + assert any("slot sets" in f.message for f in _errors(lint_locale(locale))) + + +def test_mixing_slotted_and_slotless_lines_in_dialog_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.dialog", + "Hello {name}.\nWelcome back.\n") + assert any("slot sets" in f.message for f in _errors(lint_locale(locale))) + + +# --- robustness -------------------------------------------------------------- + +def test_non_utf8_file_is_reported_not_crashed(tmp_path): + lang = tmp_path / "locale" / "en-US" + lang.mkdir(parents=True) + (lang / "bad.intent").write_bytes(b"\xff\xfe not valid utf-8\n") + findings = lint_locale(tmp_path / "locale") + assert any("cannot read" in f.message for f in findings) + + +def test_empty_language_directory_warns(tmp_path): + locale = tmp_path / "locale" + (locale / "en-US").mkdir(parents=True) + assert any("no resource files" in f.message + for f in _warnings(lint_locale(locale))) + + +# --- .blacklist pairing (OVOS-INTENT-2 §4.3) -------------------------------- + +def test_orphan_blacklist_warns(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "ghost.blacklist", "spam words\n") + assert any("no matching" in f.message + for f in _warnings(lint_locale(locale))) + + +def test_blacklist_with_a_matching_intent_is_clean(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "play music\n") + _write(locale / "en-US" / "play.blacklist", "trailer\n") + assert lint_locale(locale) == [] + + +# --- the --spec-version flag ------------------------------------------------- + +def test_spec_version_0_flags_the_blacklist_role(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "play music\n") + _write(locale / "en-US" / "play.blacklist", "trailer\n") + findings = lint_locale(locale, spec_version=0) + assert any("requires spec version" in f.message for f in findings) + + +def test_spec_version_1_flags_a_vocabulary_reference(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greeting.voc", "hello\nhi\n") + _write(locale / "en-US" / "greet.intent", " there\n") + errors = _errors(lint_locale(locale, spec_version=1)) + assert any("vocabulary reference" in f.message for f in errors) + + +def test_default_spec_version_flags_neither(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greeting.voc", "hello\nhi\n") + _write(locale / "en-US" / "greet.intent", " there\n") + _write(locale / "en-US" / "greet.blacklist", "spam\n") + findings = lint_locale(locale) # default spec-version 2 + assert not any("spec version" in f.message for f in findings) + + +def test_main_honors_spec_version(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greeting.voc", "hello\nhi\n") + _write(locale / "en-US" / "greet.intent", " there\n") + assert main([str(locale)]) == 0 # v3 — fine + assert main([str(locale), "--spec-version", "1"]) == 1 # is v2 + + +# --- the .prompt role (OVOS-INTENT-2 §4.4) ---------------------------------- + +def test_prompt_file_is_a_recognized_role(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "system.prompt", "You are {assistant}.\n") + assert lint_locale(locale) == [] + + +def test_empty_prompt_is_an_error(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "system.prompt", " \n\n") + assert any("empty" in f.message for f in _errors(lint_locale(locale))) + + +def test_prompt_is_not_template_checked(tmp_path): + """A `.prompt` is plain text — content that would be a malformed template + if it were one is perfectly valid.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "sys.prompt", "turn (on|off the lights {a}{b}\n") + assert lint_locale(locale) == [] + + +def test_non_utf8_prompt_is_reported_not_crashed(tmp_path): + lang = tmp_path / "locale" / "en-US" + lang.mkdir(parents=True) + (lang / "sys.prompt").write_bytes(b"\xff\xfe not valid utf-8") + assert any("cannot read" in f.message + for f in lint_locale(tmp_path / "locale")) + + +def test_spec_version_2_flags_the_prompt_role(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "system.prompt", "You are helpful.\n") + findings = lint_locale(locale, spec_version=2) + assert any("requires spec version 3" in f.message for f in findings) + + +def test_default_spec_version_does_not_flag_prompt(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "system.prompt", "You are helpful.\n") + assert not any("spec version" in f.message for f in lint_locale(locale)) + + +# --- required_slots validation (OVOS-INTENT-3 §5.3) ---------------------- + +def test_declared_slots_is_the_union_across_templates(): + templates = [ + "(play|put on) {query}", + "(play|put on) {query} (on|using) {engine}", + "i want to listen to {query}", + ] + assert declared_slots(templates) == frozenset({"query", "engine"}) + + +def test_declared_slots_folds_double_brace_spelling(): + # {{name}} and {name} are the same slot (OVOS-INTENT-1 §3.4). + assert declared_slots(["say {{name}}", "say {name}!"]) == frozenset({"name"}) + + +def test_required_slot_declared_by_a_template_is_accepted(): + templates = [ + "(play|put on) {query}", + "(play|put on) {query} (on|using) {engine}", + ] + # both required slots are declared by at least one template — no raise. + validate_required_slots(["query", "engine"], templates) + + +def test_required_slot_declared_by_no_template_is_rejected(): + templates = ["(play|put on) {query}"] + with pytest.raises(MalformedTemplate) as exc: + validate_required_slots(["query", "engine"], templates) + assert "engine" in str(exc.value) + assert "§5.3" in str(exc.value) + + +def test_required_slot_in_only_one_of_several_templates_is_accepted(): + # the engine extracts only the matched template's slots, so a required slot + # declared by a *single* template still satisfies §5.3 (it can fire). + templates = [ + "i want to listen to {query}", + "(play|put on) {query} (on|using) {engine}", + ] + validate_required_slots(["engine"], templates) + + +def test_no_required_slots_is_always_accepted(): + validate_required_slots([], ["(play|put on) {query}"]) + + +def test_required_slot_against_slotless_templates_is_rejected(): + with pytest.raises(MalformedTemplate): + validate_required_slots(["query"], ["just hello", "say hi"]) + + +def test_required_slot_declared_only_in_double_brace_is_accepted(): + # {{engine}} declares the slot just as {engine} would (§3.4 fold). + validate_required_slots(["engine"], ["play {query} on {{engine}}"]) + + +def test_lint_required_slots_returns_an_error_finding(): + findings = lint_required_slots( + "play.intent", ["query", "engine"], ["(play|put on) {query}"]) + assert len(findings) == 1 + assert findings[0].severity == ERROR + assert findings[0].path == "play.intent" + assert "engine" in findings[0].message + + +def test_lint_required_slots_clean_returns_no_findings(): + findings = lint_required_slots( + "play.intent", ["query"], ["(play|put on) {query}"]) + assert findings == [] diff --git a/test/test_message.py b/test/test_message.py new file mode 100644 index 00000000..66c4a5f7 --- /dev/null +++ b/test/test_message.py @@ -0,0 +1,452 @@ +"""Conformance tests for the OVOS-MSG-1 :class:`Message` envelope. + +Each test class targets one numbered section of the spec +(``architecture/message-object.md``). Test names are the ``MUST`` / +``SHOULD`` rule they pin so failures point at the spec sentence. +""" +import json + +import pytest + +from ovos_spec_tools import ( + DEFAULT_SESSION_ID, + MalformedMessage, + Message, +) + + +# --- §2 envelope ------------------------------------------------------------ + +class TestEnvelope: + def test_constructs_with_only_msg_type(self): + """§2: ``data`` and ``context`` MAY be empty.""" + m = Message("ovos.test") + assert m.msg_type == "ovos.test" + assert m.data == {} + assert m.context == {} + + def test_constructs_with_data_and_context(self): + m = Message("ovos.test", {"a": 1}, {"source": "x"}) + assert m.data == {"a": 1} + assert m.context == {"source": "x"} + + def test_accepts_empty_msg_type_at_construction(self): + """§2.1 (non-empty ``type``) gates the **wire** form, not + intermediate objects. The ``Message("").forward(real_type)`` + pattern is widely used to build a routing scaffold before the + real topic is known.""" + m = Message("") + assert m.msg_type == "" + + def test_rejects_non_string_msg_type(self): + with pytest.raises(MalformedMessage): + Message(123) # type: ignore[arg-type] + + def test_rejects_non_dict_data(self): + """§2.2: ``data`` is a JSON object.""" + with pytest.raises(MalformedMessage): + Message("ovos.test", "not a dict") # type: ignore[arg-type] + + def test_rejects_non_dict_context(self): + """§2.3: ``context`` is a JSON object.""" + with pytest.raises(MalformedMessage): + Message("ovos.test", {}, "not a dict") # type: ignore[arg-type] + + +# --- §6 serialization + deserialization round-trip -------------------------- + +class TestSerialization: + def test_round_trip_preserves_envelope(self): + original = Message( + "ovos.test", {"x": 1, "y": ["a", "b"]}, + {"source": "skill", "session": {"session_id": "s1"}}) + round_tripped = Message.deserialize(original.serialize()) + assert round_tripped == original + + def test_serialize_emits_a_single_json_object(self): + m = Message("ovos.test") + parsed = json.loads(m.serialize()) + assert isinstance(parsed, dict) + assert set(parsed.keys()) == {"type", "data", "context"} + + def test_deserialize_rejects_unknown_top_level_keys(self): + """§2: 'Other top-level keys MUST NOT appear; consumers MUST + reject any Message with unknown top-level keys.'""" + payload = json.dumps({ + "type": "ovos.test", "data": {}, "context": {}, + "extra": "field"}) + with pytest.raises(MalformedMessage): + Message.deserialize(payload) + + def test_deserialize_rejects_missing_type(self): + with pytest.raises(MalformedMessage): + Message.deserialize(json.dumps({"data": {}, "context": {}})) + + def test_deserialize_rejects_unparsable_payload(self): + with pytest.raises(MalformedMessage): + Message.deserialize("not json {") + + def test_deserialize_rejects_non_object_root(self): + """§2: 'A Message is a JSON object'.""" + with pytest.raises(MalformedMessage): + Message.deserialize(json.dumps(["not", "an", "object"])) + + def test_deserialize_accepts_already_parsed_dict(self): + m = Message.deserialize({"type": "ovos.test", "data": {}, "context": {}}) + assert m == Message("ovos.test") + + def test_deserialize_accepts_bytes(self): + m = Message.deserialize(b'{"type": "ovos.test", "data": {}, "context": {}}') + assert m.msg_type == "ovos.test" + + def test_serialize_forbids_nan_and_infinity(self): + """§6: 'Numbers MUST be finite. NaN, +Infinity, -Infinity are + forbidden.'""" + m = Message("ovos.test", {"x": float("nan")}) + with pytest.raises(ValueError): + m.serialize() + + def test_serialize_calls_dot_serialize_on_nested_objects(self): + """Carrier objects (Session, nested Messages, …) exposing a + ``.serialize()`` method are converted before JSON encoding so + callers don't have to pre-dictify their context.""" + + class _Carrier: + def __init__(self, sid): + self.sid = sid + + def serialize(self): + return {"session_id": self.sid, "lang": "en-US"} + + m = Message("ovos.test", {}, {"session": _Carrier("s-42")}) + parsed = json.loads(m.serialize()) + assert parsed["context"]["session"] == { + "session_id": "s-42", "lang": "en-US"} + + def test_serialize_walks_nested_lists_and_dicts(self): + class _C: + def serialize(self): + return "C-payload" + m = Message("ovos.test", {"items": [_C(), {"k": _C()}]}) + parsed = json.loads(m.serialize()) + assert parsed["data"]["items"] == ["C-payload", {"k": "C-payload"}] + + def test_serialize_data_can_be_empty(self): + """§2.2: ``data`` MAY be empty (`{}`).""" + parsed = json.loads(Message("ovos.test").serialize()) + assert parsed["data"] == {} + assert parsed["context"] == {} + + def test_serialize_rejects_empty_type(self): + """§2.1/§7 producer MUST: ``serialize`` is the conformance gate and + MUST refuse to emit an empty ``type``.""" + with pytest.raises(MalformedMessage): + Message("").serialize() + + def test_serialize_after_forward_from_empty_scaffold_ok(self): + """The construct-then-forward scaffold becomes serializable once it + has a real topic.""" + m = Message("").forward("ovos.real", {"k": "v"}) + parsed = json.loads(m.serialize()) + assert parsed["type"] == "ovos.real" + + def test_serialize_accepts_valid_type_syntax(self): + """§2.1: ASCII letters, digits, ``.`` ``:`` ``_`` ``-`` are all valid + topic characters; a dot/colon-segmented topic serializes unchanged.""" + m = Message("assistant.intent.register.keyword") + assert json.loads(m.serialize())["type"] == \ + "assistant.intent.register.keyword" + # the full permitted charset, including the colon separator and a digit + assert json.loads(Message("Foo_bar-1:Baz").serialize())["type"] == \ + "Foo_bar-1:Baz" + + def test_serialize_rejects_type_with_whitespace(self): + """§2.1 producer MUST: a topic carries no whitespace, so an embedded + space MUST be rejected at the wire gate rather than emitted.""" + with pytest.raises(MalformedMessage): + Message("a b").serialize() + + def test_serialize_rejects_type_with_illegal_char(self): + """§2.1: characters outside [A-Za-z0-9.:_-] (here a slash) are not + permitted in a topic.""" + with pytest.raises(MalformedMessage): + Message("ovos/test").serialize() + + def test_deserialize_rejects_wrong_type_data(self): + """§2.2/§6: a present-but-non-object ``data`` MUST be rejected, not + silently coerced to ``{}``.""" + for bad in ([], 0, False, "x"): + with pytest.raises(MalformedMessage): + Message.deserialize({"type": "ovos.test", "data": bad}) + + def test_deserialize_rejects_wrong_type_context(self): + """§2.3/§6: a present-but-non-object ``context`` MUST be rejected.""" + for bad in ([], 0, False, "x"): + with pytest.raises(MalformedMessage): + Message.deserialize({"type": "ovos.test", "context": bad}) + + def test_deserialize_absent_or_null_data_defaults_to_empty(self): + """§2.2/§2.3: an absent (or explicit null) ``data``/``context`` is + equivalent to ``{}`` — that is still allowed.""" + m = Message.deserialize({"type": "ovos.test"}) + assert m.data == {} and m.context == {} + m2 = Message.deserialize({"type": "ovos.test", "data": None, + "context": None}) + assert m2.data == {} and m2.context == {} + + def test_deserialize_preserves_empty_dict_data(self): + m = Message.deserialize({"type": "ovos.test", "data": {}}) + assert m.data == {} + + +# --- §5.1 forward ----------------------------------------------------------- + +class TestForward: + def test_preserves_context_unchanged(self): + """§5.1: ``context = C`` (preserved unchanged, including + ``source``, ``destination``, and ``session``).""" + m = Message("ovos.a", {}, {"source": "x", "destination": "y", + "session": {"session_id": "s1"}}) + f = m.forward("ovos.b", {"k": "v"}) + assert f.msg_type == "ovos.b" + assert f.data == {"k": "v"} + assert f.context == {"source": "x", "destination": "y", + "session": {"session_id": "s1"}} + + def test_forwarder_does_not_become_new_source(self): + """§5.1: 'The forwarder does not become the new source — the + original producer remains named.'""" + m = Message("ovos.a", {}, {"source": "original-producer"}) + assert m.forward("ovos.b").context["source"] == "original-producer" + + def test_forward_context_is_deep_copied(self): + """Mutating the source context after forward MUST NOT affect + the forwarded Message. Uses a named id the registry never folded, so + the session is carried verbatim (a session without a session_id would + normalize to the default per §4.3).""" + m = Message("ovos.a", {}, {"source": "x", + "session": {"session_id": "relay-deepcopy", + "lang": "en-US"}}) + f = m.forward("ovos.b") + m.context["session"]["lang"] = "MUTATED" + assert f.context["session"]["lang"] == "en-US" + + def test_forward_omits_data_defaults_to_empty(self): + m = Message("ovos.a", {"k": "v"}) + assert m.forward("ovos.b").data == {} + + +# --- §5.2 reply ------------------------------------------------------------- + +class TestReply: + def test_swaps_source_and_destination(self): + """§5.2: routing keys reversed.""" + m = Message("ovos.req", {}, {"source": "A", "destination": "B"}) + r = m.reply("ovos.ack") + assert r.context["source"] == "B" + assert r.context["destination"] == "A" + + def test_preserves_session_unchanged(self): + """§5.2 (3): 'All other ``context`` keys, including ``session``, + are preserved unchanged.'""" + sess = {"session_id": "s1", "lang": "en-US"} + m = Message("ovos.req", {}, {"source": "A", "destination": "B", + "session": sess}) + r = m.reply("ovos.ack") + assert r.context["session"] == sess + + def test_destination_array_picks_one_for_source(self): + """§5.2 (2): array form — implementation chooses; consumers + MUST NOT rely on a particular member being chosen.""" + m = Message("ovos.req", {}, {"source": "A", + "destination": ["B", "C"]}) + r = m.reply("ovos.ack") + # the exact choice is implementation-defined; ours is index 0 + assert r.context["source"] in ("B", "C") + assert r.context["destination"] == "A" + + def test_provided_context_keys_are_swapped_too(self): + """``context`` overlays land before the §5.2 swap (matches the + historical ``ovos_bus_client.Message.reply`` behaviour); routing + keys passed in get swapped along with everything else.""" + m = Message("ovos.req", {}, {"source": "A", "destination": "B"}) + r = m.reply("ovos.ack", + context={"source": "C", "destination": "D"}) + # overlay sets src=C, dst=D — then the swap flips them + assert r.context["source"] == "D" + assert r.context["destination"] == "C" + + def test_provided_non_routing_context_keys_pass_through(self): + """Keys that are not ``source`` / ``destination`` overlay + without being swapped — handy for custom session shapes or + tracing identifiers.""" + m = Message("ovos.req", {}, {"source": "A", "destination": "B"}) + r = m.reply("ovos.ack", context={"trace_id": "abc"}) + assert r.context["trace_id"] == "abc" + assert r.context["source"] == "B" + assert r.context["destination"] == "A" + + +# --- §5.3 response ---------------------------------------------------------- + +class TestResponse: + def test_topic_is_source_plus_dot_response(self): + """§5.3: 'A ``response`` is a ``reply`` whose topic is the source + topic suffixed with ``.response``.'""" + m = Message("ovos.intent.list", {}, {"source": "A", "destination": "B"}) + r = m.response({"intents": []}) + assert r.msg_type == "ovos.intent.list.response" + + def test_response_swaps_routing_like_reply(self): + m = Message("ovos.intent.list", {}, {"source": "A", "destination": "B"}) + r = m.response() + assert r.context["source"] == "B" + assert r.context["destination"] == "A" + + +# --- §4 session carrier ----------------------------------------------------- + +class TestSessionCarrier: + def test_default_session_id_constant_matches_spec(self): + """§4.1: 'The value "default" is reserved...'""" + assert DEFAULT_SESSION_ID == "default" + + def test_absent_session_is_well_formed(self): + """§4.3 / §7: 'a Message without [session] is well-formed'.""" + m = Message("ovos.test") + assert "session" not in m.context + + def test_context_session_is_opaque_dict(self): + """§4: 'session is mostly opaque under this specification.' The + envelope does not parse it — any dict shape conforms.""" + m = Message("ovos.test", {}, + {"session": {"weird": "shape", "session_id": "s1"}}) + round_tripped = Message.deserialize(m.serialize()) + assert round_tripped.context["session"]["weird"] == "shape" + + +# --- §3 routing key opacity ------------------------------------------------- + +class TestRoutingOpacity: + def test_source_and_destination_can_be_arbitrary_strings(self): + """§3.4: 'source and destination are opaque strings.'""" + m = Message("ovos.test", {}, + {"source": "anything-goes-here", + "destination": ["a@b", "c#d"]}) + assert Message.deserialize(m.serialize()) == m + + +# --- subclass-friendliness -------------------------------------------------- + +class TestSubclassDerivations: + """Derivations and ``deserialize`` must return instances of the + runtime class so ``ovos-bus-client``'s subclass propagates naturally + through forward / reply / response chains.""" + + class _BusClientMessage(Message): + pass + + def test_forward_returns_subclass(self): + m = self._BusClientMessage("ovos.a") + assert isinstance(m.forward("ovos.b"), self._BusClientMessage) + + def test_reply_returns_subclass(self): + m = self._BusClientMessage("ovos.a", {}, {"source": "x", "destination": "y"}) + assert isinstance(m.reply("ovos.b"), self._BusClientMessage) + + def test_response_returns_subclass(self): + m = self._BusClientMessage("ovos.a", {}, {"source": "x", "destination": "y"}) + assert isinstance(m.response(), self._BusClientMessage) + + def test_deserialize_returns_subclass(self): + payload = '{"type": "ovos.a", "data": {}, "context": {}}' + assert isinstance( + self._BusClientMessage.deserialize(payload), + self._BusClientMessage) + + +def test_malformed_message_is_catchable_as_assertion_error(): + """Legacy ``ovos_bus_client.Message`` raised ``AssertionError`` from + bare ``assert`` constructor checks; downstream code that catches + that type must keep working through the migration.""" + with pytest.raises(AssertionError): + Message("ok", data="not a dict") + + +# --- as_dict ---------------------------------------------------------------- + +def test_as_dict_returns_envelope_as_dict(): + m = Message("ovos.test", {"a": 1}, {"source": "skill"}) + d = m.as_dict + assert isinstance(d, dict) + assert d == {"type": "ovos.test", "data": {"a": 1}, + "context": {"source": "skill"}} + + +def test_as_dict_walks_carrier_serialize_protocol(): + """``as_dict`` round-trips through ``serialize`` so nested objects + with a ``.serialize()`` method are converted, same as on the wire.""" + + class _Carrier: + def serialize(self): + return {"session_id": "s-7"} + + m = Message("ovos.test", {}, {"session": _Carrier()}) + assert m.as_dict["context"]["session"] == {"session_id": "s-7"} + + +# --- hashability (library-usefulness; not a spec rule) ---------------------- + +class TestHashable: + def test_equal_messages_hash_equal(self): + a = Message("ovos.test", {"x": 1}, {"source": "skills"}) + b = Message("ovos.test", {"x": 1}, {"source": "skills"}) + assert a == b + assert hash(a) == hash(b) + + def test_usable_as_dict_key_and_set_member(self): + a = Message("ovos.test", {"x": 1}) + b = Message("ovos.test", {"x": 1}) + c = Message("ovos.other", {"x": 1}) + d = {a: "value"} + assert d[b] == "value" # equal key hits the same bucket + assert len({a, b, c}) == 2 # a and b collapse, c distinct + + def test_usable_in_lru_cache(self): + import functools + + calls = [] + + @functools.lru_cache(maxsize=None) + def handler(msg): + calls.append(msg.msg_type) + return msg.msg_type.upper() + + m1 = Message("ovos.test", {"x": 1}, {"source": "a"}) + m2 = Message("ovos.test", {"x": 1}, {"source": "a"}) # equal to m1 + assert handler(m1) == "OVOS.TEST" + assert handler(m2) == "OVOS.TEST" + assert calls == ["ovos.test"] # cached: only one underlying call + + def test_nested_dict_and_list_data_hashes(self): + m = Message("ovos.test", + {"items": [1, 2, {"k": "v"}], "meta": {"a": [3, 4]}}, + {"session": {"session_id": "s", "langs": ["en", "pt"]}}) + # must not raise, and must be stable across repeated calls + assert hash(m) == hash(m) + + def test_int_and_float_data_hash_equal(self): + # value-equality invariant: {"x": 1} == {"x": 1.0} so they MUST + # hash equal (a json.dumps digest would break this). + a = Message("ovos.test", {"x": 1}) + b = Message("ovos.test", {"x": 1.0}) + assert a == b + assert hash(a) == hash(b) + + def test_mutation_then_rehash_changes_hash(self): + # documented behavior: the hash is a point-in-time snapshot of the + # mutable payload; mutating data after hashing changes the hash. + m = Message("ovos.test", {"x": 1}) + h1 = hash(m) + m.data["x"] = 2 + assert hash(m) != h1 diff --git a/test/test_messages.py b/test/test_messages.py new file mode 100644 index 00000000..14cc9698 --- /dev/null +++ b/test/test_messages.py @@ -0,0 +1,440 @@ +"""Tests for the spec bus-message registry.""" +import unittest + +from ovos_spec_tools import ( + MIGRATION_MAP, + MIGRATION_PAYLOAD_TRANSFORMS, + SPEC_TO_LEGACY, + NamespaceTranslator, + SpecMessage, + migration_counterpart, +) + + +class _Msg: + def __init__(self, msg_type, data=None, context=None): + self.msg_type = msg_type + self.data = data or {} + self.context = context or {} + + +class TestNamespaceTranslator(unittest.TestCase): + def test_counterpart_topics_directions(self): + t = NamespaceTranslator(modernize=True, emit_legacy=True) + self.assertEqual(t.counterpart_topics("speak"), ["ovos.utterance.speak"]) + self.assertEqual(t.counterpart_topics("ovos.utterance.handle"), + ["recognizer_loop:utterance"]) + self.assertEqual(t.counterpart_topics("some.topic"), []) + + def test_flags_gate_each_direction(self): + only_mod = NamespaceTranslator(modernize=True, emit_legacy=False) + self.assertEqual(only_mod.counterpart_topics("speak"), ["ovos.utterance.speak"]) + self.assertEqual(only_mod.counterpart_topics("ovos.utterance.speak"), []) + only_leg = NamespaceTranslator(modernize=False, emit_legacy=True) + self.assertEqual(only_leg.counterpart_topics("speak"), []) + self.assertEqual(only_leg.counterpart_topics("ovos.utterance.speak"), ["speak"]) + + def test_is_migrated(self): + t = NamespaceTranslator() + self.assertTrue(t.is_migrated("speak")) + self.assertTrue(t.is_migrated("ovos.utterance.speak")) + self.assertFalse(t.is_migrated("random.topic")) + + def test_mirror_guard_drops_counterpart_pair(self): + guard = NamespaceTranslator().new_mirror_guard() + data = {"utterance": "hi"} + self.assertFalse(guard(_Msg("speak", data))) + self.assertTrue(guard(_Msg("ovos.utterance.speak", data))) # mirror + + def test_mirror_guard_keeps_same_topic_repeats(self): + guard = NamespaceTranslator().new_mirror_guard() + data = {"utterance": "ok"} + self.assertFalse(guard(_Msg("speak", data))) + self.assertFalse(guard(_Msg("speak", data))) # genuine repeat, same topic + + def test_mirror_guard_distinct_context_not_collapsed(self): + guard = NamespaceTranslator().new_mirror_guard() + data = {"utterance": "hi"} + self.assertFalse(guard(_Msg("speak", data, {"session": {"session_id": "A"}}))) + self.assertFalse(guard(_Msg("ovos.utterance.speak", data, + {"session": {"session_id": "B"}}))) + + def test_mirror_guard_window_expiry(self): + clk = {"t": 0.0} + guard = NamespaceTranslator(window=1.0).new_mirror_guard(clock=lambda: clk["t"]) + data = {"utterance": "hi"} + self.assertFalse(guard(_Msg("speak", data))) + clk["t"] = 2.0 + self.assertFalse(guard(_Msg("ovos.utterance.speak", data))) # window passed + + +class TestSpecMessage(unittest.TestCase): + def test_members_are_ovos_namespaced_strings(self): + for m in SpecMessage: + self.assertTrue(m.value.startswith("ovos."), m.value) + + def test_usable_as_plain_string(self): + self.assertEqual(SpecMessage.SPEAK, "ovos.utterance.speak") + self.assertEqual(f"{SpecMessage.UTTERANCE}", "ovos.utterance.handle") + + def test_no_duplicate_values(self): + values = [m.value for m in SpecMessage] + self.assertEqual(len(values), len(set(values))) + + def test_audio1_bus_surface_members(self): + # OVOS-AUDIO-1 §7 bus surface members. + self.assertEqual(SpecMessage.SPEAK_B64, "ovos.utterance.speak.b64") # §3.4 + self.assertEqual(SpecMessage.AUDIO_SPEECH, "ovos.audio.speech") # §4.3 + self.assertEqual(SpecMessage.AUDIO_QUEUE, "ovos.audio.queue") # §4.1 + self.assertEqual(SpecMessage.AUDIO_PLAY_SOUND, "ovos.audio.play_sound") # §4.2 + self.assertEqual(SpecMessage.AUDIO_STOP, "ovos.audio.stop") # §6 + self.assertEqual(SpecMessage.AUDIO_IS_SPEAKING, "ovos.audio.is_speaking") # §5.3 + + def test_audio1_output_topics_are_payload_compatible_renames(self): + # The AUDIO-1 §7 output topics are payload-compatible 1:1 renames of the + # Mycroft-era handler names (verified against ovos-audio + # register_handlers), so they migrate via MIGRATION_MAP with no payload + # transform (identity). + expected = { + "speak:b64_audio": SpecMessage.SPEAK_B64, + "speak:b64_audio.response": SpecMessage.AUDIO_SPEECH, + "mycroft.audio.queue": SpecMessage.AUDIO_QUEUE, + "mycroft.audio.play_sound": SpecMessage.AUDIO_PLAY_SOUND, + "mycroft.audio.speak.status": SpecMessage.AUDIO_IS_SPEAKING, + "mycroft.audio.speech.stop": SpecMessage.AUDIO_STOP, + } + for legacy, spec in expected.items(): + self.assertEqual(MIGRATION_MAP[legacy], spec) + # payload-compatible: identity transform (no per-topic entry) + self.assertNotIn(legacy, MIGRATION_PAYLOAD_TRANSFORMS) + + def test_audio1_output_topics_round_trip(self): + # legacy -> spec -> legacy and spec -> legacy round-trip. + for legacy in ("speak:b64_audio", "speak:b64_audio.response", + "mycroft.audio.queue", "mycroft.audio.play_sound", + "mycroft.audio.speak.status", "mycroft.audio.speech.stop"): + spec = migration_counterpart(legacy) + self.assertIsNotNone(spec) + self.assertEqual(migration_counterpart(spec), legacy) + + +#: The complete set of STATIC (non-templated) ``ovos.*`` topics the OVOS +#: architecture specs normatively define, each mapped to its owning spec + +#: section. This is the golden reference: ``SpecMessage`` MUST contain exactly +#: this set — every spec topic has a member (completeness) and every member is a +#: spec topic (spec-only-ness). Runtime-templated topics (MSG-1 §2.1.1: +#: ``:…`` / ``…`` / per-skill ping placeholders) are +#: deliberately excluded, as are bus-client internals no spec defines +#: (``ovos.session.update_default``, ``ovos.session.start``, ``ovos.context.*``). +SPEC_STATIC_TOPICS = { + # OVOS-PIPELINE-1 §8/§9 utterance + intent layer + "ovos.utterance.handle": "PIPELINE-1 §9.1", + "ovos.utterance.speak": "PIPELINE-1 §9.6", + "ovos.utterance.handled": "PIPELINE-1 §9.5", + "ovos.intent.matched": "PIPELINE-1 §9.2", + "ovos.intent.unmatched": "PIPELINE-1 §9.3", + "ovos.intent.handler.start": "PIPELINE-1 §8.1", + "ovos.intent.handler.complete": "PIPELINE-1 §8.1", + "ovos.intent.handler.error": "PIPELINE-1 §8.1", + # OVOS-TRANSFORM-1 + "ovos.utterance.cancelled": "TRANSFORM-1 §8.2", + "ovos.transformer.audio.list": "TRANSFORM-1 §6", + "ovos.transformer.audio.list.response": "TRANSFORM-1 §6", + "ovos.transformer.utterance.list": "TRANSFORM-1 §6", + "ovos.transformer.utterance.list.response": "TRANSFORM-1 §6", + "ovos.transformer.metadata.list": "TRANSFORM-1 §6", + "ovos.transformer.metadata.list.response": "TRANSFORM-1 §6", + "ovos.transformer.intent.list": "TRANSFORM-1 §6", + "ovos.transformer.intent.list.response": "TRANSFORM-1 §6", + "ovos.transformer.dialog.list": "TRANSFORM-1 §6", + "ovos.transformer.dialog.list.response": "TRANSFORM-1 §6", + "ovos.transformer.tts.list": "TRANSFORM-1 §6", + "ovos.transformer.tts.list.response": "TRANSFORM-1 §6", + # OVOS-INTENT-4 + "ovos.intent.register.keyword": "INTENT-4 §5", + "ovos.intent.register.template": "INTENT-4 §6", + "ovos.entity.register": "INTENT-4 §7", + "ovos.intent.deregister": "INTENT-4 §8.2", + "ovos.entity.deregister": "INTENT-4 §8.3", + "ovos.skill.deregister": "INTENT-4 §8.4", + "ovos.intent.enable": "INTENT-4 §8.5", + "ovos.intent.disable": "INTENT-4 §8.5", + "ovos.intent.list": "INTENT-4 §10.1", + "ovos.intent.list.response": "INTENT-4 §10.1", + "ovos.intent.describe": "INTENT-4 §10.2", + "ovos.intent.describe.response": "INTENT-4 §10.2", + # OVOS-STOP-1 + "ovos.stop.ping": "STOP-1 §4.2", + "ovos.stop.pong": "STOP-1 §4.2", + "ovos.stop": "STOP-1 §5.3", + # OVOS-AUDIO-1 (audio output) + "ovos.utterance.speak.b64": "AUDIO-1 §3.4", + "ovos.audio.speech": "AUDIO-1 §4.3", + "ovos.audio.queue": "AUDIO-1 §4.1", + "ovos.audio.play_sound": "AUDIO-1 §4.2", + "ovos.audio.stop": "AUDIO-1 §6", + "ovos.audio.is_speaking": "AUDIO-1 §5.3", + "ovos.audio.output.started": "AUDIO-1 §5.1", + "ovos.audio.output.ended": "AUDIO-1 §5.2", + "ovos.mic.listen": "AUDIO-1 §4.4", + # OVOS-AUDIO-IN-1 (listener) + "ovos.listener.record.started": "AUDIO-IN-1 §6.1", + "ovos.listener.record.ended": "AUDIO-IN-1 §6.2", + "ovos.listener.sleep": "AUDIO-IN-1 §6.3", + "ovos.listener.awoken": "AUDIO-IN-1 §6.4", + # OVOS-SESSION-2 + "ovos.session.sync": "SESSION-2 §2.7", + # OVOS-CONVERSE-1 + "ovos.converse.active.list": "CONVERSE-1 §6.1", + "ovos.converse.active.list.response": "CONVERSE-1 §6.1", + # OVOS-PERSONA-1 §11 bus surface + "ovos.persona.query": "PERSONA-1 §8.5", + "ovos.persona.answer": "PERSONA-1 §8.5", + "ovos.persona.list": "PERSONA-1 §8.7", + "ovos.persona.list.response": "PERSONA-1 §8.7", + "ovos.persona.register": "PERSONA-1 §9", + "ovos.persona.deregister": "PERSONA-1 §9", + "ovos.persona.activated": "PERSONA-1 §11", + "ovos.persona.dismissed": "PERSONA-1 §11", + # OVOS-FALLBACK-1 §9 bus surface + "ovos.fallback.register": "FALLBACK-1 §3.1", + "ovos.fallback.deregister": "FALLBACK-1 §3.2", + # OVOS-COMMON-QUERY-1 §13 bus surface + "ovos.common_query.ping": "COMMON-QUERY-1 §6.1", + "ovos.common_query.pong": "COMMON-QUERY-1 §6.2", + # OVOS-OCP-1 §4 Virtual Media Player + "ovos.common_play.play": "OCP-1 §4.2", + "ovos.common_play.search": "OCP-1 §4.2", + "ovos.common_play.pause": "OCP-1 §4.3", + "ovos.common_play.resume": "OCP-1 §4.3", + "ovos.common_play.stop": "OCP-1 §4.3", + "ovos.common_play.next": "OCP-1 §4.3", + "ovos.common_play.previous": "OCP-1 §4.3", + "ovos.common_play.seek": "OCP-1 §4.3", + "ovos.common_play.player.state": "OCP-1 §4.4", + "ovos.common_play.media.state": "OCP-1 §4.4", + "ovos.common_play.track.state": "OCP-1 §4.4", +} + +#: Topics ``ovos-bus-client`` uses that NO spec defines, so they MUST NOT be +#: enum members (SESSION-2 §1 defers lifecycle topics; CONTEXT-1 §5 routes +#: context mutations through ``ovos.session.sync``, not ``ovos.context.*``). +NON_SPEC_BUS_CLIENT_TOPICS = { + "ovos.session.update_default", + "ovos.session.start", + "ovos.context.set", + "ovos.context.unset", + "ovos.context.clear", +} + + +class TestSpecCompleteness(unittest.TestCase): + """The enum is the single source of truth: spec-complete and spec-only.""" + + def test_enum_covers_every_spec_static_topic(self): + # Completeness: every normative static spec topic has an enum member. + members = {m.value for m in SpecMessage} + missing = set(SPEC_STATIC_TOPICS) - members + self.assertFalse(missing, f"spec topics with no enum member: {missing}") + + def test_enum_is_spec_only(self): + # Spec-only-ness: every enum member is a normative spec topic. + members = {m.value for m in SpecMessage} + extra = members - set(SPEC_STATIC_TOPICS) + self.assertFalse(extra, f"enum members not traced to a spec: {extra}") + + def test_enum_matches_spec_set_exactly(self): + self.assertEqual({m.value for m in SpecMessage}, + set(SPEC_STATIC_TOPICS)) + + def test_bus_client_internals_are_not_enum_members(self): + # session.sync IS spec (SESSION-2 §2.7); update_default/start/context.* + # are NOT and must stay out of the enum. + members = {m.value for m in SpecMessage} + for topic in NON_SPEC_BUS_CLIENT_TOPICS: + self.assertNotIn(topic, members, + f"{topic} is not spec-defined; must not be a member") + self.assertIn("ovos.session.sync", members) # SESSION-2 §2.7 + + def test_no_templated_topics_are_members(self): + # Runtime-assembled topics (MSG-1 §2.1.1) are never static members. + for m in SpecMessage: + self.assertNotIn("<", m.value, m.value) + self.assertNotIn(":", m.value, m.value) + + +class TestMigrationMap(unittest.TestCase): + def test_maps_legacy_to_specmessage(self): + self.assertEqual(MIGRATION_MAP["speak"], SpecMessage.SPEAK) + self.assertEqual(MIGRATION_MAP["recognizer_loop:utterance"], + SpecMessage.UTTERANCE) + + def test_reverse_is_consistent(self): + for legacy, spec in MIGRATION_MAP.items(): + self.assertEqual(SPEC_TO_LEGACY[str(spec)], legacy) + + def test_values_are_spec_members(self): + for spec in MIGRATION_MAP.values(): + self.assertIsInstance(spec, SpecMessage) + + +class TestMigrationCounterpart(unittest.TestCase): + def test_legacy_returns_spec(self): + self.assertEqual(migration_counterpart("speak"), "ovos.utterance.speak") + + def test_spec_returns_legacy(self): + self.assertEqual(migration_counterpart("ovos.utterance.speak"), "speak") + + def test_unmapped_returns_none(self): + self.assertIsNone(migration_counterpart("some.random.topic")) + + def test_roundtrip(self): + for legacy in MIGRATION_MAP: + spec = migration_counterpart(legacy) + self.assertEqual(migration_counterpart(spec), legacy) + + +class TestPayloadTransforms(unittest.TestCase): + """Per-topic payload translation (MIGRATION_PAYLOAD_TRANSFORMS).""" + + def setUp(self): + self.t = NamespaceTranslator() + + # --- structure invariants --- + def test_transform_keys_are_legacy_topics(self): + for legacy in MIGRATION_PAYLOAD_TRANSFORMS: + self.assertIn(legacy, MIGRATION_MAP, + f"{legacy} is not a known legacy topic") + + def test_only_shape_changing_topics_have_transforms(self): + # The 3 shape-changing entries: detach + enable/disable. The handler + # trio is orchestrator-owned and intentionally NOT migrated, so it has + # no transform (see MIGRATION_MAP). + self.assertEqual(set(MIGRATION_PAYLOAD_TRANSFORMS), { + "detach_intent", + "mycroft.skill.enable_intent", + "mycroft.skill.disable_intent", + }) + + def test_transforms_do_not_mutate_input(self): + for legacy, (l2s, s2l) in MIGRATION_PAYLOAD_TRANSFORMS.items(): + src = {"intent_name": "a:b", "handler": "h", "skill_id": "s", + "duration": 1, "traceback": "tb", "exception": "ex", + "lang": "en-US"} + before = dict(src) + l2s(src) + s2l(src) + self.assertEqual(src, before, f"{legacy} mutated its input") + + # --- detach_intent: cleanly bidirectional --- + def test_detach_spec_to_legacy_joins(self): + out = self.t.translate_payload( + "ovos.intent.deregister", "detach_intent", + {"skill_id": "music.skill", "intent_name": "play", "lang": "en-US"}) + self.assertEqual(out, {"intent_name": "music.skill:play"}) + + def test_detach_legacy_to_spec_splits(self): + out = self.t.translate_payload( + "detach_intent", "ovos.intent.deregister", + {"intent_name": "music.skill:play"}) + self.assertEqual(out, {"skill_id": "music.skill", + "intent_name": "play"}) + + def test_detach_roundtrip_spec_legacy_spec(self): + spec = {"skill_id": "music.skill", "intent_name": "play"} + legacy = self.t.translate_payload( + "ovos.intent.deregister", "detach_intent", spec) + back = self.t.translate_payload( + "detach_intent", "ovos.intent.deregister", legacy) + self.assertEqual(back, spec) + + def test_detach_split_on_first_colon_only(self): + out = self.t.translate_payload( + "detach_intent", "ovos.intent.deregister", + {"intent_name": "skill:a:b"}) + self.assertEqual(out, {"skill_id": "skill", "intent_name": "a:b"}) + + # --- handler trio: orchestrator-owned, intentionally NOT migrated --- + def test_handler_trio_is_not_migrated(self): + # PIPELINE-1 §8: the orchestrator emits the spec trio authoritatively; + # the skill framework keeps the legacy topics as a private done-signal. + # They must NOT bridge — bridging would double-emit and reshape a + # shape-changing event. + for legacy, spec in ( + ("mycroft.skill.handler.start", "ovos.intent.handler.start"), + ("mycroft.skill.handler.complete", "ovos.intent.handler.complete"), + ("mycroft.skill.handler.error", "ovos.intent.handler.error"), + ): + self.assertNotIn(legacy, MIGRATION_MAP) + self.assertNotIn(legacy, MIGRATION_PAYLOAD_TRANSFORMS) + self.assertFalse(self.t.is_migrated(legacy)) + self.assertFalse(self.t.is_migrated(spec)) + self.assertFalse(self.t.counterpart_topics(legacy)) + self.assertFalse(self.t.counterpart_topics(spec)) + + # --- enable/disable: documented loss (no skill_id on legacy side) --- + def test_toggle_spec_to_legacy_drops_skill_and_lang(self): + out = self.t.translate_payload( + "ovos.intent.enable", "mycroft.skill.enable_intent", + {"skill_id": "music.skill", "intent_name": "play", "lang": "en-US"}) + self.assertEqual(out, {"intent_name": "play"}) + + def test_toggle_legacy_to_spec_cannot_recover_skill_id(self): + out = self.t.translate_payload( + "mycroft.skill.disable_intent", "ovos.intent.disable", + {"intent_name": "play"}) + # documented limitation: no skill_id / lang available. + self.assertEqual(out, {"intent_name": "play"}) + self.assertNotIn("skill_id", out) + + +class TestTranslatePayload(unittest.TestCase): + """translate_payload direction selection + identity behaviour.""" + + def setUp(self): + self.t = NamespaceTranslator() + + def test_identity_for_payload_compatible_rename(self): + data = {"utterances": ["hello"], "lang": "en-US"} + # speak <-> ovos.utterance.speak has no transform entry -> identity. + self.assertEqual( + self.t.translate_payload("speak", "ovos.utterance.speak", data), + data) + self.assertEqual( + self.t.translate_payload("ovos.utterance.speak", "speak", data), + data) + + def test_identity_returns_new_dict(self): + data = {"a": 1} + out = self.t.translate_payload("speak", "ovos.utterance.speak", data) + self.assertIsNot(out, data) + + def test_non_migrating_topic_returns_copy(self): + data = {"x": 1} + out = self.t.translate_payload("random.a", "random.b", data) + self.assertEqual(out, data) + self.assertIsNot(out, data) + + def test_direction_legacy_source_uses_legacy_to_spec(self): + # from_topic legacy -> applies legacy_to_spec (split form). + out = self.t.translate_payload( + "detach_intent", "ovos.intent.deregister", + {"intent_name": "s:n"}) + self.assertEqual(out, {"skill_id": "s", "intent_name": "n"}) + + def test_direction_spec_source_uses_spec_to_legacy(self): + # from_topic spec -> applies spec_to_legacy (join form). + out = self.t.translate_payload( + "ovos.intent.deregister", "detach_intent", + {"skill_id": "s", "intent_name": "n"}) + self.assertEqual(out, {"intent_name": "s:n"}) + + def test_handles_empty_data(self): + self.assertEqual( + self.t.translate_payload("detach_intent", + "ovos.intent.deregister", {}), + {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_prompt.py b/test/test_prompt.py new file mode 100644 index 00000000..3234589c --- /dev/null +++ b/test/test_prompt.py @@ -0,0 +1,182 @@ +"""Conformance tests for the OVOS-INTENT-2 §4.4 reference prompt renderer. + +A ``.prompt`` substitutes the **double-brace** ``{{name}}`` form only. A single +``{name}`` and any lone brace are literal pass-through text — a prompt is +free-form LLM text that routinely contains literal single braces (JSON, code). +""" +import pytest + +from ovos_spec_tools import ( + LocaleResources, + MalformedResource, + PromptRenderer, + render_prompt, +) + + +def _resources(tmp_path, files): + """Build a LocaleResources over a locale with the given {relpath: text}.""" + locale = tmp_path / "locale" + for relpath, text in files.items(): + path = locale / relpath + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return LocaleResources(str(locale)) + + +# --- render_prompt() — double-brace substitution ----------------------------- + +def test_render_prompt_fills_a_double_brace_slot(): + assert render_prompt("Hello {{name}}.", {"name": "Sam"}) == "Hello Sam." + + +def test_single_brace_is_literal_and_never_substituted(): + """A single ``{name}`` is literal text in a .prompt — never substituted.""" + assert render_prompt("Hello {name}.", {"name": "Sam"}) == "Hello {name}." + assert render_prompt("Hello {name}.", {}) == "Hello {name}." + + +def test_unfilled_double_brace_slot_is_left_literal(): + """A ``{{name}}`` slot is optional — unfilled means literal text, not error.""" + assert render_prompt("Hello {{name}}.", {}) == "Hello {{name}}." + assert render_prompt("Hello {{name}}.") == "Hello {{name}}." + + +def test_partial_fill_leaves_the_rest_literal(): + out = render_prompt("{{greeting}} {{name}}", {"greeting": "Hi"}) + assert out == "Hi {{name}}" + + +def test_literal_json_stays_literal_while_double_brace_substitutes(): + """The motivating case: literal JSON single braces survive, {{name}} fills.""" + text = 'Reply as {"key": "val"} for {{name}}.' + out = render_prompt(text, {"name": "Sam", "key": "K"}) + assert out == 'Reply as {"key": "val"} for Sam.' + + +@pytest.mark.parametrize("text", [ + 'json {"key": 1}', # JSON object braces (single) + "empty {} and { } braces", # not a name + "code {x+1} here", # not a valid name + "single {name} stays", # single-brace is literal in a prompt + "upper {{Name}} stays", # uppercase — outside the charset + "digit {{1st}} stays", # begins with a digit + "empty double {{}} stays", # no name + "spaced double {{ }} stays", # whitespace is not a name +]) +def test_literal_braces_are_untouched(text): + assert render_prompt(text, {"key": "K", "name": "N", "x": "X"}) == text + + +def test_no_substitution_inside_a_fenced_code_block(): + text = ( + "Fill {{greeting}} here.\n" + "```\n" + "{{greeting}} stays literal in the code block\n" + "```\n" + "And {{greeting}} again here.\n" + ) + out = render_prompt(text, {"greeting": "HI"}) + assert out == ( + "Fill HI here.\n" + "```\n" + "{{greeting}} stays literal in the code block\n" + "```\n" + "And HI again here.\n" + ) + + +def test_fence_with_an_info_string_is_recognized(): + text = "```json\n{{x}}\n```\n{{x}}" + assert render_prompt(text, {"x": "V"}) == "```json\n{{x}}\n```\nV" + + +def test_whole_file_is_verbatim_apart_from_substitution(): + """`#` lines and blank lines are kept — a prompt is not line-filtered.""" + text = "# A heading\n\nBody with {{slot}}.\n\n indented line\n" + out = render_prompt(text, {"slot": "X"}) + assert out == "# A heading\n\nBody with X.\n\n indented line\n" + + +def test_html_comment_is_literal_pass_through(): + """A .prompt has no comment syntax — ```` is literal text.""" + text = " Body for {{name}}." + assert render_prompt(text, {"name": "Sam"}) == " Body for Sam." + + +def test_numeric_slot_value_is_stringified(): + assert render_prompt("count {{n}}", {"n": 0}) == "count 0" + + +def test_filled_value_is_not_re_scanned_for_slots(): + """A value containing `{{...}}` is inserted literally, not substituted again.""" + assert render_prompt("say {{x}}", {"x": "{{y}}"}) == "say {{y}}" + + +def test_text_with_no_trailing_newline_is_preserved(): + assert render_prompt("no newline {{x}}", {"x": "here"}) == "no newline here" + + +# --- PromptRenderer — stateful, multilingual --------------------------------- + +def test_renderer_renders_from_resources(tmp_path): + res = _resources(tmp_path, {"en-US/sys.prompt": "You are {{role}}."}) + renderer = PromptRenderer(res, "sys") + assert renderer.render("en-US", {"role": "a helper"}) == "You are a helper." + + +def test_renderer_serves_multiple_languages(tmp_path): + res = _resources(tmp_path, { + "en-US/sys.prompt": "Answer in English.", + "pt-PT/sys.prompt": "Responde em português.", + }) + renderer = PromptRenderer(res, "sys") + assert renderer.render("en-US") == "Answer in English." + assert renderer.render("pt-PT") == "Responde em português." + + +def test_renderer_default_slots_are_reused(tmp_path): + res = _resources(tmp_path, {"en-US/g.prompt": "You are {{assistant}}."}) + renderer = PromptRenderer(res, "g", slots={"assistant": "OVOS"}) + assert renderer.render("en-US") == "You are OVOS." + assert renderer.render("en-US") == "You are OVOS." + + +def test_per_call_slot_overrides_a_default(tmp_path): + res = _resources(tmp_path, {"en-US/g.prompt": "You are {{assistant}}."}) + renderer = PromptRenderer(res, "g", slots={"assistant": "OVOS"}) + assert renderer.render("en-US", {"assistant": "Mycroft"}) == "You are Mycroft." + + +def test_renderer_missing_prompt_raises(tmp_path): + res = _resources(tmp_path, {"en-US/x.prompt": "hi"}) + renderer = PromptRenderer(res, "absent") + with pytest.raises(FileNotFoundError): + renderer.render("en-US") + + +# --- load_prompt() ----------------------------------------------------------- + +def test_load_prompt_returns_the_whole_file(tmp_path): + text = "# Heading\n\nBody {{slot}} text.\n" + res = _resources(tmp_path, {"en-US/p.prompt": text}) + assert res.load_prompt("p", "en-US") == text + + +def test_load_prompt_passes_single_brace_and_comments_verbatim(tmp_path): + """read_prompt_file/load_prompt does no stripping — all text is literal.""" + text = ' {"json": 1} single {slot} double {{slot}}\n' + res = _resources(tmp_path, {"en-US/p.prompt": text}) + assert res.load_prompt("p", "en-US") == text + + +def test_empty_prompt_is_malformed(tmp_path): + res = _resources(tmp_path, {"en-US/p.prompt": " \n\n"}) + with pytest.raises(MalformedResource): + res.load_prompt("p", "en-US") + + +def test_load_prompt_missing_raises(tmp_path): + res = _resources(tmp_path, {"en-US/other.prompt": "hi"}) + with pytest.raises(FileNotFoundError): + res.load_prompt("p", "en-US") diff --git a/test/test_resources.py b/test/test_resources.py new file mode 100644 index 00000000..a03f28a4 --- /dev/null +++ b/test/test_resources.py @@ -0,0 +1,744 @@ +"""Conformance tests for the OVOS-INTENT-2 reference loader.""" +import pytest + +from ovos_spec_tools import ( + LocaleResources, + MalformedResource, + read_resource_file, +) + + +def _write(path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +# --- §3 common reader -------------------------------------------------------- + +def test_common_reader_skips_blanks_and_comments(tmp_path): + f = tmp_path / "x.voc" + _write(f, "# a comment\n\nyes\n yeah \n# another\nyep\n") + assert read_resource_file(f) == ["yes", "yeah", "yep"] + + +def test_common_reader_accepts_crlf_and_bom(tmp_path): + f = tmp_path / "x.voc" + f.write_bytes(b"\xef\xbb\xbfyes\r\nyeah\r\n") # BOM + CRLF + assert read_resource_file(f) == ["yes", "yeah"] + + +# --- §4.1 .intent ------------------------------------------------------------ + +def test_load_intent_expands_and_keeps_slots(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "(play|put on) {query}\n") + res = LocaleResources(str(locale)) + assert sorted(res.load_intent("play", "en-US")) == [ + "play {query}", "put on {query}", + ] + + +def test_load_intent_resolves_voc_references(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greeting.voc", "hello\nhi\n") + _write(locale / "en-US" / "greet.intent", " {name}\n") + res = LocaleResources(str(locale)) + assert sorted(res.load_intent("greet", "en-US")) == [ + "hello {name}", "hi {name}", + ] + + +# --- one instance, many languages ------------------------------------------- + +def test_one_instance_serves_multiple_languages(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "hi.intent", "hello\n") + _write(locale / "pt-BR" / "hi.intent", "ola\n") + res = LocaleResources(str(locale)) + assert res.load_intent("hi", "en-US") == ["hello"] + assert res.load_intent("hi", "pt-BR") == ["ola"] + + +# --- §4.2 .dialog ------------------------------------------------------------ + +def test_load_dialog_returns_unexpanded_phrases(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "hi.dialog", "Hello {name}!\n(Hi|Hey) {name}.\n") + res = LocaleResources(str(locale)) + assert res.load_dialog("hi", "en-US") == ["Hello {name}!", "(Hi|Hey) {name}."] + + +# --- §4.3 slot-free roles ---------------------------------------------------- + +def test_load_entity_and_blacklist(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "weekday.entity", "monday\n(tues|wednes)day\n") + _write(locale / "en-US" / "play.blacklist", "trailer\n") + res = LocaleResources(str(locale)) + assert sorted(res.load_entity("weekday", "en-US")) == [ + "monday", "tuesday", "wednesday", + ] + assert res.load_blacklist("play", "en-US") == ["trailer"] + + +def test_slot_free_role_rejects_a_named_slot(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "bad.voc", "a slot {here}\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedResource): + res.load_vocabulary("bad", "en-US") + + +# --- §2 layout --------------------------------------------------------------- + +def test_subdirectories_are_searched_recursively(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "intents" / "deep.intent", "do the thing\n") + res = LocaleResources(str(locale)) + assert res.load_intent("deep", "en-US") == ["do the thing"] + + +def test_language_tag_is_case_insensitive(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-us" / "x.intent", "hello world\n") + res = LocaleResources(str(locale)) + assert res.load_intent("x", "en-US") == ["hello world"] + + +def test_underscore_tag_matches_hyphen_tag(tmp_path): + """Tags are standardized before comparison, so en_US finds en-US.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.intent", "hello world\n") + res = LocaleResources(str(locale)) + assert res.load_intent("x", "en_US") == ["hello world"] + + +def test_duplicate_resource_in_one_tree_is_malformed(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "a" / "dup.intent", "first\n") + _write(locale / "en-US" / "b" / "dup.intent", "second\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedResource): + res.load_intent("dup", "en-US") + + +# --- §2.1 resolution precedence --------------------------------------------- + +def test_user_override_wins_over_skill(tmp_path): + skill_locale = tmp_path / "skill" / "locale" + user_locale = tmp_path / "user" / "locale" + _write(skill_locale / "en-US" / "x.intent", "skill version\n") + _write(user_locale / "en-US" / "x.intent", "user version\n") + res = LocaleResources(str(skill_locale), user_locale=str(user_locale)) + assert res.load_intent("x", "en-US") == ["user version"] + + +# --- §5 empty files ---------------------------------------------------------- + +def test_empty_file_is_malformed(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "empty.intent", "# only a comment\n\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedResource): + res.load_intent("empty", "en-US") + + +def test_missing_resource_raises(tmp_path): + locale = tmp_path / "locale" + locale.mkdir() + res = LocaleResources(str(locale)) + with pytest.raises(FileNotFoundError): + res.load_intent("nope", "en-US") + + +# --- §2.2 smart language fallback ------------------------------------------- + +def test_fallback_resolves_a_near_language(tmp_path): + pytest.importorskip("langcodes") + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.intent", "hello world\n") + res = LocaleResources(str(locale)) + assert res.load_intent("x", "en-AU") == ["hello world"] + + +def test_fallback_rejects_a_too_distant_language(tmp_path): + pytest.importorskip("langcodes") + locale = tmp_path / "locale" + _write(locale / "fr-FR" / "x.intent", "bonjour\n") + res = LocaleResources(str(locale)) + with pytest.raises(FileNotFoundError): + res.load_intent("x", "en-US") + + +def test_fallback_disabled_with_zero_max_distance(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.intent", "hello\n") + res = LocaleResources(str(locale), max_language_distance=0) + with pytest.raises(FileNotFoundError): + res.load_intent("x", "en-AU") + + +def test_fallback_resolved_per_query_not_once(tmp_path): + """One instance: an exact tag and a fallback tag resolve independently.""" + pytest.importorskip("langcodes") + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.intent", "us text\n") + res = LocaleResources(str(locale)) + assert res.load_intent("x", "en-US") == ["us text"] # exact + assert res.load_intent("x", "en-AU") == ["us text"] # fallback + + +def test_custom_lang_resolver_is_honored(tmp_path): + """A caller may inject its own language resolver.""" + locale = tmp_path / "locale" + _write(locale / "en-GB" / "x.intent", "british\n") + + def resolver(target, available, max_distance): + return "en-GB" if "en-GB" in available else None + + res = LocaleResources(str(locale), lang_resolver=resolver) + assert res.load_intent("x", "zz-ZZ") == ["british"] + + +# --- edge cases -------------------------------------------------------------- + +def test_core_resources_are_a_fallback(tmp_path): + skill = tmp_path / "skill" / "locale" + core = tmp_path / "core" / "locale" + _write(core / "en-US" / "x.intent", "core version\n") + res = LocaleResources(str(skill), core_locale=str(core)) + assert res.load_intent("x", "en-US") == ["core version"] + + +def test_skill_resources_override_core(tmp_path): + skill = tmp_path / "skill" / "locale" + core = tmp_path / "core" / "locale" + _write(skill / "en-US" / "x.intent", "skill version\n") + _write(core / "en-US" / "x.intent", "core version\n") + res = LocaleResources(str(skill), core_locale=str(core)) + assert res.load_intent("x", "en-US") == ["skill version"] + + +def test_empty_dialog_is_malformed(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "e.dialog", "# nothing here\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedResource): + res.load_dialog("e", "en-US") + + +def test_empty_voc_is_malformed(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "e.voc", "\n\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedResource): + res.load_vocabulary("e", "en-US") + + +def test_vocabularies_collects_every_voc(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "yes\n") + _write(locale / "en-US" / "no.voc", "no\n") + res = LocaleResources(str(locale)) + assert set(res.vocabularies("en-US")) == {"yes", "no"} + + +def test_entities_collects_every_entity(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "day.entity", "monday\n(tues|wednes)day\n") + res = LocaleResources(str(locale)) + assert sorted(res.entities("en-US")["day"]) == [ + "monday", "tuesday", "wednesday", + ] + + +def test_indented_comment_is_skipped(tmp_path): + f = tmp_path / "x.voc" + _write(f, " # an indented comment\nyes\n") + assert read_resource_file(f) == ["yes"] + + +def test_hash_mid_line_is_literal_not_a_comment(tmp_path): + """Only a leading # starts a comment — there are no inline comments (§3).""" + f = tmp_path / "x.voc" + _write(f, "channel # five\n") + assert read_resource_file(f) == ["channel # five"] + + +def test_nonexistent_skill_locale_raises_on_load(tmp_path): + res = LocaleResources(str(tmp_path / "does-not-exist")) + with pytest.raises(FileNotFoundError): + res.load_intent("x", "en-US") + + +def test_intent_with_undefined_voc_reference_is_rejected(tmp_path): + from ovos_spec_tools import MalformedTemplate + locale = tmp_path / "locale" + _write(locale / "en-US" / "g.intent", " hello\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedTemplate): + res.load_intent("g", "en-US") + + +# --- LocaleResources.find ---------------------------------------------------- + +def test_find_returns_path_when_resource_exists(tmp_path): + locale = tmp_path / "locale" + f = locale / "en-US" / "play.intent" + _write(f, "play {query}\n") + res = LocaleResources(str(locale)) + assert res.find("play", ".intent", "en-US") == f + + +def test_find_returns_none_when_resource_missing(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "play.intent", "play {query}\n") + res = LocaleResources(str(locale)) + assert res.find("stop", ".intent", "en-US") is None + + +def test_find_recurses_into_subdirectories(tmp_path): + """A resource may live in a nested category folder under /.""" + locale = tmp_path / "locale" + f = locale / "en-US" / "media" / "play.intent" + _write(f, "play {query}\n") + res = LocaleResources(str(locale)) + assert res.find("play", ".intent", "en-US") == f + + +def test_find_resolves_lang_via_closest_match(tmp_path): + """A request for ``en`` matches an ``en-US/`` tree.""" + pytest.importorskip("langcodes") + locale = tmp_path / "locale" + f = locale / "en-US" / "play.intent" + _write(f, "play {query}\n") + res = LocaleResources(str(locale)) + assert res.find("play", ".intent", "en") == f + + +def test_find_walks_override_precedence(tmp_path): + """user > skill > core: the first source carrying the file wins.""" + core = tmp_path / "core" + skill = tmp_path / "skill" + user = tmp_path / "user" + _write(core / "en-US" / "play.intent", "core-version\n") + _write(skill / "en-US" / "play.intent", "skill-version\n") + _write(user / "en-US" / "play.intent", "user-version\n") + res = LocaleResources(str(skill), + core_locale=str(core), + user_locale=str(user)) + assert res.find("play", ".intent", "en-US").parent == user / "en-US" + + +def test_find_raises_on_duplicate_within_language_tree(tmp_path): + """OVOS-INTENT-2 §2: a (role, base name) must be unique per language tree.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "a" / "play.intent", "play one\n") + _write(locale / "en-US" / "b" / "play.intent", "play two\n") + res = LocaleResources(str(locale)) + with pytest.raises(MalformedResource): + res.find("play", ".intent", "en-US") + + +# --- keyword_form ------------------------------------------------------------ + +def test_keyword_form_groups_alternatives_into_entity_and_aliases(): + from ovos_spec_tools import keyword_form + entity, aliases = keyword_form("(hi|hello|hey)") + # sorted, lowercased, first is canonical + assert (entity, aliases) == ("hello", ["hey", "hi"]) + + +def test_keyword_form_lowercases_and_dedupes(): + from ovos_spec_tools import keyword_form + entity, aliases = keyword_form("(YES|Yes|yes)") + assert (entity, aliases) == ("yes", []) + + +def test_keyword_form_empty_input_returns_empty_pair(): + from ovos_spec_tools import keyword_form + assert keyword_form("") == ("", []) + + +def test_keyword_form_resolves_voc_references(): + """`` references in a template expand against the supplied vocab.""" + from ovos_spec_tools import keyword_form + entity, aliases = keyword_form( + " friend", vocabularies={"greet": ["hi", "hello"]}) + assert sorted([entity, *aliases]) == ["hello friend", "hi friend"] + + +# --- LocaleResources.vocabulary_keywords / entity_keywords ------------------- + +def test_vocabulary_keywords_yields_one_triple_per_template_line(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.voc", + "(hi|hello)\n(yo|hey)\n") + res = LocaleResources(str(locale)) + triples = sorted(res.vocabulary_keywords("en-US")) + assert triples == [ + ("greet", "hello", ["hi"]), + ("greet", "hey", ["yo"]), + ] + + +def test_vocabulary_keywords_skips_blank_lines(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "\nyes\n# a comment\n") + res = LocaleResources(str(locale)) + assert list(res.vocabulary_keywords("en-US")) == [("yes", "yes", [])] + + +def test_entity_keywords_uses_same_shape(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "color.entity", "(red|crimson)\nblue\n") + res = LocaleResources(str(locale)) + triples = sorted(res.entity_keywords("en-US")) + assert triples == [ + ("color", "blue", []), + ("color", "crimson", ["red"]), + ] + + +# --- utterance_contains ------------------------------------------------------ + +def test_utterance_contains_whole_word_substring_default(): + from ovos_spec_tools import utterance_contains + assert utterance_contains("yes, please", ["yes"]) # default whole-word + assert not utterance_contains("yesterday", ["yes"]) # substring not allowed + + +def test_utterance_contains_exact_mode(): + from ovos_spec_tools import utterance_contains + assert utterance_contains("Yes", ["yes"], exact=True) + assert not utterance_contains("yes please", ["yes"], exact=True) + + +def test_utterance_contains_folds_accents_and_punct_by_default(): + from ovos_spec_tools import utterance_contains + # á → a, ! stripped, comparison succeeds + assert utterance_contains("Olá!", ["ola"]) + + +def test_utterance_contains_keeps_diacritics_when_disabled(): + from ovos_spec_tools import utterance_contains + assert utterance_contains("ola", ["olá"], strip_diacritics=False) is False + assert utterance_contains("olá", ["olá"], strip_diacritics=False) + + +def test_utterance_contains_keeps_punct_when_disabled(): + """Punctuation stays significant when strip_punct=False.""" + from ovos_spec_tools import utterance_contains + assert utterance_contains("i love c++", ["c++"], strip_punct=False) + assert utterance_contains( + "i love c plus plus", ["c++"], strip_punct=False) is False + + +def test_utterance_contains_punct_and_diacritics_flags_compose(): + """The two flags are independent — any combination is valid.""" + from ovos_spec_tools import utterance_contains + # both stripped (default) — matches across accents and punctuation + assert utterance_contains("olá!", ["ola"]) + # diacritics stripped, punctuation kept — `!` becomes a word boundary + assert utterance_contains( + "olá!", ["ola"], strip_diacritics=True, strip_punct=False) + # punct stripped, diacritics kept — `olá!` → `olá`; sample `ola` misses + assert utterance_contains( + "olá!", ["ola"], strip_diacritics=False, strip_punct=True) is False + + +def test_utterance_contains_exact_mode_normalizes_both_sides(): + """Exact comparison still applies the configured normalization.""" + from ovos_spec_tools import utterance_contains + assert utterance_contains(" Olá! ", ["ola"], exact=True) + assert utterance_contains( + " Olá! ", ["ola"], exact=True, strip_diacritics=False) is False + + +def test_utterance_contains_empty_inputs_return_false(): + from ovos_spec_tools import utterance_contains + assert utterance_contains("", ["yes"]) is False + assert utterance_contains("yes", []) is False + + +# --- strip_samples ----------------------------------------------------------- + +def test_strip_samples_removes_whole_word_matches(): + from ovos_spec_tools import strip_samples + out = strip_samples("set volume to maximum", ["set", "volume"]) + assert "set" not in out.split() + assert "volume" not in out.split() + assert "maximum" in out.split() + + +def test_strip_samples_longest_first_consumes_composite_before_parts(): + """A composite phrase is removed before its shorter constituents.""" + from ovos_spec_tools import strip_samples + out = strip_samples("give it up now", ["give up", "up"]) + # ``give it up`` does not contain the contiguous phrase "give up", + # so the longer pattern misses; "up" is still stripped as a fallback. + assert "up" not in out.split() + + +def test_strip_samples_case_insensitive(): + from ovos_spec_tools import strip_samples + assert "Yes" not in strip_samples("Yes please", ["yes"]).split() + + +def test_strip_samples_no_match_returns_input_unchanged(): + from ovos_spec_tools import strip_samples + assert strip_samples("no match here", ["xyzzy"]) == "no match here" + + +# --- normalize_for_match ----------------------------------------------------- + +def test_normalize_for_match_lowercases_and_trims(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match(" YES ") == "yes" + + +def test_normalize_for_match_strips_diacritics_by_default(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("Olá") == "ola" + assert normalize_for_match("über") == "uber" + + +def test_normalize_for_match_strips_punct_by_default(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("yes, please!") == "yes please" + + +def test_normalize_for_match_preserves_slot_markers(): + """``{`` and ``}`` survive punctuation stripping so pre-render-pass + slot markers stay intact.""" + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("play {song}") == "play {song}" + + +def test_normalize_for_match_keeps_punct_when_disabled(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("yes, please!", strip_punct=False) == "yes, please!" + + +def test_normalize_for_match_keeps_diacritics_when_disabled(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("Olá", strip_diacritics=False) == "olá" + + +def test_normalize_for_match_both_flags_off_only_lowercases_and_trims(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match( + " Olá, World! ", + strip_diacritics=False, strip_punct=False) == "olá, world!" + + +def test_normalize_for_match_empty_string(): + from ovos_spec_tools import normalize_for_match + assert normalize_for_match("") == "" + assert normalize_for_match(" ") == "" + + +def test_normalize_for_match_is_keyword_only_for_flags(): + """The two flags are keyword-only — positional misuse can't silently + flip the wrong knob.""" + from ovos_spec_tools import normalize_for_match + import pytest as _pytest + with _pytest.raises(TypeError): + normalize_for_match("hi", False) # would have to be keyword + + +# --- strip_samples extra coverage -------------------------------------------- + +def test_strip_samples_empty_samples_returns_input_unchanged(): + from ovos_spec_tools import strip_samples + assert strip_samples("hello world", []) == "hello world" + + +def test_strip_samples_ignores_blank_samples(): + """A blank or whitespace-only sample is dropped silently.""" + from ovos_spec_tools import strip_samples + assert strip_samples("hello world", ["", " ", "world"]).strip() == "hello" + + +def test_strip_samples_escapes_regex_metacharacters_in_samples(): + """A sample like ``c++`` must not be treated as a regex pattern.""" + from ovos_spec_tools import strip_samples + out = strip_samples("i love c++ programming", ["c++"]) + assert "c++" not in out + + +def test_strip_samples_preserves_casing_of_remaining_text(): + from ovos_spec_tools import strip_samples + out = strip_samples("Yes Please", ["yes"]) + assert "Please" in out # capital P preserved + assert "Yes" not in out.split() + + +# --- vocabulary_keywords / entity_keywords extra coverage -------------------- + +def test_vocabulary_keywords_walks_override_precedence(tmp_path): + """A user-locale .voc overrides the same-name skill-locale one.""" + skill = tmp_path / "skill" + user = tmp_path / "user" + _write(skill / "en-US" / "color.voc", "(red|crimson)\n") + _write(user / "en-US" / "color.voc", "(blue|azure)\n") + res = LocaleResources(str(skill), user_locale=str(user)) + triples = sorted(res.vocabulary_keywords("en-US")) + # both files contribute — user overrides core, but both are reachable + # because each source is walked for its own .voc files + names = {entity for _, entity, _ in triples} + assert "azure" in names + + +def test_vocabulary_keywords_handles_missing_locale_gracefully(tmp_path): + """A skill that ships no .voc files yields nothing — no exception.""" + locale = tmp_path / "locale" + (locale / "en-US").mkdir(parents=True) + res = LocaleResources(str(locale)) + assert list(res.vocabulary_keywords("en-US")) == [] + + +def test_vocabulary_keywords_resolves_voc_references_in_template(tmp_path): + """A ```` reference in one .voc is resolved against the lang's + full vocab map at expansion time.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.voc", "hi\nhello\n") + _write(locale / "en-US" / "greet_friend.voc", " friend\n") + res = LocaleResources(str(locale)) + triples = sorted(res.vocabulary_keywords("en-US")) + # the greet_friend.voc line expands its reference + friend_triples = [t for t in triples if t[0] == "greet_friend"] + assert friend_triples == [("greet_friend", "hello friend", ["hi friend"])] + + +def test_vocabulary_keywords_skips_malformed_lines(tmp_path): + """A malformed template inside a .voc yields no keyword for that line + rather than raising — well-formed neighbours still register.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "things.voc", + "(hi|hello)\n(broken\nfine\n") + res = LocaleResources(str(locale)) + entities = {e for _, e, _ in res.vocabulary_keywords("en-US")} + assert "fine" in entities # well-formed line survived + # ``(broken`` does not yield (entity, []) + assert "(broken" not in entities + + +# --- keyword_form extra coverage --------------------------------------------- + +def test_keyword_form_blank_line_returns_empty_pair(): + from ovos_spec_tools import keyword_form + assert keyword_form(" \n ") == ("", []) + + +def test_keyword_form_malformed_template_returns_empty_pair(): + """A line that expansion rejects yields ('', []) — never propagates.""" + from ovos_spec_tools import keyword_form + assert keyword_form("(unclosed") == ("", []) + + +def test_keyword_form_single_alternative_yields_no_aliases(): + """A line without alternatives has the line itself as the entity.""" + from ovos_spec_tools import keyword_form + assert keyword_form("hello") == ("hello", []) + + +def test_vocabulary_keywords_returns_empty_when_language_has_no_dir(tmp_path): + """Asking for a language with no directory yields nothing (no exception) + — covers the `lang_dir is None: continue` branch in _keywords_for.""" + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.voc", "yes\n") + res = LocaleResources(str(locale), max_language_distance=0) + assert list(res.vocabulary_keywords("ja-JP")) == [] + assert list(res.entity_keywords("ja-JP")) == [] + + +def test_strip_samples_handles_unicode_samples(): + """Unicode samples that need escaping in regex still strip cleanly.""" + from ovos_spec_tools import strip_samples + assert "olá" not in strip_samples("ola olá", ["olá"]).split() + + +# --- LocaleResources.voc_match / .remove_voc --------------------------------- + +def test_voc_match_whole_word_default(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "(yes|yeah|yep)\n") + res = LocaleResources(str(locale)) + assert res.voc_match("yes, please", "yes", "en-US") + assert res.voc_match("yeah whatever", "yes", "en-US") + assert res.voc_match("yesterday", "yes", "en-US") is False + + +def test_voc_match_exact_mode(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "yes\n") + res = LocaleResources(str(locale)) + assert res.voc_match("Yes", "yes", "en-US", exact=True) + assert res.voc_match("yes please", "yes", "en-US", exact=True) is False + + +def test_voc_match_missing_resource_returns_false(tmp_path): + locale = tmp_path / "locale" + (locale / "en-US").mkdir(parents=True) + res = LocaleResources(str(locale)) + assert res.voc_match("anything", "nope", "en-US") is False + + +def test_voc_match_forwards_normalization_flags(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "greet.voc", "ola\n") + res = LocaleResources(str(locale)) + # default: diacritics stripped, "Olá" matches + assert res.voc_match("Olá", "greet", "en-US") + # with strip_diacritics=False, distinct + assert res.voc_match("Olá", "greet", "en-US", + strip_diacritics=False) is False + + +def test_remove_voc_strips_voc_phrases(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "fillers.voc", "(please|kindly)\n") + res = LocaleResources(str(locale)) + out = res.remove_voc("please set volume to ten kindly", "fillers", "en-US") + assert "please" not in out.split() + assert "kindly" not in out.split() + assert "volume" in out.split() + + +def test_remove_voc_missing_resource_returns_input_unchanged(tmp_path): + locale = tmp_path / "locale" + (locale / "en-US").mkdir(parents=True) + res = LocaleResources(str(locale)) + assert res.remove_voc("hello world", "nope", "en-US") == "hello world" + + +def test_remove_voc_empty_utterance_returns_empty(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "x.voc", "yes\n") + res = LocaleResources(str(locale)) + assert res.remove_voc("", "x", "en-US") == "" + + +def test_voc_match_and_remove_voc_resolve_smart_lang_fallback(tmp_path): + """en-AU falls back to en-US for both methods.""" + pytest.importorskip("langcodes") + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "yes\n") + res = LocaleResources(str(locale)) + assert res.voc_match("yes please", "yes", "en-AU") + assert "yes" not in res.remove_voc("yes please", "yes", "en-AU").split() + + +def test_voc_list_returns_expanded_samples(tmp_path): + locale = tmp_path / "locale" + _write(locale / "en-US" / "yes.voc", "(yes|yeah)\n") + res = LocaleResources(str(locale)) + assert sorted(res.voc_list("yes", "en-US")) == ["yeah", "yes"] + + +def test_voc_list_missing_resource_returns_empty(tmp_path): + locale = tmp_path / "locale" + (locale / "en-US").mkdir(parents=True) + res = LocaleResources(str(locale)) + assert res.voc_list("nope", "en-US") == [] diff --git a/test/test_session.py b/test/test_session.py new file mode 100644 index 00000000..170602da --- /dev/null +++ b/test/test_session.py @@ -0,0 +1,612 @@ +"""OVOS-SESSION-1 conformance tests for :class:`ovos_spec_tools.Session`. + +Every section heading below cites the spec section under test. The +handler-list sections additionally cite OVOS-PIPELINE-1 §7.1 and +OVOS-CONVERSE-1 §2.1 / §2.2 / §3, whose fields SESSION-1 §3 registers.""" +import json +import unittest + +from ovos_spec_tools import ( + DEFAULT_CONVERSE_HANDLERS_CAP, DEFAULT_SESSION_ID, MalformedSession, + SESSION1_OWNED_FIELDS, SESSION1_REGISTERED_FIELDS, Session) + + +# --- §2 wire shape --------------------------------------------------------- + +class TestWireShape(unittest.TestCase): + def test_empty_session_is_wellformed(self): + s = Session() + self.assertEqual(s.to_dict(), {}) + self.assertEqual(s.serialize(), "{}") + + def test_only_session_id_is_wellformed(self): + s = Session(session_id="abc") + self.assertEqual(s.to_dict(), {"session_id": "abc"}) + + def test_omitted_field_never_serializes_as_null(self): + s = Session(session_id="abc", lang=None) + self.assertNotIn("lang", s.to_dict()) + self.assertNotIn("null", s.serialize()) + + def test_explicit_null_on_wire_is_logged_and_treated_as_omitted(self): + with self.assertLogs("ovos_spec_tools.session", level="WARNING"): + s = Session.from_dict({"session_id": "abc", "lang": None}) + self.assertIsNone(s.lang) + self.assertEqual(s.session_id, "abc") + + def test_explicit_null_on_registered_other_spec_field_treated_as_omitted(self): + # §2.1 — null on a field claimed by another spec is also omitted. + with self.assertLogs("ovos_spec_tools.session", level="WARNING"): + s = Session.from_dict({"session_id": "abc", + "active_handlers": None, + "response_mode": None}) + self.assertEqual(s.active_handlers, []) + self.assertIsNone(s.response_mode) + self.assertNotIn("active_handlers", s.to_dict()) + self.assertNotIn("response_mode", s.to_dict()) + + def test_unknown_field_passes_through(self): + # §2.4 unknown-field tolerance + §4 propagation + s = Session.from_dict({"session_id": "abc", + "novel_future_field": 42}) + self.assertEqual(s.extras["novel_future_field"], 42) + self.assertEqual(s.to_dict()["novel_future_field"], 42) + + def test_registered_other_spec_field_is_first_class(self): + # §3 — `pipeline` is registered (owner OVOS-PIPELINE-1), so it + # lands as a first-class attribute, not in `extras`. + s = Session.from_dict({"pipeline": ["padatious_high"]}) + self.assertEqual(s.pipeline, ["padatious_high"]) + self.assertNotIn("pipeline", s.extras) + self.assertEqual(s.to_dict()["pipeline"], ["padatious_high"]) + + def test_non_dict_payload_is_malformed(self): + with self.assertRaises(MalformedSession): + Session.from_dict(["not", "an", "object"]) # type: ignore[arg-type] + + def test_unparsable_json_is_malformed(self): + with self.assertRaises(MalformedSession): + Session.deserialize("{not valid") + + def test_none_payload_yields_empty_session(self): + # §2.1 — absent session ≡ empty object + self.assertEqual(Session.deserialize(None), Session()) + self.assertEqual(Session.from_dict(None), Session()) + + +# --- §3.1 session_id semantics -------------------------------------------- + +class TestSessionId(unittest.TestCase): + def test_omitted_resolves_to_default(self): + # §2.1 + §3.1 — three forms map to the same identity + self.assertEqual(Session().resolved_session_id(), DEFAULT_SESSION_ID) + self.assertEqual( + Session.from_dict({}).resolved_session_id(), DEFAULT_SESSION_ID) + self.assertEqual( + Session(session_id=DEFAULT_SESSION_ID).resolved_session_id(), + DEFAULT_SESSION_ID) + + def test_is_default_predicate(self): + self.assertTrue(Session().is_default) + self.assertTrue(Session(session_id=DEFAULT_SESSION_ID).is_default) + self.assertFalse(Session(session_id="remote-42").is_default) + + def test_empty_session_id_rejected(self): + with self.assertRaises(MalformedSession): + Session(session_id="") + + def test_non_string_session_id_rejected(self): + with self.assertRaises(MalformedSession): + Session(session_id=123) # type: ignore[arg-type] + + +# --- §3.2 language signals ------------------------------------------------- + +class TestLanguageSignals(unittest.TestCase): + def test_all_six_lang_fields_round_trip(self): + s = Session( + lang="en-US", secondary_langs=["es-ES", "fr-FR"], + output_lang="de-DE", stt_lang="en-GB", + request_lang="en-US", detected_lang="fr-FR") + self.assertEqual(Session.from_dict(s.to_dict()), s) + + def test_secondary_langs_must_not_contain_lang(self): + # §3.2.2 + with self.assertRaises(MalformedSession): + Session(lang="en-US", secondary_langs=["en-US", "fr-FR"]) + + def test_secondary_langs_no_duplicates(self): + with self.assertRaises(MalformedSession): + Session(secondary_langs=["fr-FR", "fr-FR"]) + + def test_secondary_langs_rejects_empty_string(self): + with self.assertRaises(MalformedSession): + Session(secondary_langs=[""]) + + def test_lang_must_be_string(self): + with self.assertRaises(MalformedSession): + Session(lang=42) # type: ignore[arg-type] + + +# --- §3.3 site_id ---------------------------------------------------------- + +class TestSiteId(unittest.TestCase): + def test_site_id_round_trip(self): + s = Session(site_id="kitchen") + self.assertEqual(s.to_dict(), {"site_id": "kitchen"}) + self.assertEqual(Session.from_dict(s.to_dict()), s) + + def test_empty_site_id_rejected(self): + with self.assertRaises(MalformedSession): + Session(site_id="") + + def test_unknown_site_id_carries_no_meaning(self): + # §3.3 — "unknown" is a normal opaque value, not reserved. + s = Session(site_id="unknown") + self.assertEqual(s.to_dict(), {"site_id": "unknown"}) + + +# --- OVOS-PERSONA-1 registered persona_id ---------------------------------- + +class TestPersonaId(unittest.TestCase): + def test_persona_id_round_trip(self): + s = Session(persona_id="default") + self.assertEqual(s.to_dict(), {"persona_id": "default"}) + self.assertEqual(Session.from_dict(s.to_dict()), s) + + def test_persona_id_omitted_not_null_when_none(self): + # §2.1 omission-not-null: absent persona_id never serializes. + s = Session() + self.assertIsNone(s.persona_id) + self.assertNotIn("persona_id", s.to_dict()) + self.assertNotIn("persona_id", s.serialize()) + + def test_persona_id_is_registered_not_owned(self): + # OVOS-PERSONA-1 registers it; OVOS-SESSION-1 does not own it. + self.assertIn("persona_id", SESSION1_REGISTERED_FIELDS) + self.assertNotIn("persona_id", SESSION1_OWNED_FIELDS) + + def test_persona_id_is_first_class_not_extra(self): + # Registered ⇒ lands as a first-class attribute, not in `extras`. + s = Session.from_dict({"persona_id": "assistant"}) + self.assertEqual(s.persona_id, "assistant") + self.assertNotIn("persona_id", s.extras) + + def test_empty_persona_id_rejected(self): + with self.assertRaises(MalformedSession): + Session(persona_id="") + + def test_explicit_null_persona_id_treated_as_omitted(self): + # §2.1: explicit null on a registered field ⇒ omitted, not error. + s = Session.from_dict({"persona_id": None}) + self.assertIsNone(s.persona_id) + self.assertNotIn("persona_id", s.to_dict()) + + +class TestFallbackHandlers(unittest.TestCase): + """OVOS-SESSION-1 §3 registers ``fallback_handlers`` (array of string, + owner OVOS-FALLBACK-1 §4). It is an array-of-string override field — + same bucket / wire rules as the blacklists and transformer chains.""" + + def test_fallback_handlers_round_trip(self): + s = Session(fallback_handlers=["skill-a", "skill-b"]) + self.assertEqual(s.to_dict(), + {"fallback_handlers": ["skill-a", "skill-b"]}) + self.assertEqual(Session.from_dict(s.to_dict()), s) + self.assertEqual(Session.deserialize(s.serialize()), s) + + def test_fallback_handlers_omitted_not_null_when_none(self): + # §2.1 omission-not-null: absent fallback_handlers never serializes. + s = Session() + self.assertIsNone(s.fallback_handlers) + self.assertNotIn("fallback_handlers", s.to_dict()) + self.assertNotIn("fallback_handlers", s.serialize()) + + def test_empty_fallback_handlers_is_wire_equivalent_to_omission(self): + # §3.4 — an empty array on a list override is dropped to omission. + s = Session(fallback_handlers=[]) + self.assertIsNone(s.fallback_handlers) + self.assertEqual(s.to_dict(), {}) + + def test_fallback_handlers_is_registered_not_owned(self): + # OVOS-FALLBACK-1 registers it; OVOS-SESSION-1 does not own it. + self.assertIn("fallback_handlers", SESSION1_REGISTERED_FIELDS) + self.assertNotIn("fallback_handlers", SESSION1_OWNED_FIELDS) + + def test_fallback_handlers_is_first_class_not_extra(self): + # Registered ⇒ lands as a first-class attribute, not in `extras`. + s = Session.from_dict({"fallback_handlers": ["skill-x"]}) + self.assertEqual(s.fallback_handlers, ["skill-x"]) + self.assertNotIn("fallback_handlers", s.extras) + + def test_explicit_null_fallback_handlers_treated_as_omitted(self): + # §2.1: explicit null on a registered field ⇒ omitted, not error. + s = Session.from_dict({"fallback_handlers": None}) + self.assertIsNone(s.fallback_handlers) + self.assertNotIn("fallback_handlers", s.to_dict()) + + def test_non_list_fallback_handlers_rejected(self): + # Wrong wire type for an array-of-string override is malformed, + # same as the blacklists / transformer chains (§3). + with self.assertRaises(MalformedSession): + Session(fallback_handlers="not-a-list") + + +# --- §3 / §3.4 other-spec override fields ---------------------------------- + +class TestOverrideFields(unittest.TestCase): + def test_full_section_3_field_set_round_trips(self): + payload = { + "session_id": "abc", + "lang": "en-US", + "pipeline": ["stop_high", "converse", "adapt_high"], + "intent_context": {"frame_stack": [], "timeout": 120}, + "blacklisted_skills": ["skill-a"], + "blacklisted_intents": ["skill-b:foo"], + "blacklisted_pipelines": ["fallback_low"], + "audio_transformers": ["a"], + "utterance_transformers": ["b"], + "metadata_transformers": ["c"], + "intent_transformers": ["d"], + "dialog_transformers": ["e"], + "tts_transformers": ["f"], + "blacklisted_audio_transformers": ["ba"], + "blacklisted_utterance_transformers": ["bb"], + "blacklisted_metadata_transformers": ["bc"], + "blacklisted_intent_transformers": ["bd"], + "blacklisted_dialog_transformers": ["be"], + "blacklisted_tts_transformers": ["bf"], + } + s = Session.from_dict(payload) + self.assertEqual(s.to_dict(), payload) + self.assertEqual(Session.deserialize(s.serialize()), s) + + def test_empty_list_override_is_wire_equivalent_to_omission(self): + # §3.4 — an empty array on a list override is dropped. + s = Session(pipeline=[], blacklisted_skills=[], + audio_transformers=[]) + self.assertEqual(s.to_dict(), {}) + self.assertEqual(s.pipeline, None) + + def test_intent_context_object_round_trips(self): + s = Session(intent_context={"frame_stack": [["x", 1.0]]}) + self.assertEqual(s.to_dict()["intent_context"], + {"frame_stack": [["x", 1.0]]}) + + def test_context_key_is_not_registered_passes_through_as_extra(self): + # The spec field is `intent_context`, not `context`; a stray + # `context` key is unknown and rides in `extras` (§2.4). + s = Session.from_dict({"context": {"k": "v"}}) + self.assertEqual(s.extras["context"], {"k": "v"}) + self.assertEqual(s.to_dict()["context"], {"k": "v"}) + + +# --- OVOS-PIPELINE-1 §7.1 active_handlers ---------------------------------- + +class TestActiveHandlers(unittest.TestCase): + def test_add_active_handler_head_first(self): + s = Session() + s.add_active_handler("a", activated_at=1.0) + s.add_active_handler("b", activated_at=2.0) + self.assertEqual([h["skill_id"] for h in s.active_handlers], + ["b", "a"]) + + def test_add_active_handler_dedup_promotes_to_head(self): + # §7.1 — re-activation evicts the prior entry and re-inserts at head. + s = Session() + s.add_active_handler("a", activated_at=1.0) + s.add_active_handler("b", activated_at=2.0) + s.add_active_handler("a", activated_at=3.0) + self.assertEqual([h["skill_id"] for h in s.active_handlers], + ["a", "b"]) + self.assertEqual(s.active_handlers[0]["activated_at"], 3.0) + # no duplicate entry for "a" + self.assertEqual(len([h for h in s.active_handlers + if h["skill_id"] == "a"]), 1) + + def test_remove_active_handler(self): + s = Session() + s.add_active_handler("a") + s.add_active_handler("b") + s.remove_active_handler("a") + self.assertEqual([h["skill_id"] for h in s.active_handlers], ["b"]) + + def test_active_predicate(self): + s = Session() + self.assertFalse(s.active) + s.add_active_handler("a") + self.assertTrue(s.active) + + def test_active_handlers_round_trip(self): + s = Session() + s.add_active_handler("a", activated_at=1.0) + self.assertEqual(Session.from_dict(s.to_dict()), s) + + def test_legacy_pair_shape_coerced_on_deserialize(self): + # tolerant input — the legacy [skill_id, ts] pair shape coerces + # into the spec object shape. + s = Session.from_dict({"active_handlers": [["a", 1.0], ["b", 2.0]]}) + self.assertEqual(s.active_handlers, + [{"skill_id": "a", "activated_at": 1.0}, + {"skill_id": "b", "activated_at": 2.0}]) + + +# --- OVOS-CONVERSE-1 §2.1 converse_handlers -------------------------------- + +class TestConverseHandlers(unittest.TestCase): + def test_default_cap_is_64(self): + # The cap is the spec's §2.1 default for the add_converse_handler + # argument — NOT a session field. + self.assertEqual(DEFAULT_CONVERSE_HANDLERS_CAP, 64) + + def test_cap_is_not_session_state(self): + # §2.1 — the cap is a deployment value the orchestrator applies at + # insertion time, never an attribute carried on the session. + s = Session() + self.assertFalse(hasattr(s, "converse_handlers_cap")) + self.assertNotIn("converse_handlers_cap", s.to_dict()) + + def test_add_converse_handler_head_first_dedup(self): + s = Session() + s.add_converse_handler("a", activated_at=1.0) + s.add_converse_handler("b", activated_at=2.0) + s.add_converse_handler("a", activated_at=3.0) + self.assertEqual([h["skill_id"] for h in s.converse_handlers], + ["a", "b"]) + + def test_cap_evicts_tail(self): + # §2.1 — when the cap would be exceeded, drop the least-recent tail. + # The cap is supplied per insertion by the orchestrator. + s = Session() + for i in range(5): + s.add_converse_handler(f"s{i}", activated_at=float(i), cap=3) + self.assertEqual([h["skill_id"] for h in s.converse_handlers], + ["s4", "s3", "s2"]) + self.assertEqual(len(s.converse_handlers), 3) + + def test_not_capped_on_construction(self): + # §2.1 — a constructed session with an over-cap converse_handlers + # list is NOT auto-truncated on load; the cap applies only at the + # next insertion. + seed = [{"skill_id": f"s{i}", "activated_at": float(i)} + for i in range(10)] + s = Session(converse_handlers=seed) + self.assertEqual(len(s.converse_handlers), 10) + # the cap is enforced only on the next add_converse_handler. + s.add_converse_handler("new", activated_at=99.0, cap=4) + self.assertEqual(len(s.converse_handlers), 4) + self.assertEqual(s.converse_handlers[0]["skill_id"], "new") + + def test_not_capped_on_deserialize(self): + # §2.1 — a deserialized over-cap list is preserved verbatim; the + # cap is applied only on the next capped insertion. + seed = {"converse_handlers": [ + {"skill_id": f"s{i}", "activated_at": float(i)} + for i in range(10)]} + s = Session.deserialize(json.dumps(seed)) + self.assertEqual(len(s.converse_handlers), 10) + + def test_cap_unbounded_when_non_positive(self): + s = Session() + for i in range(100): + s.add_converse_handler(f"s{i}", activated_at=float(i), cap=0) + self.assertEqual(len(s.converse_handlers), 100) + + def test_cap_unbounded_when_none(self): + s = Session() + for i in range(100): + s.add_converse_handler(f"s{i}", activated_at=float(i), cap=None) + self.assertEqual(len(s.converse_handlers), 100) + + def test_prune_drops_stale_entries(self): + # §3.2 — entries older than ttl are pruned at now - activated_at > ttl. + s = Session() + s.add_converse_handler("fresh", activated_at=100.0) + s.add_converse_handler("stale", activated_at=10.0) + s.prune_converse_handlers(ttl=50, now=100.0) + self.assertEqual([h["skill_id"] for h in s.converse_handlers], + ["fresh"]) + + def test_prune_noop_when_ttl_non_positive(self): + s = Session() + s.add_converse_handler("a", activated_at=1.0) + s.prune_converse_handlers(ttl=0, now=1e9) + self.assertEqual(len(s.converse_handlers), 1) + + def test_remove_converse_handler(self): + s = Session() + s.add_converse_handler("a") + s.add_converse_handler("b") + s.remove_converse_handler("a") + self.assertEqual([h["skill_id"] for h in s.converse_handlers], ["b"]) + + def test_converse_handlers_round_trip(self): + s = Session() + s.add_converse_handler("a", activated_at=1.0) + self.assertEqual(Session.from_dict(s.to_dict()), s) + + +# --- OVOS-CONVERSE-1 §2.2 response_mode ------------------------------------ + +class TestResponseMode(unittest.TestCase): + def test_absent_by_default(self): + s = Session() + self.assertIsNone(s.response_mode) + self.assertNotIn("response_mode", s.to_dict()) + + def test_set_response_mode_shape(self): + s = Session() + s.set_response_mode("skill-a", expires_at=123.0) + self.assertEqual(s.response_mode, + {"skill_id": "skill-a", "expires_at": 123.0}) + + def test_single_holder_overwrite(self): + # §2.2 — setting while another holds overwrites silently. + s = Session() + s.set_response_mode("skill-a", expires_at=1.0) + s.set_response_mode("skill-b", expires_at=2.0) + self.assertEqual(s.response_mode["skill_id"], "skill-b") + + def test_clear_unconditional(self): + s = Session() + s.set_response_mode("skill-a", expires_at=1.0) + s.clear_response_mode() + self.assertIsNone(s.response_mode) + + def test_clear_only_own_holder(self): + # a skill MUST NOT clear another's hold. + s = Session() + s.set_response_mode("skill-a", expires_at=1.0) + s.clear_response_mode("skill-b") + self.assertEqual(s.response_mode["skill_id"], "skill-a") + s.clear_response_mode("skill-a") + self.assertIsNone(s.response_mode) + + def test_malformed_response_mode_resolves_to_none(self): + # §2.1 — a holder with no skill_id is not a valid window. + self.assertIsNone(Session(response_mode={}).response_mode) + self.assertIsNone( + Session.from_dict({"response_mode": {"expires_at": 1}}).response_mode) + + def test_response_mode_round_trip(self): + s = Session() + s.set_response_mode("skill-a", expires_at=99.0) + self.assertEqual(Session.from_dict(s.to_dict()), s) + + +# --- §4 propagation -------------------------------------------------------- + +class TestPropagation(unittest.TestCase): + def test_propagate_is_deep_copy(self): + s = Session(session_id="abc", pipeline=["padatious_high"]) + copy = s.propagate() + self.assertEqual(copy, s) + copy.pipeline.append("adapt_high") + self.assertEqual(s.pipeline, ["padatious_high"]) + + def test_propagate_carries_handlers(self): + s = Session() + s.add_active_handler("a", activated_at=1.0) + s.add_converse_handler("b", activated_at=2.0) + s.set_response_mode("c", expires_at=3.0) + copy = s.propagate() + self.assertEqual(copy, s) + # the cap is not session state — nothing to carry across propagation. + self.assertFalse(hasattr(copy, "converse_handlers_cap")) + self.assertEqual(copy.active_handlers, s.active_handlers) + # deep copy — mutating the twin does not touch the source + copy.active_handlers[0]["activated_at"] = 999.0 + self.assertEqual(s.active_handlers[0]["activated_at"], 1.0) + + def test_unknown_field_survives_propagation(self): + s = Session.from_dict({"session_id": "abc", + "novel_future_field": [1, 2, 3]}) + self.assertEqual(s.propagate().to_dict()["novel_future_field"], + [1, 2, 3]) + + def test_materialize_default_sets_only_session_id(self): + # §4.1 + s = Session.materialize_default() + self.assertEqual(s.to_dict(), {"session_id": DEFAULT_SESSION_ID}) + + +# --- §5 serialization ------------------------------------------------------ + +class TestSerialization(unittest.TestCase): + def test_serialize_round_trip_via_json_str(self): + s = Session(session_id="abc", lang="en-US", + pipeline=["padatious_high"]) + wire = s.serialize() + self.assertEqual(Session.deserialize(wire), s) + + def test_serialize_rejects_nan(self): + # §5 + OVOS-MSG-1 §6 — numbers MUST be finite + s = Session(extras={"weird": float("nan")}) + with self.assertRaises(ValueError): + s.serialize() + + def test_session_id_not_distinguished_from_other_strings(self): + # §3.1 — "default" is a normal session value, structurally + self.assertEqual(Session(session_id=DEFAULT_SESSION_ID).to_dict(), + {"session_id": "default"}) + + def test_no_null_anywhere_on_a_fully_populated_session(self): + # §2.1 — nothing serializes as null even when many fields set. + s = Session(session_id="abc", lang="en-US", site_id="kitchen", + pipeline=["adapt_high"]) + s.add_active_handler("a", activated_at=1.0) + s.set_response_mode("a", expires_at=2.0) + self.assertNotIn("null", s.serialize()) + + +# --- §6 conformance, registry -------------------------------------------- + +class TestRegistry(unittest.TestCase): + def test_owned_fields_registry(self): + # The fields whose semantics SESSION-1 itself owns: §3.1 session_id, + # §3.2 the six language signals, §3.3 site_id. + self.assertEqual(SESSION1_OWNED_FIELDS, frozenset({ + "session_id", "site_id", "lang", "secondary_langs", "output_lang", + "stt_lang", "request_lang", "detected_lang", + })) + + def test_registered_fields_superset_of_owned(self): + self.assertTrue(SESSION1_OWNED_FIELDS.issubset( + SESSION1_REGISTERED_FIELDS)) + # the full §3 roster includes the other-spec registered fields + for f in ("pipeline", "intent_context", "active_handlers", + "converse_handlers", "response_mode", + "blacklisted_skills", "audio_transformers"): + self.assertIn(f, SESSION1_REGISTERED_FIELDS) + + +# --- hashability (library-usefulness; not a spec rule) --------------------- + +class TestHashable(unittest.TestCase): + def test_equal_sessions_hash_equal(self): + a = Session(session_id="abc", lang="en-US") + b = Session(session_id="abc", lang="en-US") + self.assertEqual(a, b) + self.assertEqual(hash(a), hash(b)) + + def test_usable_as_dict_key_and_set_member(self): + a = Session(session_id="abc") + b = Session(session_id="abc") + c = Session(session_id="xyz") + d = {a: "value"} + self.assertEqual(d[b], "value") + self.assertEqual(len({a, b, c}), 2) + + def test_usable_in_lru_cache(self): + import functools + + calls = [] + + @functools.lru_cache(maxsize=None) + def resolve(sess): + calls.append(sess.session_id) + return sess.session_id + + s1 = Session(session_id="abc", lang="en-US") + s2 = Session(session_id="abc", lang="en-US") # equal to s1 + self.assertEqual(resolve(s1), "abc") + self.assertEqual(resolve(s2), "abc") + self.assertEqual(calls, ["abc"]) # cached: only one underlying call + + def test_nested_override_fields_hash(self): + s = Session(session_id="abc", + secondary_langs=["pt-PT", "es-ES"], + pipeline=["padatious_high"]) + self.assertEqual(hash(s), hash(s)) # stable, must not raise + + def test_mutation_then_rehash_changes_hash(self): + # documented behavior: the hash snapshots the mutable Session at the + # moment of hashing; mutating a field afterwards changes the hash. + s = Session(session_id="abc") + h1 = hash(s) + s.lang = "en-US" + self.assertNotEqual(hash(s), h1) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_session_manager.py b/test/test_session_manager.py new file mode 100644 index 00000000..59656692 --- /dev/null +++ b/test/test_session_manager.py @@ -0,0 +1,93 @@ +"""SessionManager (singleton registry) + forward/reply session stamping. + +SessionManager is an implementation detail enforcing the OVOS-SESSION-1 §4 +value-passing contract; these tests pin its core invariants. +""" +import unittest + +from ovos_spec_tools.session import Session, SessionManager +from ovos_spec_tools.message import Message + + +class TestSessionManagerRegistry(unittest.TestCase): + def setUp(self): + SessionManager.sessions = {} + SessionManager.default_session = None + + def test_one_live_object_per_id(self): + a = SessionManager.update(Session("s1")) + b = SessionManager.get(Message("x", context={"session": {"session_id": "s1"}})) + self.assertIs(a, b) # same live object, folded not replaced + + def test_held_reference_observes_later_fold(self): + held = SessionManager.update(Session("s2")) + # a later snapshot for the same id is folded onto the held object + snap = {"session_id": "s2", "active_handlers": [{"skill_id": "k", "activated_at": 1.0}]} + SessionManager.get(Message("x", context={"session": snap})) + self.assertEqual([h["skill_id"] for h in held.active_handlers], ["k"]) + + def test_default_folds_like_any_session(self): + # the default session is a normal session per §4 — no owner-only + # reservation; a message carrying it folds onto the live default. + d = SessionManager.get_default_session() + SessionManager.get(Message("x", context={"session": {"session_id": "default", + "lang": "pt-PT"}})) + self.assertEqual(d.lang, "pt-PT") + self.assertIs(d, SessionManager.get_default_session()) + + +class TestForwardReplyStamping(unittest.TestCase): + def setUp(self): + SessionManager.sessions = {} + SessionManager.default_session = None + + def test_forward_refreshes_to_live_session(self): + # get -> mutate -> forward: the derived message carries the live state, + # not the originating message's pre-mutation snapshot. + live = SessionManager.update(Session("123")) + live.add_active_handler("my.skill") + orig = Message("utt", context={"session": {"session_id": "123"}}) + fwd = orig.forward("my.skill.activate") + ah = fwd.context["session"]["active_handlers"] + self.assertEqual([h["skill_id"] for h in ah], ["my.skill"]) + + def test_reply_refreshes_to_live_session(self): + live = SessionManager.update(Session("123")) + live.add_active_handler("my.skill") + orig = Message("ask", context={"session": {"session_id": "123"}, + "source": "A", "destination": "B"}) + rep = orig.reply("ask.response") + ah = rep.context["session"]["active_handlers"] + self.assertEqual([h["skill_id"] for h in ah], ["my.skill"]) + # §5.2 routing still reversed + self.assertEqual(rep.context["source"], "B") + self.assertEqual(rep.context["destination"], "A") + + def test_unowned_id_left_untouched(self): + # a session id this process never folded is carried verbatim (§5) + snap = {"session_id": "remote", "active_handlers": []} + fwd = Message("u", context={"session": dict(snap)}).forward("x") + self.assertEqual(fwd.context["session"], snap) + + def test_session_less_message_stays_session_less(self): + fwd = Message("u", context={}).forward("x") + self.assertNotIn("session", fwd.context) + + def test_idless_session_normalizes_to_default(self): + # §4.1/§4.3: a session that names no id IS the default session. + SessionManager.get_default_session().add_active_handler("d.skill") + fwd = Message("u", context={"session": {"lang": "en-US"}}).forward("x") + self.assertEqual(fwd.context["session"]["session_id"], "default") + + def test_stamp_is_noop_when_nothing_changed(self): + # no intervening update -> stamp re-serializes the same state + live = SessionManager.update(Session("123")) + live.add_active_handler("a") + orig = Message("u", context={"session": live.to_dict()}) + fwd = orig.forward("x") + self.assertEqual([h["skill_id"] for h in fwd.context["session"]["active_handlers"]], + ["a"]) + + +if __name__ == "__main__": + unittest.main()