From 19ec10b4eae2754c1b8aac5821cbc0865812e97c Mon Sep 17 00:00:00 2001 From: Alejandro Giacometti Date: Wed, 8 Jul 2026 23:51:10 +0100 Subject: [PATCH] Add a tree-sitter query language plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Index .scm files — the tree-sitter query language rbtr's own plugins are written in. Each top-level pattern becomes a doc_section chunk, named by its own outer capture where the author gave one (query.scm pairs an optional @_section_name to a direct-child capture), else a legitimately anonymous section; nested captures stay full-text-searchable either way. No name_extractor — the query resolves the name. Adds the tree-sitter-query dependency and a comprehensive sample exercising every construct. rbtr now indexes its own query files. Update docs (rbtr-languages skill, README, ARCHITECTURE): add the query plugin to the primary-path list and note that rbtr now indexes .scm files (including third-party tags/highlights/injections). --- .agents/skills/rbtr-languages/SKILL.md | 9 +- packages/rbtr/ARCHITECTURE.md | 3 +- packages/rbtr/README.md | 6 +- packages/rbtr/pyproject.toml | 1 + packages/rbtr/src/rbtr/languages/__init__.py | 2 + .../rbtr/src/rbtr/languages/query/__init__.py | 1 + .../rbtr/src/rbtr/languages/query/plugin.py | 43 ++++ .../rbtr/src/rbtr/languages/query/query.scm | 11 + ...st_sample_edges_match_snapshot[query].json | 1 + ...le_extraction_matches_snapshot[query].json | 200 ++++++++++++++++++ .../src/rbtr/tests/languages/cases_samples.py | 13 ++ .../languages/samples/query/highlights.scm | 39 ++++ uv.lock | 18 ++ 13 files changed, 341 insertions(+), 6 deletions(-) create mode 100644 packages/rbtr/src/rbtr/languages/query/__init__.py create mode 100644 packages/rbtr/src/rbtr/languages/query/plugin.py create mode 100644 packages/rbtr/src/rbtr/languages/query/query.scm create mode 100644 packages/rbtr/src/rbtr/tests/languages/__snapshots__/test_samples/test_sample_edges_match_snapshot[query].json create mode 100644 packages/rbtr/src/rbtr/tests/languages/__snapshots__/test_samples/test_sample_extraction_matches_snapshot[query].json create mode 100644 packages/rbtr/src/rbtr/tests/languages/samples/query/highlights.scm diff --git a/.agents/skills/rbtr-languages/SKILL.md b/.agents/skills/rbtr-languages/SKILL.md index 04324ed..f9921d8 100644 --- a/.agents/skills/rbtr-languages/SKILL.md +++ b/.agents/skills/rbtr-languages/SKILL.md @@ -84,9 +84,11 @@ The orchestrator's primary path is one of three: as an injection target for an embedded block (see below). 2. **Query** — `reg.grammar_module` + `reg.query`: code, plus config/data whose scope a query can express (python, rust, …, json, css, html, toml, - yaml, hcl). Goes through `extract_symbols`. + yaml, hcl, query). Goes through `extract_symbols`. HTML captures its semantic elements (`head`, `body`, sectioning content, landmarks) as doc sections, named by `id` else tag via a `name_extractor`. + The `query` plugin indexes `.scm` files themselves: each top-level pattern + is a `@doc_section`, named by its own outer capture else anonymous. 3. **Plaintext fallback** — no grammar/detection: fixed-size raw chunks. `extract_symbols` is the query engine: *parse → run query → captures → @@ -252,7 +254,10 @@ itself a tree-sitter query, so it is **inspiration for ours, not a drop-in**: Distinct from **rbtr's own** `.scm` files (see *Where queries live*): we load those at runtime (`reg.query` / `injection_query`) as the source of truth, -whereas `tags.scm` we only mine for ideas. +whereas `tags.scm` we only mine for ideas. And the `query` plugin now +*indexes* `.scm` files found in a repo — including third-party `tags.scm` / +`highlights.scm` / `injections.scm` — as content, orthogonal to whether we +run them. Curated per-language verdicts (take / modify / ignore) and the `@definition.* → ChunkKind` mapping live in diff --git a/packages/rbtr/ARCHITECTURE.md b/packages/rbtr/ARCHITECTURE.md index 55a2fe4..67a43f1 100644 --- a/packages/rbtr/ARCHITECTURE.md +++ b/packages/rbtr/ARCHITECTURE.md @@ -1091,7 +1091,8 @@ capture conventions), see language has one (`chunker(path, sha, content, grammar, ranges)`; markdown, rst, svelte, vue), else tree-sitter `extract_symbols` for a `grammar` + `query` (python, - rust, go, ...; json, css, html, toml, yaml, hcl). + rust, go, ...; json, css, html, toml, yaml, hcl; and + `query` itself, indexing `.scm` files). 2. **Injection (additive)** - if the registration has an `injection_query`, the engine *also* runs `extract_injections`: it matches embedded blocks diff --git a/packages/rbtr/README.md b/packages/rbtr/README.md index 91c6760..f763ab0 100644 --- a/packages/rbtr/README.md +++ b/packages/rbtr/README.md @@ -358,9 +358,9 @@ extraction (symbol-level chunks, import metadata, scope detection). Everything else gets line-based chunking. Built-in: bash, c, cpp, css, go, hcl, html, java, -javascript, json, less, markdown, python, rst, ruby, -rust, scss, sql, svelte, toml, tsx, typescript, vue, -yaml. +javascript, json, less, markdown, python, query, rst, +ruby, rust, scss, sql, svelte, toml, tsx, typescript, +vue, yaml. Code embedded in another file is indexed in its own language. A fenced code block in Markdown is extracted as diff --git a/packages/rbtr/pyproject.toml b/packages/rbtr/pyproject.toml index efad210..d3f9fcd 100644 --- a/packages/rbtr/pyproject.toml +++ b/packages/rbtr/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ "tree-sitter-less", "tree-sitter-markdown", "tree-sitter-python", + "tree-sitter-query", "tree-sitter-rst", "tree-sitter-ruby", "tree-sitter-rust", diff --git a/packages/rbtr/src/rbtr/languages/__init__.py b/packages/rbtr/src/rbtr/languages/__init__.py index 617f287..de9bdee 100644 --- a/packages/rbtr/src/rbtr/languages/__init__.py +++ b/packages/rbtr/src/rbtr/languages/__init__.py @@ -55,6 +55,7 @@ from rbtr.languages.less.plugin import LessPlugin from rbtr.languages.markdown.plugin import MarkdownPlugin from rbtr.languages.python.plugin import PythonPlugin +from rbtr.languages.query.plugin import QueryPlugin from rbtr.languages.rst.plugin import RstPlugin from rbtr.languages.ruby.plugin import RubyPlugin from rbtr.languages.rust.plugin import RustPlugin @@ -120,6 +121,7 @@ def _register_builtins(self) -> None: LessPlugin, MarkdownPlugin, PythonPlugin, + QueryPlugin, RstPlugin, RubyPlugin, RustPlugin, diff --git a/packages/rbtr/src/rbtr/languages/query/__init__.py b/packages/rbtr/src/rbtr/languages/query/__init__.py new file mode 100644 index 0000000..2d86509 --- /dev/null +++ b/packages/rbtr/src/rbtr/languages/query/__init__.py @@ -0,0 +1 @@ +"""tree-sitter query language plugin package.""" diff --git a/packages/rbtr/src/rbtr/languages/query/plugin.py b/packages/rbtr/src/rbtr/languages/query/plugin.py new file mode 100644 index 0000000..3868ef8 --- /dev/null +++ b/packages/rbtr/src/rbtr/languages/query/plugin.py @@ -0,0 +1,43 @@ +"""tree-sitter query language plugin. + +Indexes `.scm` files — the tree-sitter query language that rbtr's own +language plugins are written in (see the `.scm` files beside each +`plugin.py`). Third-party grammars ship `.scm` too (`tags.scm`, +`highlights.scm`, `injections.scm`), so cloned repos are covered as well. + +Each top-level pattern becomes a `doc_section` chunk, named by its own +outer capture where the author gave one (`query.scm` pairs it as +`@_section_name`): + + (function_definition name: (identifier) @_fn_name) @function + → section "function" + + ["if" "else"] @keyword + → section "keyword" + +Patterns with no outer label of their own — a predicate-wrapped injection +rule, or one anchored at a structural wrapper — are anonymous sections; +their nested captures stay full-text-searchable. +""" + +from __future__ import annotations + +from rbtr.languages._queries import load_query +from rbtr.languages.hookspec import LanguageRegistration, hookimpl + + +class QueryPlugin: + """tree-sitter query (`.scm`) language support.""" + + @hookimpl + def rbtr_register_languages(self) -> list[LanguageRegistration]: + return [ + LanguageRegistration( + id="query", + extensions=frozenset({".scm"}), + grammar_module="tree_sitter_query", + query=load_query(__package__, "query"), + doc_comment_node_types=frozenset({"comment"}), + language_plugin_version=1, + ), + ] diff --git a/packages/rbtr/src/rbtr/languages/query/query.scm b/packages/rbtr/src/rbtr/languages/query/query.scm new file mode 100644 index 0000000..160ac8c --- /dev/null +++ b/packages/rbtr/src/rbtr/languages/query/query.scm @@ -0,0 +1,11 @@ +; Every top-level pattern is a section. A pattern is named by its own outer +; label — a capture that is a direct child of the pattern node (e.g. +; `(function_definition …) @function` → "function"; `["if" "else"] @keyword` +; → "keyword"). The `?` makes that label optional: predicate-wrapped rules +; and structural wrappers carry no label of their own and stay anonymous. +; Captures, predicates, and matched node types inside remain full-text +; searchable within the section either way. +(program (named_node (capture (identifier) @_section_name)?) @doc_section) +(program (list (capture (identifier) @_section_name)?) @doc_section) +(program (grouping (capture (identifier) @_section_name)?) @doc_section) +(program (anonymous_node (capture (identifier) @_section_name)?) @doc_section) diff --git a/packages/rbtr/src/rbtr/tests/languages/__snapshots__/test_samples/test_sample_edges_match_snapshot[query].json b/packages/rbtr/src/rbtr/tests/languages/__snapshots__/test_samples/test_sample_edges_match_snapshot[query].json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/packages/rbtr/src/rbtr/tests/languages/__snapshots__/test_samples/test_sample_edges_match_snapshot[query].json @@ -0,0 +1 @@ +[] diff --git a/packages/rbtr/src/rbtr/tests/languages/__snapshots__/test_samples/test_sample_extraction_matches_snapshot[query].json b/packages/rbtr/src/rbtr/tests/languages/__snapshots__/test_samples/test_sample_extraction_matches_snapshot[query].json new file mode 100644 index 0000000..8ca315f --- /dev/null +++ b/packages/rbtr/src/rbtr/tests/languages/__snapshots__/test_samples/test_sample_extraction_matches_snapshot[query].json @@ -0,0 +1,200 @@ +[ + { + "id": "95bcbb40fa68e03d", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "definition.function", + "scope": "", + "language": "query", + "content": "; Named-node patterns with a field and an outer definition capture.\n(function_definition\n name: (identifier) @function\n body: (_) @_body) @definition.function", + "line_start": 6, + "line_end": 9, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + }, + { + "id": "b9d016d76766b006", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "definition.class", + "scope": "", + "language": "query", + "content": "(class_definition\n name: (identifier) @type) @definition.class", + "line_start": 11, + "line_end": 12, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + }, + { + "id": "12955e88d71fa702", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "keyword", + "scope": "", + "language": "query", + "content": "; Alternation list and a bare anonymous-node matcher.\n[\"if\" \"else\" \"while\" \"return\"] @keyword", + "line_start": 14, + "line_end": 15, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + }, + { + "id": "0a7aae7f30c848e6", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "keyword.coroutine", + "scope": "", + "language": "query", + "content": "\"async\" @keyword.coroutine", + "line_start": 17, + "line_end": 17, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + }, + { + "id": "9fd153dd462c97fe", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "", + "scope": "", + "language": "query", + "content": "; Wildcard child, a negated field, anchors, and quantifiers.\n(call_expression\n function: (_) @call\n !type_arguments)", + "line_start": 19, + "line_end": 22, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + }, + { + "id": "08bf92e385e51759", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "", + "scope": "", + "language": "query", + "content": "(array . (identifier) @first (_)* @rest)", + "line_start": 24, + "line_end": 24, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + }, + { + "id": "d4b473df03d82d82", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "", + "scope": "", + "language": "query", + "content": "; Groupings with predicates: regex match, alternation, and an injection.\n((identifier) @constant\n (#match? @constant \"^[A-Z][A-Z_]+$\"))", + "line_start": 26, + "line_end": 28, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + }, + { + "id": "fa51260b21d08e68", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "", + "scope": "", + "language": "query", + "content": "((identifier) @type.builtin\n (#any-of? @type.builtin \"int\" \"str\" \"bool\"))", + "line_start": 30, + "line_end": 31, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + }, + { + "id": "005bca176ae4b980", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "", + "scope": "", + "language": "query", + "content": "((comment) @injection.content\n (#set! injection.language \"markdown\"))", + "line_start": 33, + "line_end": 34, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + }, + { + "id": "cf19ec3f8c8c0c41", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "definition.function", + "scope": "", + "language": "query", + "content": "; Supertype/subtype match and a MISSING-node pattern.\n(declaration/function_definition) @definition.function", + "line_start": 36, + "line_end": 37, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + }, + { + "id": "c001bdd71bc27782", + "blob_sha": "sha1", + "file_path": "highlights.scm", + "kind": "doc_section", + "name": "", + "scope": "", + "language": "query", + "content": "(binary_expression (MISSING identifier))", + "line_start": 39, + "line_end": 39, + "metadata": { + "module": "", + "names": "", + "dots": "", + "language_hint": "" + } + } +] diff --git a/packages/rbtr/src/rbtr/tests/languages/cases_samples.py b/packages/rbtr/src/rbtr/tests/languages/cases_samples.py index 0c346de..777a06e 100644 --- a/packages/rbtr/src/rbtr/tests/languages/cases_samples.py +++ b/packages/rbtr/src/rbtr/tests/languages/cases_samples.py @@ -143,6 +143,19 @@ def case_hcl() -> SampleCase: ) +@case(id="query", tags=["sample"]) +def case_query() -> SampleCase: + """tree-sitter query (`.scm`): every top-level pattern is a doc section, + named by its definition capture (else its matched node type). Exercises + named nodes, alternation lists, groupings, predicates, wildcards, negated + fields, anchors, quantifiers, supertypes, and a MISSING node. + """ + return ( + "query", + {ChunkKind.DOC_SECTION}, + ) + + @case(id="rst", tags=["sample"]) def case_rst() -> SampleCase: """reStructuredText: heading hierarchy from adornment order (as doc diff --git a/packages/rbtr/src/rbtr/tests/languages/samples/query/highlights.scm b/packages/rbtr/src/rbtr/tests/languages/samples/query/highlights.scm new file mode 100644 index 0000000..28e39c9 --- /dev/null +++ b/packages/rbtr/src/rbtr/tests/languages/samples/query/highlights.scm @@ -0,0 +1,39 @@ +; A tree-sitter query file exercising every construct rbtr should index: +; named nodes with fields, anonymous nodes, alternation lists, groupings, +; predicates, wildcards, negated fields, anchors, quantifiers, supertypes, +; and a MISSING-node pattern. Every top-level pattern is one doc section. + +; Named-node patterns with a field and an outer definition capture. +(function_definition + name: (identifier) @function + body: (_) @_body) @definition.function + +(class_definition + name: (identifier) @type) @definition.class + +; Alternation list and a bare anonymous-node matcher. +["if" "else" "while" "return"] @keyword + +"async" @keyword.coroutine + +; Wildcard child, a negated field, anchors, and quantifiers. +(call_expression + function: (_) @call + !type_arguments) + +(array . (identifier) @first (_)* @rest) + +; Groupings with predicates: regex match, alternation, and an injection. +((identifier) @constant + (#match? @constant "^[A-Z][A-Z_]+$")) + +((identifier) @type.builtin + (#any-of? @type.builtin "int" "str" "bool")) + +((comment) @injection.content + (#set! injection.language "markdown")) + +; Supertype/subtype match and a MISSING-node pattern. +(declaration/function_definition) @definition.function + +(binary_expression (MISSING identifier)) diff --git a/uv.lock b/uv.lock index fb6dd8a..ba14b6d 100644 --- a/uv.lock +++ b/uv.lock @@ -3237,6 +3237,7 @@ dependencies = [ { name = "tree-sitter-less" }, { name = "tree-sitter-markdown" }, { name = "tree-sitter-python" }, + { name = "tree-sitter-query" }, { name = "tree-sitter-rst" }, { name = "tree-sitter-ruby" }, { name = "tree-sitter-rust" }, @@ -3290,6 +3291,7 @@ requires-dist = [ { name = "tree-sitter-less" }, { name = "tree-sitter-markdown" }, { name = "tree-sitter-python" }, + { name = "tree-sitter-query" }, { name = "tree-sitter-rst" }, { name = "tree-sitter-ruby" }, { name = "tree-sitter-rust" }, @@ -4246,6 +4248,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/19/4b5569d9b1ebebb5907d11554a96ef3fa09364a30fcfabeff587495b512f/tree_sitter_python-0.25.0-cp310-abi3-win_arm64.whl", hash = "sha256:0fbf6a3774ad7e89ee891851204c2e2c47e12b63a5edbe2e9156997731c128bb", size = 74169, upload-time = "2025-09-11T06:47:56.747Z" }, ] +[[package]] +name = "tree-sitter-query" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/50/0a1b27bc366b8131d3dd44c96130ec7e134473f0432473cc242b69c92ff7/tree_sitter_query-0.8.0.tar.gz", hash = "sha256:723c9f65eca3b84c7f1193dee301fbdde330ac4f2551754f6b0623392a62eedb", size = 22821, upload-time = "2025-11-27T17:32:16.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/3e/22f4d69c281fff8adeca1b5e8fbb8c52a5918fb91bee1ef5991824f69e89/tree_sitter_query-0.8.0-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:f21363ca53e3dbf4b08739ac205fa90df86ea8ed2a4ad74103a59cc509b098b9", size = 16278, upload-time = "2025-11-27T17:32:09.114Z" }, + { url = "https://files.pythonhosted.org/packages/2c/d6/687c23f735d59f623b50022935338d2a28bbf7d6e1226551e5b760d6d83d/tree_sitter_query-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffbca269b3614f22d73eafd02d16b0323c29f9ae6d6494bb332354e8a9d06d4c", size = 16695, upload-time = "2025-11-27T17:32:09.988Z" }, + { url = "https://files.pythonhosted.org/packages/f9/d6/7f154c368062e68d330bd60205d25ad72ba16e515b22aea9fa98bfb097f6/tree_sitter_query-0.8.0-cp310-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0100980f402a53067b15247e7bc1d744692b63c24a5d2f6c48e144da5f70d76c", size = 24989, upload-time = "2025-11-27T17:32:10.986Z" }, + { url = "https://files.pythonhosted.org/packages/08/dd/223f40d8bb5b825167c06a361f7b0eebe44970ca83b61230e3995c1c638b/tree_sitter_query-0.8.0-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d648962f81de56b5c74cc39aa9cc9e1c16f7139e4c60a3051b8467369967b6de", size = 26688, upload-time = "2025-11-27T17:32:11.804Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c1/06bf51f9ba4d1c42997b6716b5908ba7ff17cf63dbbbdd199b931db0e5f1/tree_sitter_query-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9709f2d0fb8251406c1abb8f18831cfb58c091a437f411dee57dacdca95d343d", size = 25942, upload-time = "2025-11-27T17:32:12.582Z" }, + { url = "https://files.pythonhosted.org/packages/d7/25/33e887a16f9159e576a58eedccbdcbe260b64589d89b13bb63ce046226f5/tree_sitter_query-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:93a0be1858970ca73a22bba3e61d89cc1b25ac1bf45d24a9f6f32c28cea9f129", size = 24921, upload-time = "2025-11-27T17:32:13.32Z" }, + { url = "https://files.pythonhosted.org/packages/69/94/ba01308f99cf1b61115f57fd59b2e9c530169a1c4e98cc4754dc0402c97b/tree_sitter_query-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:f65e779ecf33f22f7bcd8f24644559bafb8d4428731170d0cc6a0a1337db07fe", size = 18414, upload-time = "2025-11-27T17:32:14.043Z" }, + { url = "https://files.pythonhosted.org/packages/ae/7d/7a213e1bf9d2fa50dd48be90a7463419e0b9d5299c0695382832ebce714b/tree_sitter_query-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:085efb6d59f8dac1347f935dfa2d86367daecbd68b676d20720732ae706c07a4", size = 17333, upload-time = "2025-11-27T17:32:14.935Z" }, +] + [[package]] name = "tree-sitter-rst" version = "0.2.0"