Skip to content

Commit 6591c29

Browse files
jwilsjwils
committed
Add elasticgraph-protobuf schema artifacts
1 parent 40815e1 commit 6591c29

48 files changed

Lines changed: 3475 additions & 1 deletion

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CODEBASE_OVERVIEW.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,13 +195,14 @@ graph LR;
195195
click opensearch-ruby href "https://rubygems.org/gems/opensearch-ruby" "Open on RubyGems.org" _blank;
196196
```
197197

198-
### Extensions (6 gems)
198+
### Extensions (7 gems)
199199

200200
These libraries extend ElasticGraph to provide optional but commonly needed functionality.
201201

202202
* [elasticgraph-apollo](elasticgraph-apollo/README.md): Transforms an ElasticGraph project into an Apollo subgraph.
203203
* [elasticgraph-health_check](elasticgraph-health_check/README.md): Provides a health check for high availability ElasticGraph deployments.
204204
* [elasticgraph-json_ingestion](elasticgraph-json_ingestion/README.md): Pluggable JSON Schema ingestion serializer for ElasticGraph.
205+
* [elasticgraph-protobuf](elasticgraph-protobuf/README.md): Generates Protocol Buffers schema artifacts from ElasticGraph schemas.
205206
* [elasticgraph-query_interceptor](elasticgraph-query_interceptor/README.md): Intercepts ElasticGraph datastore queries.
206207
* [elasticgraph-query_registry](elasticgraph-query_registry/README.md): Provides a source-controlled query registry for ElasticGraph applications.
207208
* [elasticgraph-warehouse](elasticgraph-warehouse/README.md): Extends ElasticGraph to support ingestion into a data warehouse.
@@ -221,6 +222,7 @@ graph LR;
221222
elasticgraph-health_check["eg-health_check"];
222223
elasticgraph-datastore_core["eg-datastore_core"];
223224
elasticgraph-json_ingestion["eg-json_ingestion"];
225+
elasticgraph-protobuf["eg-protobuf"];
224226
elasticgraph-query_interceptor["eg-query_interceptor"];
225227
elasticgraph-schema_artifacts["eg-schema_artifacts"];
226228
elasticgraph-query_registry["eg-query_registry"];
@@ -234,6 +236,7 @@ graph LR;
234236
elasticgraph-health_check --> elasticgraph-graphql;
235237
elasticgraph-health_check --> elasticgraph-support;
236238
elasticgraph-json_ingestion --> elasticgraph-support;
239+
elasticgraph-protobuf --> elasticgraph-support;
237240
elasticgraph-query_interceptor --> elasticgraph-graphql;
238241
elasticgraph-query_interceptor --> elasticgraph-schema_artifacts;
239242
elasticgraph-query_registry --> elasticgraph-graphql;
@@ -249,6 +252,7 @@ graph LR;
249252
class elasticgraph-health_check targetGemStyle;
250253
class elasticgraph-datastore_core otherEgGemStyle;
251254
class elasticgraph-json_ingestion targetGemStyle;
255+
class elasticgraph-protobuf targetGemStyle;
252256
class elasticgraph-query_interceptor targetGemStyle;
253257
class elasticgraph-schema_artifacts otherEgGemStyle;
254258
class elasticgraph-query_registry targetGemStyle;

‎Gemfile.lock‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,12 @@ PATH
163163
faraday-retry (~> 2.4)
164164
opensearch-ruby (~> 3.4)
165165

166+
PATH
167+
remote: elasticgraph-protobuf
168+
specs:
169+
elasticgraph-protobuf (1.1.1.pre)
170+
elasticgraph-support (= 1.1.1.pre)
171+
166172
PATH
167173
remote: elasticgraph-query_interceptor
168174
specs:
@@ -707,6 +713,7 @@ DEPENDENCIES
707713
elasticgraph-lambda_support (= 1.1.1.pre)!
708714
elasticgraph-local (= 1.1.1.pre)!
709715
elasticgraph-opensearch (= 1.1.1.pre)!
716+
elasticgraph-protobuf (= 1.1.1.pre)!
710717
elasticgraph-query_interceptor (= 1.1.1.pre)!
711718
elasticgraph-query_registry (= 1.1.1.pre)!
712719
elasticgraph-rack (= 1.1.1.pre)!
@@ -799,6 +806,7 @@ CHECKSUMS
799806
elasticgraph-lambda_support (1.1.1.pre)
800807
elasticgraph-local (1.1.1.pre)
801808
elasticgraph-opensearch (1.1.1.pre)
809+
elasticgraph-protobuf (1.1.1.pre)
802810
elasticgraph-query_interceptor (1.1.1.pre)
803811
elasticgraph-query_registry (1.1.1.pre)
804812
elasticgraph-rack (1.1.1.pre)

‎config/docker_demo/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ COPY elasticgraph-indexer elasticgraph-indexer/
1919
COPY elasticgraph-json_ingestion elasticgraph-json_ingestion/
2020
COPY elasticgraph-local elasticgraph-local/
2121
COPY elasticgraph-opensearch elasticgraph-opensearch/
22+
COPY elasticgraph-protobuf elasticgraph-protobuf/
2223
COPY elasticgraph-query_registry elasticgraph-query_registry/
2324
COPY elasticgraph-rack elasticgraph-rack/
2425
COPY elasticgraph-schema_artifacts elasticgraph-schema_artifacts/

‎elasticgraph-protobuf/.rspec‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../spec_support/subdir_dot_rspec

‎elasticgraph-protobuf/.yardopts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../config/site/yardopts

‎elasticgraph-protobuf/Gemfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../Gemfile

‎elasticgraph-protobuf/LICENSE.txt‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
The MIT License (MIT)
2+
3+
Copyright (c) 2024 - 2026 Block, Inc.
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in
13+
all copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21+
THE SOFTWARE.

‎elasticgraph-protobuf/README.md‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
# ElasticGraph::Protobuf
2+
3+
An ElasticGraph extension that generates Protocol Buffers (`proto3`) schema artifacts from ElasticGraph schemas.
4+
5+
## Dependency Diagram
6+
7+
```mermaid
8+
graph LR;
9+
classDef targetGemStyle fill:#FADBD8,stroke:#EC7063,color:#000,stroke-width:2px;
10+
classDef otherEgGemStyle fill:#A9DFBF,stroke:#2ECC71,color:#000;
11+
classDef externalGemStyle fill:#E0EFFF,stroke:#70A1D7,color:#2980B9;
12+
elasticgraph-protobuf["elasticgraph-protobuf"];
13+
class elasticgraph-protobuf targetGemStyle;
14+
elasticgraph-support["elasticgraph-support"];
15+
elasticgraph-protobuf --> elasticgraph-support;
16+
class elasticgraph-support otherEgGemStyle;
17+
```
18+
19+
## Usage
20+
21+
First, add `elasticgraph-protobuf` to your `Gemfile`, alongside the other ElasticGraph gems:
22+
23+
```diff
24+
diff --git a/Gemfile b/Gemfile
25+
index 4a5ef1e..5c16c2b 100644
26+
--- a/Gemfile
27+
+++ b/Gemfile
28+
@@ -8,6 +8,7 @@ gem "elasticgraph-query_registry", *elasticgraph_details
29+
30+
# Can be elasticgraph-elasticsearch or elasticgraph-opensearch based on the datastore you want to use.
31+
gem "elasticgraph-opensearch", *elasticgraph_details
32+
+gem "elasticgraph-protobuf", *elasticgraph_details
33+
34+
gem "httpx", "~> 1.3"
35+
36+
```
37+
38+
Next, update your `Rakefile` so that `ElasticGraph::Protobuf::SchemaDefinition::APIExtension` is
39+
included in the schema-definition extension modules:
40+
41+
```diff
42+
diff --git a/Rakefile b/Rakefile
43+
index 2943335..26633c3 100644
44+
--- a/Rakefile
45+
+++ b/Rakefile
46+
@@ -1,5 +1,6 @@
47+
project_root = File.expand_path(__dir__)
48+
49+
+require "elastic_graph/protobuf/schema_definition/api_extension"
50+
require "elastic_graph/local/rake_tasks"
51+
require "elastic_graph/query_registry/rake_tasks"
52+
require "rspec/core/rake_task"
53+
@@ -12,6 +13,8 @@ ElasticGraph::Local::RakeTasks.new(
54+
local_config_yaml: settings_file,
55+
path_to_schema: "#{project_root}/config/schema.rb"
56+
) do |tasks|
57+
+ tasks.schema_definition_extension_modules << ElasticGraph::Protobuf::SchemaDefinition::APIExtension
58+
+
59+
# Set this to true once you're beyond the prototyping stage.
60+
tasks.enforce_json_schema_version = false
61+
62+
```
63+
64+
Then opt into proto generation from your schema definition:
65+
66+
```ruby
67+
# in config/schema/protobuf.rb
68+
69+
ElasticGraph.define_schema do |schema|
70+
schema.proto_schema_artifacts package_name: "myapp.events.v1"
71+
end
72+
```
73+
74+
After running `bundle exec rake schema_artifacts:dump`, ElasticGraph will generate:
75+
76+
- `schema.proto`
77+
- `proto_field_numbers.yaml`
78+
79+
## Schema Definition Options
80+
81+
### Custom Scalar Types
82+
83+
Built-in ElasticGraph scalar types are automatically mapped to proto scalar types.
84+
For custom scalar types, the generator infers proto scalar types from `json_schema type:` when it is one
85+
of `string`, `boolean`, `number`, or `integer`. You can override inference with `proto_field`:
86+
87+
```ruby
88+
# in config/schema/money.rb
89+
90+
ElasticGraph.define_schema do |schema|
91+
schema.scalar_type "Money" do |t|
92+
t.mapping type: "long"
93+
t.json_schema type: "integer"
94+
t.proto_field type: "int64"
95+
end
96+
end
97+
```
98+
99+
### Sourcing Enum Values From Existing Protobuf Mappings
100+
101+
If your project already maintains GraphQL-to-proto enum mappings (for example in tests),
102+
you can reuse them for proto schema generation:
103+
104+
```ruby
105+
# in config/schema/proto_enum_mappings.rb
106+
107+
ElasticGraph.define_schema do |schema|
108+
schema.proto_enum_mappings(
109+
SalesEg::ProtoEnumMappings::PROTO_ENUMS_BY_GRAPHQL_ENUM
110+
) if defined?(SalesEg::ProtoEnumMappings)
111+
end
112+
```
113+
114+
When a mapping exists for an enum, `elasticgraph-protobuf` uses the mapped proto enum(s)
115+
as the source of enum values (respecting `exclusions`, `expected_extras`, and `name_transform`).
116+
117+
### Stable Field Numbers
118+
119+
`schema_artifacts:dump` automatically reads and writes `proto_field_numbers.yaml`
120+
in the schema artifacts directory. Existing numbers stay fixed even if field order
121+
changes, and new fields get the next available numbers.
122+
123+
`schema.proto` always uses the public GraphQL field names. When a field uses a
124+
different `name_in_index`, the sidecar YAML stores that override privately:
125+
126+
```yaml
127+
messages:
128+
Widget:
129+
fields:
130+
id: 1
131+
display_name:
132+
field_number: 2
133+
name_in_index: displayName
134+
```
135+
136+
If a field is renamed with `field.renamed_from`, `elasticgraph-protobuf` reuses the
137+
existing field number under the new public field name.
138+
139+
## Type Mappings
140+
141+
The generated `schema.proto` uses these built-in scalar mappings:
142+
143+
| ElasticGraph Type | Protobuf Type |
144+
|-------------------|------------|
145+
| `Boolean` | `bool` |
146+
| `Cursor` | `string` |
147+
| `Date` | `string` |
148+
| `DateTime` | `string` |
149+
| `Float` | `double` |
150+
| `ID` | `string` |
151+
| `Int` | `int32` |
152+
| `JsonSafeLong` | `int64` |
153+
| `LocalTime` | `string` |
154+
| `LongString` | `int64` |
155+
| `String` | `string` |
156+
| `TimeZone` | `string` |
157+
| `Untyped` | `string` |
158+
159+
Additionally:
160+
- List types become `repeated` fields.
161+
- Nested list types generate wrapper messages so the output remains valid `proto3`.
162+
- Enum types generate `enum` definitions whose values are prefixed with the enum type name in `UPPER_SNAKE_CASE`, including a zero-valued `*_UNSPECIFIED` entry.

‎elasticgraph-protobuf/elasticgraph-protobuf.gemspec‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright 2024 - 2026 Block, Inc.
2+
#
3+
# Use of this source code is governed by an MIT-style
4+
# license that can be found in the LICENSE file or at
5+
# https://opensource.org/licenses/MIT.
6+
#
7+
# frozen_string_literal: true
8+
9+
require_relative "../elasticgraph-support/lib/elastic_graph/version"
10+
11+
Gem::Specification.new do |spec|
12+
spec.name = "elasticgraph-protobuf"
13+
spec.version = ElasticGraph::VERSION
14+
spec.authors = ["Josh Wilson", "Myron Marston", "Block Engineering"]
15+
spec.email = ["joshuaw@squareup.com"]
16+
spec.homepage = "https://block.github.io/elasticgraph/"
17+
spec.license = "MIT"
18+
spec.summary = "Generates Protocol Buffers schema artifacts from ElasticGraph schemas."
19+
20+
spec.metadata = {
21+
"bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
22+
"changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
23+
"documentation_uri" => "https://block.github.io/elasticgraph/api-docs/v#{ElasticGraph::VERSION}/",
24+
"homepage_uri" => "https://block.github.io/elasticgraph/",
25+
"source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
26+
"gem_category" => "extension"
27+
}
28+
29+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
30+
`git ls-files -z`.split("\x0").reject do |f|
31+
(f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features|sig)/|\.(?:git|travis|circleci)|appveyor)})
32+
end - [".rspec", "Gemfile", ".yardopts"]
33+
end
34+
35+
spec.required_ruby_version = [">= 3.4", "< 4.1"]
36+
37+
spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION
38+
39+
spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
40+
end

‎elasticgraph-protobuf/lib/elastic_graph/protobuf.rb‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Copyright 2024 - 2026 Block, Inc.
2+
#
3+
# Use of this source code is governed by an MIT-style
4+
# license that can be found in the LICENSE file or at
5+
# https://opensource.org/licenses/MIT.
6+
#
7+
# frozen_string_literal: true
8+
9+
module ElasticGraph
10+
# Namespace for Protocol Buffers schema artifact generation extensions.
11+
module Protobuf
12+
# The name of the generated Protocol Buffers schema file.
13+
PROTO_SCHEMA_FILE = "schema.proto"
14+
15+
# The name of the generated proto field-number mapping file.
16+
PROTO_FIELD_NUMBERS_FILE = "proto_field_numbers.yaml"
17+
end
18+
end

0 commit comments

Comments
 (0)